[verified] Gate adoption of delegated evidence
This commit is contained in:
@@ -0,0 +1,322 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCHEMA_C = "gitea-agent.delegation-evidence-context/v1"
|
||||
SCHEMA_R = "gitea-agent.delegation-evidence-result/v1"
|
||||
|
||||
|
||||
class DelegationEvidenceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.tmp.name) / "bundle"
|
||||
self.root.mkdir()
|
||||
(self.root / "tests").mkdir()
|
||||
(self.root / "src").mkdir()
|
||||
(self.root / "tests/sample_test.py").write_bytes(b"def test_sample():\n assert True\n")
|
||||
(self.root / "src/app.py").write_bytes(b"VALUE = 1\n")
|
||||
self.files = {p: hashlib.sha256((self.root / p).read_bytes()).hexdigest()
|
||||
for p in ("src/app.py", "tests/sample_test.py")}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
(self.root / "evidence").mkdir()
|
||||
(self.root / "evidence/run.txt").write_bytes(b"exit 0\n")
|
||||
evidence_sha = hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest()
|
||||
self.context = {
|
||||
"schema": SCHEMA_C,
|
||||
"source": {"kind": "captured_worktree", "base_revision": "a" * 40},
|
||||
"stage": "implementation",
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"files": self.files,
|
||||
"allowed_actions": ["static", "execute"],
|
||||
"obligations": [
|
||||
{"id": "static-source", "kind": "static"},
|
||||
{"id": "run-sample", "kind": "execution"},
|
||||
],
|
||||
"executions": [{"id": "exec-1", "test_id": "", "bundle_sha256": self.bundle_sha,
|
||||
"test_file_sha256": "0" * 64, "argv": ["python3", "-c", "pass"],
|
||||
"exit_code": 0, "failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": evidence_sha, "origin": "direct"}],
|
||||
}
|
||||
self.result = {
|
||||
"schema": SCHEMA_R,
|
||||
"source": copy.deepcopy(self.context["source"]),
|
||||
"stage": self.context["stage"],
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"claims": [
|
||||
{"id": "static-source", "status": "supported", "anchors": [
|
||||
{"kind": "lines", "path": "src/app.py", "start_line": 1,
|
||||
"end_line": 1, "text_sha256": self.line_hash("src/app.py", 1, 1)}
|
||||
], "execution_ids": []},
|
||||
{"id": "run-sample", "status": "supported", "anchors": [], "execution_ids": ["exec-1"]},
|
||||
],
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
def line_hash(self, path, start, end):
|
||||
lines = (self.root / path).read_bytes().splitlines(keepends=True)
|
||||
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
||||
|
||||
def make_git_repo(self, files=None, name="repo"):
|
||||
repo = Path(self.tmp.name) / name
|
||||
repo.mkdir()
|
||||
for path in (files or self.files):
|
||||
target = repo / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((self.root / path).read_bytes())
|
||||
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "fixture"], check=True)
|
||||
revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
||||
return repo / ".git", revision
|
||||
|
||||
def git_context(self, git_revision):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["source"] = {"kind": "git_tree", "revision": git_revision}
|
||||
result = copy.deepcopy(self.result)
|
||||
result["source"] = copy.deepcopy(context["source"])
|
||||
return context, result
|
||||
|
||||
def test_worker_source_stage_bundle_mismatches_are_detected_blocks(self):
|
||||
for key, value in [('source', {'kind': 'captured_worktree', 'base_revision': 'b' * 40}),
|
||||
('stage', 'stale-stage'), ('bundle_sha256', '0' * 64)]:
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad[key] = value
|
||||
with self.subTest(field=key):
|
||||
try:
|
||||
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
except evidence.EvidenceError:
|
||||
self.fail('syntactically valid provenance disagreement must return BLOCK')
|
||||
self.assertEqual(out['status'], 'block')
|
||||
self.assertIn('result_' + key + '_mismatch', out['reason_codes'])
|
||||
|
||||
def test_valid_static_lines_and_bounded_counts(self):
|
||||
out = evidence.verify_evidence(str(self.root), self.context, self.result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
self.assertEqual(out["reason_codes"], [])
|
||||
self.assertEqual(out["counts"], {"files": 2, "obligations": 2, "claims": 2, "anchors": 1, "executions": 1})
|
||||
|
||||
def test_wrong_line_hash_blocks(self):
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
||||
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_indeterminate_claim_blocks(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["status"] = "indeterminate"
|
||||
out = evidence.verify_evidence(str(self.root), self.context, result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("indeterminate_claim", out["reason_codes"])
|
||||
|
||||
def test_test_anchor_requires_real_python_function(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"] = [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}]
|
||||
out = evidence.verify_evidence(str(self.root), self.context, result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
result["claims"][0]["anchors"][0]["symbol"] = "missing"
|
||||
self.assertIn("test_anchor_missing", evidence.verify_evidence(str(self.root), self.context, result)["reason_codes"])
|
||||
|
||||
def test_claim_ids_must_exactly_match_obligations(self):
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad["claims"][1]["id"] = "extra"
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
|
||||
def test_captured_worktree_absence_is_indeterminate(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"] = {"src/app.py": self.files["src/app.py"]}
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
||||
"bundle_sha256": context["bundle_sha256"], "claims": [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}]}
|
||||
out = evidence.verify_evidence(str(self.root), context, result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("absence_indeterminate", out["reason_codes"])
|
||||
|
||||
def test_git_tree_matching_frozen_bundle_passes(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_git_tree_altered_frozen_snapshot_blocks(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
(self.root / "src/app.py").write_bytes(b"VALUE = 2\n")
|
||||
context, result = self.git_context(revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_git_tree_source_mismatch_blocks(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
repo = Path(self.tmp.name) / "other"
|
||||
repo.mkdir()
|
||||
for path in self.files:
|
||||
target = repo / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((self.root / path).read_bytes())
|
||||
(repo / "src/app.py").write_bytes(b"VALUE = 9\n")
|
||||
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "other"], check=True)
|
||||
other_revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
||||
context, result = self.git_context(other_revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
||||
context_path = self.root / "context.json"
|
||||
result_path = self.root / "result.json"
|
||||
context_path.write_text(json.dumps(context))
|
||||
result_path.write_text(json.dumps(result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(context_path),
|
||||
"--result", str(result_path), "--git-dir", str(git_dir)],
|
||||
text=True, capture_output=True)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertEqual(json.loads(proc.stdout)["status"], "block")
|
||||
|
||||
def test_git_tree_present_path_cannot_be_claimed_absent(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/sample_test.py"}],
|
||||
"execution_ids": []}]
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("absence_not_proven", out["reason_codes"])
|
||||
|
||||
def test_git_tree_genuine_missing_path_is_authoritative(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
||||
"execution_ids": []}]
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_git_tree_absence_uses_original_directory_fd_after_replacement(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
replacement, _ = self.make_git_repo(name="replacement")
|
||||
replacement_repo = replacement.parent
|
||||
(replacement_repo / "tests/missing.py").write_bytes(b"present\n")
|
||||
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "commit", "-q", "-m", "replacement"], check=True)
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
||||
"execution_ids": []}]
|
||||
original_git = evidence._git
|
||||
moved = {"done": False}
|
||||
|
||||
def replacing_git(git_path, args, fd):
|
||||
if args[0] == "ls-tree" and args[-1] == "tests/missing.py" and not moved["done"]:
|
||||
moved["done"] = True
|
||||
backup = Path(self.tmp.name) / "original.git"
|
||||
os.rename(git_dir, backup)
|
||||
os.rename(replacement, git_dir)
|
||||
return original_git(git_path, args, fd)
|
||||
|
||||
evidence._git = replacing_git
|
||||
try:
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
finally:
|
||||
evidence._git = original_git
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_inherited_execution_is_allowed_for_static_only_worker_but_direct_is_not(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
context["executions"][0]["origin"] = "inherited"
|
||||
out = evidence.verify_evidence(str(self.root), context, self.result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
context["executions"][0]["origin"] = "direct"
|
||||
out = evidence.verify_evidence(str(self.root), context, self.result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unauthorized_execute", out["reason_codes"])
|
||||
|
||||
def test_execution_origin_is_required_and_closed(self):
|
||||
for origin in ("worker", None):
|
||||
context = copy.deepcopy(self.context)
|
||||
if origin is None:
|
||||
del context["executions"][0]["origin"]
|
||||
else:
|
||||
context["executions"][0]["origin"] = origin
|
||||
with self.subTest(origin=origin), self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), context, self.result)
|
||||
|
||||
def test_inherited_red_green_is_accepted_without_execute_permission(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
context["obligations"] = [{"id": "tdd", "kind": "red_green"}]
|
||||
test_sha = self.files["tests/sample_test.py"]
|
||||
context["executions"] = [
|
||||
{"id": "red", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": "b" * 64,
|
||||
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 1,
|
||||
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
||||
"origin": "inherited"},
|
||||
{"id": "green", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": self.bundle_sha,
|
||||
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 0,
|
||||
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
||||
"origin": "inherited"},
|
||||
]
|
||||
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
||||
"bundle_sha256": self.bundle_sha, "claims": [{"id": "tdd", "status": "supported",
|
||||
"anchors": [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}],
|
||||
"execution_ids": ["red", "green"]}]}
|
||||
self.assertEqual(evidence.verify_evidence(str(self.root), context, result)["status"], "pass")
|
||||
|
||||
def test_rejected_input_and_unknown_cli_are_bounded(self):
|
||||
bad = copy.deepcopy(self.context)
|
||||
bad["files"]["../secret"] = "0" * 64
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), bad, self.result)
|
||||
c = self.root / "context.json"; r = self.root / "result.json"
|
||||
c.write_text(json.dumps(self.context)); r.write_text(json.dumps(self.result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r), "--unknown=secret"],
|
||||
text=True, capture_output=True)
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertNotIn("secret", proc.stdout + proc.stderr)
|
||||
self.assertEqual(json.loads(proc.stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
||||
|
||||
def test_duplicate_json_keys_and_symlink_are_rejected(self):
|
||||
c = self.root / "context.json"; r = self.root / "result.json"
|
||||
c.write_text('{"schema":"x","schema":"secret"}')
|
||||
r.write_text(json.dumps(self.result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r)], capture_output=True, text=True)
|
||||
self.assertEqual(proc.returncode, 2); self.assertNotIn("secret", proc.stdout + proc.stderr)
|
||||
real = self.root / "real.json"; real.write_text(json.dumps(self.context))
|
||||
alias = self.root / "alias.json"; alias.symlink_to(real)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence._json_file(str(alias))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Procedural consumer wiring is not proof of live agent execution."""
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ROLES = [
|
||||
'agents/impl/skills/gitea-issue-implementation/SKILL.md',
|
||||
'agents/impl/skills/gitea-issue-rework/SKILL.md',
|
||||
'agents/spec/skills/gitea-implementation-spec-review/SKILL.md',
|
||||
'agents/review/skills/gitea-code-review/SKILL.md',
|
||||
]
|
||||
|
||||
|
||||
class DelegationEvidenceContractTests(unittest.TestCase):
|
||||
def test_consumers_require_gate_before_adoption(self):
|
||||
for path in ROLES:
|
||||
with self.subTest(path=path):
|
||||
text = (ROOT / path).read_text()
|
||||
for required in ['## Delegation evidence acceptance gate',
|
||||
'delegation_evidence.py', '위임 결과 채택 이전',
|
||||
'main이 독립적으로 고정한 context',
|
||||
'worker 자기 선언으로 context를 만들지 않는다']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
def test_canonical_contract_names_cli_and_schema(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
for required in ['### Delegation evidence acceptance preflight',
|
||||
'--bundle-root', '--context', '--result', '--git-dir',
|
||||
'gitea-agent.delegation-evidence-context/v1',
|
||||
'gitea-agent.delegation-evidence-result/v1']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
def test_source_modes_and_coverage_limits_are_explicit(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
for required in ['captured_worktree', 'base_revision', 'git_tree',
|
||||
'bundle에 없다는 이유로 repository 부재로 판정하지 않는다',
|
||||
'named symbol 존재는 assertion 검출력을 인증하지 않는다',
|
||||
'제품의 추가 evidence commit이나 전체 테스트 재실행 의무를 만들지 않는다']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
def test_trusted_execution_catalog_and_failure_origin_limits(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
for required in ['독립 실행 catalog', 'import·환경 오류는 RED로 인정하지 않는다',
|
||||
'허용 작업', '실제 command/output/exit와 실패 원인',
|
||||
'gate는 context의 신뢰성을 스스로 인증하지 않는다']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Independent safety/regression coverage for delegation evidence acceptance."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
HELPER = ROOT / "harness" / "lib" / "delegation_evidence.py"
|
||||
CONTEXT_SCHEMA = "gitea-agent.delegation-evidence-context/v1"
|
||||
RESULT_SCHEMA = "gitea-agent.delegation-evidence-result/v1"
|
||||
REVISION = "a" * 40
|
||||
|
||||
|
||||
class DelegationEvidenceSafetyTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch")
|
||||
self.base = Path(self.tmp.name)
|
||||
self.bundle = self.base / "bundle"
|
||||
(self.bundle / "tests").mkdir(parents=True)
|
||||
(self.bundle / "src").mkdir()
|
||||
(self.bundle / "evidence").mkdir()
|
||||
(self.bundle / "tests/example.py").write_bytes(
|
||||
b"def test_fn():\n assert True\n"
|
||||
)
|
||||
(self.bundle / "src/value.py").write_bytes(b"VALUE = 1\n")
|
||||
(self.bundle / "evidence/run.txt").write_bytes(b"recorded output\n")
|
||||
self.files = {
|
||||
path: hashlib.sha256((self.bundle / path).read_bytes()).hexdigest()
|
||||
for path in ("src/value.py", "tests/example.py")
|
||||
}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
self.evidence_sha = hashlib.sha256(
|
||||
(self.bundle / "evidence/run.txt").read_bytes()
|
||||
).hexdigest()
|
||||
self.context = self.make_context()
|
||||
self.result = self.make_result()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
|
||||
def make_context(self, *, actions=None, obligations=None, executions=None):
|
||||
return {
|
||||
"schema": CONTEXT_SCHEMA,
|
||||
"source": {"kind": "captured_worktree", "base_revision": REVISION},
|
||||
"stage": "implementation",
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"files": copy.deepcopy(self.files),
|
||||
"allowed_actions": ["static", "execute"] if actions is None else actions,
|
||||
"obligations": [
|
||||
{"id": "static-src", "kind": "static"},
|
||||
{"id": "run-test", "kind": "execution"},
|
||||
] if obligations is None else obligations,
|
||||
"executions": [{
|
||||
"id": "exec-1", "origin": "direct", "test_id": "",
|
||||
"bundle_sha256": self.bundle_sha, "test_file_sha256": "0" * 64,
|
||||
"argv": ["python3", "-B", "-c", "pass"], "exit_code": 0,
|
||||
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": self.evidence_sha,
|
||||
}] if executions is None else executions,
|
||||
}
|
||||
|
||||
def make_result(self, *, source=None, stage=None, bundle_sha=None, claims=None):
|
||||
return {
|
||||
"schema": RESULT_SCHEMA,
|
||||
"source": copy.deepcopy(self.context["source"] if source is None else source),
|
||||
"stage": self.context["stage"] if stage is None else stage,
|
||||
"bundle_sha256": self.bundle_sha if bundle_sha is None else bundle_sha,
|
||||
"claims": [
|
||||
{"id": "static-src", "status": "supported", "anchors": [{
|
||||
"kind": "lines", "path": "src/value.py", "start_line": 1,
|
||||
"end_line": 1, "text_sha256": self.line_hash("src/value.py", 1, 1),
|
||||
}], "execution_ids": []},
|
||||
{"id": "run-test", "status": "supported", "anchors": [],
|
||||
"execution_ids": ["exec-1"]},
|
||||
] if claims is None else claims,
|
||||
}
|
||||
|
||||
def line_hash(self, path, start, end):
|
||||
lines = (self.bundle / path).read_bytes().splitlines(keepends=True)
|
||||
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
||||
|
||||
def api(self, context=None, result=None, root=None):
|
||||
return evidence.verify_evidence(
|
||||
str(self.bundle if root is None else root),
|
||||
self.context if context is None else context,
|
||||
self.result if result is None else result,
|
||||
)
|
||||
|
||||
def cli(self, context_text=None, result=None, *, root=None, context_path=None,
|
||||
result_path=None, extra=None, timeout=3):
|
||||
root = self.bundle if root is None else Path(root)
|
||||
context_path = self.base / "context.json" if context_path is None else Path(context_path)
|
||||
result_path = self.base / "result.json" if result_path is None else Path(result_path)
|
||||
if context_text is None:
|
||||
context_path.write_text(json.dumps(self.context), encoding="utf-8")
|
||||
elif isinstance(context_text, bytes):
|
||||
context_path.write_bytes(context_text)
|
||||
else:
|
||||
context_path.write_text(context_text, encoding="utf-8")
|
||||
if result is not None:
|
||||
result_path.write_text(json.dumps(result), encoding="utf-8")
|
||||
else:
|
||||
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
||||
command = [sys.executable, "-B", str(HELPER), "--bundle-root", str(root),
|
||||
"--context", str(context_path), "--result", str(result_path)]
|
||||
if extra:
|
||||
command.extend(extra)
|
||||
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
def assert_rejected(self, value, *, field=None):
|
||||
bad = copy.deepcopy(self.context)
|
||||
if field is None:
|
||||
bad = value
|
||||
else:
|
||||
target = bad
|
||||
for part in field[:-1]:
|
||||
target = target[part]
|
||||
target[field[-1]] = value
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=bad)
|
||||
|
||||
def test_real_paths_reject_root_file_ancestor_and_canonical_aliases(self):
|
||||
alias = self.base / "bundle-alias"
|
||||
alias.symlink_to(self.bundle, target_is_directory=True)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(root=alias)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(root=self.base / "missing" / ".." / "bundle")
|
||||
outside = self.base / "outside.txt"
|
||||
outside.write_bytes(b"outside")
|
||||
(self.bundle / "src" / "alias.py").symlink_to(outside)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/alias.py"] = hashlib.sha256(outside.read_bytes()).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_json_input_symlink_fifo_and_source_fifo_are_rejected_without_hang(self):
|
||||
real_context = self.base / "real-context.json"
|
||||
real_context.write_text(json.dumps(self.context), encoding="utf-8")
|
||||
alias = self.base / "context.json"
|
||||
alias.symlink_to(real_context)
|
||||
result_path = self.base / "result.json"
|
||||
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence._json_file(str(alias))
|
||||
|
||||
fifo = self.base / "context-fifo"
|
||||
os.mkfifo(fifo)
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-B", str(HELPER), "--bundle-root", str(self.bundle),
|
||||
"--context", str(fifo), "--result", str(result_path)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
stdout, stderr = proc.communicate(timeout=3)
|
||||
self.assertEqual(proc.returncode, 2, (stdout, stderr))
|
||||
self.assertEqual(json.loads(stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
||||
|
||||
source_fifo = self.bundle / "src" / "fifo.py"
|
||||
os.mkfifo(source_fifo)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/fifo.py"] = "0" * 64
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_oversized_source_and_json_are_bounded(self):
|
||||
huge = self.bundle / "src" / "huge.py"
|
||||
huge.write_bytes(b"x" * (1024 * 1024 + 1))
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/huge.py"] = hashlib.sha256(huge.read_bytes()).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
proc = self.cli(context_text=b"{" + b"\"x\":\"" + b"a" * (1024 * 1024) + b"\"}")
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertEqual(json.loads(proc.stdout)["reason_codes"], ["rejected_input"])
|
||||
|
||||
def test_duplicate_unknown_and_wrong_typed_json_fields_are_rejected(self):
|
||||
cases = [
|
||||
b'{"schema":"x","schema":"y"}',
|
||||
json.dumps({**self.context, "unknown": 1}).encode(),
|
||||
]
|
||||
for payload in cases:
|
||||
with self.subTest(case=payload[:20]):
|
||||
proc = self.cli(context_text=payload)
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
for path, value in [
|
||||
(("stage",), 1), (("source", "revision"), True),
|
||||
(("files",), []), (("allowed_actions",), "static"),
|
||||
(("obligations",), {}), (("executions",), "none"),
|
||||
]:
|
||||
context = copy.deepcopy(self.context)
|
||||
target = context
|
||||
for part in path[:-1]:
|
||||
target = target[part]
|
||||
if path == ("source", "revision"):
|
||||
context["source"] = {"kind": "git_tree", "revision": value}
|
||||
else:
|
||||
target[path[-1]] = value
|
||||
with self.subTest(path=path):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_malformed_digests_revisions_and_boolean_integer_fields_are_rejected(self):
|
||||
cases = [
|
||||
(("bundle_sha256",), "not-a-digest"),
|
||||
(("source", "base_revision"), "b" * 39),
|
||||
(("files", "src/value.py"), "A" * 64),
|
||||
]
|
||||
for path, value in cases:
|
||||
context = copy.deepcopy(self.context)
|
||||
if path[0] == "files":
|
||||
context[path[0]][path[1]] = value
|
||||
elif len(path) == 2:
|
||||
context[path[0]][path[1]] = value
|
||||
else:
|
||||
context[path[0]] = value
|
||||
with self.subTest(path=path):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["executions"][0]["exit_code"] = True
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_duplicate_missing_and_extra_claim_or_obligation_ids_block_input(self):
|
||||
cases = []
|
||||
duplicate_claim = copy.deepcopy(self.result)
|
||||
duplicate_claim["claims"][1]["id"] = duplicate_claim["claims"][0]["id"]
|
||||
cases.append(duplicate_claim)
|
||||
missing_claim = copy.deepcopy(self.result)
|
||||
missing_claim["claims"] = missing_claim["claims"][:1]
|
||||
cases.append(missing_claim)
|
||||
extra_claim = copy.deepcopy(self.result)
|
||||
extra_claim["claims"].append({"id": "extra", "status": "supported", "anchors": [], "execution_ids": []})
|
||||
cases.append(extra_claim)
|
||||
for result in cases:
|
||||
with self.subTest(ids=[claim["id"] for claim in result["claims"]]):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(result=result)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["obligations"].append({"id": "static-src", "kind": "static"})
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_stale_source_stage_and_bundle_claims_block(self):
|
||||
stale_context = copy.deepcopy(self.context)
|
||||
stale_context["stage"] = "review"
|
||||
self.assertIn("result_stage_mismatch", self.api(context=stale_context)["reason_codes"])
|
||||
for field, value in [("stage", "review"), ("bundle_sha256", "f" * 64),
|
||||
("source", {"kind": "captured_worktree", "base_revision": "b" * 40})]:
|
||||
stale_result = copy.deepcopy(self.result)
|
||||
stale_result[field] = value
|
||||
with self.subTest(field=field):
|
||||
out = self.api(result=stale_result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("result_" + field + "_mismatch", out["reason_codes"])
|
||||
|
||||
def test_line_anchor_rejects_zero_negative_reversed_out_of_range_and_wrong_hash(self):
|
||||
for start, end in [(0, 1), (-1, 1), (2, 1), (1, 99)]:
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"][0].update(start_line=start, end_line=end)
|
||||
with self.subTest(start=start, end=end):
|
||||
if end == 99:
|
||||
out = self.api(result=result)
|
||||
self.assertIn("anchor_range_invalid", out["reason_codes"])
|
||||
else:
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(result=result)
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
||||
out = self.api(result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_ast_anchor_rejects_nested_local_dynamic_and_syntax_error_sources(self):
|
||||
variants = {
|
||||
"nested": b"def outer():\n def test_fn():\n pass\n",
|
||||
"dynamic": b"globals()['test_fn'] = lambda: None\n",
|
||||
"syntax": b"def test_fn(:\n",
|
||||
}
|
||||
for name, source in variants.items():
|
||||
path = self.bundle / "tests/example.py"
|
||||
path.write_bytes(source)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["tests/example.py"] = hashlib.sha256(source).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
result = copy.deepcopy(self.result)
|
||||
result["bundle_sha256"] = context["bundle_sha256"]
|
||||
result["claims"][0]["anchors"] = [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}]
|
||||
with self.subTest(source=name):
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("test_anchor_missing", out["reason_codes"])
|
||||
|
||||
def test_unknown_execution_reference_and_authorization_origin_are_blocked(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][1]["execution_ids"] = ["missing"]
|
||||
out = self.api(result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unknown_execution_id", out["reason_codes"])
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
out = self.api(context=context)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unauthorized_execute", out["reason_codes"])
|
||||
context["executions"][0]["origin"] = "inherited"
|
||||
self.assertEqual(self.api(context=context)["status"], "pass")
|
||||
|
||||
def test_red_green_rejects_import_environment_timeout_and_mismatch_records(self):
|
||||
test_sha = self.files["tests/example.py"]
|
||||
obligations = [{"id": "tdd", "kind": "red_green"}]
|
||||
def execution(eid, exit_code, failure_kind, bundle, test_file=test_sha):
|
||||
return {
|
||||
"id": eid, "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
||||
"bundle_sha256": bundle, "test_file_sha256": test_file,
|
||||
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": exit_code,
|
||||
"failure_kind": failure_kind, "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": self.evidence_sha,
|
||||
}
|
||||
for failure_kind in ("import", "environment", "timeout"):
|
||||
context = self.make_context(actions=["static"], obligations=obligations,
|
||||
executions=[execution("red", 1, failure_kind, "b" * 64),
|
||||
execution("green", 0, "none", self.bundle_sha)])
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red", "green"],
|
||||
}])
|
||||
with self.subTest(failure_kind=failure_kind):
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("red_green_missing_red", out["reason_codes"])
|
||||
context = self.make_context(actions=["static"], obligations=obligations,
|
||||
executions=[execution("red", 1, "assertion", "b" * 64, "c" * 64),
|
||||
execution("green", 0, "none", self.bundle_sha)])
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red", "green"],
|
||||
}])
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertIn("red_green_test_scope_mismatch", out["reason_codes"])
|
||||
|
||||
def test_missing_green_and_evidence_hash_mismatch_block(self):
|
||||
obligations = [{"id": "tdd", "kind": "red_green"}]
|
||||
test_sha = self.files["tests/example.py"]
|
||||
executions = [{
|
||||
"id": "red", "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
||||
"bundle_sha256": "b" * 64, "test_file_sha256": test_sha,
|
||||
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": 1,
|
||||
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": "0" * 64,
|
||||
}]
|
||||
context = self.make_context(actions=["static"], obligations=obligations, executions=executions)
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red"],
|
||||
}])
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertIn("red_green_missing_green", out["reason_codes"])
|
||||
self.assertIn("evidence_hash_mismatch", out["reason_codes"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user