from harness.tests._support.paths import ROOT as SOURCE_ROOT import copy from harness.tests._support.data import canonical_digest import hashlib import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from harness.lib import delegation_evidence as evidence ROOT = SOURCE_ROOT SCHEMA_C = "gitea-agent.delegation-evidence-context/v1" SCHEMA_R = "gitea-agent.delegation-evidence-result/v1" class DelegationEvidenceTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) / "bundle" self.root.mkdir() (self.root / "tests").mkdir() (self.root / "src").mkdir() (self.root / "tests/sample_test.py").write_bytes(b"def test_sample():\n assert True\n") (self.root / "src/app.py").write_bytes(b"VALUE = 1\n") self.files = {p: hashlib.sha256((self.root / p).read_bytes()).hexdigest() for p in ("src/app.py", "tests/sample_test.py")} self.bundle_sha = self.digest(self.files) (self.root / "evidence").mkdir() (self.root / "evidence/run.txt").write_bytes(b"exit 0\n") evidence_sha = hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest() self.context = { "schema": SCHEMA_C, "source": {"kind": "captured_worktree", "base_revision": "a" * 40}, "stage": "implementation", "bundle_sha256": self.bundle_sha, "files": self.files, "allowed_actions": ["static", "execute"], "obligations": [ {"id": "static-source", "kind": "static"}, {"id": "run-sample", "kind": "execution"}, ], "executions": [{"id": "exec-1", "test_id": "", "bundle_sha256": self.bundle_sha, "test_file_sha256": "0" * 64, "argv": ["python3", "-c", "pass"], "exit_code": 0, "failure_kind": "none", "evidence_path": "evidence/run.txt", "evidence_sha256": evidence_sha, "origin": "direct"}], } self.result = { "schema": SCHEMA_R, "source": copy.deepcopy(self.context["source"]), "stage": self.context["stage"], "bundle_sha256": self.bundle_sha, "claims": [ {"id": "static-source", "status": "supported", "anchors": [ {"kind": "lines", "path": "src/app.py", "start_line": 1, "end_line": 1, "text_sha256": self.line_hash("src/app.py", 1, 1)} ], "execution_ids": []}, {"id": "run-sample", "status": "supported", "anchors": [], "execution_ids": ["exec-1"]}, ], } def tearDown(self): self.tmp.cleanup() digest = staticmethod(canonical_digest) def line_hash(self, path, start, end): lines = (self.root / path).read_bytes().splitlines(keepends=True) return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest() def make_git_repo(self, files=None, name="repo"): repo = Path(self.tmp.name) / name repo.mkdir() for path in (files or self.files): target = repo / path target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes((self.root / path).read_bytes()) subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "fixture"], check=True) revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip() return repo / ".git", revision def git_context(self, git_revision): context = copy.deepcopy(self.context) context["source"] = {"kind": "git_tree", "revision": git_revision} result = copy.deepcopy(self.result) result["source"] = copy.deepcopy(context["source"]) return context, result def test_worker_source_stage_bundle_mismatches_are_detected_blocks(self): for key, value in [('source', {'kind': 'captured_worktree', 'base_revision': 'b' * 40}), ('stage', 'stale-stage'), ('bundle_sha256', '0' * 64)]: bad = copy.deepcopy(self.result) bad[key] = value with self.subTest(field=key): try: out = evidence.verify_evidence(str(self.root), self.context, bad) except evidence.EvidenceError: self.fail('syntactically valid provenance disagreement must return BLOCK') self.assertEqual(out['status'], 'block') self.assertIn('result_' + key + '_mismatch', out['reason_codes']) def test_valid_static_lines_and_bounded_counts(self): out = evidence.verify_evidence(str(self.root), self.context, self.result) self.assertEqual(out["status"], "pass") self.assertEqual(out["reason_codes"], []) self.assertEqual(out["counts"], {"files": 2, "obligations": 2, "claims": 2, "anchors": 1, "executions": 1}) def test_wrong_line_hash_blocks(self): bad = copy.deepcopy(self.result) bad["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64 out = evidence.verify_evidence(str(self.root), self.context, bad) self.assertEqual(out["status"], "block") self.assertIn("anchor_hash_mismatch", out["reason_codes"]) def test_indeterminate_claim_blocks(self): result = copy.deepcopy(self.result) result["claims"][0]["status"] = "indeterminate" out = evidence.verify_evidence(str(self.root), self.context, result) self.assertEqual(out["status"], "block") self.assertIn("indeterminate_claim", out["reason_codes"]) def test_test_anchor_requires_real_python_function(self): result = copy.deepcopy(self.result) result["claims"][0]["anchors"] = [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}] out = evidence.verify_evidence(str(self.root), self.context, result) self.assertEqual(out["status"], "pass") result["claims"][0]["anchors"][0]["symbol"] = "missing" self.assertIn("test_anchor_missing", evidence.verify_evidence(str(self.root), self.context, result)["reason_codes"]) def test_claim_ids_must_exactly_match_obligations(self): bad = copy.deepcopy(self.result) bad["claims"][1]["id"] = "extra" with self.assertRaises(evidence.EvidenceError): evidence.verify_evidence(str(self.root), self.context, bad) def test_captured_worktree_absence_is_indeterminate(self): context = copy.deepcopy(self.context) context["files"] = {"src/app.py": self.files["src/app.py"]} context["bundle_sha256"] = self.digest(context["files"]) context["obligations"] = [{"id": "absent", "kind": "static"}] result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"], "bundle_sha256": context["bundle_sha256"], "claims": [{"id": "absent", "status": "supported", "anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}]} out = evidence.verify_evidence(str(self.root), context, result) self.assertEqual(out["status"], "block") self.assertIn("absence_indeterminate", out["reason_codes"]) def test_git_tree_matching_frozen_bundle_passes(self): git_dir, revision = self.make_git_repo() context, result = self.git_context(revision) out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) self.assertEqual(out["status"], "pass") def test_git_tree_altered_frozen_snapshot_blocks(self): git_dir, revision = self.make_git_repo() (self.root / "src/app.py").write_bytes(b"VALUE = 2\n") context, result = self.git_context(revision) out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) self.assertEqual(out["status"], "block") self.assertIn("source_hash_mismatch", out["reason_codes"]) def test_git_tree_source_mismatch_blocks(self): git_dir, revision = self.make_git_repo() repo = Path(self.tmp.name) / "other" repo.mkdir() for path in self.files: target = repo / path target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes((self.root / path).read_bytes()) (repo / "src/app.py").write_bytes(b"VALUE = 9\n") subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True) subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "other"], check=True) other_revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip() context, result = self.git_context(other_revision) out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) self.assertEqual(out["status"], "block") self.assertIn("source_hash_mismatch", out["reason_codes"]) context_path = self.root / "context.json" result_path = self.root / "result.json" context_path.write_text(json.dumps(context)) result_path.write_text(json.dumps(result)) proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"), "--bundle-root", str(self.root), "--context", str(context_path), "--result", str(result_path), "--git-dir", str(git_dir)], text=True, capture_output=True) self.assertEqual(proc.returncode, 1) self.assertEqual(json.loads(proc.stdout)["status"], "block") def test_git_tree_present_path_cannot_be_claimed_absent(self): git_dir, revision = self.make_git_repo() context, result = self.git_context(revision) context["obligations"] = [{"id": "absent", "kind": "static"}] result["claims"] = [{"id": "absent", "status": "supported", "anchors": [{"kind": "absence", "path": "tests/sample_test.py"}], "execution_ids": []}] out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) self.assertEqual(out["status"], "block") self.assertIn("absence_not_proven", out["reason_codes"]) def test_git_tree_genuine_missing_path_is_authoritative(self): git_dir, revision = self.make_git_repo() context, result = self.git_context(revision) context["obligations"] = [{"id": "absent", "kind": "static"}] result["claims"] = [{"id": "absent", "status": "supported", "anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}] out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) self.assertEqual(out["status"], "pass") def test_git_tree_absence_uses_original_directory_fd_after_replacement(self): git_dir, revision = self.make_git_repo() replacement, _ = self.make_git_repo(name="replacement") replacement_repo = replacement.parent (replacement_repo / "tests/missing.py").write_bytes(b"present\n") subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "add", "."], check=True) subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "commit", "-q", "-m", "replacement"], check=True) context, result = self.git_context(revision) context["obligations"] = [{"id": "absent", "kind": "static"}] result["claims"] = [{"id": "absent", "status": "supported", "anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}] original_git = evidence._git moved = {"done": False} def replacing_git(git_path, args, fd): if args[0] == "ls-tree" and args[-1] == "tests/missing.py" and not moved["done"]: moved["done"] = True backup = Path(self.tmp.name) / "original.git" os.rename(git_dir, backup) os.rename(replacement, git_dir) return original_git(git_path, args, fd) evidence._git = replacing_git try: out = evidence.verify_evidence(str(self.root), context, result, str(git_dir)) finally: evidence._git = original_git self.assertEqual(out["status"], "pass") def test_inherited_execution_is_allowed_for_static_only_worker_but_direct_is_not(self): context = copy.deepcopy(self.context) context["allowed_actions"] = ["static"] context["executions"][0]["origin"] = "inherited" out = evidence.verify_evidence(str(self.root), context, self.result) self.assertEqual(out["status"], "pass") context["executions"][0]["origin"] = "direct" out = evidence.verify_evidence(str(self.root), context, self.result) self.assertEqual(out["status"], "block") self.assertIn("unauthorized_execute", out["reason_codes"]) def test_execution_origin_is_required_and_closed(self): for origin in ("worker", None): context = copy.deepcopy(self.context) if origin is None: del context["executions"][0]["origin"] else: context["executions"][0]["origin"] = origin with self.subTest(origin=origin), self.assertRaises(evidence.EvidenceError): evidence.verify_evidence(str(self.root), context, self.result) def test_inherited_red_green_is_accepted_without_execute_permission(self): context = copy.deepcopy(self.context) context["allowed_actions"] = ["static"] context["obligations"] = [{"id": "tdd", "kind": "red_green"}] test_sha = self.files["tests/sample_test.py"] context["executions"] = [ {"id": "red", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": "b" * 64, "test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 1, "failure_kind": "assertion", "evidence_path": "evidence/run.txt", "evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(), "origin": "inherited"}, {"id": "green", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": self.bundle_sha, "test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 0, "failure_kind": "none", "evidence_path": "evidence/run.txt", "evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(), "origin": "inherited"}, ] result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"], "bundle_sha256": self.bundle_sha, "claims": [{"id": "tdd", "status": "supported", "anchors": [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}], "execution_ids": ["red", "green"]}]} self.assertEqual(evidence.verify_evidence(str(self.root), context, result)["status"], "pass") def test_rejected_input_and_unknown_cli_are_bounded(self): bad = copy.deepcopy(self.context) bad["files"]["../secret"] = "0" * 64 with self.assertRaises(evidence.EvidenceError): evidence.verify_evidence(str(self.root), bad, self.result) c = self.root / "context.json"; r = self.root / "result.json" c.write_text(json.dumps(self.context)); r.write_text(json.dumps(self.result)) proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"), "--bundle-root", str(self.root), "--context", str(c), "--result", str(r), "--unknown=secret"], text=True, capture_output=True) self.assertEqual(proc.returncode, 2) self.assertNotIn("secret", proc.stdout + proc.stderr) self.assertEqual(json.loads(proc.stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}}) def test_duplicate_json_keys_and_symlink_are_rejected(self): c = self.root / "context.json"; r = self.root / "result.json" c.write_text('{"schema":"x","schema":"secret"}') r.write_text(json.dumps(self.result)) proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"), "--bundle-root", str(self.root), "--context", str(c), "--result", str(r)], capture_output=True, text=True) self.assertEqual(proc.returncode, 2); self.assertNotIn("secret", proc.stdout + proc.stderr) real = self.root / "real.json"; real.write_text(json.dumps(self.context)) alias = self.root / "alias.json"; alias.symlink_to(real) with self.assertRaises(evidence.EvidenceError): evidence._json_file(str(alias)) if __name__ == "__main__": unittest.main()