323 lines
17 KiB
Python
323 lines
17 KiB
Python
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from harness.lib import delegation_evidence as evidence
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
SCHEMA_C = "gitea-agent.delegation-evidence-context/v1"
|
|
SCHEMA_R = "gitea-agent.delegation-evidence-result/v1"
|
|
|
|
|
|
class DelegationEvidenceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name) / "bundle"
|
|
self.root.mkdir()
|
|
(self.root / "tests").mkdir()
|
|
(self.root / "src").mkdir()
|
|
(self.root / "tests/sample_test.py").write_bytes(b"def test_sample():\n assert True\n")
|
|
(self.root / "src/app.py").write_bytes(b"VALUE = 1\n")
|
|
self.files = {p: hashlib.sha256((self.root / p).read_bytes()).hexdigest()
|
|
for p in ("src/app.py", "tests/sample_test.py")}
|
|
self.bundle_sha = self.digest(self.files)
|
|
(self.root / "evidence").mkdir()
|
|
(self.root / "evidence/run.txt").write_bytes(b"exit 0\n")
|
|
evidence_sha = hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest()
|
|
self.context = {
|
|
"schema": SCHEMA_C,
|
|
"source": {"kind": "captured_worktree", "base_revision": "a" * 40},
|
|
"stage": "implementation",
|
|
"bundle_sha256": self.bundle_sha,
|
|
"files": self.files,
|
|
"allowed_actions": ["static", "execute"],
|
|
"obligations": [
|
|
{"id": "static-source", "kind": "static"},
|
|
{"id": "run-sample", "kind": "execution"},
|
|
],
|
|
"executions": [{"id": "exec-1", "test_id": "", "bundle_sha256": self.bundle_sha,
|
|
"test_file_sha256": "0" * 64, "argv": ["python3", "-c", "pass"],
|
|
"exit_code": 0, "failure_kind": "none", "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": evidence_sha, "origin": "direct"}],
|
|
}
|
|
self.result = {
|
|
"schema": SCHEMA_R,
|
|
"source": copy.deepcopy(self.context["source"]),
|
|
"stage": self.context["stage"],
|
|
"bundle_sha256": self.bundle_sha,
|
|
"claims": [
|
|
{"id": "static-source", "status": "supported", "anchors": [
|
|
{"kind": "lines", "path": "src/app.py", "start_line": 1,
|
|
"end_line": 1, "text_sha256": self.line_hash("src/app.py", 1, 1)}
|
|
], "execution_ids": []},
|
|
{"id": "run-sample", "status": "supported", "anchors": [], "execution_ids": ["exec-1"]},
|
|
],
|
|
}
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
@staticmethod
|
|
def digest(value):
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
|
|
def line_hash(self, path, start, end):
|
|
lines = (self.root / path).read_bytes().splitlines(keepends=True)
|
|
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
|
|
|
def make_git_repo(self, files=None, name="repo"):
|
|
repo = Path(self.tmp.name) / name
|
|
repo.mkdir()
|
|
for path in (files or self.files):
|
|
target = repo / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes((self.root / path).read_bytes())
|
|
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "fixture"], check=True)
|
|
revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
|
return repo / ".git", revision
|
|
|
|
def git_context(self, git_revision):
|
|
context = copy.deepcopy(self.context)
|
|
context["source"] = {"kind": "git_tree", "revision": git_revision}
|
|
result = copy.deepcopy(self.result)
|
|
result["source"] = copy.deepcopy(context["source"])
|
|
return context, result
|
|
|
|
def test_worker_source_stage_bundle_mismatches_are_detected_blocks(self):
|
|
for key, value in [('source', {'kind': 'captured_worktree', 'base_revision': 'b' * 40}),
|
|
('stage', 'stale-stage'), ('bundle_sha256', '0' * 64)]:
|
|
bad = copy.deepcopy(self.result)
|
|
bad[key] = value
|
|
with self.subTest(field=key):
|
|
try:
|
|
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
|
except evidence.EvidenceError:
|
|
self.fail('syntactically valid provenance disagreement must return BLOCK')
|
|
self.assertEqual(out['status'], 'block')
|
|
self.assertIn('result_' + key + '_mismatch', out['reason_codes'])
|
|
|
|
def test_valid_static_lines_and_bounded_counts(self):
|
|
out = evidence.verify_evidence(str(self.root), self.context, self.result)
|
|
self.assertEqual(out["status"], "pass")
|
|
self.assertEqual(out["reason_codes"], [])
|
|
self.assertEqual(out["counts"], {"files": 2, "obligations": 2, "claims": 2, "anchors": 1, "executions": 1})
|
|
|
|
def test_wrong_line_hash_blocks(self):
|
|
bad = copy.deepcopy(self.result)
|
|
bad["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
|
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
|
|
|
def test_indeterminate_claim_blocks(self):
|
|
result = copy.deepcopy(self.result)
|
|
result["claims"][0]["status"] = "indeterminate"
|
|
out = evidence.verify_evidence(str(self.root), self.context, result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("indeterminate_claim", out["reason_codes"])
|
|
|
|
def test_test_anchor_requires_real_python_function(self):
|
|
result = copy.deepcopy(self.result)
|
|
result["claims"][0]["anchors"] = [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}]
|
|
out = evidence.verify_evidence(str(self.root), self.context, result)
|
|
self.assertEqual(out["status"], "pass")
|
|
result["claims"][0]["anchors"][0]["symbol"] = "missing"
|
|
self.assertIn("test_anchor_missing", evidence.verify_evidence(str(self.root), self.context, result)["reason_codes"])
|
|
|
|
def test_claim_ids_must_exactly_match_obligations(self):
|
|
bad = copy.deepcopy(self.result)
|
|
bad["claims"][1]["id"] = "extra"
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
evidence.verify_evidence(str(self.root), self.context, bad)
|
|
|
|
def test_captured_worktree_absence_is_indeterminate(self):
|
|
context = copy.deepcopy(self.context)
|
|
context["files"] = {"src/app.py": self.files["src/app.py"]}
|
|
context["bundle_sha256"] = self.digest(context["files"])
|
|
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
|
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
|
"bundle_sha256": context["bundle_sha256"], "claims": [{"id": "absent", "status": "supported",
|
|
"anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}]}
|
|
out = evidence.verify_evidence(str(self.root), context, result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("absence_indeterminate", out["reason_codes"])
|
|
|
|
def test_git_tree_matching_frozen_bundle_passes(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
context, result = self.git_context(revision)
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
self.assertEqual(out["status"], "pass")
|
|
|
|
def test_git_tree_altered_frozen_snapshot_blocks(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
(self.root / "src/app.py").write_bytes(b"VALUE = 2\n")
|
|
context, result = self.git_context(revision)
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
|
|
|
def test_git_tree_source_mismatch_blocks(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
repo = Path(self.tmp.name) / "other"
|
|
repo.mkdir()
|
|
for path in self.files:
|
|
target = repo / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes((self.root / path).read_bytes())
|
|
(repo / "src/app.py").write_bytes(b"VALUE = 9\n")
|
|
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "other"], check=True)
|
|
other_revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
|
context, result = self.git_context(other_revision)
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
|
context_path = self.root / "context.json"
|
|
result_path = self.root / "result.json"
|
|
context_path.write_text(json.dumps(context))
|
|
result_path.write_text(json.dumps(result))
|
|
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
|
"--bundle-root", str(self.root), "--context", str(context_path),
|
|
"--result", str(result_path), "--git-dir", str(git_dir)],
|
|
text=True, capture_output=True)
|
|
self.assertEqual(proc.returncode, 1)
|
|
self.assertEqual(json.loads(proc.stdout)["status"], "block")
|
|
|
|
def test_git_tree_present_path_cannot_be_claimed_absent(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
context, result = self.git_context(revision)
|
|
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
|
result["claims"] = [{"id": "absent", "status": "supported",
|
|
"anchors": [{"kind": "absence", "path": "tests/sample_test.py"}],
|
|
"execution_ids": []}]
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("absence_not_proven", out["reason_codes"])
|
|
|
|
def test_git_tree_genuine_missing_path_is_authoritative(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
context, result = self.git_context(revision)
|
|
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
|
result["claims"] = [{"id": "absent", "status": "supported",
|
|
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
|
"execution_ids": []}]
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
self.assertEqual(out["status"], "pass")
|
|
|
|
def test_git_tree_absence_uses_original_directory_fd_after_replacement(self):
|
|
git_dir, revision = self.make_git_repo()
|
|
replacement, _ = self.make_git_repo(name="replacement")
|
|
replacement_repo = replacement.parent
|
|
(replacement_repo / "tests/missing.py").write_bytes(b"present\n")
|
|
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "add", "."], check=True)
|
|
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "commit", "-q", "-m", "replacement"], check=True)
|
|
context, result = self.git_context(revision)
|
|
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
|
result["claims"] = [{"id": "absent", "status": "supported",
|
|
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
|
"execution_ids": []}]
|
|
original_git = evidence._git
|
|
moved = {"done": False}
|
|
|
|
def replacing_git(git_path, args, fd):
|
|
if args[0] == "ls-tree" and args[-1] == "tests/missing.py" and not moved["done"]:
|
|
moved["done"] = True
|
|
backup = Path(self.tmp.name) / "original.git"
|
|
os.rename(git_dir, backup)
|
|
os.rename(replacement, git_dir)
|
|
return original_git(git_path, args, fd)
|
|
|
|
evidence._git = replacing_git
|
|
try:
|
|
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
|
finally:
|
|
evidence._git = original_git
|
|
self.assertEqual(out["status"], "pass")
|
|
|
|
def test_inherited_execution_is_allowed_for_static_only_worker_but_direct_is_not(self):
|
|
context = copy.deepcopy(self.context)
|
|
context["allowed_actions"] = ["static"]
|
|
context["executions"][0]["origin"] = "inherited"
|
|
out = evidence.verify_evidence(str(self.root), context, self.result)
|
|
self.assertEqual(out["status"], "pass")
|
|
context["executions"][0]["origin"] = "direct"
|
|
out = evidence.verify_evidence(str(self.root), context, self.result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("unauthorized_execute", out["reason_codes"])
|
|
|
|
def test_execution_origin_is_required_and_closed(self):
|
|
for origin in ("worker", None):
|
|
context = copy.deepcopy(self.context)
|
|
if origin is None:
|
|
del context["executions"][0]["origin"]
|
|
else:
|
|
context["executions"][0]["origin"] = origin
|
|
with self.subTest(origin=origin), self.assertRaises(evidence.EvidenceError):
|
|
evidence.verify_evidence(str(self.root), context, self.result)
|
|
|
|
def test_inherited_red_green_is_accepted_without_execute_permission(self):
|
|
context = copy.deepcopy(self.context)
|
|
context["allowed_actions"] = ["static"]
|
|
context["obligations"] = [{"id": "tdd", "kind": "red_green"}]
|
|
test_sha = self.files["tests/sample_test.py"]
|
|
context["executions"] = [
|
|
{"id": "red", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": "b" * 64,
|
|
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 1,
|
|
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
|
"origin": "inherited"},
|
|
{"id": "green", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": self.bundle_sha,
|
|
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 0,
|
|
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
|
"origin": "inherited"},
|
|
]
|
|
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
|
"bundle_sha256": self.bundle_sha, "claims": [{"id": "tdd", "status": "supported",
|
|
"anchors": [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}],
|
|
"execution_ids": ["red", "green"]}]}
|
|
self.assertEqual(evidence.verify_evidence(str(self.root), context, result)["status"], "pass")
|
|
|
|
def test_rejected_input_and_unknown_cli_are_bounded(self):
|
|
bad = copy.deepcopy(self.context)
|
|
bad["files"]["../secret"] = "0" * 64
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
evidence.verify_evidence(str(self.root), bad, self.result)
|
|
c = self.root / "context.json"; r = self.root / "result.json"
|
|
c.write_text(json.dumps(self.context)); r.write_text(json.dumps(self.result))
|
|
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
|
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r), "--unknown=secret"],
|
|
text=True, capture_output=True)
|
|
self.assertEqual(proc.returncode, 2)
|
|
self.assertNotIn("secret", proc.stdout + proc.stderr)
|
|
self.assertEqual(json.loads(proc.stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
|
|
|
def test_duplicate_json_keys_and_symlink_are_rejected(self):
|
|
c = self.root / "context.json"; r = self.root / "result.json"
|
|
c.write_text('{"schema":"x","schema":"secret"}')
|
|
r.write_text(json.dumps(self.result))
|
|
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
|
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r)], capture_output=True, text=True)
|
|
self.assertEqual(proc.returncode, 2); self.assertNotIn("secret", proc.stdout + proc.stderr)
|
|
real = self.root / "real.json"; real.write_text(json.dumps(self.context))
|
|
alias = self.root / "alias.json"; alias.symlink_to(real)
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
evidence._json_file(str(alias))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|