384 lines
18 KiB
Python
384 lines
18 KiB
Python
"""Independent safety/regression coverage for delegation evidence acceptance."""
|
|
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
|
|
|
import copy
|
|
from harness.tests._support.data import canonical_digest
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from harness.lib import delegation_evidence as evidence
|
|
|
|
ROOT = SOURCE_ROOT
|
|
HELPER = ROOT / "harness" / "lib" / "delegation_evidence.py"
|
|
CONTEXT_SCHEMA = "gitea-agent.delegation-evidence-context/v1"
|
|
RESULT_SCHEMA = "gitea-agent.delegation-evidence-result/v1"
|
|
REVISION = "a" * 40
|
|
|
|
|
|
class DelegationEvidenceSafetyTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch")
|
|
self.base = Path(self.tmp.name)
|
|
self.bundle = self.base / "bundle"
|
|
(self.bundle / "tests").mkdir(parents=True)
|
|
(self.bundle / "src").mkdir()
|
|
(self.bundle / "evidence").mkdir()
|
|
(self.bundle / "tests/example.py").write_bytes(
|
|
b"def test_fn():\n assert True\n"
|
|
)
|
|
(self.bundle / "src/value.py").write_bytes(b"VALUE = 1\n")
|
|
(self.bundle / "evidence/run.txt").write_bytes(b"recorded output\n")
|
|
self.files = {
|
|
path: hashlib.sha256((self.bundle / path).read_bytes()).hexdigest()
|
|
for path in ("src/value.py", "tests/example.py")
|
|
}
|
|
self.bundle_sha = self.digest(self.files)
|
|
self.evidence_sha = hashlib.sha256(
|
|
(self.bundle / "evidence/run.txt").read_bytes()
|
|
).hexdigest()
|
|
self.context = self.make_context()
|
|
self.result = self.make_result()
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
digest = staticmethod(canonical_digest)
|
|
|
|
def make_context(self, *, actions=None, obligations=None, executions=None):
|
|
return {
|
|
"schema": CONTEXT_SCHEMA,
|
|
"source": {"kind": "captured_worktree", "base_revision": REVISION},
|
|
"stage": "implementation",
|
|
"bundle_sha256": self.bundle_sha,
|
|
"files": copy.deepcopy(self.files),
|
|
"allowed_actions": ["static", "execute"] if actions is None else actions,
|
|
"obligations": [
|
|
{"id": "static-src", "kind": "static"},
|
|
{"id": "run-test", "kind": "execution"},
|
|
] if obligations is None else obligations,
|
|
"executions": [{
|
|
"id": "exec-1", "origin": "direct", "test_id": "",
|
|
"bundle_sha256": self.bundle_sha, "test_file_sha256": "0" * 64,
|
|
"argv": ["python3", "-B", "-c", "pass"], "exit_code": 0,
|
|
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": self.evidence_sha,
|
|
}] if executions is None else executions,
|
|
}
|
|
|
|
def make_result(self, *, source=None, stage=None, bundle_sha=None, claims=None):
|
|
return {
|
|
"schema": RESULT_SCHEMA,
|
|
"source": copy.deepcopy(self.context["source"] if source is None else source),
|
|
"stage": self.context["stage"] if stage is None else stage,
|
|
"bundle_sha256": self.bundle_sha if bundle_sha is None else bundle_sha,
|
|
"claims": [
|
|
{"id": "static-src", "status": "supported", "anchors": [{
|
|
"kind": "lines", "path": "src/value.py", "start_line": 1,
|
|
"end_line": 1, "text_sha256": self.line_hash("src/value.py", 1, 1),
|
|
}], "execution_ids": []},
|
|
{"id": "run-test", "status": "supported", "anchors": [],
|
|
"execution_ids": ["exec-1"]},
|
|
] if claims is None else claims,
|
|
}
|
|
|
|
def line_hash(self, path, start, end):
|
|
lines = (self.bundle / path).read_bytes().splitlines(keepends=True)
|
|
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
|
|
|
def api(self, context=None, result=None, root=None):
|
|
return evidence.verify_evidence(
|
|
str(self.bundle if root is None else root),
|
|
self.context if context is None else context,
|
|
self.result if result is None else result,
|
|
)
|
|
|
|
def cli(self, context_text=None, result=None, *, root=None, context_path=None,
|
|
result_path=None, extra=None, timeout=3):
|
|
root = self.bundle if root is None else Path(root)
|
|
context_path = self.base / "context.json" if context_path is None else Path(context_path)
|
|
result_path = self.base / "result.json" if result_path is None else Path(result_path)
|
|
if context_text is None:
|
|
context_path.write_text(json.dumps(self.context), encoding="utf-8")
|
|
elif isinstance(context_text, bytes):
|
|
context_path.write_bytes(context_text)
|
|
else:
|
|
context_path.write_text(context_text, encoding="utf-8")
|
|
if result is not None:
|
|
result_path.write_text(json.dumps(result), encoding="utf-8")
|
|
else:
|
|
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
|
command = [sys.executable, "-B", str(HELPER), "--bundle-root", str(root),
|
|
"--context", str(context_path), "--result", str(result_path)]
|
|
if extra:
|
|
command.extend(extra)
|
|
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
|
|
|
def assert_rejected(self, value, *, field=None):
|
|
bad = copy.deepcopy(self.context)
|
|
if field is None:
|
|
bad = value
|
|
else:
|
|
target = bad
|
|
for part in field[:-1]:
|
|
target = target[part]
|
|
target[field[-1]] = value
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=bad)
|
|
|
|
def test_real_paths_reject_root_file_ancestor_and_canonical_aliases(self):
|
|
alias = self.base / "bundle-alias"
|
|
alias.symlink_to(self.bundle, target_is_directory=True)
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(root=alias)
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(root=self.base / "missing" / ".." / "bundle")
|
|
outside = self.base / "outside.txt"
|
|
outside.write_bytes(b"outside")
|
|
(self.bundle / "src" / "alias.py").symlink_to(outside)
|
|
context = copy.deepcopy(self.context)
|
|
context["files"]["src/alias.py"] = hashlib.sha256(outside.read_bytes()).hexdigest()
|
|
context["bundle_sha256"] = self.digest(context["files"])
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
|
|
def test_json_input_symlink_fifo_and_source_fifo_are_rejected_without_hang(self):
|
|
real_context = self.base / "real-context.json"
|
|
real_context.write_text(json.dumps(self.context), encoding="utf-8")
|
|
alias = self.base / "context.json"
|
|
alias.symlink_to(real_context)
|
|
result_path = self.base / "result.json"
|
|
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
evidence._json_file(str(alias))
|
|
|
|
fifo = self.base / "context-fifo"
|
|
os.mkfifo(fifo)
|
|
proc = subprocess.Popen(
|
|
[sys.executable, "-B", str(HELPER), "--bundle-root", str(self.bundle),
|
|
"--context", str(fifo), "--result", str(result_path)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
|
)
|
|
stdout, stderr = proc.communicate(timeout=3)
|
|
self.assertEqual(proc.returncode, 2, (stdout, stderr))
|
|
self.assertEqual(json.loads(stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
|
|
|
source_fifo = self.bundle / "src" / "fifo.py"
|
|
os.mkfifo(source_fifo)
|
|
context = copy.deepcopy(self.context)
|
|
context["files"]["src/fifo.py"] = "0" * 64
|
|
context["bundle_sha256"] = self.digest(context["files"])
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
|
|
def test_oversized_source_and_json_are_bounded(self):
|
|
huge = self.bundle / "src" / "huge.py"
|
|
huge.write_bytes(b"x" * (1024 * 1024 + 1))
|
|
context = copy.deepcopy(self.context)
|
|
context["files"]["src/huge.py"] = hashlib.sha256(huge.read_bytes()).hexdigest()
|
|
context["bundle_sha256"] = self.digest(context["files"])
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
proc = self.cli(context_text=b"{" + b"\"x\":\"" + b"a" * (1024 * 1024) + b"\"}")
|
|
self.assertEqual(proc.returncode, 2)
|
|
self.assertEqual(json.loads(proc.stdout)["reason_codes"], ["rejected_input"])
|
|
|
|
def test_duplicate_unknown_and_wrong_typed_json_fields_are_rejected(self):
|
|
cases = [
|
|
b'{"schema":"x","schema":"y"}',
|
|
json.dumps({**self.context, "unknown": 1}).encode(),
|
|
]
|
|
for payload in cases:
|
|
with self.subTest(case=payload[:20]):
|
|
proc = self.cli(context_text=payload)
|
|
self.assertEqual(proc.returncode, 2)
|
|
for path, value in [
|
|
(("stage",), 1), (("source", "revision"), True),
|
|
(("files",), []), (("allowed_actions",), "static"),
|
|
(("obligations",), {}), (("executions",), "none"),
|
|
]:
|
|
context = copy.deepcopy(self.context)
|
|
target = context
|
|
for part in path[:-1]:
|
|
target = target[part]
|
|
if path == ("source", "revision"):
|
|
context["source"] = {"kind": "git_tree", "revision": value}
|
|
else:
|
|
target[path[-1]] = value
|
|
with self.subTest(path=path):
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
|
|
def test_malformed_digests_revisions_and_boolean_integer_fields_are_rejected(self):
|
|
cases = [
|
|
(("bundle_sha256",), "not-a-digest"),
|
|
(("source", "base_revision"), "b" * 39),
|
|
(("files", "src/value.py"), "A" * 64),
|
|
]
|
|
for path, value in cases:
|
|
context = copy.deepcopy(self.context)
|
|
if path[0] == "files":
|
|
context[path[0]][path[1]] = value
|
|
elif len(path) == 2:
|
|
context[path[0]][path[1]] = value
|
|
else:
|
|
context[path[0]] = value
|
|
with self.subTest(path=path):
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
context = copy.deepcopy(self.context)
|
|
context["executions"][0]["exit_code"] = True
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
|
|
def test_duplicate_missing_and_extra_claim_or_obligation_ids_block_input(self):
|
|
cases = []
|
|
duplicate_claim = copy.deepcopy(self.result)
|
|
duplicate_claim["claims"][1]["id"] = duplicate_claim["claims"][0]["id"]
|
|
cases.append(duplicate_claim)
|
|
missing_claim = copy.deepcopy(self.result)
|
|
missing_claim["claims"] = missing_claim["claims"][:1]
|
|
cases.append(missing_claim)
|
|
extra_claim = copy.deepcopy(self.result)
|
|
extra_claim["claims"].append({"id": "extra", "status": "supported", "anchors": [], "execution_ids": []})
|
|
cases.append(extra_claim)
|
|
for result in cases:
|
|
with self.subTest(ids=[claim["id"] for claim in result["claims"]]):
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(result=result)
|
|
context = copy.deepcopy(self.context)
|
|
context["obligations"].append({"id": "static-src", "kind": "static"})
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(context=context)
|
|
|
|
def test_stale_source_stage_and_bundle_claims_block(self):
|
|
stale_context = copy.deepcopy(self.context)
|
|
stale_context["stage"] = "review"
|
|
self.assertIn("result_stage_mismatch", self.api(context=stale_context)["reason_codes"])
|
|
for field, value in [("stage", "review"), ("bundle_sha256", "f" * 64),
|
|
("source", {"kind": "captured_worktree", "base_revision": "b" * 40})]:
|
|
stale_result = copy.deepcopy(self.result)
|
|
stale_result[field] = value
|
|
with self.subTest(field=field):
|
|
out = self.api(result=stale_result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("result_" + field + "_mismatch", out["reason_codes"])
|
|
|
|
def test_line_anchor_rejects_zero_negative_reversed_out_of_range_and_wrong_hash(self):
|
|
for start, end in [(0, 1), (-1, 1), (2, 1), (1, 99)]:
|
|
result = copy.deepcopy(self.result)
|
|
result["claims"][0]["anchors"][0].update(start_line=start, end_line=end)
|
|
with self.subTest(start=start, end=end):
|
|
if end == 99:
|
|
out = self.api(result=result)
|
|
self.assertIn("anchor_range_invalid", out["reason_codes"])
|
|
else:
|
|
with self.assertRaises(evidence.EvidenceError):
|
|
self.api(result=result)
|
|
result = copy.deepcopy(self.result)
|
|
result["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
|
out = self.api(result=result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
|
|
|
def test_ast_anchor_rejects_nested_local_dynamic_and_syntax_error_sources(self):
|
|
variants = {
|
|
"nested": b"def outer():\n def test_fn():\n pass\n",
|
|
"dynamic": b"globals()['test_fn'] = lambda: None\n",
|
|
"syntax": b"def test_fn(:\n",
|
|
}
|
|
for name, source in variants.items():
|
|
path = self.bundle / "tests/example.py"
|
|
path.write_bytes(source)
|
|
context = copy.deepcopy(self.context)
|
|
context["files"]["tests/example.py"] = hashlib.sha256(source).hexdigest()
|
|
context["bundle_sha256"] = self.digest(context["files"])
|
|
result = copy.deepcopy(self.result)
|
|
result["bundle_sha256"] = context["bundle_sha256"]
|
|
result["claims"][0]["anchors"] = [{
|
|
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
|
}]
|
|
with self.subTest(source=name):
|
|
out = self.api(context=context, result=result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("test_anchor_missing", out["reason_codes"])
|
|
|
|
def test_unknown_execution_reference_and_authorization_origin_are_blocked(self):
|
|
result = copy.deepcopy(self.result)
|
|
result["claims"][1]["execution_ids"] = ["missing"]
|
|
out = self.api(result=result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("unknown_execution_id", out["reason_codes"])
|
|
context = copy.deepcopy(self.context)
|
|
context["allowed_actions"] = ["static"]
|
|
out = self.api(context=context)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("unauthorized_execute", out["reason_codes"])
|
|
context["executions"][0]["origin"] = "inherited"
|
|
self.assertEqual(self.api(context=context)["status"], "pass")
|
|
|
|
def test_red_green_rejects_import_environment_timeout_and_mismatch_records(self):
|
|
test_sha = self.files["tests/example.py"]
|
|
obligations = [{"id": "tdd", "kind": "red_green"}]
|
|
def execution(eid, exit_code, failure_kind, bundle, test_file=test_sha):
|
|
return {
|
|
"id": eid, "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
|
"bundle_sha256": bundle, "test_file_sha256": test_file,
|
|
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": exit_code,
|
|
"failure_kind": failure_kind, "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": self.evidence_sha,
|
|
}
|
|
for failure_kind in ("import", "environment", "timeout"):
|
|
context = self.make_context(actions=["static"], obligations=obligations,
|
|
executions=[execution("red", 1, failure_kind, "b" * 64),
|
|
execution("green", 0, "none", self.bundle_sha)])
|
|
result = self.make_result(claims=[{
|
|
"id": "tdd", "status": "supported", "anchors": [{
|
|
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
|
}], "execution_ids": ["red", "green"],
|
|
}])
|
|
with self.subTest(failure_kind=failure_kind):
|
|
out = self.api(context=context, result=result)
|
|
self.assertEqual(out["status"], "block")
|
|
self.assertIn("red_green_missing_red", out["reason_codes"])
|
|
context = self.make_context(actions=["static"], obligations=obligations,
|
|
executions=[execution("red", 1, "assertion", "b" * 64, "c" * 64),
|
|
execution("green", 0, "none", self.bundle_sha)])
|
|
result = self.make_result(claims=[{
|
|
"id": "tdd", "status": "supported", "anchors": [{
|
|
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
|
}], "execution_ids": ["red", "green"],
|
|
}])
|
|
out = self.api(context=context, result=result)
|
|
self.assertIn("red_green_test_scope_mismatch", out["reason_codes"])
|
|
|
|
def test_missing_green_and_evidence_hash_mismatch_block(self):
|
|
obligations = [{"id": "tdd", "kind": "red_green"}]
|
|
test_sha = self.files["tests/example.py"]
|
|
executions = [{
|
|
"id": "red", "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
|
"bundle_sha256": "b" * 64, "test_file_sha256": test_sha,
|
|
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": 1,
|
|
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
|
"evidence_sha256": "0" * 64,
|
|
}]
|
|
context = self.make_context(actions=["static"], obligations=obligations, executions=executions)
|
|
result = self.make_result(claims=[{
|
|
"id": "tdd", "status": "supported", "anchors": [{
|
|
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
|
}], "execution_ids": ["red"],
|
|
}])
|
|
out = self.api(context=context, result=result)
|
|
self.assertIn("red_green_missing_green", out["reason_codes"])
|
|
self.assertIn("evidence_hash_mismatch", out["reason_codes"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|