test(harness): reorganize regression suites and preserve contract coverage
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""delegation regression tests."""
|
||||
@@ -0,0 +1,207 @@
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from harness.lib import delegation_adoption as adoption
|
||||
|
||||
|
||||
class DelegationAdoptionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch")
|
||||
self.base = Path(self.tmp.name)
|
||||
self.bundle = self.base / "bundle"
|
||||
(self.bundle / "src").mkdir(parents=True)
|
||||
(self.bundle / "evidence").mkdir()
|
||||
(self.bundle / "src/value.py").write_bytes(b"VALUE = 1\n")
|
||||
(self.bundle / "evidence/run.txt").write_bytes(b"run\n")
|
||||
self.files = {"src/value.py": hashlib.sha256((self.bundle / "src/value.py").read_bytes()).hexdigest()}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
self.context = {
|
||||
"schema": "gitea-agent.delegation-evidence-context/v1",
|
||||
"source": {"kind": "captured_worktree", "base_revision": "a" * 40},
|
||||
"stage": "implementation",
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"files": self.files,
|
||||
"allowed_actions": ["static"],
|
||||
"obligations": [{"id": "static", "kind": "static"}],
|
||||
"executions": [],
|
||||
}
|
||||
self.result = {
|
||||
"schema": "gitea-agent.delegation-evidence-result/v1",
|
||||
"source": copy.deepcopy(self.context["source"]),
|
||||
"stage": self.context["stage"],
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"claims": [{"id": "static", "status": "supported", "anchors": [{
|
||||
"kind": "lines", "path": "src/value.py", "start_line": 1,
|
||||
"end_line": 1, "text_sha256": hashlib.sha256(b"VALUE = 1\n").hexdigest(),
|
||||
}], "execution_ids": []}],
|
||||
}
|
||||
self.context_path = self.base / "context.json"
|
||||
self.result_path = self.base / "result.json"
|
||||
self.context_path.write_text(json.dumps(self.context))
|
||||
self.result_path.write_text(json.dumps(self.result))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def declaration(self):
|
||||
return {
|
||||
"mode": "delegated",
|
||||
"bundle_root": str(self.bundle),
|
||||
"context_path": str(self.context_path),
|
||||
"result_path": str(self.result_path),
|
||||
"git_dir": None,
|
||||
}
|
||||
|
||||
def test_valid_delegated_declaration_returns_fresh_pass_result(self):
|
||||
expected_source = copy.deepcopy(self.context["source"])
|
||||
out = adoption.require_adoption(
|
||||
self.declaration(), expected_source=expected_source, expected_stage="implementation"
|
||||
)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
self.assertEqual(out["counts"]["obligations"], 1)
|
||||
|
||||
def test_valid_git_tree_declaration_uses_expected_revision(self):
|
||||
repo = self.base / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "src").mkdir()
|
||||
(repo / "src/value.py").write_bytes(b"VALUE = 1\n")
|
||||
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "fixture"], check=True)
|
||||
revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
||||
context = copy.deepcopy(self.context)
|
||||
context["source"] = {"kind": "git_tree", "revision": revision}
|
||||
result = copy.deepcopy(self.result)
|
||||
result["source"] = copy.deepcopy(context["source"])
|
||||
self.context_path.write_text(json.dumps(context))
|
||||
self.result_path.write_text(json.dumps(result))
|
||||
declaration = self.declaration()
|
||||
declaration["git_dir"] = str(repo / ".git")
|
||||
out = adoption.require_adoption(declaration, expected_source=context["source"], expected_stage="implementation")
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_explicit_none_is_the_only_non_delegated_success(self):
|
||||
self.assertIsNone(adoption.require_adoption(
|
||||
{"mode": "none"}, expected_source=self.context["source"], expected_stage="implementation"
|
||||
))
|
||||
|
||||
def test_missing_declaration_is_rejected_with_sanitized_error(self):
|
||||
with self.assertRaisesRegex(adoption.AdoptionError, "^delegation adoption rejected$"):
|
||||
adoption.require_adoption(None, expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_unsupported_or_extra_declaration_fields_are_rejected(self):
|
||||
for declaration in (
|
||||
{"mode": "worker"},
|
||||
{"mode": "none", "bundle_root": str(self.bundle)},
|
||||
{**self.declaration(), "extra": 1},
|
||||
):
|
||||
with self.subTest(declaration=declaration):
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(declaration, expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_expected_source_and_stage_are_independent_and_exact(self):
|
||||
for source, stage in (
|
||||
({"kind": "captured_worktree", "base_revision": "b" * 40}, "implementation"),
|
||||
(self.context["source"], "review"),
|
||||
):
|
||||
with self.subTest(source=source, stage=stage), self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=source, expected_stage=stage)
|
||||
|
||||
def test_stale_context_result_or_bundle_is_rejected(self):
|
||||
for field, value in (("stage", "review"), ("bundle_sha256", "0" * 64)):
|
||||
context = copy.deepcopy(self.context)
|
||||
context[field] = value
|
||||
self.context_path.write_text(json.dumps(context))
|
||||
with self.subTest(field=field), self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
self.context_path.write_text(json.dumps(self.context))
|
||||
self.result_path.write_text(json.dumps({**self.result, "stage": "review"}))
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
self.result_path.write_text(json.dumps(self.result))
|
||||
(self.bundle / "src/value.py").write_bytes(b"VALUE = 2\n")
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_empty_obligations_or_claims_cannot_be_adopted(self):
|
||||
for context, result in (
|
||||
({**self.context, "obligations": []}, self.result),
|
||||
(self.context, {**self.result, "claims": []}),
|
||||
):
|
||||
self.context_path.write_text(json.dumps(context))
|
||||
self.result_path.write_text(json.dumps(result))
|
||||
with self.subTest(context=context, result=result), self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
self.context_path.write_text(json.dumps(self.context))
|
||||
self.result_path.write_text(json.dumps(self.result))
|
||||
|
||||
def test_block_and_verifier_exception_fail_closed(self):
|
||||
with mock.patch.object(adoption.delegation_evidence, "verify_evidence", return_value={"status": "block"}):
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
with mock.patch.object(adoption.delegation_evidence, "verify_evidence", side_effect=RuntimeError("secret")):
|
||||
with self.assertRaisesRegex(adoption.AdoptionError, "^delegation adoption rejected$"):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_verifier_is_called_again_after_a_prior_pass(self):
|
||||
with mock.patch.object(adoption.delegation_evidence, "verify_evidence", wraps=adoption.delegation_evidence.verify_evidence) as verifier:
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
self.assertEqual(verifier.call_count, 2)
|
||||
|
||||
def test_malformed_expected_inputs_and_paths_fail_closed(self):
|
||||
cases = (
|
||||
({"kind": "captured_worktree", "base_revision": "A" * 40}, "implementation"),
|
||||
(self.context["source"], ""),
|
||||
(self.context["source"], None),
|
||||
)
|
||||
for source, stage in cases:
|
||||
with self.subTest(source=source, stage=stage), self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption({"mode": "none"}, expected_source=source, expected_stage=stage)
|
||||
for key, value in (("bundle_root", str(self.base / ".." / self.base.name / "bundle")),
|
||||
("context_path", "relative.json"), ("result_path", str(self.result_path) + "\x00"),
|
||||
("git_dir", 3)):
|
||||
declaration = self.declaration()
|
||||
declaration[key] = value
|
||||
with self.subTest(key=key), self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(declaration, expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_all_git_declaration_paths_reject_symlinks_at_runtime(self):
|
||||
from harness.tests._support.delegation import RealEvidence
|
||||
evidence = RealEvidence(self.base / 'safe-path-fixture')
|
||||
declaration = evidence.declaration()
|
||||
for field in ('bundle_root', 'context_path', 'result_path', 'git_dir'):
|
||||
alias = self.base / ('alias-' + field)
|
||||
alias.symlink_to(declaration[field], target_is_directory=field in ('bundle_root', 'git_dir'))
|
||||
bad = {**declaration, field: str(alias)}
|
||||
with self.subTest(field=field), self.assertRaisesRegex(adoption.AdoptionError, '^delegation adoption rejected$'):
|
||||
adoption.require_adoption(bad, expected_source=evidence.context['source'], expected_stage='spec-review')
|
||||
alias_parent = self.base / 'alias-parent'
|
||||
alias_parent.symlink_to(evidence.root, target_is_directory=True)
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption({**declaration, 'context_path': str(alias_parent / 'context.json')},
|
||||
expected_source=evidence.context['source'], expected_stage='spec-review')
|
||||
|
||||
def test_duplicate_json_and_missing_files_fail_closed(self):
|
||||
self.context_path.write_text('{"schema":"x","schema":"y"}')
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
self.context_path.write_text(json.dumps(self.context))
|
||||
self.result_path.unlink()
|
||||
with self.assertRaises(adoption.AdoptionError):
|
||||
adoption.require_adoption(self.declaration(), expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import importlib.util
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = SOURCE_ROOT
|
||||
PATH = ROOT / "harness/agents/impl/skills/gitea-issue-implementation/scripts/delivery_adoption.py"
|
||||
spec = importlib.util.spec_from_file_location("delivery_adoption", PATH)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
|
||||
|
||||
class DeliveryConsumerTests(unittest.TestCase):
|
||||
def test_delivery_requires_adoption_then_calls_adapter_once(self):
|
||||
calls = []
|
||||
with patch.object(mod, "require_adoption", return_value={"status": "pass"}) as adopted:
|
||||
result = mod.deliver(lambda: {"adapter": "actual"}, head="a" * 40,
|
||||
declaration={"mode": "delegated"})
|
||||
adopted.assert_called_once_with({"mode": "delegated"}, expected_source={"kind": "git_tree", "revision": "a" * 40}, expected_stage="implementation-delivery")
|
||||
self.assertEqual(result, {"adapter": "actual"})
|
||||
|
||||
def test_rejected_adoption_blocks_adapter(self):
|
||||
calls = []
|
||||
with patch.object(mod, "require_adoption", side_effect=mod.AdoptionError("no")):
|
||||
with self.assertRaises(mod.AdoptionError):
|
||||
mod.deliver(lambda: calls.append("called"), head="a" * 40,
|
||||
declaration={"mode": "delegated"})
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_missing_declaration_blocks_adapter(self):
|
||||
calls = []
|
||||
with self.assertRaises(mod.AdoptionError):
|
||||
mod.deliver(lambda: calls.append("called"), head="a" * 40, declaration=None)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_explicit_none_still_validates_head_and_calls_adapter(self):
|
||||
calls = []
|
||||
with patch.object(mod, "require_adoption", return_value={"status": "none"}):
|
||||
with self.assertRaises(mod.AdoptionError):
|
||||
mod.deliver(lambda: calls.append("called"), head="bad", declaration={"mode": "none"})
|
||||
mod.deliver(lambda: calls.append("called"), head="a" * 40, declaration={"mode": "none"})
|
||||
self.assertEqual(calls, ["called"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,246 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = SOURCE_ROOT
|
||||
SCRATCH = Path('/opt/data/cache/scratch/delegation-adoption-integration')
|
||||
SPEC = ROOT / 'harness/agents/spec/skills/gitea-implementation-spec-review/scripts/review_support.py'
|
||||
REWORK = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/scripts/rework_comment.py'
|
||||
DELIVERY = ROOT / 'harness/agents/impl/skills/gitea-issue-implementation/scripts/delivery_adoption.py'
|
||||
NATIVE = ROOT / 'harness/agents/review/skills/gitea-code-review/scripts/native_review.py'
|
||||
|
||||
def load(path, name):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
support = load(SPEC, 'integration_spec_support')
|
||||
delivery = load(DELIVERY, 'integration_delivery')
|
||||
native = load(NATIVE, 'integration_native')
|
||||
rework = load(REWORK, 'integration_rework')
|
||||
|
||||
from harness.tests._support.delegation import RealEvidence
|
||||
|
||||
class IntegrationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
SCRATCH.mkdir(parents=True, exist_ok=True)
|
||||
self.tmp = tempfile.TemporaryDirectory(dir=SCRATCH)
|
||||
self.ev = RealEvidence(Path(self.tmp.name))
|
||||
from unittest.mock import patch as mock_patch
|
||||
self.registry_patch = mock_patch.object(native._journal, 'root_for',
|
||||
return_value=Path(self.tmp.name) / 'native-review-journals')
|
||||
self.registry_patch.start()
|
||||
self.addCleanup(self.registry_patch.stop)
|
||||
|
||||
def tearDown(self): self.tmp.cleanup()
|
||||
|
||||
def test_spec_real_git_pass_roundtrip_and_blocks_drift(self):
|
||||
self.ev.set_stage('spec-review')
|
||||
result = {'schema': support.SCHEMA, 'target': {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pull': 7, 'head': self.ev.revision},
|
||||
'contract_sha256': hashlib.sha256(b'contract').hexdigest(), 'verdict': 'approve',
|
||||
'criteria': [{'criterion': 'REQ-1', 'contract': 'contract', 'implementation_paths': ['src/a.py'], 'test_paths': ['tests/test_a.py'], 'evidence': 'verified', 'result': 'satisfied', 'reason': 'ok'}],
|
||||
'scope': {'missing': [], 'excess': [], 'spec_conflicts': []}, 'conclusion': 'ok'}
|
||||
body = support.render_review(result, {'src/a.py', 'tests/test_a.py'}, declaration=self.ev.declaration())
|
||||
self.assertEqual(support.parse_review(body, {'src/a.py', 'tests/test_a.py'}), result)
|
||||
self.ev.set_stage('other'); self.ev.write()
|
||||
with self.assertRaises(support.ValidationError): support.render_review(result, {'src/a.py', 'tests/test_a.py'}, declaration=self.ev.declaration())
|
||||
|
||||
def test_spec_cli_delegated_import_without_pythonpath_and_block_is_quiet(self):
|
||||
self.ev.set_stage('spec-review')
|
||||
result = {'schema': support.SCHEMA, 'target': {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pull': 7, 'head': self.ev.revision},
|
||||
'contract_sha256': hashlib.sha256(b'contract').hexdigest(), 'verdict': 'approve',
|
||||
'criteria': [{'criterion': 'REQ-1', 'contract': 'contract', 'implementation_paths': ['src/a.py'], 'test_paths': [], 'evidence': 'verified', 'result': 'satisfied', 'reason': 'ok'}],
|
||||
'scope': {'missing': [], 'excess': [], 'spec_conflicts': []}, 'conclusion': 'ok'}
|
||||
result_path = Path(self.tmp.name) / 'spec-result.json'; paths_path = Path(self.tmp.name) / 'paths.json'; decl_path = Path(self.tmp.name) / 'decl.json'; result_path.write_text(json.dumps(result)); paths_path.write_text(json.dumps(['src/a.py'])); decl_path.write_text(json.dumps(self.ev.declaration()))
|
||||
env = os.environ.copy(); env.pop('PYTHONPATH', None)
|
||||
cmd = [sys.executable, str(SPEC), 'render', '--result', str(result_path), '--allowed-paths', str(paths_path), '--delegation-adoption', str(decl_path)]
|
||||
run = subprocess.run(cmd, cwd=self.tmp.name, env=env, text=True, capture_output=True)
|
||||
self.assertEqual(run.returncode, 0, run.stderr); self.assertIn('implementation-spec-review/v1', run.stdout)
|
||||
self.ev.context['stage'] = 'drift'; self.ev.write()
|
||||
blocked = subprocess.run(cmd, cwd=self.tmp.name, env=env, text=True, capture_output=True)
|
||||
self.assertNotEqual(blocked.returncode, 0); self.assertEqual(blocked.stdout, '')
|
||||
|
||||
def test_delivery_real_pass_returns_adapter_and_blocks_without_write(self):
|
||||
self.ev.set_stage('implementation-delivery')
|
||||
calls = []
|
||||
out = delivery.deliver(lambda: {'receipt': 'actual'}, head=self.ev.revision, declaration=self.ev.declaration())
|
||||
self.assertEqual(out, {'receipt': 'actual'})
|
||||
self.ev.bundle.joinpath('src/a.py').write_text('VALUE = 2\n')
|
||||
with self.assertRaises(delivery.AdoptionError): delivery.deliver(lambda: calls.append(1), head=self.ev.revision, declaration=self.ev.declaration())
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def bound_publication_fixture(self):
|
||||
from harness.tests._support import review as fixture
|
||||
plan, before, after, exact = fixture.publication_fixture()
|
||||
old_head = plan['target']['head']
|
||||
def replace(value):
|
||||
if isinstance(value, str): return self.ev.revision if value == old_head else value
|
||||
if isinstance(value, list): return [replace(x) for x in value]
|
||||
if isinstance(value, dict): return {k: replace(v) for k, v in value.items()}
|
||||
return value
|
||||
plan, before, after, exact = map(replace, (plan, before, after, exact))
|
||||
plan['body'] = native.marker(plan) + '\n- Verdict: `approve`\n검토 완료.\n'
|
||||
exact['body'] = plan['body']
|
||||
for review in before['reviews'] + after['reviews']:
|
||||
if review['id'] == plan['spec_review_id']:
|
||||
original = fixture.publication_fixture()[1]['reviews'][1]['body']
|
||||
sr = support.parse_review(original, {'src/a.py'})
|
||||
sr['target']['head'] = self.ev.revision
|
||||
review['body'] = support.render_review(sr, {'src/a.py'}, declaration={'mode': 'none'})
|
||||
review['commit_id'] = self.ev.revision
|
||||
elif review['id'] == 10:
|
||||
review['body'] = plan['body']
|
||||
exact['commit_id'] = self.ev.revision
|
||||
native.validate_snapshot(plan, before)
|
||||
return fixture, plan, before, after, exact
|
||||
|
||||
def test_review_real_git_adoption_posts_once_then_noops(self):
|
||||
self.ev.set_stage('code-review')
|
||||
fixture, plan, before, after, exact = self.bound_publication_fixture()
|
||||
api = fixture.FakeAPI(plan, before, after, exact)
|
||||
journal = native.canonical_journal_path(plan)
|
||||
first = native.publish(plan, before, api, fixture.FakeLease(), journal, declaration=self.ev.declaration())
|
||||
second = native.publish(plan, before, api, fixture.FakeLease(), journal, declaration=self.ev.declaration())
|
||||
self.assertEqual(first['mode'], 'created')
|
||||
self.assertEqual(second['mode'], 'no_op')
|
||||
self.assertEqual(api.posts, 1)
|
||||
|
||||
def test_review_fresh_adoption_blocks_post_if_bundle_changes_before_create(self):
|
||||
self.ev.set_stage('code-review')
|
||||
fixture, plan, before, after, exact = self.bound_publication_fixture()
|
||||
api = fixture.FakeAPI(plan, before, after, exact)
|
||||
original_snapshot = api.snapshot
|
||||
calls = {'n': 0}
|
||||
def mutating_snapshot(target):
|
||||
calls['n'] += 1
|
||||
out = original_snapshot(target)
|
||||
if calls['n'] == 2:
|
||||
self.ev.bundle.joinpath('src/a.py').write_text('VALUE = 9\n')
|
||||
return out
|
||||
api.snapshot = mutating_snapshot
|
||||
journal = native.canonical_journal_path(plan)
|
||||
with self.assertRaisesRegex(native.ValidationError, 'delegation adoption'):
|
||||
native.publish(plan, before, api, fixture.FakeLease(), journal, declaration=self.ev.declaration())
|
||||
self.assertEqual(calls['n'], 2, 'must reach both provider reads and final adoption check')
|
||||
self.assertTrue((Path(journal) / 'intent.json').exists())
|
||||
self.assertEqual(api.posts, 0)
|
||||
# Conservatively preserve durable intent: restoring evidence cannot trigger POST.
|
||||
self.ev.bundle.joinpath('src/a.py').write_text('VALUE = 1\n')
|
||||
with self.assertRaises(native.UncertainPublication):
|
||||
native.publish(plan, before, api, fixture.FakeLease(), journal, declaration=self.ev.declaration())
|
||||
self.assertEqual(api.posts, 0)
|
||||
|
||||
def spec_result(self):
|
||||
return {'schema': support.SCHEMA,
|
||||
'target': {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pull': 7, 'head': self.ev.revision},
|
||||
'contract_sha256': hashlib.sha256(b'contract').hexdigest(), 'verdict': 'approve',
|
||||
'criteria': [{'criterion': 'REQ-1', 'contract': 'contract', 'implementation_paths': ['src/a.py'],
|
||||
'test_paths': [], 'evidence': 'verified', 'result': 'satisfied', 'reason': 'ok'}],
|
||||
'scope': {'missing': [], 'excess': [], 'spec_conflicts': []}, 'conclusion': 'ok'}
|
||||
|
||||
def rework_args(self, declaration):
|
||||
from types import SimpleNamespace
|
||||
root = Path(self.tmp.name)
|
||||
data = {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pr': 7, 'review': 9,
|
||||
'reviewed_head': self.ev.revision, 'final_head': self.ev.revision,
|
||||
'contract_sha256': hashlib.sha256(b'contract').hexdigest(), 'result': 'rework_delivered',
|
||||
'findings': [{'id': 'F-1', 'status': 'resolved', 'classification': 'valid_fix',
|
||||
'changes': 'fixed', 'verification': 'passed', 'reason': 'ok'}],
|
||||
'local_checks': 'passed', 'runtime_browser': 'passed', 'commit_status': 'success', 'actions_runs': '없음'}
|
||||
(root / 'input.json').write_text(json.dumps(data))
|
||||
path = root / 'declaration.json'
|
||||
if declaration is not None:
|
||||
path.write_text(json.dumps(declaration))
|
||||
return SimpleNamespace(input=str(root / 'input.json'), comment=str(root / 'denied.md'),
|
||||
expected=str(root / 'denied.json'), no_delegated_evidence=False,
|
||||
delegation_adoption=str(path) if declaration is not None else None)
|
||||
|
||||
def test_all_consumers_reject_missing_malformed_and_indeterminate_without_effects(self):
|
||||
for mode in ('missing', 'malformed', 'indeterminate'):
|
||||
self.ev.result['claims'][0]['status'] = 'indeterminate' if mode == 'indeterminate' else 'supported'
|
||||
self.ev.write()
|
||||
declaration = None if mode == 'missing' else {'mode': 'bad'} if mode == 'malformed' else self.ev.declaration()
|
||||
with self.subTest(mode=mode, consumer='spec'):
|
||||
self.ev.set_stage('spec-review')
|
||||
with self.assertRaises(support.ValidationError):
|
||||
support.render_review(self.spec_result(), {'src/a.py'}, declaration=declaration)
|
||||
self.ev.set_stage('implementation-delivery')
|
||||
calls = []
|
||||
with self.subTest(mode=mode, consumer='delivery'):
|
||||
with self.assertRaises(delivery.AdoptionError):
|
||||
delivery.deliver(lambda: calls.append(1), head=self.ev.revision, declaration=declaration)
|
||||
self.assertEqual(calls, [])
|
||||
with self.subTest(mode=mode, consumer='rework'):
|
||||
args = self.rework_args(declaration)
|
||||
with self.assertRaises(Exception):
|
||||
rework.do_render(args)
|
||||
self.assertFalse(Path(args.comment).exists())
|
||||
self.assertFalse(Path(args.expected).exists())
|
||||
self.ev.set_stage('code-review')
|
||||
fixture, plan, before, after, exact = self.bound_publication_fixture()
|
||||
api = fixture.FakeAPI(plan, before, after, exact)
|
||||
journal = native.canonical_journal_path(plan)
|
||||
with self.subTest(mode=mode, consumer='review'):
|
||||
with self.assertRaises(native.ValidationError):
|
||||
native.publish(plan, before, api, fixture.FakeLease(), journal, declaration=declaration)
|
||||
self.assertEqual(api.posts, 0)
|
||||
self.assertEqual(api.events, [])
|
||||
self.assertFalse(journal.exists())
|
||||
|
||||
def test_rework_explicit_none_does_not_bypass_checker_failure(self):
|
||||
from unittest.mock import patch
|
||||
from harness.lib.delegation_adoption import AdoptionError
|
||||
args = self.rework_args(None)
|
||||
args.no_delegated_evidence = True
|
||||
with patch('harness.lib.delegation_adoption.require_adoption', side_effect=AdoptionError('rejected')):
|
||||
try:
|
||||
rework.do_render(args)
|
||||
except rework.Bad:
|
||||
pass
|
||||
else:
|
||||
self.fail('explicit none must still invoke common checker')
|
||||
self.assertFalse(Path(args.comment).exists())
|
||||
self.assertFalse(Path(args.expected).exists())
|
||||
|
||||
def test_spec_cli_rejects_symlink_declaration(self):
|
||||
self.ev.set_stage('spec-review')
|
||||
root = Path(self.tmp.name)
|
||||
(root / 'spec.json').write_text(json.dumps(self.spec_result()))
|
||||
(root / 'paths.json').write_text(json.dumps(['src/a.py']))
|
||||
(root / 'real-decl.json').write_text(json.dumps(self.ev.declaration()))
|
||||
(root / 'alias.json').symlink_to(root / 'real-decl.json')
|
||||
env = os.environ.copy(); env.pop('PYTHONPATH', None)
|
||||
cmd = [sys.executable, str(SPEC), 'render', '--result', str(root / 'spec.json'),
|
||||
'--allowed-paths', str(root / 'paths.json'), '--delegation-adoption', str(root / 'alias.json')]
|
||||
proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True, timeout=10)
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
self.assertEqual(proc.stdout, '')
|
||||
|
||||
def test_cli_import_and_safe_declaration_reader_without_pythonpath(self):
|
||||
self.ev.set_stage('implementation-delivery')
|
||||
inp = Path(self.tmp.name) / 'rework.json'; comment = Path(self.tmp.name) / 'comment.md'; expected = Path(self.tmp.name) / 'expected.json'
|
||||
data = {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pr': 7, 'review': 9, 'reviewed_head': self.ev.revision, 'final_head': self.ev.revision,
|
||||
'contract_sha256': hashlib.sha256(b'contract').hexdigest(), 'result': 'rework_delivered',
|
||||
'findings': [{'id': 'F-1', 'status': 'resolved', 'classification': 'valid_fix', 'changes': 'fixed', 'verification': 'passed', 'reason': 'ok'}],
|
||||
'local_checks': 'passed', 'runtime_browser': 'passed', 'commit_status': 'success', 'actions_runs': '없음'}
|
||||
inp.write_text(json.dumps(data)); decl = Path(self.tmp.name) / 'decl.json'; decl.write_text(json.dumps(self.ev.declaration()))
|
||||
env = os.environ.copy(); env.pop('PYTHONPATH', None)
|
||||
cmd = [sys.executable, str(REWORK), 'render', '--input', str(inp), '--comment', str(comment), '--expected', str(expected), '--delegation-adoption', str(decl)]
|
||||
run = subprocess.run(cmd, cwd=self.tmp.name, env=env, text=True, capture_output=True)
|
||||
self.assertEqual(run.returncode, 0, run.stderr)
|
||||
self.assertTrue(comment.exists() and expected.exists())
|
||||
os.symlink(self.ev.result_path, Path(self.tmp.name) / 'evil.json')
|
||||
bad = subprocess.run([sys.executable, str(REWORK), 'render', '--input', str(inp), '--comment', str(comment), '--expected', str(expected), '--delegation-adoption', str(self.tmp.name) + '/evil.json'], cwd=self.tmp.name, env=env, text=True, capture_output=True)
|
||||
self.assertNotEqual(bad.returncode, 0); self.assertEqual(bad.stdout, '')
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,322 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = SOURCE_ROOT
|
||||
SCHEMA_C = "gitea-agent.delegation-evidence-context/v1"
|
||||
SCHEMA_R = "gitea-agent.delegation-evidence-result/v1"
|
||||
|
||||
|
||||
class DelegationEvidenceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.tmp.name) / "bundle"
|
||||
self.root.mkdir()
|
||||
(self.root / "tests").mkdir()
|
||||
(self.root / "src").mkdir()
|
||||
(self.root / "tests/sample_test.py").write_bytes(b"def test_sample():\n assert True\n")
|
||||
(self.root / "src/app.py").write_bytes(b"VALUE = 1\n")
|
||||
self.files = {p: hashlib.sha256((self.root / p).read_bytes()).hexdigest()
|
||||
for p in ("src/app.py", "tests/sample_test.py")}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
(self.root / "evidence").mkdir()
|
||||
(self.root / "evidence/run.txt").write_bytes(b"exit 0\n")
|
||||
evidence_sha = hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest()
|
||||
self.context = {
|
||||
"schema": SCHEMA_C,
|
||||
"source": {"kind": "captured_worktree", "base_revision": "a" * 40},
|
||||
"stage": "implementation",
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"files": self.files,
|
||||
"allowed_actions": ["static", "execute"],
|
||||
"obligations": [
|
||||
{"id": "static-source", "kind": "static"},
|
||||
{"id": "run-sample", "kind": "execution"},
|
||||
],
|
||||
"executions": [{"id": "exec-1", "test_id": "", "bundle_sha256": self.bundle_sha,
|
||||
"test_file_sha256": "0" * 64, "argv": ["python3", "-c", "pass"],
|
||||
"exit_code": 0, "failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": evidence_sha, "origin": "direct"}],
|
||||
}
|
||||
self.result = {
|
||||
"schema": SCHEMA_R,
|
||||
"source": copy.deepcopy(self.context["source"]),
|
||||
"stage": self.context["stage"],
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"claims": [
|
||||
{"id": "static-source", "status": "supported", "anchors": [
|
||||
{"kind": "lines", "path": "src/app.py", "start_line": 1,
|
||||
"end_line": 1, "text_sha256": self.line_hash("src/app.py", 1, 1)}
|
||||
], "execution_ids": []},
|
||||
{"id": "run-sample", "status": "supported", "anchors": [], "execution_ids": ["exec-1"]},
|
||||
],
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def line_hash(self, path, start, end):
|
||||
lines = (self.root / path).read_bytes().splitlines(keepends=True)
|
||||
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
||||
|
||||
def make_git_repo(self, files=None, name="repo"):
|
||||
repo = Path(self.tmp.name) / name
|
||||
repo.mkdir()
|
||||
for path in (files or self.files):
|
||||
target = repo / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((self.root / path).read_bytes())
|
||||
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "fixture"], check=True)
|
||||
revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
||||
return repo / ".git", revision
|
||||
|
||||
def git_context(self, git_revision):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["source"] = {"kind": "git_tree", "revision": git_revision}
|
||||
result = copy.deepcopy(self.result)
|
||||
result["source"] = copy.deepcopy(context["source"])
|
||||
return context, result
|
||||
|
||||
def test_worker_source_stage_bundle_mismatches_are_detected_blocks(self):
|
||||
for key, value in [('source', {'kind': 'captured_worktree', 'base_revision': 'b' * 40}),
|
||||
('stage', 'stale-stage'), ('bundle_sha256', '0' * 64)]:
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad[key] = value
|
||||
with self.subTest(field=key):
|
||||
try:
|
||||
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
except evidence.EvidenceError:
|
||||
self.fail('syntactically valid provenance disagreement must return BLOCK')
|
||||
self.assertEqual(out['status'], 'block')
|
||||
self.assertIn('result_' + key + '_mismatch', out['reason_codes'])
|
||||
|
||||
def test_valid_static_lines_and_bounded_counts(self):
|
||||
out = evidence.verify_evidence(str(self.root), self.context, self.result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
self.assertEqual(out["reason_codes"], [])
|
||||
self.assertEqual(out["counts"], {"files": 2, "obligations": 2, "claims": 2, "anchors": 1, "executions": 1})
|
||||
|
||||
def test_wrong_line_hash_blocks(self):
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
||||
out = evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_indeterminate_claim_blocks(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["status"] = "indeterminate"
|
||||
out = evidence.verify_evidence(str(self.root), self.context, result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("indeterminate_claim", out["reason_codes"])
|
||||
|
||||
def test_test_anchor_requires_real_python_function(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"] = [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}]
|
||||
out = evidence.verify_evidence(str(self.root), self.context, result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
result["claims"][0]["anchors"][0]["symbol"] = "missing"
|
||||
self.assertIn("test_anchor_missing", evidence.verify_evidence(str(self.root), self.context, result)["reason_codes"])
|
||||
|
||||
def test_claim_ids_must_exactly_match_obligations(self):
|
||||
bad = copy.deepcopy(self.result)
|
||||
bad["claims"][1]["id"] = "extra"
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), self.context, bad)
|
||||
|
||||
def test_captured_worktree_absence_is_indeterminate(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"] = {"src/app.py": self.files["src/app.py"]}
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
||||
"bundle_sha256": context["bundle_sha256"], "claims": [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}], "execution_ids": []}]}
|
||||
out = evidence.verify_evidence(str(self.root), context, result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("absence_indeterminate", out["reason_codes"])
|
||||
|
||||
def test_git_tree_matching_frozen_bundle_passes(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_git_tree_altered_frozen_snapshot_blocks(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
(self.root / "src/app.py").write_bytes(b"VALUE = 2\n")
|
||||
context, result = self.git_context(revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_git_tree_source_mismatch_blocks(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
repo = Path(self.tmp.name) / "other"
|
||||
repo.mkdir()
|
||||
for path in self.files:
|
||||
target = repo / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((self.root / path).read_bytes())
|
||||
(repo / "src/app.py").write_bytes(b"VALUE = 9\n")
|
||||
subprocess.run(["/usr/bin/git", "init", "-q", str(repo)], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "test@example.invalid"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(repo), "commit", "-q", "-m", "other"], check=True)
|
||||
other_revision = subprocess.check_output(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
|
||||
context, result = self.git_context(other_revision)
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("source_hash_mismatch", out["reason_codes"])
|
||||
context_path = self.root / "context.json"
|
||||
result_path = self.root / "result.json"
|
||||
context_path.write_text(json.dumps(context))
|
||||
result_path.write_text(json.dumps(result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(context_path),
|
||||
"--result", str(result_path), "--git-dir", str(git_dir)],
|
||||
text=True, capture_output=True)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertEqual(json.loads(proc.stdout)["status"], "block")
|
||||
|
||||
def test_git_tree_present_path_cannot_be_claimed_absent(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/sample_test.py"}],
|
||||
"execution_ids": []}]
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("absence_not_proven", out["reason_codes"])
|
||||
|
||||
def test_git_tree_genuine_missing_path_is_authoritative(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
||||
"execution_ids": []}]
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_git_tree_absence_uses_original_directory_fd_after_replacement(self):
|
||||
git_dir, revision = self.make_git_repo()
|
||||
replacement, _ = self.make_git_repo(name="replacement")
|
||||
replacement_repo = replacement.parent
|
||||
(replacement_repo / "tests/missing.py").write_bytes(b"present\n")
|
||||
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "add", "."], check=True)
|
||||
subprocess.run(["/usr/bin/git", "-C", str(replacement_repo), "commit", "-q", "-m", "replacement"], check=True)
|
||||
context, result = self.git_context(revision)
|
||||
context["obligations"] = [{"id": "absent", "kind": "static"}]
|
||||
result["claims"] = [{"id": "absent", "status": "supported",
|
||||
"anchors": [{"kind": "absence", "path": "tests/missing.py"}],
|
||||
"execution_ids": []}]
|
||||
original_git = evidence._git
|
||||
moved = {"done": False}
|
||||
|
||||
def replacing_git(git_path, args, fd):
|
||||
if args[0] == "ls-tree" and args[-1] == "tests/missing.py" and not moved["done"]:
|
||||
moved["done"] = True
|
||||
backup = Path(self.tmp.name) / "original.git"
|
||||
os.rename(git_dir, backup)
|
||||
os.rename(replacement, git_dir)
|
||||
return original_git(git_path, args, fd)
|
||||
|
||||
evidence._git = replacing_git
|
||||
try:
|
||||
out = evidence.verify_evidence(str(self.root), context, result, str(git_dir))
|
||||
finally:
|
||||
evidence._git = original_git
|
||||
self.assertEqual(out["status"], "pass")
|
||||
|
||||
def test_inherited_execution_is_allowed_for_static_only_worker_but_direct_is_not(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
context["executions"][0]["origin"] = "inherited"
|
||||
out = evidence.verify_evidence(str(self.root), context, self.result)
|
||||
self.assertEqual(out["status"], "pass")
|
||||
context["executions"][0]["origin"] = "direct"
|
||||
out = evidence.verify_evidence(str(self.root), context, self.result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unauthorized_execute", out["reason_codes"])
|
||||
|
||||
def test_execution_origin_is_required_and_closed(self):
|
||||
for origin in ("worker", None):
|
||||
context = copy.deepcopy(self.context)
|
||||
if origin is None:
|
||||
del context["executions"][0]["origin"]
|
||||
else:
|
||||
context["executions"][0]["origin"] = origin
|
||||
with self.subTest(origin=origin), self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), context, self.result)
|
||||
|
||||
def test_inherited_red_green_is_accepted_without_execute_permission(self):
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
context["obligations"] = [{"id": "tdd", "kind": "red_green"}]
|
||||
test_sha = self.files["tests/sample_test.py"]
|
||||
context["executions"] = [
|
||||
{"id": "red", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": "b" * 64,
|
||||
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 1,
|
||||
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
||||
"origin": "inherited"},
|
||||
{"id": "green", "test_id": "tests/sample_test.py::test_sample", "bundle_sha256": self.bundle_sha,
|
||||
"test_file_sha256": test_sha, "argv": ["python3", "-m", "unittest"], "exit_code": 0,
|
||||
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": hashlib.sha256((self.root / "evidence/run.txt").read_bytes()).hexdigest(),
|
||||
"origin": "inherited"},
|
||||
]
|
||||
result = {"schema": SCHEMA_R, "source": context["source"], "stage": context["stage"],
|
||||
"bundle_sha256": self.bundle_sha, "claims": [{"id": "tdd", "status": "supported",
|
||||
"anchors": [{"kind": "test", "path": "tests/sample_test.py", "symbol": "test_sample"}],
|
||||
"execution_ids": ["red", "green"]}]}
|
||||
self.assertEqual(evidence.verify_evidence(str(self.root), context, result)["status"], "pass")
|
||||
|
||||
def test_rejected_input_and_unknown_cli_are_bounded(self):
|
||||
bad = copy.deepcopy(self.context)
|
||||
bad["files"]["../secret"] = "0" * 64
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence.verify_evidence(str(self.root), bad, self.result)
|
||||
c = self.root / "context.json"; r = self.root / "result.json"
|
||||
c.write_text(json.dumps(self.context)); r.write_text(json.dumps(self.result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r), "--unknown=secret"],
|
||||
text=True, capture_output=True)
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertNotIn("secret", proc.stdout + proc.stderr)
|
||||
self.assertEqual(json.loads(proc.stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
||||
|
||||
def test_duplicate_json_keys_and_symlink_are_rejected(self):
|
||||
c = self.root / "context.json"; r = self.root / "result.json"
|
||||
c.write_text('{"schema":"x","schema":"secret"}')
|
||||
r.write_text(json.dumps(self.result))
|
||||
proc = subprocess.run([sys.executable, str(ROOT / "harness/lib/delegation_evidence.py"),
|
||||
"--bundle-root", str(self.root), "--context", str(c), "--result", str(r)], capture_output=True, text=True)
|
||||
self.assertEqual(proc.returncode, 2); self.assertNotIn("secret", proc.stdout + proc.stderr)
|
||||
real = self.root / "real.json"; real.write_text(json.dumps(self.context))
|
||||
alias = self.root / "alias.json"; alias.symlink_to(real)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence._json_file(str(alias))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,383 @@
|
||||
"""Independent safety/regression coverage for delegation evidence acceptance."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = SOURCE_ROOT
|
||||
HELPER = ROOT / "harness" / "lib" / "delegation_evidence.py"
|
||||
CONTEXT_SCHEMA = "gitea-agent.delegation-evidence-context/v1"
|
||||
RESULT_SCHEMA = "gitea-agent.delegation-evidence-result/v1"
|
||||
REVISION = "a" * 40
|
||||
|
||||
|
||||
class DelegationEvidenceSafetyTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch")
|
||||
self.base = Path(self.tmp.name)
|
||||
self.bundle = self.base / "bundle"
|
||||
(self.bundle / "tests").mkdir(parents=True)
|
||||
(self.bundle / "src").mkdir()
|
||||
(self.bundle / "evidence").mkdir()
|
||||
(self.bundle / "tests/example.py").write_bytes(
|
||||
b"def test_fn():\n assert True\n"
|
||||
)
|
||||
(self.bundle / "src/value.py").write_bytes(b"VALUE = 1\n")
|
||||
(self.bundle / "evidence/run.txt").write_bytes(b"recorded output\n")
|
||||
self.files = {
|
||||
path: hashlib.sha256((self.bundle / path).read_bytes()).hexdigest()
|
||||
for path in ("src/value.py", "tests/example.py")
|
||||
}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
self.evidence_sha = hashlib.sha256(
|
||||
(self.bundle / "evidence/run.txt").read_bytes()
|
||||
).hexdigest()
|
||||
self.context = self.make_context()
|
||||
self.result = self.make_result()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def make_context(self, *, actions=None, obligations=None, executions=None):
|
||||
return {
|
||||
"schema": CONTEXT_SCHEMA,
|
||||
"source": {"kind": "captured_worktree", "base_revision": REVISION},
|
||||
"stage": "implementation",
|
||||
"bundle_sha256": self.bundle_sha,
|
||||
"files": copy.deepcopy(self.files),
|
||||
"allowed_actions": ["static", "execute"] if actions is None else actions,
|
||||
"obligations": [
|
||||
{"id": "static-src", "kind": "static"},
|
||||
{"id": "run-test", "kind": "execution"},
|
||||
] if obligations is None else obligations,
|
||||
"executions": [{
|
||||
"id": "exec-1", "origin": "direct", "test_id": "",
|
||||
"bundle_sha256": self.bundle_sha, "test_file_sha256": "0" * 64,
|
||||
"argv": ["python3", "-B", "-c", "pass"], "exit_code": 0,
|
||||
"failure_kind": "none", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": self.evidence_sha,
|
||||
}] if executions is None else executions,
|
||||
}
|
||||
|
||||
def make_result(self, *, source=None, stage=None, bundle_sha=None, claims=None):
|
||||
return {
|
||||
"schema": RESULT_SCHEMA,
|
||||
"source": copy.deepcopy(self.context["source"] if source is None else source),
|
||||
"stage": self.context["stage"] if stage is None else stage,
|
||||
"bundle_sha256": self.bundle_sha if bundle_sha is None else bundle_sha,
|
||||
"claims": [
|
||||
{"id": "static-src", "status": "supported", "anchors": [{
|
||||
"kind": "lines", "path": "src/value.py", "start_line": 1,
|
||||
"end_line": 1, "text_sha256": self.line_hash("src/value.py", 1, 1),
|
||||
}], "execution_ids": []},
|
||||
{"id": "run-test", "status": "supported", "anchors": [],
|
||||
"execution_ids": ["exec-1"]},
|
||||
] if claims is None else claims,
|
||||
}
|
||||
|
||||
def line_hash(self, path, start, end):
|
||||
lines = (self.bundle / path).read_bytes().splitlines(keepends=True)
|
||||
return hashlib.sha256(b"".join(lines[start - 1:end])).hexdigest()
|
||||
|
||||
def api(self, context=None, result=None, root=None):
|
||||
return evidence.verify_evidence(
|
||||
str(self.bundle if root is None else root),
|
||||
self.context if context is None else context,
|
||||
self.result if result is None else result,
|
||||
)
|
||||
|
||||
def cli(self, context_text=None, result=None, *, root=None, context_path=None,
|
||||
result_path=None, extra=None, timeout=3):
|
||||
root = self.bundle if root is None else Path(root)
|
||||
context_path = self.base / "context.json" if context_path is None else Path(context_path)
|
||||
result_path = self.base / "result.json" if result_path is None else Path(result_path)
|
||||
if context_text is None:
|
||||
context_path.write_text(json.dumps(self.context), encoding="utf-8")
|
||||
elif isinstance(context_text, bytes):
|
||||
context_path.write_bytes(context_text)
|
||||
else:
|
||||
context_path.write_text(context_text, encoding="utf-8")
|
||||
if result is not None:
|
||||
result_path.write_text(json.dumps(result), encoding="utf-8")
|
||||
else:
|
||||
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
||||
command = [sys.executable, "-B", str(HELPER), "--bundle-root", str(root),
|
||||
"--context", str(context_path), "--result", str(result_path)]
|
||||
if extra:
|
||||
command.extend(extra)
|
||||
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
def assert_rejected(self, value, *, field=None):
|
||||
bad = copy.deepcopy(self.context)
|
||||
if field is None:
|
||||
bad = value
|
||||
else:
|
||||
target = bad
|
||||
for part in field[:-1]:
|
||||
target = target[part]
|
||||
target[field[-1]] = value
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=bad)
|
||||
|
||||
def test_real_paths_reject_root_file_ancestor_and_canonical_aliases(self):
|
||||
alias = self.base / "bundle-alias"
|
||||
alias.symlink_to(self.bundle, target_is_directory=True)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(root=alias)
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(root=self.base / "missing" / ".." / "bundle")
|
||||
outside = self.base / "outside.txt"
|
||||
outside.write_bytes(b"outside")
|
||||
(self.bundle / "src" / "alias.py").symlink_to(outside)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/alias.py"] = hashlib.sha256(outside.read_bytes()).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_json_input_symlink_fifo_and_source_fifo_are_rejected_without_hang(self):
|
||||
real_context = self.base / "real-context.json"
|
||||
real_context.write_text(json.dumps(self.context), encoding="utf-8")
|
||||
alias = self.base / "context.json"
|
||||
alias.symlink_to(real_context)
|
||||
result_path = self.base / "result.json"
|
||||
result_path.write_text(json.dumps(self.result), encoding="utf-8")
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
evidence._json_file(str(alias))
|
||||
|
||||
fifo = self.base / "context-fifo"
|
||||
os.mkfifo(fifo)
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-B", str(HELPER), "--bundle-root", str(self.bundle),
|
||||
"--context", str(fifo), "--result", str(result_path)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
stdout, stderr = proc.communicate(timeout=3)
|
||||
self.assertEqual(proc.returncode, 2, (stdout, stderr))
|
||||
self.assertEqual(json.loads(stdout), {"status": "block", "reason_codes": ["rejected_input"], "counts": {}})
|
||||
|
||||
source_fifo = self.bundle / "src" / "fifo.py"
|
||||
os.mkfifo(source_fifo)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/fifo.py"] = "0" * 64
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_oversized_source_and_json_are_bounded(self):
|
||||
huge = self.bundle / "src" / "huge.py"
|
||||
huge.write_bytes(b"x" * (1024 * 1024 + 1))
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["src/huge.py"] = hashlib.sha256(huge.read_bytes()).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
proc = self.cli(context_text=b"{" + b"\"x\":\"" + b"a" * (1024 * 1024) + b"\"}")
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertEqual(json.loads(proc.stdout)["reason_codes"], ["rejected_input"])
|
||||
|
||||
def test_duplicate_unknown_and_wrong_typed_json_fields_are_rejected(self):
|
||||
cases = [
|
||||
b'{"schema":"x","schema":"y"}',
|
||||
json.dumps({**self.context, "unknown": 1}).encode(),
|
||||
]
|
||||
for payload in cases:
|
||||
with self.subTest(case=payload[:20]):
|
||||
proc = self.cli(context_text=payload)
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
for path, value in [
|
||||
(("stage",), 1), (("source", "revision"), True),
|
||||
(("files",), []), (("allowed_actions",), "static"),
|
||||
(("obligations",), {}), (("executions",), "none"),
|
||||
]:
|
||||
context = copy.deepcopy(self.context)
|
||||
target = context
|
||||
for part in path[:-1]:
|
||||
target = target[part]
|
||||
if path == ("source", "revision"):
|
||||
context["source"] = {"kind": "git_tree", "revision": value}
|
||||
else:
|
||||
target[path[-1]] = value
|
||||
with self.subTest(path=path):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_malformed_digests_revisions_and_boolean_integer_fields_are_rejected(self):
|
||||
cases = [
|
||||
(("bundle_sha256",), "not-a-digest"),
|
||||
(("source", "base_revision"), "b" * 39),
|
||||
(("files", "src/value.py"), "A" * 64),
|
||||
]
|
||||
for path, value in cases:
|
||||
context = copy.deepcopy(self.context)
|
||||
if path[0] == "files":
|
||||
context[path[0]][path[1]] = value
|
||||
elif len(path) == 2:
|
||||
context[path[0]][path[1]] = value
|
||||
else:
|
||||
context[path[0]] = value
|
||||
with self.subTest(path=path):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["executions"][0]["exit_code"] = True
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_duplicate_missing_and_extra_claim_or_obligation_ids_block_input(self):
|
||||
cases = []
|
||||
duplicate_claim = copy.deepcopy(self.result)
|
||||
duplicate_claim["claims"][1]["id"] = duplicate_claim["claims"][0]["id"]
|
||||
cases.append(duplicate_claim)
|
||||
missing_claim = copy.deepcopy(self.result)
|
||||
missing_claim["claims"] = missing_claim["claims"][:1]
|
||||
cases.append(missing_claim)
|
||||
extra_claim = copy.deepcopy(self.result)
|
||||
extra_claim["claims"].append({"id": "extra", "status": "supported", "anchors": [], "execution_ids": []})
|
||||
cases.append(extra_claim)
|
||||
for result in cases:
|
||||
with self.subTest(ids=[claim["id"] for claim in result["claims"]]):
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(result=result)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["obligations"].append({"id": "static-src", "kind": "static"})
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(context=context)
|
||||
|
||||
def test_stale_source_stage_and_bundle_claims_block(self):
|
||||
stale_context = copy.deepcopy(self.context)
|
||||
stale_context["stage"] = "review"
|
||||
self.assertIn("result_stage_mismatch", self.api(context=stale_context)["reason_codes"])
|
||||
for field, value in [("stage", "review"), ("bundle_sha256", "f" * 64),
|
||||
("source", {"kind": "captured_worktree", "base_revision": "b" * 40})]:
|
||||
stale_result = copy.deepcopy(self.result)
|
||||
stale_result[field] = value
|
||||
with self.subTest(field=field):
|
||||
out = self.api(result=stale_result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("result_" + field + "_mismatch", out["reason_codes"])
|
||||
|
||||
def test_line_anchor_rejects_zero_negative_reversed_out_of_range_and_wrong_hash(self):
|
||||
for start, end in [(0, 1), (-1, 1), (2, 1), (1, 99)]:
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"][0].update(start_line=start, end_line=end)
|
||||
with self.subTest(start=start, end=end):
|
||||
if end == 99:
|
||||
out = self.api(result=result)
|
||||
self.assertIn("anchor_range_invalid", out["reason_codes"])
|
||||
else:
|
||||
with self.assertRaises(evidence.EvidenceError):
|
||||
self.api(result=result)
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][0]["anchors"][0]["text_sha256"] = "0" * 64
|
||||
out = self.api(result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("anchor_hash_mismatch", out["reason_codes"])
|
||||
|
||||
def test_ast_anchor_rejects_nested_local_dynamic_and_syntax_error_sources(self):
|
||||
variants = {
|
||||
"nested": b"def outer():\n def test_fn():\n pass\n",
|
||||
"dynamic": b"globals()['test_fn'] = lambda: None\n",
|
||||
"syntax": b"def test_fn(:\n",
|
||||
}
|
||||
for name, source in variants.items():
|
||||
path = self.bundle / "tests/example.py"
|
||||
path.write_bytes(source)
|
||||
context = copy.deepcopy(self.context)
|
||||
context["files"]["tests/example.py"] = hashlib.sha256(source).hexdigest()
|
||||
context["bundle_sha256"] = self.digest(context["files"])
|
||||
result = copy.deepcopy(self.result)
|
||||
result["bundle_sha256"] = context["bundle_sha256"]
|
||||
result["claims"][0]["anchors"] = [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}]
|
||||
with self.subTest(source=name):
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("test_anchor_missing", out["reason_codes"])
|
||||
|
||||
def test_unknown_execution_reference_and_authorization_origin_are_blocked(self):
|
||||
result = copy.deepcopy(self.result)
|
||||
result["claims"][1]["execution_ids"] = ["missing"]
|
||||
out = self.api(result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unknown_execution_id", out["reason_codes"])
|
||||
context = copy.deepcopy(self.context)
|
||||
context["allowed_actions"] = ["static"]
|
||||
out = self.api(context=context)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("unauthorized_execute", out["reason_codes"])
|
||||
context["executions"][0]["origin"] = "inherited"
|
||||
self.assertEqual(self.api(context=context)["status"], "pass")
|
||||
|
||||
def test_red_green_rejects_import_environment_timeout_and_mismatch_records(self):
|
||||
test_sha = self.files["tests/example.py"]
|
||||
obligations = [{"id": "tdd", "kind": "red_green"}]
|
||||
def execution(eid, exit_code, failure_kind, bundle, test_file=test_sha):
|
||||
return {
|
||||
"id": eid, "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
||||
"bundle_sha256": bundle, "test_file_sha256": test_file,
|
||||
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": exit_code,
|
||||
"failure_kind": failure_kind, "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": self.evidence_sha,
|
||||
}
|
||||
for failure_kind in ("import", "environment", "timeout"):
|
||||
context = self.make_context(actions=["static"], obligations=obligations,
|
||||
executions=[execution("red", 1, failure_kind, "b" * 64),
|
||||
execution("green", 0, "none", self.bundle_sha)])
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red", "green"],
|
||||
}])
|
||||
with self.subTest(failure_kind=failure_kind):
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertEqual(out["status"], "block")
|
||||
self.assertIn("red_green_missing_red", out["reason_codes"])
|
||||
context = self.make_context(actions=["static"], obligations=obligations,
|
||||
executions=[execution("red", 1, "assertion", "b" * 64, "c" * 64),
|
||||
execution("green", 0, "none", self.bundle_sha)])
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red", "green"],
|
||||
}])
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertIn("red_green_test_scope_mismatch", out["reason_codes"])
|
||||
|
||||
def test_missing_green_and_evidence_hash_mismatch_block(self):
|
||||
obligations = [{"id": "tdd", "kind": "red_green"}]
|
||||
test_sha = self.files["tests/example.py"]
|
||||
executions = [{
|
||||
"id": "red", "origin": "inherited", "test_id": "tests/example.py::test_fn",
|
||||
"bundle_sha256": "b" * 64, "test_file_sha256": test_sha,
|
||||
"argv": ["python3", "-B", "-m", "unittest"], "exit_code": 1,
|
||||
"failure_kind": "assertion", "evidence_path": "evidence/run.txt",
|
||||
"evidence_sha256": "0" * 64,
|
||||
}]
|
||||
context = self.make_context(actions=["static"], obligations=obligations, executions=executions)
|
||||
result = self.make_result(claims=[{
|
||||
"id": "tdd", "status": "supported", "anchors": [{
|
||||
"kind": "test", "path": "tests/example.py", "symbol": "test_fn",
|
||||
}], "execution_ids": ["red"],
|
||||
}])
|
||||
out = self.api(context=context, result=result)
|
||||
self.assertIn("red_green_missing_green", out["reason_codes"])
|
||||
self.assertIn("evidence_hash_mismatch", out["reason_codes"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user