[verified] Add executable reviewer evidence manifest recipe

2026-10-12 06:49:44 +00:00
parent 4acc28563c
commit d4db3f6a8e
2 changed files with 137 additions and 0 deletions
@@ -0,0 +1,110 @@
+"""Reviewer recipe contracts; no live provider or publication calls."""
+import json
+import shlex
+import stat
+import string
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+from harness.tests.impl import test_evidence_manifest as fixtures
+
+ROOT = Path(__file__).resolve().parents[3]
+REFERENCE = ROOT / "harness/agents/review/skills/gitea-code-review/references/native-review.md"
+HELPER_RELATIVE = "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
+HEADING = "## Reviewer verification evidence recipe"
+
+
+class NativeEvidenceRecipeTests(unittest.TestCase):
+    def recipe(self):
+        text = REFERENCE.read_text(encoding="utf-8")
+        self.assertTrue(HEADING in text, "reviewer evidence-manifest recipe is missing")
+        return text.split(HEADING, 1)[1].split("\n## ", 1)[0]
+
+    def test_current_recipe_preserves_canonical_path_and_archive_contracts(self):
+        section = self.recipe()
+        for required in (HELPER_RELATIVE, "verification_root.py prepare", "evidence-manifest/v3",
+                         "validate-archived-v2", "read-only", "run-relative", "clean checkout",
+                         "no source/evidence exclusions", "0700", "0600"):
+            self.assertIn(required, section)
+        block = section.split("```sh\n", 1)[1].split("```", 1)[0]
+        lines = block.replace("\\\n", "").splitlines()
+        commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")]
+        self.assertEqual(len(commands), 2)
+        self.assertEqual([command[:4] for command in commands],
+                         [["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "run"],
+                          ["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "validate"]])
+        for command in commands:
+            self.assertEqual(command[command.index("--contract-path") + 1],
+                             "$HARNESS_ROOT/harness/operations/repository-cache.md")
+            self.assertEqual(command[command.index("--repo-root") + 1], "$REPO_ROOT")
+            self.assertEqual(command[command.index("--artifact-root") + 1], "$VERIFICATION_RUN_ROOT")
+        self.assertEqual(commands[0][commands[0].index("--plan") + 1], "evidence-plan.json")
+        self.assertEqual(commands[0][commands[0].index("--output") + 1], "evidence-manifest.json")
+        self.assertEqual(commands[1][commands[1].index("--manifest") + 1], "evidence-manifest.json")
+
+    def test_documented_commands_run_validate_and_reject_absolute_artifact_paths(self):
+        section = self.recipe()
+        fixture = fixtures.EvidenceManifestTests()
+        try:
+            fixture.setUp()
+            # Install exact current helper bytes into an isolated canonical-shaped
+            # synthetic harness. No fork, wrapper, live lease or provider is used.
+            source = ROOT / HELPER_RELATIVE
+            helper = fixture.root / HELPER_RELATIVE
+            helper.parent.mkdir(parents=True)
+            helper.write_bytes(source.read_bytes())
+            self.assertEqual(helper.read_bytes(), source.read_bytes())
+            fixture.plan().rename(fixture.artifacts / "evidence-plan.json")
+            variables = {"HARNESS_ROOT": str(fixture.root), "REPO_ROOT": str(fixture.repo),
+                         "VERIFICATION_RUN_ROOT": str(fixture.artifacts)}
+            block = section.split("```sh\n", 1)[1].split("```", 1)[0]
+            lines = string.Template(block).substitute(variables).replace("\\\n", "").splitlines()
+            commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")]
+            self.assertEqual(len(commands), 2)
+            self.assertEqual([command[:4] for command in commands],
+                             [["python3", "-B", str(helper), "run"],
+                              ["python3", "-B", str(helper), "validate"]])
+            for command in commands:
+                command[0] = sys.executable
+
+            def invoke(command):
+                return subprocess.run(command, cwd=fixture.repo, text=True, capture_output=True, timeout=15)
+
+            run = invoke(commands[0])
+            self.assertEqual(run.returncode, 0, run.stderr)
+            manifest_path = fixture.artifacts / "evidence-manifest.json"
+            manifest = json.loads(manifest_path.read_text())
+            self.assertEqual(manifest["schema"], "evidence-manifest/v3")
+            self.assertEqual(manifest["source_revision"], fixtures.run_git(fixture.repo, "rev-parse", "HEAD").stdout.strip())
+            self.assertEqual(manifest["plan"]["path"], "evidence-plan.json")
+            self.assertEqual(manifest["commands"][0]["argv"], ["git", "show", "HEAD:input.txt"])
+            self.assertEqual(manifest["commands"][0]["exit_code"], 0)
+            self.assertEqual(manifest["commands"][0]["output"], "hello\n")
+            validated = invoke(commands[1])
+            self.assertEqual(validated.returncode, 0, validated.stderr)
+            before = manifest_path.read_bytes()
+            mode = stat.S_IMODE(manifest_path.stat().st_mode)
+            self.assertEqual(mode, 0o600)
+            for flag, base_command, filename in (
+                    ("--plan", commands[0], "evidence-plan.json"),
+                    ("--output", commands[0], "bad-output.json"),
+                    ("--manifest", commands[1], "evidence-manifest.json")):
+                with self.subTest(flag=flag):
+                    invalid = list(base_command)
+                    invalid[invalid.index(flag) + 1] = str(fixture.artifacts / filename)
+                    if flag == "--plan":
+                        # Use an absent output so collision cannot mask the path rejection.
+                        invalid[invalid.index("--output") + 1] = "bad-plan.json"
+                    rejected = invoke(invalid)
+                    self.assertEqual(rejected.returncode, 2)
+                    self.assertIn("artifact path must be run-relative", rejected.stderr)
+                    self.assertEqual(manifest_path.read_bytes(), before)
+                    self.assertEqual(stat.S_IMODE(manifest_path.stat().st_mode), mode)
+            self.assertFalse((fixture.artifacts / "bad-plan.json").exists())
+            self.assertFalse((fixture.artifacts / "bad-output.json").exists())
+            self.assertEqual(fixtures.run_git(fixture.repo, "status", "--porcelain").stdout, "")
+        finally:
+            if hasattr(fixture, "tmp"):
+                fixture.tearDown()