111 lines
6.3 KiB
Python
111 lines
6.3 KiB
Python
"""Reviewer recipe contracts; no live provider or publication calls."""
|
|
import json
|
|
import shlex
|
|
import stat
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from harness.tests.impl import test_evidence_manifest as fixtures
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
REFERENCE = ROOT / "harness/agents/review/skills/gitea-code-review/references/native-review.md"
|
|
HELPER_RELATIVE = "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
|
HEADING = "## Reviewer verification evidence recipe"
|
|
|
|
|
|
class NativeEvidenceRecipeTests(unittest.TestCase):
|
|
def recipe(self):
|
|
text = REFERENCE.read_text(encoding="utf-8")
|
|
self.assertTrue(HEADING in text, "reviewer evidence-manifest recipe is missing")
|
|
return text.split(HEADING, 1)[1].split("\n## ", 1)[0]
|
|
|
|
def test_current_recipe_preserves_canonical_path_and_archive_contracts(self):
|
|
section = self.recipe()
|
|
for required in (HELPER_RELATIVE, "verification_root.py prepare", "evidence-manifest/v3",
|
|
"validate-archived-v2", "read-only", "run-relative", "clean checkout",
|
|
"no source/evidence exclusions", "0700", "0600"):
|
|
self.assertIn(required, section)
|
|
block = section.split("```sh\n", 1)[1].split("```", 1)[0]
|
|
lines = block.replace("\\\n", "").splitlines()
|
|
commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")]
|
|
self.assertEqual(len(commands), 2)
|
|
self.assertEqual([command[:4] for command in commands],
|
|
[["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "run"],
|
|
["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "validate"]])
|
|
for command in commands:
|
|
self.assertEqual(command[command.index("--contract-path") + 1],
|
|
"$HARNESS_ROOT/harness/operations/repository-cache.md")
|
|
self.assertEqual(command[command.index("--repo-root") + 1], "$REPO_ROOT")
|
|
self.assertEqual(command[command.index("--artifact-root") + 1], "$VERIFICATION_RUN_ROOT")
|
|
self.assertEqual(commands[0][commands[0].index("--plan") + 1], "evidence-plan.json")
|
|
self.assertEqual(commands[0][commands[0].index("--output") + 1], "evidence-manifest.json")
|
|
self.assertEqual(commands[1][commands[1].index("--manifest") + 1], "evidence-manifest.json")
|
|
|
|
def test_documented_commands_run_validate_and_reject_absolute_artifact_paths(self):
|
|
section = self.recipe()
|
|
fixture = fixtures.EvidenceManifestTests()
|
|
try:
|
|
fixture.setUp()
|
|
# Install exact current helper bytes into an isolated canonical-shaped
|
|
# synthetic harness. No fork, wrapper, live lease or provider is used.
|
|
source = ROOT / HELPER_RELATIVE
|
|
helper = fixture.root / HELPER_RELATIVE
|
|
helper.parent.mkdir(parents=True)
|
|
helper.write_bytes(source.read_bytes())
|
|
self.assertEqual(helper.read_bytes(), source.read_bytes())
|
|
fixture.plan().rename(fixture.artifacts / "evidence-plan.json")
|
|
variables = {"HARNESS_ROOT": str(fixture.root), "REPO_ROOT": str(fixture.repo),
|
|
"VERIFICATION_RUN_ROOT": str(fixture.artifacts)}
|
|
block = section.split("```sh\n", 1)[1].split("```", 1)[0]
|
|
lines = string.Template(block).substitute(variables).replace("\\\n", "").splitlines()
|
|
commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")]
|
|
self.assertEqual(len(commands), 2)
|
|
self.assertEqual([command[:4] for command in commands],
|
|
[["python3", "-B", str(helper), "run"],
|
|
["python3", "-B", str(helper), "validate"]])
|
|
for command in commands:
|
|
command[0] = sys.executable
|
|
|
|
def invoke(command):
|
|
return subprocess.run(command, cwd=fixture.repo, text=True, capture_output=True, timeout=15)
|
|
|
|
run = invoke(commands[0])
|
|
self.assertEqual(run.returncode, 0, run.stderr)
|
|
manifest_path = fixture.artifacts / "evidence-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
self.assertEqual(manifest["schema"], "evidence-manifest/v3")
|
|
self.assertEqual(manifest["source_revision"], fixtures.run_git(fixture.repo, "rev-parse", "HEAD").stdout.strip())
|
|
self.assertEqual(manifest["plan"]["path"], "evidence-plan.json")
|
|
self.assertEqual(manifest["commands"][0]["argv"], ["git", "show", "HEAD:input.txt"])
|
|
self.assertEqual(manifest["commands"][0]["exit_code"], 0)
|
|
self.assertEqual(manifest["commands"][0]["output"], "hello\n")
|
|
validated = invoke(commands[1])
|
|
self.assertEqual(validated.returncode, 0, validated.stderr)
|
|
before = manifest_path.read_bytes()
|
|
mode = stat.S_IMODE(manifest_path.stat().st_mode)
|
|
self.assertEqual(mode, 0o600)
|
|
for flag, base_command, filename in (
|
|
("--plan", commands[0], "evidence-plan.json"),
|
|
("--output", commands[0], "bad-output.json"),
|
|
("--manifest", commands[1], "evidence-manifest.json")):
|
|
with self.subTest(flag=flag):
|
|
invalid = list(base_command)
|
|
invalid[invalid.index(flag) + 1] = str(fixture.artifacts / filename)
|
|
if flag == "--plan":
|
|
# Use an absent output so collision cannot mask the path rejection.
|
|
invalid[invalid.index("--output") + 1] = "bad-plan.json"
|
|
rejected = invoke(invalid)
|
|
self.assertEqual(rejected.returncode, 2)
|
|
self.assertIn("artifact path must be run-relative", rejected.stderr)
|
|
self.assertEqual(manifest_path.read_bytes(), before)
|
|
self.assertEqual(stat.S_IMODE(manifest_path.stat().st_mode), mode)
|
|
self.assertFalse((fixture.artifacts / "bad-plan.json").exists())
|
|
self.assertFalse((fixture.artifacts / "bad-output.json").exists())
|
|
self.assertEqual(fixtures.run_git(fixture.repo, "status", "--porcelain").stdout, "")
|
|
finally:
|
|
if hasattr(fixture, "tmp"):
|
|
fixture.tearDown()
|