From d4db3f6a8e0669f76b8f580cbb9672c0d32863f4 Mon Sep 17 00:00:00 2001 From: gitea-spec-agent Date: Mon, 12 Oct 2026 06:49:44 +0000 Subject: [PATCH] [verified] Add executable reviewer evidence manifest recipe --- .../references/native-review.md | 27 +++++ .../review/test_native_evidence_recipe.py | 110 ++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 harness/tests/review/test_native_evidence_recipe.py diff --git a/harness/agents/review/skills/gitea-code-review/references/native-review.md b/harness/agents/review/skills/gitea-code-review/references/native-review.md index a4f156f..cb4e144 100644 --- a/harness/agents/review/skills/gitea-code-review/references/native-review.md +++ b/harness/agents/review/skills/gitea-code-review/references/native-review.md @@ -137,6 +137,33 @@ PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s harness/tests/impl -v Run the test commands from the canonical harness repository root. The existing spec suite requires `/opt/data/cache/scratch`: prepare that directory only if absent after confirming a nonsymlink, role-owned, non-group/world-writable parent, and require mode 0700 for the scratch child. Preserve existing shared parent permissions; do not chmod/chown them to satisfy a test. Use isolated/local runner variables and properly quoted command arguments rather than reusing mutable interpreter globals across review steps. Keep local stub/loopback transport checks, real temporary Git/worktree/lease checks, and saved production-wire replay clearly distinguished from a live Gitea mutation test. Introducing this helper does not authorize changing any existing product PR verdict, source, Issue state, merge or deployment. +## Reviewer verification evidence recipe + +This is the existing common evidence helper, not the `native_review.py preflight/verify` publisher replay interface. Use the canonical helper at `$HARNESS_ROOT/harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py`; do not create a reviewer wrapper or copy it into another profile. The historical v2 recipe gap is addressed using the current `evidence-manifest/v3` contract. + +First complete authenticated cache/worktree admission and retain the current review-role Issue broker lease. Prepare a fresh stage/run directory using the existing `verification_root.py prepare` procedure in the [canonical repository-cache contract](../../../../../operations/repository-cache.md). That procedure authenticates the live handle and Issue identity; the evidence helper does not acquire or certify a lease. Derive these absolute paths from the admitted harness/Issue identity, not from an arbitrary output directory: + +- `HARNESS_ROOT`: the admitted harness root containing `harness/operations/repository-cache.md`. +- `REPO_ROOT`: `$HARNESS_ROOT/repositories/$OWNER/$REPOSITORY/worktrees/issue-$ISSUE/checkout`. +- `VERIFICATION_RUN_ROOT`: `$HARNESS_ROOT/repositories/$OWNER/$REPOSITORY/worktrees/issue-$ISSUE/verification/$RUN_ID`. + +Require a **clean checkout**, **no source/evidence exclusions**, and prepared owner-only `0700` verification/run directories. Place the reviewed command plan at `$VERIFICATION_RUN_ROOT/evidence-plan.json` with mode `0600` before running; adapt the existing [evidence-plan template](../../../../impl/skills/gitea-issue-rework/templates/evidence-plan.json) to the actual reviewer commands, finite `timeout_seconds`, output bounds, source `hash_paths`, claims and limitations. `hash_paths` are checkout-relative, while plan/output/manifest CLI arguments are **run-relative**. Do not put artifacts in the checkout, weaken permissions, change the clean-tree check, or normalize rejected absolute artifact filenames. All roots/contract paths below are absolute; the three artifact filenames are not: + +```sh +python3 -B "$HARNESS_ROOT/harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py" run \ + --contract-path "$HARNESS_ROOT/harness/operations/repository-cache.md" \ + --repo-root "$REPO_ROOT" --artifact-root "$VERIFICATION_RUN_ROOT" \ + --plan evidence-plan.json --output evidence-manifest.json +python3 -B "$HARNESS_ROOT/harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py" validate \ + --contract-path "$HARNESS_ROOT/harness/operations/repository-cache.md" \ + --repo-root "$REPO_ROOT" --artifact-root "$VERIFICATION_RUN_ROOT" \ + --manifest evidence-manifest.json +``` + +Run `validate` only after `run` succeeds, using the same roots and exact source HEAD. A nonzero result, timeout or overflow blocks adoption; retain bounded failure evidence and do not automatically rerun the suite. Existing manifests are not overwritten. Local validation binds the current source, plan and actual command records; it is not live Gitea readback, reviewer approval, Native publication verification or cleanup completion. Preserve the fresh stage/run ID, source SHA, plan/manifest digests, actual results/limitations and verified reviewer-access method under the existing reporting/cleanup contract. + +Archived v2 uses only `validate-archived-v2` read-only compatibility; archived v1 uses only `validate-legacy`. Neither upgrades old records to v3 deadline/cleanup guarantees nor creates fresh publication evidence. Do not migrate archived manifests, add exclusions or change schemas to make the example pass. + ## Canonical native-review journal Native code-review publication의 정본 journal root는 `/harness/.runtime/native-review-journals`이다. Canonical publisher의 물리적인 harness 위치에서 결정하며 caller의 run-id, 임의 journal 경로, 환경 변수 변경으로 다른 registry를 선택하지 않는다. `canonical_journal_path(plan)`은 검증된 plan의 표시용 경로 계산일 뿐 인증·게시 권한의 증명이 아니다. 실제 registry claim 전에 현재 role identity와 held Issue lease 및 source/delegation gate를 검증한다. diff --git a/harness/tests/review/test_native_evidence_recipe.py b/harness/tests/review/test_native_evidence_recipe.py new file mode 100644 index 0000000..23c2757 --- /dev/null +++ b/harness/tests/review/test_native_evidence_recipe.py @@ -0,0 +1,110 @@ +"""Reviewer recipe contracts; no live provider or publication calls.""" +import json +import shlex +import stat +import string +import subprocess +import sys +import unittest +from pathlib import Path + +from harness.tests.impl import test_evidence_manifest as fixtures + +ROOT = Path(__file__).resolve().parents[3] +REFERENCE = ROOT / "harness/agents/review/skills/gitea-code-review/references/native-review.md" +HELPER_RELATIVE = "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py" +HEADING = "## Reviewer verification evidence recipe" + + +class NativeEvidenceRecipeTests(unittest.TestCase): + def recipe(self): + text = REFERENCE.read_text(encoding="utf-8") + self.assertTrue(HEADING in text, "reviewer evidence-manifest recipe is missing") + return text.split(HEADING, 1)[1].split("\n## ", 1)[0] + + def test_current_recipe_preserves_canonical_path_and_archive_contracts(self): + section = self.recipe() + for required in (HELPER_RELATIVE, "verification_root.py prepare", "evidence-manifest/v3", + "validate-archived-v2", "read-only", "run-relative", "clean checkout", + "no source/evidence exclusions", "0700", "0600"): + self.assertIn(required, section) + block = section.split("```sh\n", 1)[1].split("```", 1)[0] + lines = block.replace("\\\n", "").splitlines() + commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")] + self.assertEqual(len(commands), 2) + self.assertEqual([command[:4] for command in commands], + [["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "run"], + ["python3", "-B", "$HARNESS_ROOT/" + HELPER_RELATIVE, "validate"]]) + for command in commands: + self.assertEqual(command[command.index("--contract-path") + 1], + "$HARNESS_ROOT/harness/operations/repository-cache.md") + self.assertEqual(command[command.index("--repo-root") + 1], "$REPO_ROOT") + self.assertEqual(command[command.index("--artifact-root") + 1], "$VERIFICATION_RUN_ROOT") + self.assertEqual(commands[0][commands[0].index("--plan") + 1], "evidence-plan.json") + self.assertEqual(commands[0][commands[0].index("--output") + 1], "evidence-manifest.json") + self.assertEqual(commands[1][commands[1].index("--manifest") + 1], "evidence-manifest.json") + + def test_documented_commands_run_validate_and_reject_absolute_artifact_paths(self): + section = self.recipe() + fixture = fixtures.EvidenceManifestTests() + try: + fixture.setUp() + # Install exact current helper bytes into an isolated canonical-shaped + # synthetic harness. No fork, wrapper, live lease or provider is used. + source = ROOT / HELPER_RELATIVE + helper = fixture.root / HELPER_RELATIVE + helper.parent.mkdir(parents=True) + helper.write_bytes(source.read_bytes()) + self.assertEqual(helper.read_bytes(), source.read_bytes()) + fixture.plan().rename(fixture.artifacts / "evidence-plan.json") + variables = {"HARNESS_ROOT": str(fixture.root), "REPO_ROOT": str(fixture.repo), + "VERIFICATION_RUN_ROOT": str(fixture.artifacts)} + block = section.split("```sh\n", 1)[1].split("```", 1)[0] + lines = string.Template(block).substitute(variables).replace("\\\n", "").splitlines() + commands = [shlex.split(line) for line in lines if line.strip() and not line.lstrip().startswith("#")] + self.assertEqual(len(commands), 2) + self.assertEqual([command[:4] for command in commands], + [["python3", "-B", str(helper), "run"], + ["python3", "-B", str(helper), "validate"]]) + for command in commands: + command[0] = sys.executable + + def invoke(command): + return subprocess.run(command, cwd=fixture.repo, text=True, capture_output=True, timeout=15) + + run = invoke(commands[0]) + self.assertEqual(run.returncode, 0, run.stderr) + manifest_path = fixture.artifacts / "evidence-manifest.json" + manifest = json.loads(manifest_path.read_text()) + self.assertEqual(manifest["schema"], "evidence-manifest/v3") + self.assertEqual(manifest["source_revision"], fixtures.run_git(fixture.repo, "rev-parse", "HEAD").stdout.strip()) + self.assertEqual(manifest["plan"]["path"], "evidence-plan.json") + self.assertEqual(manifest["commands"][0]["argv"], ["git", "show", "HEAD:input.txt"]) + self.assertEqual(manifest["commands"][0]["exit_code"], 0) + self.assertEqual(manifest["commands"][0]["output"], "hello\n") + validated = invoke(commands[1]) + self.assertEqual(validated.returncode, 0, validated.stderr) + before = manifest_path.read_bytes() + mode = stat.S_IMODE(manifest_path.stat().st_mode) + self.assertEqual(mode, 0o600) + for flag, base_command, filename in ( + ("--plan", commands[0], "evidence-plan.json"), + ("--output", commands[0], "bad-output.json"), + ("--manifest", commands[1], "evidence-manifest.json")): + with self.subTest(flag=flag): + invalid = list(base_command) + invalid[invalid.index(flag) + 1] = str(fixture.artifacts / filename) + if flag == "--plan": + # Use an absent output so collision cannot mask the path rejection. + invalid[invalid.index("--output") + 1] = "bad-plan.json" + rejected = invoke(invalid) + self.assertEqual(rejected.returncode, 2) + self.assertIn("artifact path must be run-relative", rejected.stderr) + self.assertEqual(manifest_path.read_bytes(), before) + self.assertEqual(stat.S_IMODE(manifest_path.stat().st_mode), mode) + self.assertFalse((fixture.artifacts / "bad-plan.json").exists()) + self.assertFalse((fixture.artifacts / "bad-output.json").exists()) + self.assertEqual(fixtures.run_git(fixture.repo, "status", "--porcelain").stdout, "") + finally: + if hasattr(fixture, "tmp"): + fixture.tearDown()