from harness.tests._support.paths import ROOT as SOURCE_ROOT import re import subprocess import sys import unittest from pathlib import Path from harness.tests._support.contracts import read_contract ROOT = SOURCE_ROOT SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py" PREPARE_SCRIPT = ROOT / "harness/lib/verification_root.py" CACHE = ROOT / "harness/operations/repository-cache.md" SKILLS = { "implementation": ROOT / "harness/agents/impl/skills/gitea-issue-implementation/SKILL.md", "rework": ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md", "review": ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/SKILL.md", } CANONICAL = "/repositories///worktrees/issue-/verification/" class VerificationContractDocumentationTests(unittest.TestCase): def test_runtime_policy_is_broker_owned_and_separate_from_verification(self): text = CACHE.read_text(encoding='utf-8') self.assertIn('runtime/', text) self.assertIn('broker.handle', text) self.assertIn('broker.sock', text) self.assertIn('ready JSON', text) self.assertIn('active resource', text) self.assertIn('unknown', text) for path in SKILLS.values(): with self.subTest(path=path.name): role = read_contract(path) gate = role.split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0] self.assertIn('broker-generated', gate) self.assertIn('ready JSON', gate) self.assertNotIn('pre-created', gate) self.assertNotIn('pre-create', gate) def test_role_runtime_startup_precedes_and_does_not_replace_admission(self): roles = [path for path in (ROOT / 'harness/agents').rglob('SKILL.md') if 'broker-generated runtime' in read_contract(path)] self.assertEqual(len(roles), 6) for path in roles: with self.subTest(role=path.parent.name): gate = read_contract(path).split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0] self.assertIn('ready 이후 각 workflow의 진입 검증·승인 조건을 확인한다', gate) self.assertIn('ready JSON의 `handle`/`socket`', gate) def test_evidence_plan_disables_python_bytecode_in_clean_checkout(self): import json template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json" plan = json.loads(template.read_text()) self.assertIn("-B", plan["commands"][0]["argv"]) self.assertIn("PYTHONDONTWRITEBYTECODE", plan["command_env_allowlist"]) def test_command_deadline_and_archive_policy_is_explicit(self): import json template = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json' for command in json.loads(template.read_text())['commands']: self.assertIsInstance(command['timeout_seconds'], (int, float)) self.assertGreater(command['timeout_seconds'], 0) self.assertLessEqual(command['timeout_seconds'], 3600) for path in (CACHE, *SKILLS.values()): text = read_contract(path) for token in ('evidence-manifest/v3', 'timeout_seconds', 'process group', 'validate-archived-v2'): with self.subTest(path=str(path), token=token): self.assertIn(token, text) help_result = subprocess.run([sys.executable, str(SCRIPT), 'validate-archived-v2', '--help'], capture_output=True, text=True, timeout=10) self.assertEqual(help_result.returncode, 0, help_result.stderr) for option in ('--contract-path', '--repo-root', '--artifact-root', '--manifest'): self.assertIn(option, help_result.stdout) def test_all_roles_reference_canonical_external_artifacts(self): for path in (CACHE, *SKILLS.values()): with self.subTest(path=path.name): text = read_contract(path) self.assertIn(CANONICAL, text) self.assertIn("evidence_gap", text) self.assertIn("[REDACTED]", text) self.assertIn("evidence-manifest/v3", text) def test_examples_match_real_helper_cli_not_unsupported_options(self): for path in (SKILLS['implementation'], SKILLS['rework']): section = read_contract(path).split('## Canonical external verification evidence', 1)[1] match = re.search('```sh\\n(.*?)```', section, re.S) assert match is not None block = match.group(1).replace('\\\n', ' ') commands = [line for line in block.splitlines() if line.startswith('python3 ')] self.assertEqual(len(commands), 3) for op, command in zip(('prepare', 'run', 'validate'), commands): with self.subTest(skill=path.parent.name, op=op): declared = set(re.findall('--[a-z-]+', command)) if op == 'prepare': self.assertIn('harness/lib/verification_root.py', command) help_result = subprocess.run([sys.executable, str(PREPARE_SCRIPT), op, '--help'], text=True, capture_output=True) self.assertEqual(help_result.returncode, 0, help_result.stderr) supported = set(re.findall('--[a-z-]+', help_result.stdout)) required = {'--root', '--owner', '--repository', '--issue', '--run-id', '--handle'} self.assertEqual(declared, required) self.assertTrue(declared <= supported) else: help_result = subprocess.run([sys.executable, str(SCRIPT), op, '--help'], text=True, capture_output=True) self.assertEqual(help_result.returncode, 0) supported = set(re.findall('--[a-z-]+', help_result.stdout)) required = {'--contract-path', '--repo-root', '--artifact-root'} | ({'--plan', '--output'} if op == 'run' else {'--manifest'}) self.assertEqual(declared, required) self.assertTrue(declared <= supported) self.assertIn('exact current HEAD', section) self.assertIn('validate-legacy', section) def test_prepare_contract_binds_root_and_forbids_recursive_parent_creation(self): for path in (CACHE, SKILLS['implementation'], SKILLS['rework']): with self.subTest(path=path.name): text = read_contract(path) self.assertIn('exact harness root', text) self.assertIn('Path.mkdir(parents=True, mode=0o700)', text) def test_checkout_cleanup_preserves_verification_without_recursive_delete(self): text = CACHE.read_text() cleanup = text.split('## Worktree 실패 처리와 제거', 1)[1].split('## Credential 안전', 1)[0] self.assertIn('`verification/`이 남으면 Issue directory를 보존한다', cleanup) self.assertIn('mode `0700`', text) self.assertIn('mode `0600`', text) def test_spec_review_uses_immutable_source_and_not_moving_checkout(self): section = SKILLS['review'].read_text().split('## External Issue verification evidence boundary', 1)[1] self.assertIn('moving implementation checkout', section) self.assertIn('authoritative blob evidence', section) self.assertIn('trusted `harness/.runtime`', section) def test_workflow_guard_records_reuse_existing_verification_location(self): text = CACHE.read_text() guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0] self.assertIn('verification/', guard) def test_workflow_guard_snapshot_protects_scope_before_delegation(self): implementation = read_contract(SKILLS['implementation']) self.assertIn('workflow_guard.py snapshot', implementation) self.assertIn('compare-source --repo-root', implementation) def test_workflow_guard_inventory_is_checked_before_adopting_results(self): rework = read_contract(SKILLS['rework']) self.assertIn('workflow_guard.py check-tests', rework) self.assertIn('test inventory', rework) def test_workflow_guard_contract_preservation_precedes_spec_verdict(self): spec = SKILLS['review'].read_text() self.assertIn('workflow_guard.py check-contracts', spec) self.assertIn('baseline inventory', spec) def test_workflow_guard_nested_schema_is_actionable_and_claims_are_not_authenticated(self): text = CACHE.read_text() section = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0] for token in ('#### Nested input contracts', 'baseline_collection', 'inventory_sha256', 'baseline_status', 'expected_failure', 'replacement_ids', 'kind', 'delete', 'rename', 'status', 'review_evidence', 'scope_sha256', 'expected_exit_code', 'failure_kind', 'semantic_key', 'current_keys', 'observation', 'intentionally_changed', 'sort_keys=True', 'separators', 'UTF-8'): with self.subTest(token=token): self.assertIn(token, section) import json blocks = re.findall('```json\\n(.*?)```', section, re.S) self.assertEqual(len(blocks), 2, 'provide runnable test and contract input examples') self.assertEqual(set(json.loads(blocks[0])), {'baseline_ids', 'current_ids', 'baseline_status', 'current_status', 'baseline_collection', 'current_collection', 'authorized_changes', 'changed_tests', 'findings'}) self.assertEqual(set(json.loads(blocks[1])), {'baseline', 'current', 'mappings', 'stages', 'precedence'}) def test_mapping_schema_types_use_raw_source_not_redacted_tool_rendering(self): # Authorization-looking prose can be masked by read-tool transport. # Inspect the approved public schema in raw source; never disable redaction. declaration = 'authorization: string, verification: string' masked = 'authorization: ' + '*' * 3 + ' verification' helper = ROOT / 'harness/lib/workflow_guard.py' for path in (helper, CACHE): text = read_contract(path) with self.subTest(path=path.name): self.assertEqual(text.count(declaration), 1) self.assertNotIn(masked, text) def test_rework_no_longer_excludes_repo_evidence_from_clean_tree(self): text = read_contract(SKILLS['rework']) self.assertIn('archived v1 read-only', text) if __name__ == "__main__": unittest.main()