183 lines
10 KiB
Python
183 lines
10 KiB
Python
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
from harness.tests._support.contracts import read_contract
|
|
|
|
ROOT = SOURCE_ROOT
|
|
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
|
PREPARE_SCRIPT = ROOT / "harness/lib/verification_root.py"
|
|
CACHE = ROOT / "harness/operations/repository-cache.md"
|
|
SKILLS = {
|
|
"implementation": ROOT / "harness/agents/impl/skills/gitea-issue-implementation/SKILL.md",
|
|
"rework": ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md",
|
|
"review": ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/SKILL.md",
|
|
}
|
|
CANONICAL = "<harness-root>/repositories/<owner>/<repo>/worktrees/issue-<number>/verification/<run-id>"
|
|
|
|
|
|
class VerificationContractDocumentationTests(unittest.TestCase):
|
|
def test_runtime_policy_is_broker_owned_and_separate_from_verification(self):
|
|
text = CACHE.read_text(encoding='utf-8')
|
|
self.assertIn('runtime/<workflow-run-id>', text)
|
|
self.assertIn('broker.handle', text)
|
|
self.assertIn('broker.sock', text)
|
|
self.assertIn('ready JSON', text)
|
|
self.assertIn('active resource', text)
|
|
self.assertIn('unknown', text)
|
|
for path in SKILLS.values():
|
|
with self.subTest(path=path.name):
|
|
role = read_contract(path)
|
|
gate = role.split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
|
|
self.assertIn('broker-generated', gate)
|
|
self.assertIn('ready JSON', gate)
|
|
self.assertNotIn('pre-created', gate)
|
|
self.assertNotIn('pre-create', gate)
|
|
|
|
def test_role_runtime_startup_precedes_and_does_not_replace_admission(self):
|
|
roles = [path for path in (ROOT / 'harness/agents').rglob('SKILL.md') if 'broker-generated runtime' in read_contract(path)]
|
|
self.assertEqual(len(roles), 6)
|
|
for path in roles:
|
|
with self.subTest(role=path.parent.name):
|
|
gate = read_contract(path).split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
|
|
self.assertIn('ready 이후 각 workflow의 진입 검증·승인 조건을 확인한다', gate)
|
|
self.assertIn('ready JSON의 `handle`/`socket`', gate)
|
|
|
|
def test_evidence_plan_disables_python_bytecode_in_clean_checkout(self):
|
|
import json
|
|
template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json"
|
|
plan = json.loads(template.read_text())
|
|
self.assertIn("-B", plan["commands"][0]["argv"])
|
|
self.assertIn("PYTHONDONTWRITEBYTECODE", plan["command_env_allowlist"])
|
|
|
|
|
|
def test_command_deadline_and_archive_policy_is_explicit(self):
|
|
import json
|
|
template = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json'
|
|
for command in json.loads(template.read_text())['commands']:
|
|
self.assertIsInstance(command['timeout_seconds'], (int, float))
|
|
self.assertGreater(command['timeout_seconds'], 0)
|
|
self.assertLessEqual(command['timeout_seconds'], 3600)
|
|
for path in (CACHE, *SKILLS.values()):
|
|
text = read_contract(path)
|
|
for token in ('evidence-manifest/v3', 'timeout_seconds', 'process group', 'validate-archived-v2'):
|
|
with self.subTest(path=str(path), token=token):
|
|
self.assertIn(token, text)
|
|
help_result = subprocess.run([sys.executable, str(SCRIPT), 'validate-archived-v2', '--help'], capture_output=True, text=True, timeout=10)
|
|
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
|
for option in ('--contract-path', '--repo-root', '--artifact-root', '--manifest'):
|
|
self.assertIn(option, help_result.stdout)
|
|
|
|
def test_all_roles_reference_canonical_external_artifacts(self):
|
|
for path in (CACHE, *SKILLS.values()):
|
|
with self.subTest(path=path.name):
|
|
text = read_contract(path)
|
|
self.assertIn(CANONICAL, text)
|
|
self.assertIn("evidence_gap", text)
|
|
self.assertIn("[REDACTED]", text)
|
|
self.assertIn("evidence-manifest/v3", text)
|
|
|
|
def test_examples_match_real_helper_cli_not_unsupported_options(self):
|
|
for path in (SKILLS['implementation'], SKILLS['rework']):
|
|
section = read_contract(path).split('## Canonical external verification evidence', 1)[1]
|
|
match = re.search('```sh\\n(.*?)```', section, re.S)
|
|
assert match is not None
|
|
block = match.group(1).replace('\\\n', ' ')
|
|
commands = [line for line in block.splitlines() if line.startswith('python3 ')]
|
|
self.assertEqual(len(commands), 3)
|
|
for op, command in zip(('prepare', 'run', 'validate'), commands):
|
|
with self.subTest(skill=path.parent.name, op=op):
|
|
declared = set(re.findall('--[a-z-]+', command))
|
|
if op == 'prepare':
|
|
self.assertIn('harness/lib/verification_root.py', command)
|
|
help_result = subprocess.run([sys.executable, str(PREPARE_SCRIPT), op, '--help'], text=True, capture_output=True)
|
|
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
|
supported = set(re.findall('--[a-z-]+', help_result.stdout))
|
|
required = {'--root', '--owner', '--repository', '--issue', '--run-id', '--handle'}
|
|
self.assertEqual(declared, required)
|
|
self.assertTrue(declared <= supported)
|
|
else:
|
|
help_result = subprocess.run([sys.executable, str(SCRIPT), op, '--help'], text=True, capture_output=True)
|
|
self.assertEqual(help_result.returncode, 0)
|
|
supported = set(re.findall('--[a-z-]+', help_result.stdout))
|
|
required = {'--contract-path', '--repo-root', '--artifact-root'} | ({'--plan', '--output'} if op == 'run' else {'--manifest'})
|
|
self.assertEqual(declared, required)
|
|
self.assertTrue(declared <= supported)
|
|
self.assertIn('exact current HEAD', section)
|
|
self.assertIn('validate-legacy', section)
|
|
|
|
def test_prepare_contract_binds_root_and_forbids_recursive_parent_creation(self):
|
|
for path in (CACHE, SKILLS['implementation'], SKILLS['rework']):
|
|
with self.subTest(path=path.name):
|
|
text = read_contract(path)
|
|
self.assertIn('exact harness root', text)
|
|
self.assertIn('Path.mkdir(parents=True, mode=0o700)', text)
|
|
|
|
def test_checkout_cleanup_preserves_verification_without_recursive_delete(self):
|
|
text = CACHE.read_text()
|
|
cleanup = text.split('## Worktree 실패 처리와 제거', 1)[1].split('## Credential 안전', 1)[0]
|
|
self.assertIn('`verification/`이 남으면 Issue directory를 보존한다', cleanup)
|
|
self.assertIn('mode `0700`', text)
|
|
self.assertIn('mode `0600`', text)
|
|
|
|
def test_spec_review_uses_immutable_source_and_not_moving_checkout(self):
|
|
section = SKILLS['review'].read_text().split('## External Issue verification evidence boundary', 1)[1]
|
|
self.assertIn('moving implementation checkout', section)
|
|
self.assertIn('authoritative blob evidence', section)
|
|
self.assertIn('trusted `harness/.runtime`', section)
|
|
|
|
def test_workflow_guard_records_reuse_existing_verification_location(self):
|
|
text = CACHE.read_text()
|
|
guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
|
self.assertIn('verification/<run-id>', guard)
|
|
|
|
|
|
def test_workflow_guard_snapshot_protects_scope_before_delegation(self):
|
|
implementation = read_contract(SKILLS['implementation'])
|
|
self.assertIn('workflow_guard.py snapshot', implementation)
|
|
self.assertIn('compare-source --repo-root', implementation)
|
|
|
|
def test_workflow_guard_inventory_is_checked_before_adopting_results(self):
|
|
rework = read_contract(SKILLS['rework'])
|
|
self.assertIn('workflow_guard.py check-tests', rework)
|
|
self.assertIn('test inventory', rework)
|
|
|
|
def test_workflow_guard_contract_preservation_precedes_spec_verdict(self):
|
|
spec = SKILLS['review'].read_text()
|
|
self.assertIn('workflow_guard.py check-contracts', spec)
|
|
self.assertIn('baseline inventory', spec)
|
|
|
|
def test_workflow_guard_nested_schema_is_actionable_and_claims_are_not_authenticated(self):
|
|
text = CACHE.read_text()
|
|
section = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
|
for token in ('#### Nested input contracts', 'baseline_collection', 'inventory_sha256', 'baseline_status', 'expected_failure', 'replacement_ids', 'kind', 'delete', 'rename', 'status', 'review_evidence', 'scope_sha256', 'expected_exit_code', 'failure_kind', 'semantic_key', 'current_keys', 'observation', 'intentionally_changed', 'sort_keys=True', 'separators', 'UTF-8'):
|
|
with self.subTest(token=token):
|
|
self.assertIn(token, section)
|
|
import json
|
|
blocks = re.findall('```json\\n(.*?)```', section, re.S)
|
|
self.assertEqual(len(blocks), 2, 'provide runnable test and contract input examples')
|
|
self.assertEqual(set(json.loads(blocks[0])), {'baseline_ids', 'current_ids', 'baseline_status', 'current_status', 'baseline_collection', 'current_collection', 'authorized_changes', 'changed_tests', 'findings'})
|
|
self.assertEqual(set(json.loads(blocks[1])), {'baseline', 'current', 'mappings', 'stages', 'precedence'})
|
|
|
|
def test_mapping_schema_types_use_raw_source_not_redacted_tool_rendering(self):
|
|
# Authorization-looking prose can be masked by read-tool transport.
|
|
# Inspect the approved public schema in raw source; never disable redaction.
|
|
declaration = 'authorization: string, verification: string'
|
|
masked = 'authorization: ' + '*' * 3 + ' verification'
|
|
helper = ROOT / 'harness/lib/workflow_guard.py'
|
|
for path in (helper, CACHE):
|
|
text = read_contract(path)
|
|
with self.subTest(path=path.name):
|
|
self.assertEqual(text.count(declaration), 1)
|
|
self.assertNotIn(masked, text)
|
|
|
|
def test_rework_no_longer_excludes_repo_evidence_from_clean_tree(self):
|
|
text = read_contract(SKILLS['rework'])
|
|
self.assertIn('archived v1 read-only', text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|