genichin/gitea-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(harness): reorganize regression suites and preserve contract coverage

Browse Source
This commit is contained in:
gitea-spec-agent
2026-10-16 07:24:49 +00:00
parent b6f87db623
commit 33709ab873
108 changed files with 5869 additions and 5328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/common/contracts/test_verification_contract_docs.py
+182
View File
@@ -0,0 +1,182 @@
from harness.tests._support.paths import ROOT as SOURCE_ROOT
import re
import subprocess
import sys
import unittest
from pathlib import Path
from harness.tests._support.contracts import read_contract
ROOT = SOURCE_ROOT
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
PREPARE_SCRIPT = ROOT / "harness/lib/verification_root.py"
CACHE = ROOT / "harness/operations/repository-cache.md"
SKILLS = {
"implementation": ROOT / "harness/agents/impl/skills/gitea-issue-implementation/SKILL.md",
"rework": ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md",
"review": ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/SKILL.md",
}
CANONICAL = "<harness-root>/repositories/<owner>/<repo>/worktrees/issue-<number>/verification/<run-id>"
class VerificationContractDocumentationTests(unittest.TestCase):
def test_runtime_policy_is_broker_owned_and_separate_from_verification(self):
text = CACHE.read_text(encoding='utf-8')
self.assertIn('runtime/<workflow-run-id>', text)
self.assertIn('broker.handle', text)
self.assertIn('broker.sock', text)
self.assertIn('ready JSON', text)
self.assertIn('active resource', text)
self.assertIn('unknown', text)
for path in SKILLS.values():
with self.subTest(path=path.name):
role = read_contract(path)
gate = role.split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
self.assertIn('broker-generated', gate)
self.assertIn('ready JSON', gate)
self.assertNotIn('pre-created', gate)
self.assertNotIn('pre-create', gate)
def test_role_runtime_startup_precedes_and_does_not_replace_admission(self):
roles = [path for path in (ROOT / 'harness/agents').rglob('SKILL.md') if 'broker-generated runtime' in read_contract(path)]
self.assertEqual(len(roles), 6)
for path in roles:
with self.subTest(role=path.parent.name):
gate = read_contract(path).split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
self.assertIn('ready 이후 각 workflow의 진입 검증·승인 조건을 확인한다', gate)
self.assertIn('ready JSON의 `handle`/`socket`', gate)
def test_evidence_plan_disables_python_bytecode_in_clean_checkout(self):
import json
template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json"
plan = json.loads(template.read_text())
self.assertIn("-B", plan["commands"][0]["argv"])
self.assertIn("PYTHONDONTWRITEBYTECODE", plan["command_env_allowlist"])
def test_command_deadline_and_archive_policy_is_explicit(self):
import json
template = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json'
for command in json.loads(template.read_text())['commands']:
self.assertIsInstance(command['timeout_seconds'], (int, float))
self.assertGreater(command['timeout_seconds'], 0)
self.assertLessEqual(command['timeout_seconds'], 3600)
for path in (CACHE, *SKILLS.values()):
text = read_contract(path)
for token in ('evidence-manifest/v3', 'timeout_seconds', 'process group', 'validate-archived-v2'):
with self.subTest(path=str(path), token=token):
self.assertIn(token, text)
help_result = subprocess.run([sys.executable, str(SCRIPT), 'validate-archived-v2', '--help'], capture_output=True, text=True, timeout=10)
self.assertEqual(help_result.returncode, 0, help_result.stderr)
for option in ('--contract-path', '--repo-root', '--artifact-root', '--manifest'):
self.assertIn(option, help_result.stdout)
def test_all_roles_reference_canonical_external_artifacts(self):
for path in (CACHE, *SKILLS.values()):
with self.subTest(path=path.name):
text = read_contract(path)
self.assertIn(CANONICAL, text)
self.assertIn("evidence_gap", text)
self.assertIn("[REDACTED]", text)
self.assertIn("evidence-manifest/v3", text)
def test_examples_match_real_helper_cli_not_unsupported_options(self):
for path in (SKILLS['implementation'], SKILLS['rework']):
section = read_contract(path).split('## Canonical external verification evidence', 1)[1]
match = re.search('```sh\\n(.*?)```', section, re.S)
assert match is not None
block = match.group(1).replace('\\\n', ' ')
commands = [line for line in block.splitlines() if line.startswith('python3 ')]
self.assertEqual(len(commands), 3)
for op, command in zip(('prepare', 'run', 'validate'), commands):
with self.subTest(skill=path.parent.name, op=op):
declared = set(re.findall('--[a-z-]+', command))
if op == 'prepare':
self.assertIn('harness/lib/verification_root.py', command)
help_result = subprocess.run([sys.executable, str(PREPARE_SCRIPT), op, '--help'], text=True, capture_output=True)
self.assertEqual(help_result.returncode, 0, help_result.stderr)
supported = set(re.findall('--[a-z-]+', help_result.stdout))
required = {'--root', '--owner', '--repository', '--issue', '--run-id', '--handle'}
self.assertEqual(declared, required)
self.assertTrue(declared <= supported)
else:
help_result = subprocess.run([sys.executable, str(SCRIPT), op, '--help'], text=True, capture_output=True)
self.assertEqual(help_result.returncode, 0)
supported = set(re.findall('--[a-z-]+', help_result.stdout))
required = {'--contract-path', '--repo-root', '--artifact-root'} | ({'--plan', '--output'} if op == 'run' else {'--manifest'})
self.assertEqual(declared, required)
self.assertTrue(declared <= supported)
self.assertIn('exact current HEAD', section)
self.assertIn('validate-legacy', section)
def test_prepare_contract_binds_root_and_forbids_recursive_parent_creation(self):
for path in (CACHE, SKILLS['implementation'], SKILLS['rework']):
with self.subTest(path=path.name):
text = read_contract(path)
self.assertIn('exact harness root', text)
self.assertIn('Path.mkdir(parents=True, mode=0o700)', text)
def test_checkout_cleanup_preserves_verification_without_recursive_delete(self):
text = CACHE.read_text()
cleanup = text.split('## Worktree 실패 처리와 제거', 1)[1].split('## Credential 안전', 1)[0]
self.assertIn('`verification/`이 남으면 Issue directory를 보존한다', cleanup)
self.assertIn('mode `0700`', text)
self.assertIn('mode `0600`', text)
def test_spec_review_uses_immutable_source_and_not_moving_checkout(self):
section = SKILLS['review'].read_text().split('## External Issue verification evidence boundary', 1)[1]
self.assertIn('moving implementation checkout', section)
self.assertIn('authoritative blob evidence', section)
self.assertIn('trusted `harness/.runtime`', section)
def test_workflow_guard_records_reuse_existing_verification_location(self):
text = CACHE.read_text()
guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
self.assertIn('verification/<run-id>', guard)
def test_workflow_guard_snapshot_protects_scope_before_delegation(self):
implementation = read_contract(SKILLS['implementation'])
self.assertIn('workflow_guard.py snapshot', implementation)
self.assertIn('compare-source --repo-root', implementation)
def test_workflow_guard_inventory_is_checked_before_adopting_results(self):
rework = read_contract(SKILLS['rework'])
self.assertIn('workflow_guard.py check-tests', rework)
self.assertIn('test inventory', rework)
def test_workflow_guard_contract_preservation_precedes_spec_verdict(self):
spec = SKILLS['review'].read_text()
self.assertIn('workflow_guard.py check-contracts', spec)
self.assertIn('baseline inventory', spec)
def test_workflow_guard_nested_schema_is_actionable_and_claims_are_not_authenticated(self):
text = CACHE.read_text()
section = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
for token in ('#### Nested input contracts', 'baseline_collection', 'inventory_sha256', 'baseline_status', 'expected_failure', 'replacement_ids', 'kind', 'delete', 'rename', 'status', 'review_evidence', 'scope_sha256', 'expected_exit_code', 'failure_kind', 'semantic_key', 'current_keys', 'observation', 'intentionally_changed', 'sort_keys=True', 'separators', 'UTF-8'):
with self.subTest(token=token):
self.assertIn(token, section)
import json
blocks = re.findall('```json\\n(.*?)```', section, re.S)
self.assertEqual(len(blocks), 2, 'provide runnable test and contract input examples')
self.assertEqual(set(json.loads(blocks[0])), {'baseline_ids', 'current_ids', 'baseline_status', 'current_status', 'baseline_collection', 'current_collection', 'authorized_changes', 'changed_tests', 'findings'})
self.assertEqual(set(json.loads(blocks[1])), {'baseline', 'current', 'mappings', 'stages', 'precedence'})
def test_mapping_schema_types_use_raw_source_not_redacted_tool_rendering(self):
# Authorization-looking prose can be masked by read-tool transport.
# Inspect the approved public schema in raw source; never disable redaction.
declaration = 'authorization: string, verification: string'
masked = 'authorization: ' + '*' * 3 + ' verification'
helper = ROOT / 'harness/lib/workflow_guard.py'
for path in (helper, CACHE):
text = read_contract(path)
with self.subTest(path=path.name):
self.assertEqual(text.count(declaration), 1)
self.assertNotIn(masked, text)
def test_rework_no_longer_excludes_repo_evidence_from_clean_tree(self):
text = read_contract(SKILLS['rework'])
self.assertIn('archived v1 read-only', text)
if __name__ == "__main__":
unittest.main()