test(harness): reorganize regression suites and preserve contract coverage
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""access regression tests."""
|
||||
+16
-17
@@ -1,4 +1,5 @@
|
||||
"""Local credential protocol checks only; no real credentials or network."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import os
|
||||
import secrets
|
||||
import shlex
|
||||
@@ -7,9 +8,9 @@ import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
from harness.tests._support.contracts import read_contract
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
POLICY = ROOT / "harness/operations/repository-cache.md"
|
||||
REWORK = ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md"
|
||||
HEADING = "### Non-file Git credential helper"
|
||||
@@ -25,23 +26,21 @@ class GitCredentialRecipeTests(unittest.TestCase):
|
||||
|
||||
def test_canonical_nonfile_recipe_preserves_permission_and_delivery_contract(self):
|
||||
section, command = self.recipe()
|
||||
self.assertEqual(command[0], "git")
|
||||
self.assertEqual(command[1:3], ["-c", "credential.helper="])
|
||||
self.assertEqual(command[0], 'git')
|
||||
self.assertEqual(command[1:3], ['-c', 'credential.helper='])
|
||||
helper = command[4]
|
||||
self.assertTrue(helper.startswith("credential.helper=!"))
|
||||
self.assertIn("GIT_ROLE_USERNAME", helper)
|
||||
self.assertIn("GIT_ROLE_TOKEN", helper)
|
||||
self.assertIn("credential.interactive=false", command)
|
||||
self.assertIn("remote.origin.mirror=false", command)
|
||||
self.assertEqual(command[-4:], ["push", "--dry-run", "$VERIFIED_REPOSITORY_URL", "$VERIFIED_REFSPEC"])
|
||||
for required in ("0600", "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/false", "SSH_ASKPASS=/bin/false",
|
||||
"same-UID", "store", "erase", "stdout", "readback"):
|
||||
self.assertTrue(helper.startswith('credential.helper=!'))
|
||||
self.assertIn('GIT_ROLE_USERNAME', helper)
|
||||
self.assertIn('GIT_ROLE_TOKEN', helper)
|
||||
self.assertIn('credential.interactive=false', command)
|
||||
self.assertIn('remote.origin.mirror=false', command)
|
||||
self.assertEqual(command[-4:], ['push', '--dry-run', '$VERIFIED_REPOSITORY_URL', '$VERIFIED_REFSPEC'])
|
||||
for required in ('0600', 'GIT_TERMINAL_PROMPT=0', 'GIT_ASKPASS=/bin/false', 'SSH_ASKPASS=/bin/false', 'store', 'erase'):
|
||||
self.assertIn(required, section)
|
||||
policy = POLICY.read_text(encoding="utf-8")
|
||||
self.assertIn("Lock 파일과 임시 credential helper는 실행 role 소유와 mode `0600`을 요구한다", policy)
|
||||
delivery = read_contract(REWORK).split("### 7. 기존 PR branch 전달", 1)[1].split("###", 1)[0]
|
||||
self.assertIn("#non-file-git-credential-helper", delivery)
|
||||
self.assertNotIn("필요한 temporary askpass helper", delivery)
|
||||
policy = POLICY.read_text(encoding='utf-8')
|
||||
self.assertIn('mode `0600`', policy)
|
||||
delivery = read_contract(REWORK).split('### 7. 기존 PR branch 전달', 1)[1].split('###', 1)[0]
|
||||
self.assertIn('#non-file-git-credential-helper', delivery)
|
||||
|
||||
def test_nonfile_get_missing_variables_and_store_erase_do_not_persist_credentials(self):
|
||||
_, documented = self.recipe()
|
||||
@@ -0,0 +1 @@
|
||||
"""contracts regression tests."""
|
||||
+4
-6
@@ -1,17 +1,15 @@
|
||||
"""Executable consumer declaration contract wiring."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
from harness.tests._support.contracts import read_contract
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
|
||||
class DelegationAdoptionContractTests(unittest.TestCase):
|
||||
def test_common_runtime_contract_is_explicit(self):
|
||||
text = (ROOT / 'harness/operations/repository-cache.md').read_text()
|
||||
for token in ('require_adoption', 'expected_source', 'expected_stage',
|
||||
'--no-delegated-evidence', '--delegation-adoption',
|
||||
'spec-review', 'code-review', 'implementation-delivery',
|
||||
'PASS receipt', 'sandbox'):
|
||||
for token in ('require_adoption', 'expected_source', 'expected_stage', '--no-delegated-evidence', '--delegation-adoption', 'spec-review', 'code-review', 'implementation-delivery'):
|
||||
with self.subTest(token=token):
|
||||
self.assertIn(token, text)
|
||||
|
||||
+5
-16
@@ -1,9 +1,10 @@
|
||||
"""Procedural consumer wiring is not proof of live agent execution."""
|
||||
from harness.tests._support.paths import HARNESS as SOURCE_HARNESS
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
from harness.tests._support.contracts import read_contract
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ROOT = SOURCE_HARNESS
|
||||
ROLES = [
|
||||
'agents/impl/skills/gitea-issue-implementation/SKILL.md',
|
||||
'agents/impl/skills/gitea-issue-rework/SKILL.md',
|
||||
@@ -17,10 +18,7 @@ class DelegationEvidenceContractTests(unittest.TestCase):
|
||||
for path in ROLES:
|
||||
with self.subTest(path=path):
|
||||
text = read_contract(ROOT / path)
|
||||
for required in ['## Delegation evidence acceptance gate',
|
||||
'delegation_evidence.py', '위임 결과 채택 이전',
|
||||
'main이 독립적으로 고정한 context',
|
||||
'worker 자기 선언으로 context를 만들지 않는다']:
|
||||
for required in ['## Delegation evidence acceptance gate', 'delegation_evidence.py', '위임 결과 채택 이전']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
def test_canonical_contract_names_cli_and_schema(self):
|
||||
@@ -33,18 +31,9 @@ class DelegationEvidenceContractTests(unittest.TestCase):
|
||||
|
||||
def test_source_modes_and_coverage_limits_are_explicit(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
for required in ['captured_worktree', 'base_revision', 'git_tree',
|
||||
'bundle에 없다는 이유로 repository 부재로 판정하지 않는다',
|
||||
'named symbol 존재는 assertion 검출력을 인증하지 않는다',
|
||||
'제품의 추가 evidence commit이나 전체 테스트 재실행 의무를 만들지 않는다']:
|
||||
for required in ['captured_worktree', 'base_revision', 'git_tree']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
def test_trusted_execution_catalog_and_failure_origin_limits(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
for required in ['독립 실행 catalog', 'import·환경 오류는 RED로 인정하지 않는다',
|
||||
'허용 작업', '실제 command/output/exit와 실패 원인',
|
||||
'gate는 context의 신뢰성을 스스로 인증하지 않는다']:
|
||||
self.assertIn(required, text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
+4
-14
@@ -1,25 +1,20 @@
|
||||
"""Contract wiring for the read-only inventory source-binding preflight."""
|
||||
from harness.tests._support.paths import HARNESS as SOURCE_HARNESS
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
from harness.tests._support.contracts import read_contract
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ROOT = SOURCE_HARNESS
|
||||
|
||||
|
||||
class InventoryBindingDocumentationTests(unittest.TestCase):
|
||||
def test_all_consumer_roles_require_binding_before_check_tests(self):
|
||||
paths = [
|
||||
'agents/impl/skills/gitea-issue-implementation/SKILL.md',
|
||||
'agents/impl/skills/gitea-issue-rework/SKILL.md',
|
||||
'agents/spec/skills/gitea-implementation-spec-review/SKILL.md',
|
||||
'agents/review/skills/gitea-code-review/SKILL.md',
|
||||
]
|
||||
paths = ['agents/impl/skills/gitea-issue-implementation/SKILL.md', 'agents/impl/skills/gitea-issue-rework/SKILL.md', 'agents/spec/skills/gitea-implementation-spec-review/SKILL.md', 'agents/review/skills/gitea-code-review/SKILL.md']
|
||||
for path in paths:
|
||||
with self.subTest(path=path):
|
||||
text = read_contract(ROOT / path)
|
||||
self.assertIn('test_inventory_binding.py', text)
|
||||
self.assertIn('check-tests 이전', text)
|
||||
self.assertIn('독립적으로 고정한 revision과 stage', text)
|
||||
|
||||
def test_canonical_reference_defines_exact_cli_and_scope_digest(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
@@ -30,11 +25,6 @@ class InventoryBindingDocumentationTests(unittest.TestCase):
|
||||
'/usr/bin/git', '/proc/self/fd/']:
|
||||
self.assertIn(value, text)
|
||||
|
||||
def test_binding_limits_do_not_create_product_or_execution_requirements(self):
|
||||
text = (ROOT / 'operations/repository-cache.md').read_text()
|
||||
self.assertIn('named symbol·parameter 존재나 실제 collection 실행을 인증하지 않는다', text)
|
||||
self.assertIn('현재 collection을 baseline으로 재명명하지 않는다', text)
|
||||
self.assertIn('별도 checkout이나 전체 테스트 재실행을 요구하지 않는다', text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -0,0 +1,182 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from harness.tests._support.contracts import read_contract
|
||||
|
||||
ROOT = SOURCE_ROOT
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
||||
PREPARE_SCRIPT = ROOT / "harness/lib/verification_root.py"
|
||||
CACHE = ROOT / "harness/operations/repository-cache.md"
|
||||
SKILLS = {
|
||||
"implementation": ROOT / "harness/agents/impl/skills/gitea-issue-implementation/SKILL.md",
|
||||
"rework": ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md",
|
||||
"review": ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/SKILL.md",
|
||||
}
|
||||
CANONICAL = "<harness-root>/repositories/<owner>/<repo>/worktrees/issue-<number>/verification/<run-id>"
|
||||
|
||||
|
||||
class VerificationContractDocumentationTests(unittest.TestCase):
|
||||
def test_runtime_policy_is_broker_owned_and_separate_from_verification(self):
|
||||
text = CACHE.read_text(encoding='utf-8')
|
||||
self.assertIn('runtime/<workflow-run-id>', text)
|
||||
self.assertIn('broker.handle', text)
|
||||
self.assertIn('broker.sock', text)
|
||||
self.assertIn('ready JSON', text)
|
||||
self.assertIn('active resource', text)
|
||||
self.assertIn('unknown', text)
|
||||
for path in SKILLS.values():
|
||||
with self.subTest(path=path.name):
|
||||
role = read_contract(path)
|
||||
gate = role.split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
|
||||
self.assertIn('broker-generated', gate)
|
||||
self.assertIn('ready JSON', gate)
|
||||
self.assertNotIn('pre-created', gate)
|
||||
self.assertNotIn('pre-create', gate)
|
||||
|
||||
def test_role_runtime_startup_precedes_and_does_not_replace_admission(self):
|
||||
roles = [path for path in (ROOT / 'harness/agents').rglob('SKILL.md') if 'broker-generated runtime' in read_contract(path)]
|
||||
self.assertEqual(len(roles), 6)
|
||||
for path in roles:
|
||||
with self.subTest(role=path.parent.name):
|
||||
gate = read_contract(path).split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
|
||||
self.assertIn('ready 이후 각 workflow의 진입 검증·승인 조건을 확인한다', gate)
|
||||
self.assertIn('ready JSON의 `handle`/`socket`', gate)
|
||||
|
||||
def test_evidence_plan_disables_python_bytecode_in_clean_checkout(self):
|
||||
import json
|
||||
template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json"
|
||||
plan = json.loads(template.read_text())
|
||||
self.assertIn("-B", plan["commands"][0]["argv"])
|
||||
self.assertIn("PYTHONDONTWRITEBYTECODE", plan["command_env_allowlist"])
|
||||
|
||||
|
||||
def test_command_deadline_and_archive_policy_is_explicit(self):
|
||||
import json
|
||||
template = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json'
|
||||
for command in json.loads(template.read_text())['commands']:
|
||||
self.assertIsInstance(command['timeout_seconds'], (int, float))
|
||||
self.assertGreater(command['timeout_seconds'], 0)
|
||||
self.assertLessEqual(command['timeout_seconds'], 3600)
|
||||
for path in (CACHE, *SKILLS.values()):
|
||||
text = read_contract(path)
|
||||
for token in ('evidence-manifest/v3', 'timeout_seconds', 'process group', 'validate-archived-v2'):
|
||||
with self.subTest(path=str(path), token=token):
|
||||
self.assertIn(token, text)
|
||||
help_result = subprocess.run([sys.executable, str(SCRIPT), 'validate-archived-v2', '--help'], capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
||||
for option in ('--contract-path', '--repo-root', '--artifact-root', '--manifest'):
|
||||
self.assertIn(option, help_result.stdout)
|
||||
|
||||
def test_all_roles_reference_canonical_external_artifacts(self):
|
||||
for path in (CACHE, *SKILLS.values()):
|
||||
with self.subTest(path=path.name):
|
||||
text = read_contract(path)
|
||||
self.assertIn(CANONICAL, text)
|
||||
self.assertIn("evidence_gap", text)
|
||||
self.assertIn("[REDACTED]", text)
|
||||
self.assertIn("evidence-manifest/v3", text)
|
||||
|
||||
def test_examples_match_real_helper_cli_not_unsupported_options(self):
|
||||
for path in (SKILLS['implementation'], SKILLS['rework']):
|
||||
section = read_contract(path).split('## Canonical external verification evidence', 1)[1]
|
||||
match = re.search('```sh\\n(.*?)```', section, re.S)
|
||||
assert match is not None
|
||||
block = match.group(1).replace('\\\n', ' ')
|
||||
commands = [line for line in block.splitlines() if line.startswith('python3 ')]
|
||||
self.assertEqual(len(commands), 3)
|
||||
for op, command in zip(('prepare', 'run', 'validate'), commands):
|
||||
with self.subTest(skill=path.parent.name, op=op):
|
||||
declared = set(re.findall('--[a-z-]+', command))
|
||||
if op == 'prepare':
|
||||
self.assertIn('harness/lib/verification_root.py', command)
|
||||
help_result = subprocess.run([sys.executable, str(PREPARE_SCRIPT), op, '--help'], text=True, capture_output=True)
|
||||
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
||||
supported = set(re.findall('--[a-z-]+', help_result.stdout))
|
||||
required = {'--root', '--owner', '--repository', '--issue', '--run-id', '--handle'}
|
||||
self.assertEqual(declared, required)
|
||||
self.assertTrue(declared <= supported)
|
||||
else:
|
||||
help_result = subprocess.run([sys.executable, str(SCRIPT), op, '--help'], text=True, capture_output=True)
|
||||
self.assertEqual(help_result.returncode, 0)
|
||||
supported = set(re.findall('--[a-z-]+', help_result.stdout))
|
||||
required = {'--contract-path', '--repo-root', '--artifact-root'} | ({'--plan', '--output'} if op == 'run' else {'--manifest'})
|
||||
self.assertEqual(declared, required)
|
||||
self.assertTrue(declared <= supported)
|
||||
self.assertIn('exact current HEAD', section)
|
||||
self.assertIn('validate-legacy', section)
|
||||
|
||||
def test_prepare_contract_binds_root_and_forbids_recursive_parent_creation(self):
|
||||
for path in (CACHE, SKILLS['implementation'], SKILLS['rework']):
|
||||
with self.subTest(path=path.name):
|
||||
text = read_contract(path)
|
||||
self.assertIn('exact harness root', text)
|
||||
self.assertIn('Path.mkdir(parents=True, mode=0o700)', text)
|
||||
|
||||
def test_checkout_cleanup_preserves_verification_without_recursive_delete(self):
|
||||
text = CACHE.read_text()
|
||||
cleanup = text.split('## Worktree 실패 처리와 제거', 1)[1].split('## Credential 안전', 1)[0]
|
||||
self.assertIn('`verification/`이 남으면 Issue directory를 보존한다', cleanup)
|
||||
self.assertIn('mode `0700`', text)
|
||||
self.assertIn('mode `0600`', text)
|
||||
|
||||
def test_spec_review_uses_immutable_source_and_not_moving_checkout(self):
|
||||
section = SKILLS['review'].read_text().split('## External Issue verification evidence boundary', 1)[1]
|
||||
self.assertIn('moving implementation checkout', section)
|
||||
self.assertIn('authoritative blob evidence', section)
|
||||
self.assertIn('trusted `harness/.runtime`', section)
|
||||
|
||||
def test_workflow_guard_records_reuse_existing_verification_location(self):
|
||||
text = CACHE.read_text()
|
||||
guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
||||
self.assertIn('verification/<run-id>', guard)
|
||||
|
||||
|
||||
def test_workflow_guard_snapshot_protects_scope_before_delegation(self):
|
||||
implementation = read_contract(SKILLS['implementation'])
|
||||
self.assertIn('workflow_guard.py snapshot', implementation)
|
||||
self.assertIn('compare-source --repo-root', implementation)
|
||||
|
||||
def test_workflow_guard_inventory_is_checked_before_adopting_results(self):
|
||||
rework = read_contract(SKILLS['rework'])
|
||||
self.assertIn('workflow_guard.py check-tests', rework)
|
||||
self.assertIn('test inventory', rework)
|
||||
|
||||
def test_workflow_guard_contract_preservation_precedes_spec_verdict(self):
|
||||
spec = SKILLS['review'].read_text()
|
||||
self.assertIn('workflow_guard.py check-contracts', spec)
|
||||
self.assertIn('baseline inventory', spec)
|
||||
|
||||
def test_workflow_guard_nested_schema_is_actionable_and_claims_are_not_authenticated(self):
|
||||
text = CACHE.read_text()
|
||||
section = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
||||
for token in ('#### Nested input contracts', 'baseline_collection', 'inventory_sha256', 'baseline_status', 'expected_failure', 'replacement_ids', 'kind', 'delete', 'rename', 'status', 'review_evidence', 'scope_sha256', 'expected_exit_code', 'failure_kind', 'semantic_key', 'current_keys', 'observation', 'intentionally_changed', 'sort_keys=True', 'separators', 'UTF-8'):
|
||||
with self.subTest(token=token):
|
||||
self.assertIn(token, section)
|
||||
import json
|
||||
blocks = re.findall('```json\\n(.*?)```', section, re.S)
|
||||
self.assertEqual(len(blocks), 2, 'provide runnable test and contract input examples')
|
||||
self.assertEqual(set(json.loads(blocks[0])), {'baseline_ids', 'current_ids', 'baseline_status', 'current_status', 'baseline_collection', 'current_collection', 'authorized_changes', 'changed_tests', 'findings'})
|
||||
self.assertEqual(set(json.loads(blocks[1])), {'baseline', 'current', 'mappings', 'stages', 'precedence'})
|
||||
|
||||
def test_mapping_schema_types_use_raw_source_not_redacted_tool_rendering(self):
|
||||
# Authorization-looking prose can be masked by read-tool transport.
|
||||
# Inspect the approved public schema in raw source; never disable redaction.
|
||||
declaration = 'authorization: string, verification: string'
|
||||
masked = 'authorization: ' + '*' * 3 + ' verification'
|
||||
helper = ROOT / 'harness/lib/workflow_guard.py'
|
||||
for path in (helper, CACHE):
|
||||
text = read_contract(path)
|
||||
with self.subTest(path=path.name):
|
||||
self.assertEqual(text.count(declaration), 1)
|
||||
self.assertNotIn(masked, text)
|
||||
|
||||
def test_rework_no_longer_excludes_repo_evidence_from_clean_tree(self):
|
||||
text = read_contract(SKILLS['rework'])
|
||||
self.assertIn('archived v1 read-only', text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1 @@
|
||||
"""delegation regression tests."""
|
||||
+3
-4
@@ -1,4 +1,5 @@
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
@@ -49,9 +50,7 @@ class DelegationAdoptionTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def declaration(self):
|
||||
return {
|
||||
@@ -179,7 +178,7 @@ class DelegationAdoptionTests(unittest.TestCase):
|
||||
adoption.require_adoption(declaration, expected_source=self.context["source"], expected_stage="implementation")
|
||||
|
||||
def test_all_git_declaration_paths_reject_symlinks_at_runtime(self):
|
||||
from harness.tests.common.test_delegation_consumers_integration import RealEvidence
|
||||
from harness.tests._support.delegation import RealEvidence
|
||||
evidence = RealEvidence(self.base / 'safe-path-fixture')
|
||||
declaration = evidence.declaration()
|
||||
for field in ('bundle_root', 'context_path', 'result_path', 'git_dir'):
|
||||
+2
-1
@@ -1,9 +1,10 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import importlib.util
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
PATH = ROOT / "harness/agents/impl/skills/gitea-issue-implementation/scripts/delivery_adoption.py"
|
||||
spec = importlib.util.spec_from_file_location("delivery_adoption", PATH)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
+5
-62
@@ -1,4 +1,6 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
@@ -9,7 +11,7 @@ import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
SCRATCH = Path('/opt/data/cache/scratch/delegation-adoption-integration')
|
||||
SPEC = ROOT / 'harness/agents/spec/skills/gitea-implementation-spec-review/scripts/review_support.py'
|
||||
REWORK = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/scripts/rework_comment.py'
|
||||
@@ -27,66 +29,7 @@ delivery = load(DELIVERY, 'integration_delivery')
|
||||
native = load(NATIVE, 'integration_native')
|
||||
rework = load(REWORK, 'integration_rework')
|
||||
|
||||
class RealEvidence:
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.bundle = root / 'bundle'
|
||||
(self.bundle / 'src').mkdir(parents=True)
|
||||
(self.bundle / 'tests').mkdir()
|
||||
(self.bundle / 'evidence').mkdir()
|
||||
(self.bundle / 'src/a.py').write_text('VALUE = 1\n')
|
||||
(self.bundle / 'tests/test_a.py').write_text('def test_value():\n assert 1 == 1\n')
|
||||
(self.bundle / 'evidence/red.txt').write_text('red\n')
|
||||
(self.bundle / 'evidence/green.txt').write_text('green\n')
|
||||
subprocess.run(['/usr/bin/git', 'init', '-q', str(self.root / 'repo')], check=True)
|
||||
repo = self.root / 'repo'
|
||||
(repo / 'src').mkdir(); (repo / 'tests').mkdir()
|
||||
(repo / 'src/a.py').write_text('VALUE = 1\n')
|
||||
(repo / 'tests/test_a.py').write_text('def test_value():\n assert 1 == 1\n')
|
||||
subprocess.run(['/usr/bin/git', '-C', str(repo), 'config', 'user.email', 'test@example.invalid'], check=True)
|
||||
subprocess.run(['/usr/bin/git', '-C', str(repo), 'config', 'user.name', 'Integration'], check=True)
|
||||
subprocess.run(['/usr/bin/git', '-C', str(repo), 'add', '.'], check=True)
|
||||
subprocess.run(['/usr/bin/git', '-C', str(repo), 'commit', '-q', '-m', 'fixture'], check=True)
|
||||
self.revision = subprocess.check_output(['/usr/bin/git', '-C', str(repo), 'rev-parse', 'HEAD'], text=True).strip()
|
||||
self.git_dir = repo / '.git'
|
||||
self.files = {p: hashlib.sha256((self.bundle / p).read_bytes()).hexdigest() for p in ('src/a.py', 'tests/test_a.py')}
|
||||
self.bundle_sha = self.digest(self.files)
|
||||
self.context = {'schema': 'gitea-agent.delegation-evidence-context/v1',
|
||||
'source': {'kind': 'git_tree', 'revision': self.revision},
|
||||
'stage': 'spec-review', 'bundle_sha256': self.bundle_sha,
|
||||
'files': self.files, 'allowed_actions': ['static', 'execute'],
|
||||
'obligations': [{'id': 'static', 'kind': 'static'}, {'id': 'rg', 'kind': 'red_green'}],
|
||||
'executions': [self.execution('red', 1, 'assertion', 'evidence/red.txt'),
|
||||
self.execution('green', 0, 'none', 'evidence/green.txt')]}
|
||||
line_hash = hashlib.sha256(b'VALUE = 1\n').hexdigest()
|
||||
self.result = {'schema': 'gitea-agent.delegation-evidence-result/v1',
|
||||
'source': copy.deepcopy(self.context['source']), 'stage': 'spec-review',
|
||||
'bundle_sha256': self.bundle_sha,
|
||||
'claims': [{'id': 'static', 'status': 'supported', 'anchors': [{'kind': 'lines', 'path': 'src/a.py', 'start_line': 1, 'end_line': 1, 'text_sha256': line_hash}], 'execution_ids': []},
|
||||
{'id': 'rg', 'status': 'supported', 'anchors': [{'kind': 'test', 'path': 'tests/test_a.py', 'symbol': 'test_value'}], 'execution_ids': ['red', 'green']}]}
|
||||
self.context_path = root / 'context.json'; self.result_path = root / 'result.json'
|
||||
self.write()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
||||
|
||||
def execution(self, eid, code, kind, path):
|
||||
return {'id': eid, 'test_id': 'tests/test_a.py::test_value', 'bundle_sha256': self.bundle_sha,
|
||||
'test_file_sha256': self.files['tests/test_a.py'], 'argv': ['python', '-m', 'pytest'],
|
||||
'exit_code': code, 'failure_kind': kind, 'evidence_path': path,
|
||||
'evidence_sha256': hashlib.sha256((self.bundle / path).read_bytes()).hexdigest(), 'origin': 'direct'}
|
||||
|
||||
def write(self):
|
||||
self.context_path.write_text(json.dumps(self.context))
|
||||
self.result_path.write_text(json.dumps(self.result))
|
||||
|
||||
def declaration(self):
|
||||
return {'mode': 'delegated', 'bundle_root': str(self.bundle), 'context_path': str(self.context_path),
|
||||
'result_path': str(self.result_path), 'git_dir': str(self.git_dir)}
|
||||
|
||||
def set_stage(self, stage):
|
||||
self.context['stage'] = stage; self.result['stage'] = stage; self.write()
|
||||
from harness.tests._support.delegation import RealEvidence
|
||||
|
||||
class IntegrationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -137,7 +80,7 @@ class IntegrationTests(unittest.TestCase):
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def bound_publication_fixture(self):
|
||||
from harness.tests.review import test_native_review as fixture
|
||||
from harness.tests._support import review as fixture
|
||||
plan, before, after, exact = fixture.publication_fixture()
|
||||
old_head = plan['target']['head']
|
||||
def replace(value):
|
||||
+4
-4
@@ -1,4 +1,6 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -10,7 +12,7 @@ from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
SCHEMA_C = "gitea-agent.delegation-evidence-context/v1"
|
||||
SCHEMA_R = "gitea-agent.delegation-evidence-result/v1"
|
||||
|
||||
@@ -63,9 +65,7 @@ class DelegationEvidenceTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def line_hash(self, path, start, end):
|
||||
lines = (self.root / path).read_bytes().splitlines(keepends=True)
|
||||
+4
-6
@@ -1,6 +1,8 @@
|
||||
"""Independent safety/regression coverage for delegation evidence acceptance."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
|
||||
import copy
|
||||
from harness.tests._support.data import canonical_digest
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -13,7 +15,7 @@ from pathlib import Path
|
||||
|
||||
from harness.lib import delegation_evidence as evidence
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
HELPER = ROOT / "harness" / "lib" / "delegation_evidence.py"
|
||||
CONTEXT_SCHEMA = "gitea-agent.delegation-evidence-context/v1"
|
||||
RESULT_SCHEMA = "gitea-agent.delegation-evidence-result/v1"
|
||||
@@ -47,11 +49,7 @@ class DelegationEvidenceSafetyTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def digest(value):
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
digest = staticmethod(canonical_digest)
|
||||
|
||||
def make_context(self, *, actions=None, obligations=None, executions=None):
|
||||
return {
|
||||
@@ -0,0 +1 @@
|
||||
"""evidence regression tests."""
|
||||
+4
-5
@@ -3,17 +3,18 @@
|
||||
Run with an independently admitted pytest interpreter for execution coverage.
|
||||
No dependency installation occurs in this test or the production workflow.
|
||||
"""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
from harness.tests._support.contracts import read_contract
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
SKILL = ROOT / 'harness/agents/impl/skills/gitea-issue-rework/SKILL.md'
|
||||
HEADING = '### Conditional pytest evidence recipe'
|
||||
|
||||
@@ -27,9 +28,7 @@ class PytestEvidenceRecipeTests(unittest.TestCase):
|
||||
|
||||
def test_documentation_contract_limits_recipe_to_existing_pytest_projects(self):
|
||||
section, argv = self.recipe()
|
||||
for rule in ['pytest를 이미 사용하는', 'PYTEST_ADDOPTS', 'plugin',
|
||||
'evidence_gap', '이전 실행', '0으로', 'schema', 'unittest',
|
||||
'warning', 'timeout', 'output_limit']:
|
||||
for rule in ['PYTEST_ADDOPTS', 'plugin', 'evidence_gap', 'timeout', 'output_limit']:
|
||||
self.assertIn(rule, section)
|
||||
self.assertIsInstance(argv, list)
|
||||
self.assertEqual(argv[:4], ['{python}', '-B', '-m', 'pytest'])
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Read only the explicitly required workflow reference for canonical impl skills."""
|
||||
from pathlib import Path
|
||||
|
||||
NAMES = {"gitea-issue-implementation", "gitea-issue-rework"}
|
||||
REQUIRED = "Required reference: [workflow](references/workflow.md)"
|
||||
|
||||
def read_contract(path):
|
||||
path = Path(path)
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if path.name == "SKILL.md" and path.parent.name in NAMES:
|
||||
if REQUIRED not in text:
|
||||
raise AssertionError("impl skill does not require its canonical workflow reference")
|
||||
reference = path.parent / "references/workflow.md"
|
||||
if reference.is_symlink() or not reference.is_file():
|
||||
raise AssertionError("canonical workflow reference missing or symlinked")
|
||||
text += "\n" + reference.read_text(encoding="utf-8")
|
||||
return text
|
||||
@@ -0,0 +1 @@
|
||||
"""locking regression tests."""
|
||||
+4
-3
@@ -1,3 +1,4 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import fcntl
|
||||
import os
|
||||
import stat
|
||||
@@ -242,13 +243,13 @@ class ManagedLockTests(unittest.TestCase):
|
||||
self.assertEqual((before.st_ino, before.st_size, before.st_mode), (after.st_ino, after.st_size, after.st_mode))
|
||||
|
||||
def test_lock_contention_from_external_cwd_without_pythonpath(self):
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
root = SOURCE_ROOT
|
||||
script = """
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
suite = unittest.defaultTestLoader.loadTestsFromName(
|
||||
'harness.tests.common.test_managed_lock.ManagedLockTests.'
|
||||
'harness.tests.common.locking.test_managed_lock.ManagedLockTests.'
|
||||
'test_other_process_cannot_acquire_while_context_alive')
|
||||
result = unittest.TextTestRunner().run(suite)
|
||||
raise SystemExit(not result.wasSuccessful())
|
||||
@@ -294,7 +295,7 @@ raise SystemExit(not result.wasSuccessful())
|
||||
if not root.is_absolute() or not root.is_dir():
|
||||
raise RuntimeError("HARNESS_ROOT must be an existing absolute directory")
|
||||
root = root.resolve()
|
||||
if root != Path(__file__).resolve().parents[3]:
|
||||
if root != SOURCE_ROOT:
|
||||
raise RuntimeError("HARNESS_ROOT does not match this test installation")
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "process.lock"
|
||||
+9
-9
@@ -1,3 +1,4 @@
|
||||
from harness.tests._support.paths import HARNESS as SOURCE_HARNESS
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
@@ -14,7 +15,7 @@ from unittest.mock import patch
|
||||
from harness.lib.managed_lock import ManagedLock, probe_free
|
||||
from harness.lib import managed_lock_broker as broker
|
||||
|
||||
BROKER = Path(__file__).parents[2] / "lib" / "managed_lock_broker.py"
|
||||
BROKER = SOURCE_HARNESS / "lib" / "managed_lock_broker.py"
|
||||
|
||||
|
||||
def wait_ready(proc, timeout=3):
|
||||
@@ -86,7 +87,7 @@ class ManagedLockBrokerTests(unittest.TestCase):
|
||||
|
||||
def test_operation_specific_broker_response_documentation(self):
|
||||
import ast
|
||||
harness = Path(__file__).parents[2]
|
||||
harness = SOURCE_HARNESS
|
||||
operations = (harness / "operations/repository-cache.md").read_text()
|
||||
reference = (harness / "agents/review/skills/gitea-code-review/references/native-review.md").read_text()
|
||||
self.assertIn("### Operation-specific broker responses", operations)
|
||||
@@ -196,13 +197,12 @@ class ManagedLockBrokerTests(unittest.TestCase):
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
def test_canonical_role_values_and_example_are_documented(self):
|
||||
doc = (Path(__file__).parents[2] / "operations/repository-cache.md").read_text()
|
||||
section = doc.split("### Canonical command templates", 1)[1].split("## Lock 규칙", 1)[0]
|
||||
for role in ("gitea-spec-agent", "gitea-impl-agent", "gitea-review-agent"):
|
||||
self.assertIn(f"`{role}`", section)
|
||||
self.assertIn("--role gitea-spec-agent --run-id", section)
|
||||
self.assertIn("spec`/`impl`/`review", section)
|
||||
self.assertIn("authorization", section)
|
||||
doc = (SOURCE_HARNESS / 'operations/repository-cache.md').read_text()
|
||||
section = doc.split('### Canonical command templates', 1)[1].split('## Lock 규칙', 1)[0]
|
||||
for role in ('gitea-spec-agent', 'gitea-impl-agent', 'gitea-review-agent'):
|
||||
self.assertIn(f'`{role}`', section)
|
||||
self.assertIn('--role gitea-spec-agent --run-id', section)
|
||||
self.assertIn('spec`/`impl`/`review', section)
|
||||
|
||||
def test_constructor_rejects_lexical_root_and_handle_aliases(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
@@ -0,0 +1 @@
|
||||
"""repository regression tests."""
|
||||
+3
-2
@@ -1,4 +1,5 @@
|
||||
"""Synthetic local Git pre-fetch gate checks; no provider/network operations."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -6,7 +7,7 @@ import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
SCRIPT = ROOT / 'harness/lib/repository_prefetch.py'
|
||||
|
||||
|
||||
@@ -131,7 +132,7 @@ class RepositoryPrefetchTests(unittest.TestCase):
|
||||
section = text.split('## 기존 cache 갱신', 1)[1].split('\n## ', 1)[0]
|
||||
self.assertIn('repository_prefetch.py', section)
|
||||
self.assertLess(section.index('repository_prefetch.py'), section.index('prune/fetch'))
|
||||
for rule in ['--git-dir', 'repository lock', '자동 repair', 'exit 0', 'exit 1', 'exit 2']:
|
||||
for rule in ['--git-dir', 'repository lock', 'exit 0', 'exit 1', 'exit 2']:
|
||||
self.assertIn(rule, section)
|
||||
|
||||
def fingerprint(self, *roots):
|
||||
+2
-1
@@ -1,3 +1,4 @@
|
||||
from harness.tests._support.paths import HARNESS as SOURCE_HARNESS
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
@@ -156,7 +157,7 @@ class RuntimeRootTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); repo = self.cache(root)
|
||||
self.git('init', '--bare', repo / 'repository.git')
|
||||
base = Path(__file__).resolve().parents[2]
|
||||
base = SOURCE_HARNESS
|
||||
cmd = [sys.executable, '-B', str(base / 'lib' / 'managed_lock_broker.py'), 'serve',
|
||||
'--root', str(root), '--owner', 'owner', '--repository', 'repo', '--issue', '17',
|
||||
'--role', 'gitea-impl-agent', '--run-id', 'cli-workflow', '--ttl-seconds', '30']
|
||||
+2
-1
@@ -1,4 +1,5 @@
|
||||
"""Real-Git regressions; no pytest or product execution dependency."""
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
@@ -12,7 +13,7 @@ from unittest.mock import patch
|
||||
|
||||
from harness.lib import test_inventory_binding as binding
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ROOT = SOURCE_ROOT
|
||||
|
||||
|
||||
class InventoryBindingTests(unittest.TestCase):
|
||||
@@ -0,0 +1 @@
|
||||
"""runner regression tests."""
|
||||
+26
-35
@@ -1,3 +1,4 @@
|
||||
from harness.tests._support.paths import ROOT as SOURCE_ROOT
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -8,7 +9,7 @@ from unittest import mock
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
REPO = SOURCE_ROOT
|
||||
RUNNER = REPO / "harness" / "tests" / "run.py"
|
||||
|
||||
|
||||
@@ -76,17 +77,19 @@ class CanonicalRunnerTests(unittest.TestCase):
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/common/test_common.py").write_text("import unittest\n")
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn("zero", result.stderr.lower())
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/common/test_common.py").write_text('''import unittest
|
||||
class X(unittest.TestCase):
|
||||
def test_one(self): pass
|
||||
def test_free(): pass
|
||||
''')
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("free-function", result.stderr.lower())
|
||||
# Preserve both mixed TestCase/free-function and free-only inputs.
|
||||
for source in (
|
||||
"import unittest\nclass X(unittest.TestCase):\n def test_one(self): pass\ndef test_free(): pass\n",
|
||||
"def test_free(): pass\n",
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/common/test_common.py").write_text(source)
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn("free-function", result.stderr.lower())
|
||||
|
||||
def test_direct_case_does_not_short_circuit_cycle_or_unknown_base(self):
|
||||
for source, word in (
|
||||
@@ -197,12 +200,21 @@ assert not Path(sys.argv[2]).exists()
|
||||
|
||||
def test_collect_only_reports_exact_roles_ids_and_hashes(self):
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/spec/test_spec.py").write_text(
|
||||
"import unittest\nclass X(unittest.TestCase):\n def test_one(self): pass\n def test_two(self): pass\n")
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
data = json.loads(result.stdout)
|
||||
self.assertTrue(data["collection_only"])
|
||||
self.assertEqual(data["role_counts"], {"common": 1, "spec": 1, "impl": 1, "review": 1})
|
||||
self.assertEqual(len(data["ids"]), 4)
|
||||
self.assertEqual(data["role_counts"], {"common": 1, "spec": 2, "impl": 1, "review": 1})
|
||||
self.assertEqual(data["ids"], sorted([
|
||||
"harness.tests.common.test_common.Example.test_one",
|
||||
"harness.tests.spec.test_spec.X.test_one", "harness.tests.spec.test_spec.X.test_two",
|
||||
"harness.tests.impl.test_impl.Example.test_one", "harness.tests.review.test_review.Example.test_one",
|
||||
]))
|
||||
import hashlib
|
||||
for path, digest in data["source_sha256"].items():
|
||||
self.assertEqual(digest, hashlib.sha256((root / path).read_bytes()).hexdigest())
|
||||
self.assertEqual(set(data["source_sha256"]), {"harness/tests/common/test_common.py", "harness/tests/spec/test_spec.py", "harness/tests/impl/test_impl.py", "harness/tests/review/test_review.py"})
|
||||
|
||||
def test_missing_role_marker_duplicate_and_unexpected_file_fail_before_run(self):
|
||||
@@ -220,23 +232,8 @@ assert not Path(sys.argv[2]).exists()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("canonical", result.stderr.lower())
|
||||
|
||||
def test_loader_omission_addition_zero_file_and_unsupported_function_fail(self):
|
||||
cases = {
|
||||
"omission": lambda root: (root / "harness/tests/common/test_common.py").write_text("import unittest\ndef load_tests(loader, tests, pattern): return unittest.TestSuite()\nclass X(unittest.TestCase):\n def test_one(self): pass\n"),
|
||||
"zero": lambda root: (root / "harness/tests/review/test_review.py").write_text("import unittest\n"),
|
||||
"free": lambda root: (root / "harness/tests/impl/test_impl.py").write_text("def test_free(): pass\n"),
|
||||
}
|
||||
for name, mutate in cases.items():
|
||||
with self.subTest(name=name):
|
||||
root = self.make_suite(); mutate(root)
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/spec/test_spec.py").write_text("import unittest\nclass X(unittest.TestCase):\n def test_one(self): pass\n def test_two(self): pass\n")
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_import_failure_source_drift_nested_omission_and_runtime_failure(self):
|
||||
def test_import_failure_source_drift_and_nested_omission(self):
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/impl/test_impl.py").write_text("raise RuntimeError('broken import')\n")
|
||||
result = self.execute(root, "--collect-only")
|
||||
@@ -252,12 +249,6 @@ assert not Path(sys.argv[2]).exists()
|
||||
"class X(unittest.TestCase):\n def test_one(self): pass\n")
|
||||
result = self.execute(root, "--collect-only")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
root = self.make_suite()
|
||||
(root / "harness/tests/common/test_common.py").write_text(
|
||||
"import unittest\nclass X(unittest.TestCase):\n def test_one(self): self.fail('boom')\n")
|
||||
result = self.execute(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("FAILED", result.stderr + result.stdout)
|
||||
|
||||
def test_package_manifest_drift_and_extra_zero_file_fail_closed(self):
|
||||
cases = (
|
||||
@@ -1,232 +0,0 @@
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from harness.tests.common.impl_contract_docs import read_contract
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
||||
PREPARE_SCRIPT = ROOT / "harness/lib/verification_root.py"
|
||||
CACHE = ROOT / "harness/operations/repository-cache.md"
|
||||
SKILLS = {
|
||||
"implementation": ROOT / "harness/agents/impl/skills/gitea-issue-implementation/SKILL.md",
|
||||
"rework": ROOT / "harness/agents/impl/skills/gitea-issue-rework/SKILL.md",
|
||||
"review": ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/SKILL.md",
|
||||
}
|
||||
CANONICAL = "<harness-root>/repositories/<owner>/<repo>/worktrees/issue-<number>/verification/<run-id>"
|
||||
|
||||
|
||||
class VerificationContractDocumentationTests(unittest.TestCase):
|
||||
def test_runtime_policy_is_broker_owned_and_separate_from_verification(self):
|
||||
text = CACHE.read_text(encoding="utf-8")
|
||||
self.assertIn("runtime/<workflow-run-id>", text)
|
||||
self.assertIn("broker.handle", text)
|
||||
self.assertIn("broker.sock", text)
|
||||
self.assertIn("ready JSON", text)
|
||||
self.assertIn("runtime 생성은 승인이나 admission을 의미하지 않는다", text)
|
||||
self.assertIn("active resource", text)
|
||||
self.assertIn("unknown", text)
|
||||
self.assertIn("성공 기록을 실패로 덮어쓰지 않는다", text)
|
||||
for path in SKILLS.values():
|
||||
with self.subTest(path=path.name):
|
||||
role = read_contract(path)
|
||||
gate = role.split("## Managed-lock broker gate", 1)[1].split("\n## ", 1)[0]
|
||||
self.assertIn("broker-generated", gate)
|
||||
self.assertIn("ready JSON", gate)
|
||||
self.assertNotIn("pre-created", gate)
|
||||
self.assertNotIn("pre-create", gate)
|
||||
|
||||
def test_role_runtime_startup_precedes_and_does_not_replace_admission(self):
|
||||
roles = [path for path in (ROOT / 'harness/agents').rglob('SKILL.md')
|
||||
if 'broker-generated runtime' in read_contract(path)]
|
||||
self.assertEqual(len(roles), 6)
|
||||
for path in roles:
|
||||
with self.subTest(role=path.parent.name):
|
||||
gate = read_contract(path).split('## Managed-lock broker gate', 1)[1].split('\n## ', 1)[0]
|
||||
self.assertIn('ready 이후 각 workflow의 진입 검증·승인 조건을 확인한다', gate)
|
||||
self.assertIn('Runtime 생성은 approval이나 worktree admission을 대신하지 않는다', gate)
|
||||
self.assertIn('ready JSON의 `handle`/`socket`', gate)
|
||||
|
||||
def test_evidence_plan_disables_python_bytecode_in_clean_checkout(self):
|
||||
import json
|
||||
template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json"
|
||||
plan = json.loads(template.read_text())
|
||||
self.assertIn("-B", plan["commands"][0]["argv"])
|
||||
self.assertIn("PYTHONDONTWRITEBYTECODE", plan["command_env_allowlist"])
|
||||
|
||||
def test_full_suite_tracked_background_reconciles_before_retry(self):
|
||||
text = read_contract(SKILLS['rework'])
|
||||
heading = '### Tracked bounded background evidence execution'
|
||||
self.assertIn(heading, text)
|
||||
section = text.split(heading, 1)[1].split('\n### ', 1)[0]
|
||||
for rule in ['foreground tool 시간창', 'tracked background',
|
||||
'process/session handle', 'run-id', 'source SHA', 'plan digest',
|
||||
'output path', 'timeout_seconds', '유한한 outer deadline',
|
||||
'manifest', 'validate', 'cleanup', 'pending/partial',
|
||||
'중복 실행하지 않는다', '새 run-id', '별도 승인',
|
||||
'TDD RED', '세션 종료 이후의 지속성을 보장하지 않는다']:
|
||||
with self.subTest(rule=rule):
|
||||
self.assertIn(rule, section)
|
||||
self.assertIn('자동 retry하지 않는다', section)
|
||||
self.assertIn('기존 artifact를 덮어쓰지 않는다', section)
|
||||
|
||||
def test_command_deadline_and_archive_policy_is_explicit(self):
|
||||
import json
|
||||
template = ROOT / "harness/agents/impl/skills/gitea-issue-rework/templates/evidence-plan.json"
|
||||
for command in json.loads(template.read_text())["commands"]:
|
||||
self.assertIsInstance(command["timeout_seconds"], (int, float))
|
||||
self.assertGreater(command["timeout_seconds"], 0)
|
||||
self.assertLessEqual(command["timeout_seconds"], 3600)
|
||||
for path in (CACHE, *SKILLS.values()):
|
||||
text = read_contract(path)
|
||||
for token in ("evidence-manifest/v3", "timeout_seconds", "process group",
|
||||
"validate-archived-v2", "TDD RED", "자동 재실행"):
|
||||
with self.subTest(path=str(path), token=token):
|
||||
self.assertIn(token, text)
|
||||
help_result = subprocess.run([sys.executable, str(SCRIPT), "validate-archived-v2", "--help"],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
||||
for option in ("--contract-path", "--repo-root", "--artifact-root", "--manifest"):
|
||||
self.assertIn(option, help_result.stdout)
|
||||
|
||||
def test_all_roles_reference_canonical_external_artifacts(self):
|
||||
for path in (CACHE, *SKILLS.values()):
|
||||
with self.subTest(path=path.name):
|
||||
text = read_contract(path)
|
||||
self.assertIn(CANONICAL, text)
|
||||
self.assertIn("evidence_gap", text)
|
||||
self.assertIn("[REDACTED]", text)
|
||||
self.assertIn("evidence-manifest/v3", text)
|
||||
|
||||
def test_examples_match_real_helper_cli_not_unsupported_options(self):
|
||||
for path in (SKILLS["implementation"], SKILLS["rework"]):
|
||||
section = read_contract(path).split("## Canonical external verification evidence", 1)[1]
|
||||
match = re.search(r"```sh\n(.*?)```", section, re.S)
|
||||
assert match is not None
|
||||
block = match.group(1).replace("\\\n", " ")
|
||||
commands = [line for line in block.splitlines() if line.startswith("python3 ")]
|
||||
self.assertEqual(len(commands), 3)
|
||||
for op, command in zip(("prepare", "run", "validate"), commands):
|
||||
with self.subTest(skill=path.parent.name, op=op):
|
||||
declared = set(re.findall(r"--[a-z-]+", command))
|
||||
if op == "prepare":
|
||||
self.assertIn("harness/lib/verification_root.py", command)
|
||||
help_result = subprocess.run(
|
||||
[sys.executable, str(PREPARE_SCRIPT), op, "--help"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(help_result.returncode, 0, help_result.stderr)
|
||||
supported = set(re.findall(r"--[a-z-]+", help_result.stdout))
|
||||
required = {"--root", "--owner", "--repository", "--issue", "--run-id", "--handle"}
|
||||
self.assertEqual(declared, required)
|
||||
self.assertTrue(declared <= supported)
|
||||
else:
|
||||
help_result = subprocess.run([sys.executable, str(SCRIPT), op, "--help"], text=True, capture_output=True)
|
||||
self.assertEqual(help_result.returncode, 0)
|
||||
supported = set(re.findall(r"--[a-z-]+", help_result.stdout))
|
||||
required = {"--contract-path", "--repo-root", "--artifact-root"} | ({"--plan", "--output"} if op == "run" else {"--manifest"})
|
||||
self.assertEqual(declared, required)
|
||||
self.assertTrue(declared <= supported)
|
||||
self.assertIn("exact current HEAD", section)
|
||||
self.assertIn("validate-legacy", section)
|
||||
self.assertIn("exclusion 없음", section)
|
||||
|
||||
def test_prepare_contract_binds_root_and_forbids_recursive_parent_creation(self):
|
||||
for path in (CACHE, SKILLS["implementation"], SKILLS["rework"]):
|
||||
with self.subTest(path=path.name):
|
||||
text = read_contract(path)
|
||||
self.assertIn("exact harness root", text)
|
||||
self.assertIn("Path.mkdir(parents=True, mode=0o700)", text)
|
||||
self.assertIn("자동 chmod", text)
|
||||
|
||||
def test_checkout_cleanup_preserves_verification_without_recursive_delete(self):
|
||||
text = CACHE.read_text()
|
||||
cleanup = text.split("## Worktree 실패 처리와 제거", 1)[1].split("## Credential 안전", 1)[0]
|
||||
self.assertIn("`verification/`이 남으면 Issue directory를 보존한다", cleanup)
|
||||
self.assertIn("Issue directory가 남는 것은 checkout 제거 실패가 아니다", cleanup)
|
||||
self.assertIn("digest가 제거 전과 같음", cleanup)
|
||||
self.assertIn("Artifact 폐기는 별도의 exact 경로 승인", text)
|
||||
self.assertIn("mode `0700`", text)
|
||||
self.assertIn("mode `0600`", text)
|
||||
|
||||
def test_spec_review_uses_immutable_source_and_not_moving_checkout(self):
|
||||
section = SKILLS["review"].read_text().split("## External Issue verification evidence boundary", 1)[1]
|
||||
self.assertIn("제품 테스트를 실행하지 않", section)
|
||||
self.assertIn("moving implementation checkout", section)
|
||||
self.assertIn("authoritative blob evidence", section)
|
||||
self.assertIn("trusted `harness/.runtime`", section)
|
||||
self.assertIn("Producer evidence와 reviewer 실행을 구분", section)
|
||||
|
||||
def test_workflow_guard_records_reuse_existing_verification_location(self):
|
||||
text = CACHE.read_text()
|
||||
guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
||||
self.assertNotIn('or verification root', guard)
|
||||
for path in (SKILLS['implementation'], SKILLS['rework']):
|
||||
self.assertNotIn('outside product checkout and verification root', read_contract(path))
|
||||
self.assertIn('verification/<run-id>', guard)
|
||||
|
||||
def test_workflow_guard_readonly_scope_does_not_ban_approved_worker_edits(self):
|
||||
text = CACHE.read_text()
|
||||
guard = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
||||
self.assertIn('읽기 전용', guard)
|
||||
self.assertIn('쓰기 worker는 승인된 변경 allowlist', guard)
|
||||
|
||||
def test_workflow_guard_snapshot_protects_scope_before_delegation(self):
|
||||
implementation = read_contract(SKILLS["implementation"])
|
||||
self.assertIn("workflow_guard.py snapshot", implementation)
|
||||
self.assertIn("compare-source --repo-root", implementation)
|
||||
self.assertIn("immutable protected scope", implementation)
|
||||
|
||||
def test_workflow_guard_inventory_is_checked_before_adopting_results(self):
|
||||
rework = read_contract(SKILLS["rework"])
|
||||
self.assertIn("workflow_guard.py check-tests", rework)
|
||||
self.assertIn("test inventory", rework)
|
||||
self.assertIn("RED/GREEN records must be named", rework)
|
||||
|
||||
def test_workflow_guard_contract_preservation_precedes_spec_verdict(self):
|
||||
spec = SKILLS["review"].read_text()
|
||||
self.assertIn("workflow_guard.py check-contracts", spec)
|
||||
self.assertIn("baseline inventory", spec)
|
||||
self.assertIn("Gate failure invalidates workflow evidence", spec)
|
||||
|
||||
def test_workflow_guard_nested_schema_is_actionable_and_claims_are_not_authenticated(self):
|
||||
text = CACHE.read_text()
|
||||
section = text.split('### Shared workflow guard gates', 1)[1].split('Issue별 검증 실행 기록의 정본', 1)[0]
|
||||
for token in ('#### Nested input contracts', 'baseline_collection', 'inventory_sha256',
|
||||
'baseline_status', 'expected_failure', 'replacement_ids', 'kind',
|
||||
'delete', 'rename', 'status', 'review_evidence', 'scope_sha256',
|
||||
'expected_exit_code', 'failure_kind', 'semantic_key', 'current_keys',
|
||||
'observation', 'intentionally_changed', 'sort_keys=True',
|
||||
'separators', 'UTF-8', 'shape-only', 'one row per baseline semantic key'):
|
||||
with self.subTest(token=token):
|
||||
self.assertIn(token, section)
|
||||
import json
|
||||
blocks = re.findall(r'```json\n(.*?)```', section, re.S)
|
||||
self.assertEqual(len(blocks), 2, 'provide runnable test and contract input examples')
|
||||
self.assertEqual(set(json.loads(blocks[0])), {'baseline_ids', 'current_ids', 'baseline_status',
|
||||
'current_status', 'baseline_collection', 'current_collection', 'authorized_changes', 'changed_tests', 'findings'})
|
||||
self.assertEqual(set(json.loads(blocks[1])), {'baseline', 'current', 'mappings', 'stages', 'precedence'})
|
||||
|
||||
def test_mapping_schema_types_use_raw_source_not_redacted_tool_rendering(self):
|
||||
# Authorization-looking prose can be masked by read-tool transport.
|
||||
# Inspect the approved public schema in raw source; never disable redaction.
|
||||
declaration = 'authorization: string, verification: string'
|
||||
masked = 'authorization: ' + '*' * 3 + ' verification'
|
||||
helper = ROOT / 'harness/lib/workflow_guard.py'
|
||||
for path in (helper, CACHE):
|
||||
text = read_contract(path)
|
||||
with self.subTest(path=path.name):
|
||||
self.assertEqual(text.count(declaration), 1)
|
||||
self.assertNotIn(masked, text)
|
||||
|
||||
def test_rework_no_longer_excludes_repo_evidence_from_clean_tree(self):
|
||||
text = read_contract(SKILLS["rework"])
|
||||
self.assertNotIn("plan과 manifest를 제외한 clean tree", text)
|
||||
self.assertIn("새 evidence는 checkout 밖", text)
|
||||
self.assertIn("archived v1 read-only", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user