test(harness): reorganize regression suites and preserve contract coverage

This commit is contained in:
2026-10-16 07:24:49 +00:00
parent b6f87db623
commit 33709ab873
108 changed files with 5869 additions and 5328 deletions
+1
View File
@@ -0,0 +1 @@
"""Static test helpers; never declare or re-export test cases here."""
+17
View File
@@ -0,0 +1,17 @@
"""Read only the explicitly required workflow reference for canonical impl skills."""
from pathlib import Path
NAMES = {"gitea-issue-implementation", "gitea-issue-rework"}
REQUIRED = "Required reference: [workflow](references/workflow.md)"
def read_contract(path):
path = Path(path)
text = path.read_text(encoding="utf-8")
if path.name == "SKILL.md" and path.parent.name in NAMES:
if REQUIRED not in text:
raise AssertionError("impl skill does not require its canonical workflow reference")
reference = path.parent / "references/workflow.md"
if reference.is_symlink() or not reference.is_file():
raise AssertionError("canonical workflow reference missing or symlinked")
text += "\n" + reference.read_text(encoding="utf-8")
return text
+7
View File
@@ -0,0 +1,7 @@
"""Independent canonical JSON digests for synthetic evidence fixtures."""
import hashlib
import json
def canonical_digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+65
View File
@@ -0,0 +1,65 @@
"""Synthetic real-Git delegation evidence builder; no test cases."""
import copy
import hashlib
import json
import subprocess
from harness.tests._support.data import canonical_digest
class RealEvidence:
def __init__(self, root):
self.root = root
self.bundle = root / 'bundle'
(self.bundle / 'src').mkdir(parents=True)
(self.bundle / 'tests').mkdir()
(self.bundle / 'evidence').mkdir()
(self.bundle / 'src/a.py').write_text('VALUE = 1\n')
(self.bundle / 'tests/test_a.py').write_text('def test_value():\n assert 1 == 1\n')
(self.bundle / 'evidence/red.txt').write_text('red\n')
(self.bundle / 'evidence/green.txt').write_text('green\n')
subprocess.run(['/usr/bin/git', 'init', '-q', str(self.root / 'repo')], check=True)
repo = self.root / 'repo'
(repo / 'src').mkdir(); (repo / 'tests').mkdir()
(repo / 'src/a.py').write_text('VALUE = 1\n')
(repo / 'tests/test_a.py').write_text('def test_value():\n assert 1 == 1\n')
subprocess.run(['/usr/bin/git', '-C', str(repo), 'config', 'user.email', 'test@example.invalid'], check=True)
subprocess.run(['/usr/bin/git', '-C', str(repo), 'config', 'user.name', 'Integration'], check=True)
subprocess.run(['/usr/bin/git', '-C', str(repo), 'add', '.'], check=True)
subprocess.run(['/usr/bin/git', '-C', str(repo), 'commit', '-q', '-m', 'fixture'], check=True)
self.revision = subprocess.check_output(['/usr/bin/git', '-C', str(repo), 'rev-parse', 'HEAD'], text=True).strip()
self.git_dir = repo / '.git'
self.files = {p: hashlib.sha256((self.bundle / p).read_bytes()).hexdigest() for p in ('src/a.py', 'tests/test_a.py')}
self.bundle_sha = self.digest(self.files)
self.context = {'schema': 'gitea-agent.delegation-evidence-context/v1',
'source': {'kind': 'git_tree', 'revision': self.revision},
'stage': 'spec-review', 'bundle_sha256': self.bundle_sha,
'files': self.files, 'allowed_actions': ['static', 'execute'],
'obligations': [{'id': 'static', 'kind': 'static'}, {'id': 'rg', 'kind': 'red_green'}],
'executions': [self.execution('red', 1, 'assertion', 'evidence/red.txt'),
self.execution('green', 0, 'none', 'evidence/green.txt')]}
line_hash = hashlib.sha256(b'VALUE = 1\n').hexdigest()
self.result = {'schema': 'gitea-agent.delegation-evidence-result/v1',
'source': copy.deepcopy(self.context['source']), 'stage': 'spec-review',
'bundle_sha256': self.bundle_sha,
'claims': [{'id': 'static', 'status': 'supported', 'anchors': [{'kind': 'lines', 'path': 'src/a.py', 'start_line': 1, 'end_line': 1, 'text_sha256': line_hash}], 'execution_ids': []},
{'id': 'rg', 'status': 'supported', 'anchors': [{'kind': 'test', 'path': 'tests/test_a.py', 'symbol': 'test_value'}], 'execution_ids': ['red', 'green']}]}
self.context_path = root / 'context.json'; self.result_path = root / 'result.json'
self.write()
digest = staticmethod(canonical_digest)
def execution(self, eid, code, kind, path):
return {'id': eid, 'test_id': 'tests/test_a.py::test_value', 'bundle_sha256': self.bundle_sha,
'test_file_sha256': self.files['tests/test_a.py'], 'argv': ['python', '-m', 'pytest'],
'exit_code': code, 'failure_kind': kind, 'evidence_path': path,
'evidence_sha256': hashlib.sha256((self.bundle / path).read_bytes()).hexdigest(), 'origin': 'direct'}
def write(self):
self.context_path.write_text(json.dumps(self.context))
self.result_path.write_text(json.dumps(self.result))
def declaration(self):
return {'mode': 'delegated', 'bundle_root': str(self.bundle), 'context_path': str(self.context_path),
'result_path': str(self.result_path), 'git_dir': str(self.git_dir)}
def set_stage(self, stage):
self.context['stage'] = stage; self.result['stage'] = stage; self.write()
+72
View File
@@ -0,0 +1,72 @@
"""Ordinary evidence fixture: owns synthetic Git checkout and artifacts."""
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from harness.tests._support.paths import ROOT
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
def run_git(cwd, *args):
return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=True)
class EvidenceManifestFixture:
def setup(self):
try:
self._setup()
except BaseException:
self.cleanup()
raise
return self
def cleanup(self):
if hasattr(self, 'tmp'):
self.tmp.cleanup()
def _setup(self):
self.tmp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR', '/opt/data/cache/scratch'))
self.root = Path(self.tmp.name)
self.contract = self.root / 'harness/operations/repository-cache.md'
self.contract.parent.mkdir(parents=True)
self.contract.write_text('# Repository cache 운영 지침\n')
self.repo = self.root / 'repositories/owner/project/worktrees/issue-1/checkout'
self.artifacts = self.repo.parent / 'verification/run-a'
self.repo.mkdir(parents=True)
self.artifacts.mkdir(parents=True)
for path in (self.root / 'repositories', *self.repo.parents):
if path == self.root:
break
path.chmod(448)
self.repo.chmod(448)
self.artifacts.parent.chmod(448)
self.artifacts.chmod(448)
run_git(self.repo, 'init', '-q')
run_git(self.repo, 'config', 'user.email', 'test@example.invalid')
run_git(self.repo, 'config', 'user.name', 'Test')
(self.repo / 'input.txt').write_text('hello\n')
run_git(self.repo, 'add', 'input.txt')
run_git(self.repo, 'commit', '-qm', 'initial')
def invoke(self, *args, env=None):
op, *rest = args
common = [] if op == 'validate-legacy' else ['--contract-path', str(self.contract), '--artifact-root', str(self.artifacts)]
return subprocess.run([sys.executable, str(SCRIPT), str(op), *common, *map(str, rest)], cwd=self.repo, text=True, capture_output=True, env=env)
def plan(self, commands=None, **extra):
if commands is None:
commands = [{'id': 'check', 'purpose': 'show input', 'argv': ['git', 'show', 'HEAD:input.txt'], 'expected_exit_code': 0, 'claims': ['output']}]
data = {'hash_paths': ['input.txt'], 'commands': [dict({'timeout_seconds': 5}, **command) for command in commands]}
data.update(extra)
p = self.artifacts / 'plan.json'
p.write_text(json.dumps(data))
p.chmod(384)
return p
def run_manifest(self, plan=None):
if plan is None:
plan = self.plan()
out = self.artifacts / 'manifest.json'
result = self.invoke('run', '--repo-root', self.repo, '--plan', plan.name, '--output', 'manifest.json')
return (result, out)
+6
View File
@@ -0,0 +1,6 @@
"""Canonical source-tree paths, independent of cwd and environment."""
from pathlib import Path
TESTS = Path(__file__).resolve().parents[1]
HARNESS = TESTS.parent
ROOT = HARNESS.parent
+112
View File
@@ -0,0 +1,112 @@
"""Shared review fixtures; no unittest test cases or TestCase imports."""
import copy
import importlib.util
from pathlib import Path
from harness.tests._support.paths import ROOT
SPEC_SCRIPT = ROOT / 'harness/agents/spec/skills/gitea-implementation-spec-review/scripts/review_support.py'
def _load(path, name):
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
support = _load(SPEC_SCRIPT, 'spec_support_fixture')
def plan_fixture():
target = {'owner': 'acme', 'repository': 'repo', 'issue': 3, 'pull': 7,
'head': 'a' * 40, 'base': 'c' * 40}
digest = __import__('hashlib').sha256(b'approved contract').hexdigest()
marker = ('<!-- gitea-agent:code-review/v1 owner=acme repository=repo issue=3 pr=7 '
f'head={target["head"]} contract-sha256={digest} -->')
return {'schema': 'gitea-agent.code-review-publication/v1', 'target': target,
'contract_sha256': digest, 'reviewer': {'id': 42, 'login': 'gitea-review-agent'},
'spec_reviewer': {'id': 5, 'login': 'gitea-spec-agent'}, 'spec_review_id': 9, 'spec_paths': ['src/a.py'],
'verdict': 'approve', 'body': marker + '\n- Verdict: `approve`\n검토 완료.\n',
'provider_contract': {'supersede_same_author': True, 'retouch_dismissed': True,
'unordered_commit_files': True,
'evidence': ['saved observed Gitea projection and ordering']}}
def publication_fixture():
p = plan_fixture(); t = p['target']
spec_result = {'schema': support.SCHEMA,
'target': {k: t[k] for k in ['owner', 'repository', 'issue', 'pull', 'head']},
'contract_sha256': p['contract_sha256'], 'verdict': 'approve',
'criteria': [{'criterion': 'REQ-1', 'contract': '계약',
'implementation_paths': ['src/a.py'], 'test_paths': [],
'evidence': '직접 검증', 'result': 'satisfied', 'reason': '충족'}],
'scope': {'missing': [], 'excess': [], 'spec_conflicts': []}, 'conclusion': '충족'}
url = 'https://gitea.jsfamily.quest/acme/repo/pulls/7'
old_time = '2026-10-04T00:00:00Z'; new_time = '2026-10-04T00:01:00Z'
def review(rid, user, body, head, state):
return {'id': rid, 'user': copy.deepcopy(user), 'body': body, 'commit_id': head, 'state': state,
'official': True, 'dismissed': False, 'stale': head != t['head'],
'submitted_at': old_time, 'updated_at': old_time, 'comments_count': 0,
'pull_request_url': url}
sr = review(9, p['spec_reviewer'], support.render_review(spec_result, {'src/a.py'}, declaration={'mode': 'none'}),
t['head'], 'APPROVED')
previous = review(8, p['reviewer'], 'old review body', 'd' * 40, 'REQUEST_CHANGES')
s = {'identity': copy.deepcopy(p['reviewer']), 'spec_user': copy.deepcopy(p['spec_reviewer']),
'repository': {'full_name': 'acme/repo', 'default_branch': 'main'},
'issue': {'number': 3, 'body': 'approved contract', 'state': 'open',
'updated_at': old_time, 'repository': {'full_name': 'acme/repo'}},
'pr': {'number': 7, 'html_url': url, 'body': 'Closes #3', 'user': {'id': 99},
'state': 'open', 'merged': False, 'updated_at': old_time,
'review_comments': 2, 'requested_reviewers': [],
'base': {'sha': t['base'], 'ref': 'main', 'repo': {'full_name': 'acme/repo'}},
'head': {'sha': t['head'], 'ref': 'feature', 'repo': {'full_name': 'acme/repo'}}},
'files': [{'filename': 'src/a.py', 'status': 'modified'}],
'commits': [{'sha': t['head'], 'files': [{'filename': 'src/a.py', 'status': 'modified'},
{'filename': 'docs/x.md', 'status': 'added'}]}],
'reviews': [previous, sr], 'review_comments': {'8': [], '9': []},
'issue_comments': [], 'pr_comments': [], 'statuses': [], 'pagination': {}}
exact = review(10, p['reviewer'], p['body'], t['head'], 'APPROVED')
exact['submitted_at'] = exact['updated_at'] = new_time
after = copy.deepcopy(s); after['reviews'].append(exact); after['review_comments']['10'] = []
after['pr']['updated_at'] = new_time; after['pr']['review_comments'] += 1
for snap in [s, after]:
for name in ['files', 'commits', 'reviews', 'issue_comments', 'pr_comments', 'statuses']:
snap['pagination'][name] = [{'page': 1, 'count': len(snap[name]),
'declared_total': len(snap[name]), 'link': None}]
for rid, comments in snap['review_comments'].items():
snap['pagination']['review_comments_' + rid] = [{'page': 1, 'count': len(comments),
'declared_total': len(comments), 'link': None}]
return p, s, after, copy.deepcopy(exact)
class FakeAPI:
def __init__(self, plan, before, after, exact):
self.plan = plan; self.before = before; self.after = after; self.exact = exact
self.created = False; self.posts = 0; self.timeout = False; self.apply = True
self.events = []
def identity(self):
self.events.append('identity')
return copy.deepcopy(self.plan['reviewer'])
def snapshot(self, target):
self.events.append('snapshot')
return copy.deepcopy(self.after if self.created else self.before)
def create_review(self, target, payload):
self.events.append('create'); self.posts += 1
if self.apply: self.created = True
if self.timeout: raise TimeoutError('secret transport detail must not escape')
return copy.deepcopy(self.exact)
def read_review(self, target, rid):
self.events.append('exact')
return copy.deepcopy(self.exact)
class FakeLease:
def __init__(self): self.valid = True; self.checks = 0
def check(self, plan):
self.checks += 1
if not self.valid: raise support.ValidationError('expired lease')
+130
View File
@@ -0,0 +1,130 @@
"""Provider-shaped rework readback fixture with explicit resource ownership."""
import copy
import hashlib
import importlib.util
import json
import tempfile
from argparse import Namespace
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from harness.tests._support.paths import ROOT
DIRECTORY = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts"
ORIGIN = "https://gitea.jsfamily.quest"
T0 = "2026-01-01T00:00:00Z"
T1 = "2026-01-02T00:00:00Z"
class ReworkProviderFixture:
"""Own synthetic provider state and its temporary resources."""
def __init__(self):
self._temp = None
self._cleaned = False
def setup(self):
if self._temp is not None:
raise RuntimeError("fixture already set up")
self._temp = tempfile.TemporaryDirectory()
root = Path(self._temp.name)
self.legacy = self.load("rework_comment")
data = {
"owner": "acme", "repository": "demo", "issue": 7, "pr": 9, "review": 301,
"reviewed_head": "a" * 40, "final_head": "b" * 40, "contract_sha256": "c" * 64,
"result": "partial", "findings": [{"id": "F-1", "status": "resolved", "classification": "valid_fix",
"changes": "bounded fix", "verification": "local checks pass", "reason": "regression"}],
"local_checks": "pass", "runtime_browser": "not applicable",
"commit_status": "success", "actions_runs": "없음",
}
(root / "input.json").write_text(json.dumps(data))
self.legacy.do_render(Namespace(input=str(root / "input.json"), no_delegated_evidence=True,
delegation_adoption=None, comment=str(root / "comment.md"), expected=str(root / "expected.json")))
self.expected = json.loads((root / "expected.json").read_text())
self.contract = {"schema": "gitea-agent.rework-provider/v1", "version": "1.24.6",
"aggregate": {"pr": {"field": "comments", "components": ["top_level", "reviews"]},
"pr_issue": {"field": "comments", "components": ["top_level", "reviews"]}},
"inline": {"mode": "per_review", "field": "review_comments", "components": ["reviews", "inline"]}}
self.contract["aggregate"]["pr"]["components"] = ["top_level", "reviews", "inline"]
self.contract["aggregate"]["pr_issue"]["components"] = ["top_level", "reviews", "inline"]
self.repo = {"id": 3, "full_name": "acme/demo", "html_url": ORIGIN + "/acme/demo"}
self.source = {"id": 707, "number": 7, "html_url": ORIGIN + "/acme/demo/issues/7",
"body": "source contract", "title": "Source", "state": "open", "comments": 0, "updated_at": T0}
self.pr = {"id": 99, "number": 9, "html_url": ORIGIN + "/acme/demo/pulls/9",
"url": ORIGIN + "/acme/demo/pulls/9", "state": "open",
"head": {"sha": "b" * 40, "repo": {"id": 12, "full_name": "contributor/demo"}},
"base": {"repo": copy.deepcopy(self.repo)}, "title": "PR", "body": "Closes #7",
"comments": 1, "review_comments": 1, "updated_at": T0}
self.underlying = {"id": 99, "number": 9, "html_url": self.pr["html_url"],
"pull_request": {"html_url": self.pr["html_url"], "merged": False},
"title": "PR", "body": "Closes #7", "state": "open", "comments": 1, "updated_at": T0}
self.reviews = [{"id": 301, "commit_id": "a" * 40, "comments_count": 0, "state": "REQUEST_CHANGES",
"user": {"id": 33, "login": "gitea-review-agent"}}]
self.comments = []
self.source_comments = []
self.inline = []
self.overrides = {}
self.calls = []
self._request_override = None
for record in (self.source, self.pr, self.underlying):
record.update(created_at=T0, labels=[], assignees=[], milestone=None,
user={"id": 22, "login": "genichin"} if record is self.source
else {"id": 42, "login": "gitea-impl-agent"})
self.pr["base"].update(ref="main", label="main", sha="d" * 40, repo_id=3)
self.pr["head"].update(ref="fix/issue-7", label="contributor:fix/issue-7", repo_id=12)
self.reviews[0].update(body="Fix F-1", submitted_at=T0, updated_at=T0,
stale=True, dismissed=False, html_url=self.pr["html_url"] + "#pullreview-301",
pull_request_url=self.pr["html_url"])
return self
def cleanup(self):
if self._temp is not None and not self._cleaned:
self._temp.cleanup()
self._cleaned = True
def load(self, name):
path = DIRECTORY / (name + ".py")
if not path.is_file():
raise AssertionError("canonical readback core missing")
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def request(self, path):
self.calls.append(path)
if path in self.overrides:
return copy.deepcopy(self.overrides[path])
parsed = urlsplit(path)
objects = {"/user": {"id": 42, "login": "gitea-impl-agent"}, "/version": {"version": "1.24.6"},
"/repos/acme/demo": self.repo, "/repos/acme/demo/issues/7": self.source,
"/repos/acme/demo/pulls/9": self.pr, "/repos/acme/demo/issues/9": self.underlying}
if parsed.path in objects:
return copy.deepcopy(objects[parsed.path]), {}, 200
collections = {"/repos/acme/demo/issues/7/comments": self.source_comments,
"/repos/acme/demo/issues/9/comments": self.comments,
"/repos/acme/demo/pulls/9/reviews": self.reviews,
"/repos/acme/demo/pulls/9/reviews/301/comments": self.inline}
if parsed.path not in collections:
raise AssertionError("unexpected GET endpoint")
rows = collections[parsed.path]
page = int(parse_qs(parsed.query)["page"][0])
return copy.deepcopy(rows[(page - 1) * 50:page * 50]), {"X-Total-Count": str(len(rows))}, 200
def adapter(self):
module = self.load("rework_readback")
try:
return module.ReworkReadbackAdapter(self._request_override or self.request, origin=ORIGIN + "/api/v1",
credential_source="$HERMES_HOME/.env:GITEA_TOKEN", actor_id=42,
provider_contract=self.contract)
except Exception as error:
raise AssertionError("canonical CLI transport contract rejected: " + str(error)) from error
def add_result(self):
self.comments.append({"id": 501, "html_url": self.pr["html_url"] + "#issuecomment-501",
"body": self.expected["body"], "created_at": T1, "updated_at": T1,
"user": {"id": 42, "login": "gitea-impl-agent"}})
self.pr["comments"] += 1
self.underlying["comments"] += 1
self.pr["updated_at"] = self.underlying["updated_at"] = T1
+55
View File
@@ -0,0 +1,55 @@
"""Shared implementation-spec review fixtures; no test cases are declared here."""
import copy
import importlib.util
from .paths import ROOT
MODPATH = ROOT / "harness/agents/spec/skills/gitea-implementation-spec-review/scripts/review_support.py"
spec = importlib.util.spec_from_file_location("review_support", MODPATH)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
def result(verdict="approve"):
return {
"schema": "gitea-agent.implementation-spec-review-result/v1",
"target": {"owner": "acme", "repository": "repo", "issue": 3, "pull": 7,
"head": "a" * 40},
"contract_sha256": "b" * 64,
"verdict": verdict,
"criteria": [{"criterion": "REQ-1", "contract": "계약", "implementation_paths": ["src/a.py"],
"test_paths": ["tests/test_a.py"], "evidence": "증거", "result": "satisfied",
"reason": "충족"}],
"scope": {"missing": [], "excess": [], "spec_conflicts": []},
"conclusion": "모든 기준을 충족했다.",
}
def publication_fixture():
r = result(); paths = {"src/a.py", "tests/test_a.py"}
body = mod.render_review(r, paths, declaration={"mode": "none"})
url = "https://example.test/acme/repo/pulls/7"
user = {"id": 42, "login": "spec-agent"}
review = {"id": 11, "body": body, "commit_id": r["target"]["head"], "state": "APPROVED",
"user": user, "official": True, "stale": False, "dismissed": False,
"comments_count": 0, "submitted_at": "2026-10-03T01:24:00Z",
"updated_at": "2026-10-03T01:24:00Z", "pull_request_url": url}
before = {"issue": {"id": 3, "number": 3, "repository": {"full_name": "acme/repo"},
"updated_at": "2026-10-03T00:00:00Z", "title": "contract"},
"issue_comments": [], "pr": {"id": 7, "number": 7, "html_url": url,
"url": url, "base": {"repo": {"full_name": "acme/repo"}, "sha": "c" * 40},
"head": {"sha": "a" * 40}, "user": {"id": 99}, "state": "open", "merged": False,
"review_comments": 0, "updated_at": "2026-10-03T00:00:00Z", "requested_reviewers": []},
"pr_comments": [], "reviews": [], "timeline": [],
"files": [{"filename": "src/a.py", "status": "modified", "additions": 1}]}
after = copy.deepcopy(before); after["reviews"] = [review]
after["pr"]["updated_at"] = review["submitted_at"]; after["pr"]["review_comments"] = 1
after["timeline"] = [{"id": 101, "review_id": 11, "user": user, "body": body,
"type": "review", "created_at": review["submitted_at"],
"updated_at": review["submitted_at"], "pull_request_url": url}]
return before, after, review, r, paths
def verify_fixture(before, after, review, r, paths, contract=None):
return mod.verify_publication(before, after, review, r, paths, 42, [], contract,
expected_pr_url="https://example.test/acme/repo/pulls/7")