test(harness): reorganize regression suites and preserve contract coverage

2026-10-16 07:24:49 +00:00
parent b6f87db623
commit 33709ab873
108 changed files with 5869 additions and 5328 deletions
@@ -0,0 +1,147 @@
+import copy
+import json
+import re
+import unittest
+from harness.tests._support.contracts import read_contract, REQUIRED
+from harness.tests._support.paths import ROOT
+
+COMMIT_FILES_START = "<!-- commit-files-comparison-example:start -->"
+COMMIT_FILES_END = "<!-- commit-files-comparison-example:end -->"
+
+
+def _execute_documented_commit_files_comparator():
+    """Load the single trusted Python example from the canonical workflow."""
+    path = ROOT / "harness/agents/impl/skills/gitea-issue-rework/references/workflow.md"
+    text = path.read_text()
+    start = text.count(COMMIT_FILES_START)
+    end = text.count(COMMIT_FILES_END)
+    if start != 1 or end != 1:
+        raise AssertionError("workflow must contain exactly one commit-files example boundary pair")
+    section = text.split(COMMIT_FILES_START, 1)[1].split(COMMIT_FILES_END, 1)[0]
+    snippets = re.findall(r"```python\s*\n(.*?)\n```", section, re.DOTALL)
+    if len(snippets) != 1:
+        raise AssertionError("commit-files example must contain exactly one Python fenced block")
+    namespace = {}
+    exec(compile(snippets[0], str(path) + "#commit-files-example", "exec"), namespace)
+    comparator = namespace.get("normalize_commit_files")
+    if not callable(comparator):
+        raise AssertionError("Python example must define normalize_commit_files(rows)")
+    heading = "#### commit.files 순서 비교 계약\n"
+    if text.count(heading) != 1:
+        raise AssertionError("workflow must contain one scoped commit-files comparison contract")
+    contract = text.split(heading, 1)[1].split("\n### 7.", 1)[0]
+    return comparator, contract
+
+
+def _commit(sha, message, parent, files):
+    return {
+        "sha": sha,
+        "commit": {"message": message, "author": {"name": "Test Author", "email": "author@example.invalid"}},
+        "url": "https://example.invalid/commit/" + sha,
+        "parents": [{"sha": parent, "url": "https://example.invalid/commit/" + parent}],
+        "verification": {"verified": True, "reason": "valid", "arbitrary": [1, {"keep": True}]},
+        "files": files,
+        "provider_extension": {"retained": ["opaque", 7]},
+    }
+
+
+def _file(filename, status, additions):
+    return {
+        "filename": filename,
+        "status": status,
+        "additions": additions,
+        "deletions": 2,
+        "changes": additions + 2,
+        "patch": "@@ -1 +1 @@\n-old\n+new",
+        "provider_extension": {"nested": [filename, {"opaque": True}]},
+    }
+
+class DocumentedCommitFilesComparatorTests(unittest.TestCase):
+    SHA_A = '0123456789abcdef0123456789abcdef01234567'
+    SHA_B = '89abcdef0123456789abcdef0123456789abcdef'
+    PARENT_A = '1111111111111111111111111111111111111111'
+    PARENT_B = '2222222222222222222222222222222222222222'
+
+    def setUp(self):
+        self.normalize, self.workflow = _execute_documented_commit_files_comparator()
+
+    def fixture(self):
+        return [_commit(self.SHA_A, 'first message', self.PARENT_A, [_file('src/zeta.py', 'modified', 3), _file('docs/alpha.md', 'added', 5)]), _commit(self.SHA_B, 'second message', self.PARENT_B, [_file('tests/test_beta.py', 'modified', 7), _file('README.md', 'added', 11)])]
+
+    def test_inner_file_order_is_ignored_but_outer_commit_order_is_preserved(self):
+        first = self.fixture()
+        second = copy.deepcopy(first)
+        for commit in second:
+            commit['files'].reverse()
+        self.assertEqual(self.normalize(first), self.normalize(second))
+        self.assertNotEqual(self.normalize(first), self.normalize(list(reversed(second))))
+
+    def test_identity_and_all_file_metadata_remain_comparison_bound(self):
+        baseline = self.normalize(self.fixture())
+        mutations = (lambda rows: rows[0]['files'][0].update(filename='src/renamed.py'), lambda rows: rows[0]['files'][0].update(status='added'), lambda rows: rows[0]['files'][0].update(provider_extension={'changed': True}), lambda rows: rows[0]['files'].append(_file('new.txt', 'added', 1)), lambda rows: rows[0]['files'].pop(), lambda rows: rows[0].update(commit={'message': 'changed message', 'author': {'name': 'Test Author', 'email': 'author@example.invalid'}}), lambda rows: rows[0]['parents'].__setitem__(0, {'sha': self.SHA_B, 'url': 'https://example.invalid/commit/' + self.SHA_B}), lambda rows: rows[0].update(sha='3333333333333333333333333333333333333333'))
+        for mutate in mutations:
+            with self.subTest(mutation=mutate):
+                changed = self.fixture()
+                mutate(changed)
+                self.assertNotEqual(baseline, self.normalize(changed))
+
+    def test_normalization_preserves_input_and_returns_independent_deep_copy(self):
+        rows = self.fixture()
+        before = json.dumps(rows, sort_keys=True, separators=(',', ':'))
+        normalized = self.normalize(rows)
+        self.assertEqual(before, json.dumps(rows, sort_keys=True, separators=(',', ':')))
+        self.assertEqual(rows, self.fixture())
+        normalized[0]['files'][0]['provider_extension']['nested'][1]['opaque'] = False
+        normalized[0]['provider_extension']['retained'].append('output-only')
+        self.assertTrue(rows[0]['files'][0]['provider_extension']['nested'][1]['opaque'])
+        self.assertNotIn('output-only', rows[0]['provider_extension']['retained'])
+
+    def test_empty_file_list_is_valid_and_order_is_only_normalized_within_commits(self):
+        rows = self.fixture()
+        rows[0]['files'] = []
+        result = self.normalize(rows)
+        self.assertEqual([], result[0]['files'])
+        self.assertEqual([self.SHA_A, self.SHA_B], [row['sha'] for row in result])
+        self.assertEqual(sorted((file['filename'] for file in rows[1]['files'])), [file['filename'] for file in result[1]['files']])
+
+    def test_invalid_collection_commit_and_file_shapes_raise_value_error(self):
+        valid = self.fixture()
+        invalid_cases = []
+        for value in (None, {}, 'rows', (valid,)):
+            invalid_cases.append(value)
+        invalid_cases.extend(([None], ['commit'], [{'sha': self.SHA_A}], [{'sha': self.SHA_A, 'files': None}], [{'sha': self.SHA_A, 'files': {}}]))
+        for bad in invalid_cases:
+            with self.subTest(rows=bad):
+                with self.assertRaises(ValueError):
+                    self.normalize(bad)
+        for bad_sha in ('', 'A' * 40, 'a' * 39, 'g' * 40, None, 7):
+            with self.subTest(sha=bad_sha):
+                rows = self.fixture()
+                rows[0]['sha'] = bad_sha
+                with self.assertRaises(ValueError):
+                    self.normalize(rows)
+        duplicate_sha = self.fixture()
+        duplicate_sha[1]['sha'] = duplicate_sha[0]['sha']
+        with self.assertRaises(ValueError):
+            self.normalize(duplicate_sha)
+        for bad_file in (None, 'file', {}, {'filename': 'x'}):
+            with self.subTest(file=bad_file):
+                rows = self.fixture()
+                rows[0]['files'] = [bad_file]
+                with self.assertRaises(ValueError):
+                    self.normalize(rows)
+        for key in ('filename', 'status'):
+            for value in (None, 1, ''):
+                with self.subTest(key=key, value=value):
+                    rows = self.fixture()
+                    rows[0]['files'][0][key] = value
+                    with self.assertRaises(ValueError):
+                        self.normalize(rows)
+        duplicate_file = self.fixture()
+        duplicate_file[0]['files'][1]['filename'] = duplicate_file[0]['files'][0]['filename']
+        with self.assertRaises(ValueError):
+            self.normalize(duplicate_file)
+
+    def test_minimal_file_record_and_literal_filename_are_retained(self):
+        rows = [{'sha': self.SHA_A, 'files': [{'filename': ' dir/../literal name.txt ', 'status': 'modified'}]}]
+        self.assertEqual(rows, self.normalize(rows))
@@ -0,0 +1,40 @@
+"""Canonical impl contracts stay reachable without duplicated skill bodies."""
+import re
+import unittest
+from pathlib import Path
+from harness.tests._support.contracts import read_contract, REQUIRED
+from harness.tests._support.paths import ROOT
+
+class ImplArchitectureDocsTests(unittest.TestCase):
+
+    def test_skills_require_single_detailed_workflow_source(self):
+        for name in ('gitea-issue-implementation', 'gitea-issue-rework'):
+            path = ROOT / 'harness/agents/impl/skills' / name / 'SKILL.md'
+            text = path.read_text()
+            reference = path.parent / 'references/workflow.md'
+            with self.subTest(skill=name):
+                self.assertIn(REQUIRED, text)
+                self.assertLess(len(text.splitlines()), 100)
+                self.assertEqual(text.count('name: ' + name), 1)
+                self.assertNotIn('## Managed-lock broker gate', text)
+                self.assertFalse(reference.read_text().startswith('---'))
+                contract = read_contract(path)
+                self.assertEqual(contract.count('## Managed-lock broker gate'), 1)
+                for gate in ('## Delegation evidence acceptance gate', '## Test inventory source-binding gate', '## Canonical external verification evidence'):
+                    self.assertEqual(contract.count(gate), 1)
+
+    def test_all_canonical_impl_markdown_links_resolve(self):
+        for name in ('gitea-issue-implementation', 'gitea-issue-rework'):
+            directory = ROOT / 'harness/agents/impl/skills' / name
+            for path in [directory / 'SKILL.md', directory / 'references/workflow.md']:
+                for link in re.findall('\\[[^\\]]+\\]\\(([^)]+)\\)', path.read_text()):
+                    if link.startswith(('https://', 'http://', '#')):
+                        continue
+                    target = link.split('#', 1)[0]
+                    with self.subTest(path=path.name, link=link):
+                        self.assertTrue((path.parent / target).resolve().exists())
+
+    def test_correction_default_and_exception_remain_distinct(self):
+        text = (ROOT / 'harness/agents/impl/skills/gitea-issue-rework/references/readback.md').read_text()
+        for token in ('atomic_client_precondition=false',):
+            self.assertIn(token.lower(), text.lower())