[verified] Add canonical harness test runner and enforce role collection

harness/tests/README.md

@@ -0,0 +1,74 @@
+# Canonical harness tests
+
+The canonical runner is anchored to its own source tree and works from any
+current directory. From the repository root:
+
+```sh
+python3 -B harness/tests/run.py
+python3 -B harness/tests/run.py --collect-only
+```
+
+From outside the repository, use the absolute command:
+
+```sh
+python3 -B /opt/data/gitea-agent/harness/tests/run.py --collect-only
+```
+
+That absolute command is intentionally literal and may be run from `/` or any
+other working directory; do not replace it with a relative path. The canonical
+source paths are `/opt/data/gitea-agent/harness/tests/run.py` and the four role
+trees under `/opt/data/gitea-agent/harness/tests/{common,spec,impl,review}`.
+This document is owned by the canonical harness tests and must be updated in
+place when their supported discovery contract or command paths change.
+
+It validates all `test*.py` files recursively under exactly these required
+roles: `common`, `spec`, `impl`, and `review`. `harness/tests`, every role, and
+every nested test package must have an `__init__.py`; every role and every
+individual test source file must contain at least one statically inventoried
+test. Files matching `test*.py` outside a role are rejected.
+
+Before executing tests, the runner independently inventories source with the
+Python AST, collects with standard `unittest`, and requires exact qualified
+ID-set equality. It first preserves the loader's raw ID list so duplicate
+loader results cannot disappear through set conversion. It rejects missing or
+extra loader IDs, import failures and `_FailedTest` entries, duplicate class or
+method declarations, zero-test files/roles, module/package `load_tests`,
+unsupported conditional discovery declarations, unsupported bases, cyclic
+local inheritance, free-function tests, missing package markers, and raw-byte
+source/package-manifest drift during collection. The package manifest covers
+required, nested, and root markers; it is rescanned after collection so added
+or removed files/markers and initializer changes fail closed.
+
+As the first admission step, the runner rejects every symbolic link anywhere
+under `harness/tests` (including test sources, package markers, role
+directories, and nested directories) with a canonical exit-2 diagnostic. This
+boundary is checked before source bytes are read or test modules are imported;
+it prevents source-tree path escape through links. It is not a sandbox: code
+that is admitted and imported may still perform ordinary external filesystem,
+process, network, or other side effects.
+
+`--collect-only` emits JSON containing `collection_only: true`, per-role test
+counts, sorted exact unittest IDs, and source SHA-256 hashes. Collection-only
+success is not runtime success: normal unittest assertion failures and errors
+remain visible and return a nonzero status. The runner intentionally accepts
+standard `unittest.TestCase`, aliases, local mixins/inheritance, inherited
+`test*` methods, and normal decorators such as `@unittest.skip`.
+
+The inventory is deliberately conservative, not a Python semantic evaluator or
+security sandbox. Test classes/functions declared inside module/class control
+flow are unsupported and rejected; helper functions/classes nested inside an
+ordinary test method are runtime fixture code and are not inspected as
+canonical discovery declarations. Tests supplied solely by dynamic imports,
+metaclasses, or custom discovery protocols are unsupported: their collected
+IDs must not substitute for or extend the statically declared inventory.
+Imports used by ordinary fixture helpers are not prohibited. The harness does
+not claim to prevent side effects performed by imported test code; it detects
+source drift and validates collection before execution.
+
+The regression tests use bounded subprocess timeouts and clean test-owned
+`TemporaryDirectory` trees under `/opt/data/cache/scratch/canonical-tests-main/fix2/`.
+Run them with:
+
+```sh
+python3 -m unittest harness.tests.common.test_canonical_test_runner -v
+```

harness/tests/common/test_canonical_test_runner.py

@@ -0,0 +1,345 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from pathlib import Path
+
+
+REPO = Path(__file__).resolve().parents[3]
+RUNNER = REPO / "harness" / "tests" / "run.py"
+
+
+class CanonicalRunnerTests(unittest.TestCase):
+    SCRATCH = Path("/opt/data/cache/scratch/canonical-tests-main/fix/test-tmp")
+
+    def make_suite(self, *, roles=None, extra="", nested=False):
+        self.SCRATCH.mkdir(parents=True, exist_ok=True)
+        temp = tempfile.TemporaryDirectory(dir=self.SCRATCH)
+        self.addCleanup(temp.cleanup)
+        root = Path(temp.name)
+        tests = root / "harness" / "tests"
+        roles = roles or {"common": "test_common", "spec": "test_spec", "impl": "test_impl", "review": "test_review"}
+        (tests / "__init__.py").parent.mkdir(parents=True)
+        (tests / "__init__.py").write_text("")
+        (tests / "run.py").write_bytes(RUNNER.read_bytes())
+        for role, stem in roles.items():
+            d = tests / role
+            d.mkdir()
+            (d / "__init__.py").write_text("")
+            (d / f"{stem}.py").write_text(
+                "import unittest\n"
+                "class Example(unittest.TestCase):\n"
+                "    def test_one(self):\n"
+                "        self.assertTrue(True)\n"
+            )
+        if nested:
+            d = tests / "common" / "nested"
+            d.mkdir()
+            (d / "__init__.py").write_text("")
+            (d / "test_nested.py").write_text(
+                "import unittest\nclass Nested(unittest.TestCase):\n"
+                "    def test_nested(self): self.assertTrue(True)\n"
+            )
+        if extra:
+            (tests / "test_outside.py").write_text(extra)
+        return root
+
+    def execute(self, root, *args):
+        env = dict(os.environ)
+        env["PYTHONPATH"] = str(root) + os.pathsep + env.get("PYTHONPATH", "")
+        return subprocess.run([sys.executable, "-B", str(root / "harness/tests/run.py"), *args], cwd="/", env=env,
+                              capture_output=True, text=True, timeout=20)
+
+    def test_real_canonical_collect_only_succeeds_with_every_role(self):
+        result = subprocess.run([sys.executable, "-B", str(RUNNER), "--collect-only"], cwd="/",
+                                capture_output=True, text=True, timeout=30)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        data = json.loads(result.stdout)
+        self.assertEqual(set(data["role_counts"]), {"common", "spec", "impl", "review"})
+        self.assertTrue(all(data["role_counts"].values()))
+        self.assertEqual(len(data["ids"]), sum(data["role_counts"].values()))
+        self.assertTrue(data["ids"])
+
+    def test_module_load_tests_is_rejected(self):
+        root = self.make_suite()
+        (root / "harness/tests/common/test_common.py").write_text(
+            "import unittest\ndef load_tests(loader, tests, pattern): return tests\n"
+            "class X(unittest.TestCase):\n def test_one(self): pass\n")
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("load_tests", result.stderr)
+
+    def test_zero_file_and_mixed_free_function_fail(self):
+        root = self.make_suite()
+        (root / "harness/tests/common/test_common.py").write_text("import unittest\n")
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("zero", result.stderr.lower())
+        root = self.make_suite()
+        (root / "harness/tests/common/test_common.py").write_text('''import unittest
+class X(unittest.TestCase):
+    def test_one(self): pass
+def test_free(): pass
+''')
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("free-function", result.stderr.lower())
+
+    def test_direct_case_does_not_short_circuit_cycle_or_unknown_base(self):
+        for source, word in (
+            ('''import unittest
+class A(B, unittest.TestCase):
+    def test_a(self): pass
+class B(A):
+    def test_b(self): pass
+''', "cyclic"),
+            ('''import unittest
+class X(unittest.TestCase, Missing):
+    def test_x(self): pass
+''', "unknown"),
+        ):
+            root = self.make_suite()
+            (root / "harness/tests/common/test_common.py").write_text(source)
+            result = self.execute(root, "--collect-only")
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn(word, result.stderr.lower())
+
+    def test_package_load_tests_is_rejected(self):
+        root = self.make_suite(nested=True)
+        (root / "harness/tests/common/nested/__init__.py").write_text('''def load_tests(loader, tests, pattern):
+    return tests
+''')
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("load_tests", result.stderr)
+
+    def test_alias_local_inheritance_and_runtime_helpers_are_valid(self):
+        root = self.make_suite()
+        (root / "harness/tests/common/test_common.py").write_text('''import unittest as ut
+class Mixin:
+    def test_mix(self): pass
+class Base(ut.TestCase, Mixin):
+    def test_base(self):
+        class Helper:
+            def test_not_discovered(self): pass
+        self.assertTrue(Helper)
+class Child(Base):
+    def test_child(self): pass
+''')
+        result = self.execute(root, "--collect-only")
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_conditional_discovery_declarations_fail_closed(self):
+        for source in (
+            '''import unittest
+if True:
+    class X(unittest.TestCase):
+        def test_x(self): pass
+''',
+            '''import unittest
+class X(unittest.TestCase):
+    if True:
+        def test_x(self): pass
+''',
+        ):
+            root = self.make_suite()
+            (root / "harness/tests/common/test_common.py").write_text(source)
+            result = self.execute(root, "--collect-only")
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("conditional", result.stderr.lower())
+
+    def test_custom_loader_duplicate_and_omission_are_rejected(self):
+        root = self.make_suite()
+        marker = root / "admission-marker"
+        script = root / "probe.py"
+        script.write_text("""
+import sys
+from pathlib import Path
+sys.path.insert(0, sys.argv[1])
+import harness.tests.run as run
+root = Path(sys.argv[1])
+files = run.source_files()
+expected, _, _ = run.inventory(files)
+baseline, suite = run.collect(files)
+assert not any(isinstance(t, run.unittest.loader._FailedTest) for t in run.flatten(suite))
+assert baseline == expected
+items = list(run.flatten(suite))
+class Fake(run.unittest.FunctionTestCase):
+    def __init__(self, ident):
+        super().__init__(lambda: Path(sys.argv[2]).write_text('ran'))
+        self.ident = ident
+    def id(self):
+        return self.ident
+for mode in ('omission', 'addition', 'duplicate'):
+    class Loader:
+        def discover(self, *args, **kwargs):
+            selected = list(items)
+            if mode == 'omission': selected = selected[:-1]
+            elif mode == 'addition': selected.append(Fake('fake.added'))
+            else: selected = [Fake('fake.duplicate'), Fake('fake.duplicate')]
+            return run.unittest.TestSuite(selected)
+    run.unittest.defaultTestLoader = Loader()
+    try:
+        if mode == 'duplicate': run.collect(files)
+        else: run.validate()
+    except run.CanonicalError:
+        pass
+    else:
+        raise AssertionError(mode + ' was admitted')
+assert not Path(sys.argv[2]).exists()
+""")
+        result = subprocess.run([sys.executable, "-B", str(script), str(root), str(marker)], cwd="/",
+                                capture_output=True, text=True, timeout=20)
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_collect_only_reports_exact_roles_ids_and_hashes(self):
+        root = self.make_suite()
+        result = self.execute(root, "--collect-only")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        data = json.loads(result.stdout)
+        self.assertTrue(data["collection_only"])
+        self.assertEqual(data["role_counts"], {"common": 1, "spec": 1, "impl": 1, "review": 1})
+        self.assertEqual(len(data["ids"]), 4)
+        self.assertEqual(set(data["source_sha256"]), {"harness/tests/common/test_common.py", "harness/tests/spec/test_spec.py", "harness/tests/impl/test_impl.py", "harness/tests/review/test_review.py"})
+
+    def test_missing_role_marker_duplicate_and_unexpected_file_fail_before_run(self):
+        for mode in ("missing", "marker", "duplicate", "unexpected"):
+            with self.subTest(mode=mode):
+                root = self.make_suite(extra=("import unittest\nclass Example(unittest.TestCase):\n def test_one(self): pass\n" if mode == "unexpected" else ""))
+                if mode == "missing":
+                    (root / "harness/tests/review/test_review.py").unlink()
+                elif mode == "marker":
+                    (root / "harness/tests/spec/__init__.py").unlink()
+                elif mode == "duplicate":
+                    (root / "harness/tests/common/test_common.py").write_text(
+                        "import unittest\nclass Example(unittest.TestCase):\n def test_one(self): pass\nclass Example(unittest.TestCase):\n def test_two(self): pass\n")
+                result = self.execute(root, "--collect-only")
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("canonical", result.stderr.lower())
+
+    def test_loader_omission_addition_zero_file_and_unsupported_function_fail(self):
+        cases = {
+            "omission": lambda root: (root / "harness/tests/common/test_common.py").write_text("import unittest\ndef load_tests(loader, tests, pattern): return unittest.TestSuite()\nclass X(unittest.TestCase):\n def test_one(self): pass\n"),
+            "zero": lambda root: (root / "harness/tests/review/test_review.py").write_text("import unittest\n"),
+            "free": lambda root: (root / "harness/tests/impl/test_impl.py").write_text("def test_free(): pass\n"),
+        }
+        for name, mutate in cases.items():
+            with self.subTest(name=name):
+                root = self.make_suite(); mutate(root)
+                result = self.execute(root, "--collect-only")
+                self.assertNotEqual(result.returncode, 0)
+        root = self.make_suite()
+        (root / "harness/tests/spec/test_spec.py").write_text("import unittest\nclass X(unittest.TestCase):\n def test_one(self): pass\n def test_two(self): pass\n")
+        result = self.execute(root, "--collect-only")
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_import_failure_source_drift_nested_omission_and_runtime_failure(self):
+        root = self.make_suite()
+        (root / "harness/tests/impl/test_impl.py").write_text("raise RuntimeError('broken import')\n")
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("canonical", result.stderr.lower())
+        root = self.make_suite(nested=True)
+        (root / "harness/tests/common/nested/__init__.py").unlink()
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        root = self.make_suite()
+        (root / "harness/tests/review/test_review.py").write_text(
+            "import unittest\nfrom pathlib import Path\np = Path(__file__)\np.write_text(p.read_text() + '# drift\\n')\n"
+            "class X(unittest.TestCase):\n def test_one(self): pass\n")
+        result = self.execute(root, "--collect-only")
+        self.assertNotEqual(result.returncode, 0)
+        root = self.make_suite()
+        (root / "harness/tests/common/test_common.py").write_text(
+            "import unittest\nclass X(unittest.TestCase):\n def test_one(self): self.fail('boom')\n")
+        result = self.execute(root)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("FAILED", result.stderr + result.stdout)
+
+    def test_package_manifest_drift_and_extra_zero_file_fail_closed(self):
+        cases = (
+            ("initializer", "from pathlib import Path\np = Path(__file__).parent / '__init__.py'\np.write_text(p.read_text() + '# drift\\n')\n"),
+            ("add-source", "from pathlib import Path\nPath(__file__).with_name('test_added.py').write_text('import unittest\\nclass Added(unittest.TestCase):\\n def test_added(self): pass\\n')\n"),
+            ("remove-source", "from pathlib import Path\n(Path(__file__).parents[1] / 'review' / 'test_review.py').unlink()\n"),
+            ("extra-zero", "from pathlib import Path\nPath(__file__).with_name('test_empty.py').write_text('import unittest\\n')\n"),
+        )
+        for name, mutation in cases:
+            with self.subTest(name=name):
+                root = self.make_suite()
+                target = root / "harness/tests/common/test_common.py"
+                target.write_text("import unittest\n" + mutation +
+                                  "class X(unittest.TestCase):\n def test_one(self): pass\n")
+                result = self.execute(root, "--collect-only")
+                self.assertEqual(result.returncode, 2)
+                self.assertIn("source/package manifest drift", result.stderr.lower())
+
+    def test_symlinked_source_and_package_paths_fail_before_import(self):
+        for kind in ("source", "marker", "role", "nested"):
+            with self.subTest(kind=kind):
+                root = self.make_suite()
+                marker = root / "escaped-import"
+                outside = root / "outside"
+                outside.mkdir()
+                payload = ("from pathlib import Path\n"
+                           + "Path(" + repr(str(marker)) + ").write_text('imported')\n"
+                           + "import unittest\nclass Escape(unittest.TestCase):\n def test_escape(self): pass\n")
+                if kind == "source":
+                    external = outside / "escape.py"
+                    external.write_text(payload)
+                    (root / "harness/tests/common/test_escape.py").symlink_to(external)
+                elif kind == "marker":
+                    external = outside / "init.py"
+                    external.write_text(payload)
+                    path = root / "harness/tests/common/__init__.py"
+                    path.unlink()
+                    path.symlink_to(external)
+                else:
+                    (outside / "__init__.py").write_text(payload)
+                    (outside / "test_escape.py").write_text(payload)
+                    if kind == "role":
+                        path = root / "harness/tests/review"
+                        (path / "__init__.py").unlink()
+                        (path / "test_review.py").unlink()
+                        path.rmdir()
+                    else:
+                        path = root / "harness/tests/common/nested"
+                    path.symlink_to(outside, target_is_directory=True)
+                result = self.execute(root)
+                self.assertEqual(result.returncode, 2, result.stderr)
+                self.assertIn("symlink", result.stderr.lower())
+                self.assertFalse(marker.exists(), "escaped code was imported before rejection")
+
+    def test_static_extra_zero_file_fails_with_valid_sibling(self):
+        root = self.make_suite()
+        (root / "harness/tests/common/test_empty.py").write_text("# empty candidate\n")
+        result = self.execute(root, "--collect-only")
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("zero inventoried tests", result.stderr)
+
+    def test_removed_role_marker_is_manifest_failure(self):
+        root = self.make_suite()
+        source = root / "harness/tests/common/test_common.py"
+        source.write_text("import unittest\nfrom pathlib import Path\n"
+                          "(Path(__file__).parents[1] / 'review' / '__init__.py').unlink()\n"
+                          "class X(unittest.TestCase):\n def test_one(self): pass\n")
+        result = self.execute(root, "--collect-only")
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("missing package marker", result.stderr)
+
+    def test_normal_synthetic_success_and_assertion_failure_exit_statuses(self):
+        root = self.make_suite()
+        success = self.execute(root)
+        self.assertEqual(success.returncode, 0, success.stderr)
+        (root / "harness/tests/common/test_common.py").write_text(
+            "import unittest\nclass X(unittest.TestCase):\n def test_one(self): self.fail('named assertion')\n")
+        failure = self.execute(root)
+        self.assertEqual(failure.returncode, 1)
+        self.assertIn("named assertion", failure.stderr + failure.stdout)
+        (root / "harness/tests/common/test_common.py").write_text(
+            "import unittest\nclass X(unittest.TestCase):\n def test_one(self): raise RuntimeError('named error')\n")
+        error = self.execute(root)
+        self.assertEqual(error.returncode, 1)
+        self.assertIn("named error", error.stderr + error.stdout)

harness/tests/common/test_workflow_guard.py

@@ -15,7 +15,7 @@ def inventory_digest(ids, statuses):
     return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
 
 
-def test_inventory(ids=('case_a',), current=None):
+def make_test_inventory(ids=('case_a',), current=None):
     current = list(ids if current is None else current)
     ids = list(ids)
     old_status = {key: {'skip': False, 'expected_failure': False} for key in ids}
@@ -157,84 +157,84 @@ class WorkflowGuardTests(unittest.TestCase):
             self.assertEqual(guard.compare_source(root, out, ['src/a.py'])['status'], 'block')
 
     def test_new_regression_tests_do_not_require_deletion_authorization(self):
-        self.assertEqual(guard.check_tests(test_inventory(current=['case_a', 'case_b']))['status'], 'pass')
+        self.assertEqual(guard.check_tests(make_test_inventory(current=['case_a', 'case_b']))['status'], 'pass')
 
     def test_collected_inventory_requires_provenance(self):
-        value = test_inventory()
+        value = make_test_inventory()
         del value['baseline_collection']
         self.assert_block(guard.check_tests, value)
 
     def test_collected_inventory_digest_must_match_ids_and_statuses(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['baseline_collection']['inventory_sha256'] = '0' * 64
         self.assert_block(guard.check_tests, value)
 
     def test_new_skip_requires_explicit_approval(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['current_status']['case_a']['skip'] = True
         value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status'])
         self.assert_block(guard.check_tests, value)
 
     def test_new_expected_failure_requires_explicit_approval(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['current_status']['case_a']['expected_failure'] = True
         value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status'])
         self.assert_block(guard.check_tests, value)
 
     def test_rename_requires_real_replacement_and_semantic_evidence(self):
-        value = test_inventory(current=['case_b'])
+        value = make_test_inventory(current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': ['ghost'],
                                         'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assert_block(guard.check_tests, value)
 
     def test_approved_rename_preserves_mapped_test(self):
-        value = test_inventory(current=['case_b'])
+        value = make_test_inventory(current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': ['case_b'],
                                         'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assertEqual(guard.check_tests(value)['status'], 'pass')
 
     def test_modified_test_requires_semantic_review(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['changed_tests'] = [{'test_id': 'case_a', 'baseline_sha256': 'a' * 64,
                                     'current_sha256': 'b' * 64, 'review_evidence': ''}]
         self.assert_block(guard.check_tests, value)
 
     def test_finding_truthy_strings_are_not_red_green_records(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['findings'] = [{'id': 'finding-a', 'test_id': 'case_a', 'classification': 'assertion',
                               'red': 'claimed-red', 'green': 'claimed-green'}]
         self.assert_block(guard.check_tests, value)
 
     def test_red_success_exit_cannot_demonstrate_failure(self):
-        value = test_inventory()
+        value = make_test_inventory()
         finding = finding_record()
         finding['red']['exit_code'] = finding['red']['expected_exit_code'] = 0
         value['findings'] = [finding]
         self.assert_block(guard.check_tests, value)
 
     def test_green_failure_exit_cannot_demonstrate_resolution(self):
-        value = test_inventory()
+        value = make_test_inventory()
         finding = finding_record()
         finding['green']['exit_code'] = finding['green']['expected_exit_code'] = 1
         value['findings'] = [finding]
         self.assert_block(guard.check_tests, value)
 
     def test_environment_failure_is_not_valid_red(self):
-        value = test_inventory()
+        value = make_test_inventory()
         finding = finding_record()
         finding['red']['failure_kind'] = 'environment'
         value['findings'] = [finding]
         self.assert_block(guard.check_tests, value)
 
     def test_finding_must_name_present_regression_test(self):
-        value = test_inventory()
+        value = make_test_inventory()
         finding = finding_record()
         finding['test_id'] = 'ghost'
         value['findings'] = [finding]
         self.assert_block(guard.check_tests, value)
 
     def test_named_red_green_records_with_provenance_pass(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['findings'] = [finding_record()]
         self.assertEqual(guard.check_tests(value)['status'], 'pass')
 
@@ -342,31 +342,31 @@ class WorkflowGuardTests(unittest.TestCase):
             self.assertEqual(json.loads(result.stdout)['status'], 'block')
 
     def test_rename_empty_replacement_cannot_stand_in_for_deletion(self):
-        value = test_inventory(current=['case_b'])
+        value = make_test_inventory(current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': [],
             'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assert_block(guard.check_tests, value)
 
     def test_unclassified_empty_replacement_cannot_stand_in_for_deletion(self):
-        value = test_inventory(current=['case_b'])
+        value = make_test_inventory(current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'replacement_ids': [],
             'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assert_block(guard.check_tests, value)
 
     def test_explicit_approved_deletion_is_allowed(self):
-        value = test_inventory(ids=['case_a', 'case_b'], current=['case_b'])
+        value = make_test_inventory(ids=['case_a', 'case_b'], current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'delete', 'replacement_ids': [],
             'reason': 'approved obsolete case deletion', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assertEqual(guard.check_tests(value)['status'], 'pass')
 
     def test_status_approval_cannot_authorize_removal(self):
-        value = test_inventory(current=['case_b'])
+        value = make_test_inventory(current=['case_b'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'status', 'replacement_ids': ['case_b'],
             'reason': 'approved status update', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assert_block(guard.check_tests, value)
 
     def test_explicit_status_approval_maps_same_current_id(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['current_status']['case_a']['skip'] = True
         value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status'])
         value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'status', 'replacement_ids': ['case_a'],
@@ -374,7 +374,7 @@ class WorkflowGuardTests(unittest.TestCase):
         self.assertEqual(guard.check_tests(value)['status'], 'pass')
 
     def test_unhashable_authorization_id_fails_closed_api(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['authorized_changes'] = [{'old_id': ['case_a'], 'replacement_ids': [],
             'reason': 'fixture', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}]
         self.assert_block(guard.check_tests, value)
@@ -408,10 +408,10 @@ class WorkflowGuardTests(unittest.TestCase):
 
     def test_declared_evidence_shape_does_not_authenticate_records(self):
         # Deliberately synthetic fixture claims: the main agent must check actual evidence separately.
-        self.assertEqual(guard.check_tests(test_inventory())['status'], 'pass')
+        self.assertEqual(guard.check_tests(make_test_inventory())['status'], 'pass')
 
     def test_unknown_inventory_fields_are_rejected(self):
-        value = test_inventory()
+        value = make_test_inventory()
         value['pretend_verified'] = True
         self.assert_block(guard.check_tests, value)
 

harness/tests/run.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""Canonical, source-bound unittest harness."""
+import argparse
+import ast
+import hashlib
+import json
+import os
+import sys
+import unittest
+from pathlib import Path
+
+ROLES = ("common", "spec", "impl", "review")
+SOURCE_ROOT = Path(__file__).resolve().parent
+REPO_ROOT = SOURCE_ROOT.parents[1]
+
+class CanonicalError(Exception):
+    pass
+
+def fail(message):
+    raise CanonicalError("canonical harness: " + message)
+
+def rel(path):
+    return path.relative_to(REPO_ROOT).as_posix()
+
+def sha(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+def _reject_source_tree_symlinks():
+    """Reject links before any source bytes are read or imported."""
+    def visit(directory):
+        try:
+            entries = list(os.scandir(directory))
+        except OSError as exc:
+            fail(f"cannot inspect {rel(Path(directory))}: {exc}")
+            return
+        for entry in entries:
+            path = Path(entry.path)
+            if entry.is_symlink():
+                fail(f"symlinked source-tree path: {rel(path)}")
+            if entry.is_dir(follow_symlinks=False):
+                visit(path)
+    visit(SOURCE_ROOT)
+
+def module_name(path):
+    return ".".join(path.relative_to(REPO_ROOT).with_suffix("").parts)
+
+def _package_manifest():
+    """Return raw-byte fingerprints for all source and package markers."""
+    _reject_source_tree_symlinks()
+    files = {}
+    if not SOURCE_ROOT.is_dir():
+        fail("missing source root")
+    for directory in [SOURCE_ROOT, *(SOURCE_ROOT / role for role in ROLES)]:
+        marker = directory / "__init__.py"
+        if not marker.is_file():
+            fail(f"missing package marker: {rel(marker)}")
+    for path in SOURCE_ROOT.rglob("*"):
+        if path.is_file() and (path.name == "__init__.py" or path.name.startswith("test") and path.suffix == ".py"):
+            files[rel(path)] = path.read_bytes()
+    return files
+
+def _reject_package_load_tests(manifest=None):
+    manifest = _package_manifest() if manifest is None else manifest
+    for name in manifest:
+        if not name.endswith("__init__.py"):
+            continue
+        path = REPO_ROOT / name
+        try:
+            tree = ast.parse(path.read_bytes(), filename=str(path))
+        except (OSError, SyntaxError) as exc:
+            fail(f"cannot parse {rel(path)}: {exc}")
+        if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_tests" for n in tree.body):
+            fail(f"unsupported dynamic load_tests declaration in {rel(path)}")
+
+def package_dirs(manifest=None):
+    _reject_package_load_tests(manifest)
+    for path in SOURCE_ROOT.rglob("test*.py"):
+        current = path.parent
+        while current != SOURCE_ROOT.parent:
+            if current in (SOURCE_ROOT, *(SOURCE_ROOT / role for role in ROLES)):
+                current = current.parent
+                continue
+            marker = current / "__init__.py"
+            if not marker.is_file():
+                fail(f"nested test package missing marker: {rel(marker)}")
+            current = current.parent
+
+def source_files(manifest=None):
+    package_dirs(manifest)
+    files = sorted(SOURCE_ROOT.rglob("test*.py"))
+    role_roots = {SOURCE_ROOT / role for role in ROLES}
+    unexpected = [p for p in files if not any(p.is_relative_to(d) for d in role_roots)]
+    if unexpected:
+        fail("unexpected test file outside role dirs: " + ", ".join(rel(p) for p in unexpected))
+    by_role = {role: sorted((SOURCE_ROOT / role).rglob("test*.py")) for role in ROLES}
+    for role, paths in by_role.items():
+        if not paths:
+            fail(f"required role has no test files: {role}")
+    return by_role
+
+def aliases(tree):
+    names = {"TestCase", "unittest.TestCase"}
+    for node in tree.body:
+        if isinstance(node, ast.ImportFrom) and node.module == "unittest":
+            for item in node.names:
+                if item.name == "TestCase":
+                    names.add(item.asname or item.name)
+        elif isinstance(node, ast.Import):
+            for item in node.names:
+                if item.name == "unittest":
+                    names.add((item.asname or item.name) + ".TestCase")
+    return names
+
+def inventory_file(path, raw_bytes=None):
+    try:
+        raw_bytes = path.read_bytes() if raw_bytes is None else raw_bytes
+        tree = ast.parse(raw_bytes, filename=str(path))
+    except (OSError, SyntaxError) as exc:
+        fail(f"cannot parse {rel(path)}: {exc}")
+    def conditional_declarations(nodes, context):
+        controls = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.With, ast.AsyncWith, ast.Match)
+        for node in nodes:
+            if isinstance(node, controls):
+                stack = list(ast.iter_child_nodes(node))
+                while stack:
+                    child = stack.pop()
+                    if isinstance(child, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+                        if child.name.startswith("test") or isinstance(child, ast.ClassDef):
+                            fail(f"unsupported conditional test declaration in {rel(path)}: {context}.{child.name}")
+                        if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                            continue
+                    stack.extend(ast.iter_child_nodes(child))
+    conditional_declarations(tree.body, "module")
+    if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_tests" for n in tree.body):
+        fail(f"unsupported dynamic load_tests declaration in {rel(path)}")
+    testcase_names = aliases(tree)
+    classes = {}
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            if node.name in classes:
+                fail(f"duplicate test class declaration in {rel(path)}: {node.name}")
+            classes[node.name] = node
+            conditional_declarations(node.body, node.name)
+            child_names = [c.name for c in node.body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef))]
+            if len(child_names) != len(set(child_names)):
+                fail(f"duplicate declaration in {rel(path)}: {node.name}")
+    state = {}
+    memo = {}
+    def base_name(base):
+        if isinstance(base, ast.Name): return base.id
+        if isinstance(base, ast.Attribute) and isinstance(base.value, ast.Name): return base.value.id + "." + base.attr
+        fail(f"unsupported test class base in {rel(path)}: {ast.unparse(base)}")
+    def is_case(name):
+        if name in memo: return memo[name]
+        if state.get(name) == "active":
+            fail(f"cyclic test class declaration in {rel(path)}: {name}")
+        node = classes.get(name)
+        if node is None:
+            result = name in testcase_names
+            if not result and name not in testcase_names:
+                fail(f"unknown test class base in {rel(path)}: {name}")
+            return result
+        state[name] = "active"
+        result = False
+        for base in node.bases:  # eager: never short-circuit validation
+            result = is_case(base_name(base)) or result
+        state.pop(name, None)
+        memo[name] = result
+        return result
+    def local_methods(name, stack=()):
+        if name in stack:
+            fail(f"cyclic test inheritance in {rel(path)}: {' -> '.join(stack + (name,))}")
+        node = classes[name]
+        found = {c.name for c in node.body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef)) and c.name.startswith("test")}
+        for base in node.bases:
+            b = base_name(base)
+            if b in classes:
+                is_case(b)
+                found |= local_methods(b, stack + (name,))
+        return found
+    methods = {}
+    free = []
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test"):
+            free.append(node.name)
+    for name in classes:
+        case = is_case(name)
+        own = {c.name for c in classes[name].body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef)) and c.name.startswith("test")}
+        if case:
+            methods[name] = local_methods(name)
+    if free:
+        fail(f"unsupported free-function test in {rel(path)}: {free[0]}")
+    ids = {f"{module_name(path)}.{name}.{method}" for name, ms in methods.items() for method in ms}
+    if not ids:
+        fail(f"file has zero inventoried tests: {rel(path)}")
+    return ids
+
+def inventory(files, manifest=None):
+    ids, role_ids, hashes = set(), {}, {}
+    manifest = _package_manifest() if manifest is None else manifest
+    for role, paths in files.items():
+        role_ids[role] = set()
+        for path in paths:
+            raw_bytes = manifest.get(rel(path))
+            if raw_bytes is None:
+                fail(f"source missing from package manifest: {rel(path)}")
+            hashes[rel(path)] = hashlib.sha256(raw_bytes).hexdigest()
+            found = inventory_file(path, raw_bytes)
+            overlap = ids & found
+            if overlap: fail("duplicate test IDs: " + ", ".join(sorted(overlap)))
+            ids |= found; role_ids[role] |= found
+        if not role_ids[role]: fail(f"role has zero inventoried tests: {role}")
+    return ids, role_ids, hashes
+
+def flatten(suite):
+    for item in suite:
+        if isinstance(item, unittest.TestSuite): yield from flatten(item)
+        else: yield item
+
+def collect(files, loader=None):
+    loader = loader or unittest.defaultTestLoader
+    suite = loader.discover(str(SOURCE_ROOT), pattern="test*.py", top_level_dir=str(REPO_ROOT))
+    flat = list(flatten(suite))
+    failed = [test for test in flat if isinstance(test, unittest.loader._FailedTest)]
+    if failed: fail("import/loader failure: " + "; ".join(str(test) for test in failed[:20]))
+    raw = [test.id() for test in flat]
+    seen = set(); duplicates = []
+    for test_id in raw:
+        if test_id in seen and test_id not in duplicates: duplicates.append(test_id)
+        seen.add(test_id)
+    if duplicates: fail("duplicate loader test IDs: " + ", ".join(duplicates[:20]))
+    return set(raw), suite
+
+def validate():
+    before_manifest = _package_manifest()
+    files = source_files(before_manifest)
+    expected, role_ids, before = inventory(files, before_manifest)
+    actual, suite = collect(files)
+    after_manifest = _package_manifest()
+    if before_manifest != after_manifest: fail("source/package manifest drift detected during collection")
+    if expected != actual:
+        fail(f"inventory/loader mismatch: missing={sorted(expected-actual)!r} extra={sorted(actual-expected)!r}")
+    return expected, role_ids, before, suite
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(add_help=True); parser.add_argument("--collect-only", action="store_true")
+    args = parser.parse_args(argv); sys.path.insert(0, str(REPO_ROOT))
+    try: ids, role_ids, hashes, suite = validate()
+    except CanonicalError as exc: print(str(exc), file=sys.stderr); return 2
+    if args.collect_only:
+        print(json.dumps({"collection_only": True, "role_counts": {r: len(role_ids[r]) for r in ROLES}, "ids": sorted(ids), "source_sha256": hashes}, sort_keys=True)); return 0
+    result = unittest.TextTestRunner(verbosity=2).run(suite); return 0 if result.wasSuccessful() else 1
+
+if __name__ == "__main__": raise SystemExit(main())