diff --git a/harness/tests/README.md b/harness/tests/README.md new file mode 100644 index 0000000..88a6a2c --- /dev/null +++ b/harness/tests/README.md @@ -0,0 +1,74 @@ +# Canonical harness tests + +The canonical runner is anchored to its own source tree and works from any +current directory. From the repository root: + +```sh +python3 -B harness/tests/run.py +python3 -B harness/tests/run.py --collect-only +``` + +From outside the repository, use the absolute command: + +```sh +python3 -B /opt/data/gitea-agent/harness/tests/run.py --collect-only +``` + +That absolute command is intentionally literal and may be run from `/` or any +other working directory; do not replace it with a relative path. The canonical +source paths are `/opt/data/gitea-agent/harness/tests/run.py` and the four role +trees under `/opt/data/gitea-agent/harness/tests/{common,spec,impl,review}`. +This document is owned by the canonical harness tests and must be updated in +place when their supported discovery contract or command paths change. + +It validates all `test*.py` files recursively under exactly these required +roles: `common`, `spec`, `impl`, and `review`. `harness/tests`, every role, and +every nested test package must have an `__init__.py`; every role and every +individual test source file must contain at least one statically inventoried +test. Files matching `test*.py` outside a role are rejected. + +Before executing tests, the runner independently inventories source with the +Python AST, collects with standard `unittest`, and requires exact qualified +ID-set equality. It first preserves the loader's raw ID list so duplicate +loader results cannot disappear through set conversion. It rejects missing or +extra loader IDs, import failures and `_FailedTest` entries, duplicate class or +method declarations, zero-test files/roles, module/package `load_tests`, +unsupported conditional discovery declarations, unsupported bases, cyclic +local inheritance, free-function tests, missing package markers, and raw-byte +source/package-manifest drift during collection. The package manifest covers +required, nested, and root markers; it is rescanned after collection so added +or removed files/markers and initializer changes fail closed. + +As the first admission step, the runner rejects every symbolic link anywhere +under `harness/tests` (including test sources, package markers, role +directories, and nested directories) with a canonical exit-2 diagnostic. This +boundary is checked before source bytes are read or test modules are imported; +it prevents source-tree path escape through links. It is not a sandbox: code +that is admitted and imported may still perform ordinary external filesystem, +process, network, or other side effects. + +`--collect-only` emits JSON containing `collection_only: true`, per-role test +counts, sorted exact unittest IDs, and source SHA-256 hashes. Collection-only +success is not runtime success: normal unittest assertion failures and errors +remain visible and return a nonzero status. The runner intentionally accepts +standard `unittest.TestCase`, aliases, local mixins/inheritance, inherited +`test*` methods, and normal decorators such as `@unittest.skip`. + +The inventory is deliberately conservative, not a Python semantic evaluator or +security sandbox. Test classes/functions declared inside module/class control +flow are unsupported and rejected; helper functions/classes nested inside an +ordinary test method are runtime fixture code and are not inspected as +canonical discovery declarations. Tests supplied solely by dynamic imports, +metaclasses, or custom discovery protocols are unsupported: their collected +IDs must not substitute for or extend the statically declared inventory. +Imports used by ordinary fixture helpers are not prohibited. The harness does +not claim to prevent side effects performed by imported test code; it detects +source drift and validates collection before execution. + +The regression tests use bounded subprocess timeouts and clean test-owned +`TemporaryDirectory` trees under `/opt/data/cache/scratch/canonical-tests-main/fix2/`. +Run them with: + +```sh +python3 -m unittest harness.tests.common.test_canonical_test_runner -v +``` diff --git a/harness/tests/__init__.py b/harness/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/harness/tests/common/test_canonical_test_runner.py b/harness/tests/common/test_canonical_test_runner.py new file mode 100644 index 0000000..00b7991 --- /dev/null +++ b/harness/tests/common/test_canonical_test_runner.py @@ -0,0 +1,345 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[3] +RUNNER = REPO / "harness" / "tests" / "run.py" + + +class CanonicalRunnerTests(unittest.TestCase): + SCRATCH = Path("/opt/data/cache/scratch/canonical-tests-main/fix/test-tmp") + + def make_suite(self, *, roles=None, extra="", nested=False): + self.SCRATCH.mkdir(parents=True, exist_ok=True) + temp = tempfile.TemporaryDirectory(dir=self.SCRATCH) + self.addCleanup(temp.cleanup) + root = Path(temp.name) + tests = root / "harness" / "tests" + roles = roles or {"common": "test_common", "spec": "test_spec", "impl": "test_impl", "review": "test_review"} + (tests / "__init__.py").parent.mkdir(parents=True) + (tests / "__init__.py").write_text("") + (tests / "run.py").write_bytes(RUNNER.read_bytes()) + for role, stem in roles.items(): + d = tests / role + d.mkdir() + (d / "__init__.py").write_text("") + (d / f"{stem}.py").write_text( + "import unittest\n" + "class Example(unittest.TestCase):\n" + " def test_one(self):\n" + " self.assertTrue(True)\n" + ) + if nested: + d = tests / "common" / "nested" + d.mkdir() + (d / "__init__.py").write_text("") + (d / "test_nested.py").write_text( + "import unittest\nclass Nested(unittest.TestCase):\n" + " def test_nested(self): self.assertTrue(True)\n" + ) + if extra: + (tests / "test_outside.py").write_text(extra) + return root + + def execute(self, root, *args): + env = dict(os.environ) + env["PYTHONPATH"] = str(root) + os.pathsep + env.get("PYTHONPATH", "") + return subprocess.run([sys.executable, "-B", str(root / "harness/tests/run.py"), *args], cwd="/", env=env, + capture_output=True, text=True, timeout=20) + + def test_real_canonical_collect_only_succeeds_with_every_role(self): + result = subprocess.run([sys.executable, "-B", str(RUNNER), "--collect-only"], cwd="/", + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads(result.stdout) + self.assertEqual(set(data["role_counts"]), {"common", "spec", "impl", "review"}) + self.assertTrue(all(data["role_counts"].values())) + self.assertEqual(len(data["ids"]), sum(data["role_counts"].values())) + self.assertTrue(data["ids"]) + + def test_module_load_tests_is_rejected(self): + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text( + "import unittest\ndef load_tests(loader, tests, pattern): return tests\n" + "class X(unittest.TestCase):\n def test_one(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("load_tests", result.stderr) + + def test_zero_file_and_mixed_free_function_fail(self): + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text("import unittest\n") + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("zero", result.stderr.lower()) + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text('''import unittest +class X(unittest.TestCase): + def test_one(self): pass +def test_free(): pass +''') + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("free-function", result.stderr.lower()) + + def test_direct_case_does_not_short_circuit_cycle_or_unknown_base(self): + for source, word in ( + ('''import unittest +class A(B, unittest.TestCase): + def test_a(self): pass +class B(A): + def test_b(self): pass +''', "cyclic"), + ('''import unittest +class X(unittest.TestCase, Missing): + def test_x(self): pass +''', "unknown"), + ): + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text(source) + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn(word, result.stderr.lower()) + + def test_package_load_tests_is_rejected(self): + root = self.make_suite(nested=True) + (root / "harness/tests/common/nested/__init__.py").write_text('''def load_tests(loader, tests, pattern): + return tests +''') + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("load_tests", result.stderr) + + def test_alias_local_inheritance_and_runtime_helpers_are_valid(self): + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text('''import unittest as ut +class Mixin: + def test_mix(self): pass +class Base(ut.TestCase, Mixin): + def test_base(self): + class Helper: + def test_not_discovered(self): pass + self.assertTrue(Helper) +class Child(Base): + def test_child(self): pass +''') + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_conditional_discovery_declarations_fail_closed(self): + for source in ( + '''import unittest +if True: + class X(unittest.TestCase): + def test_x(self): pass +''', + '''import unittest +class X(unittest.TestCase): + if True: + def test_x(self): pass +''', + ): + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text(source) + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("conditional", result.stderr.lower()) + + def test_custom_loader_duplicate_and_omission_are_rejected(self): + root = self.make_suite() + marker = root / "admission-marker" + script = root / "probe.py" + script.write_text(""" +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import harness.tests.run as run +root = Path(sys.argv[1]) +files = run.source_files() +expected, _, _ = run.inventory(files) +baseline, suite = run.collect(files) +assert not any(isinstance(t, run.unittest.loader._FailedTest) for t in run.flatten(suite)) +assert baseline == expected +items = list(run.flatten(suite)) +class Fake(run.unittest.FunctionTestCase): + def __init__(self, ident): + super().__init__(lambda: Path(sys.argv[2]).write_text('ran')) + self.ident = ident + def id(self): + return self.ident +for mode in ('omission', 'addition', 'duplicate'): + class Loader: + def discover(self, *args, **kwargs): + selected = list(items) + if mode == 'omission': selected = selected[:-1] + elif mode == 'addition': selected.append(Fake('fake.added')) + else: selected = [Fake('fake.duplicate'), Fake('fake.duplicate')] + return run.unittest.TestSuite(selected) + run.unittest.defaultTestLoader = Loader() + try: + if mode == 'duplicate': run.collect(files) + else: run.validate() + except run.CanonicalError: + pass + else: + raise AssertionError(mode + ' was admitted') +assert not Path(sys.argv[2]).exists() +""") + result = subprocess.run([sys.executable, "-B", str(script), str(root), str(marker)], cwd="/", + capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_collect_only_reports_exact_roles_ids_and_hashes(self): + root = self.make_suite() + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads(result.stdout) + self.assertTrue(data["collection_only"]) + self.assertEqual(data["role_counts"], {"common": 1, "spec": 1, "impl": 1, "review": 1}) + self.assertEqual(len(data["ids"]), 4) + self.assertEqual(set(data["source_sha256"]), {"harness/tests/common/test_common.py", "harness/tests/spec/test_spec.py", "harness/tests/impl/test_impl.py", "harness/tests/review/test_review.py"}) + + def test_missing_role_marker_duplicate_and_unexpected_file_fail_before_run(self): + for mode in ("missing", "marker", "duplicate", "unexpected"): + with self.subTest(mode=mode): + root = self.make_suite(extra=("import unittest\nclass Example(unittest.TestCase):\n def test_one(self): pass\n" if mode == "unexpected" else "")) + if mode == "missing": + (root / "harness/tests/review/test_review.py").unlink() + elif mode == "marker": + (root / "harness/tests/spec/__init__.py").unlink() + elif mode == "duplicate": + (root / "harness/tests/common/test_common.py").write_text( + "import unittest\nclass Example(unittest.TestCase):\n def test_one(self): pass\nclass Example(unittest.TestCase):\n def test_two(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("canonical", result.stderr.lower()) + + def test_loader_omission_addition_zero_file_and_unsupported_function_fail(self): + cases = { + "omission": lambda root: (root / "harness/tests/common/test_common.py").write_text("import unittest\ndef load_tests(loader, tests, pattern): return unittest.TestSuite()\nclass X(unittest.TestCase):\n def test_one(self): pass\n"), + "zero": lambda root: (root / "harness/tests/review/test_review.py").write_text("import unittest\n"), + "free": lambda root: (root / "harness/tests/impl/test_impl.py").write_text("def test_free(): pass\n"), + } + for name, mutate in cases.items(): + with self.subTest(name=name): + root = self.make_suite(); mutate(root) + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + root = self.make_suite() + (root / "harness/tests/spec/test_spec.py").write_text("import unittest\nclass X(unittest.TestCase):\n def test_one(self): pass\n def test_two(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_import_failure_source_drift_nested_omission_and_runtime_failure(self): + root = self.make_suite() + (root / "harness/tests/impl/test_impl.py").write_text("raise RuntimeError('broken import')\n") + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + self.assertIn("canonical", result.stderr.lower()) + root = self.make_suite(nested=True) + (root / "harness/tests/common/nested/__init__.py").unlink() + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + root = self.make_suite() + (root / "harness/tests/review/test_review.py").write_text( + "import unittest\nfrom pathlib import Path\np = Path(__file__)\np.write_text(p.read_text() + '# drift\\n')\n" + "class X(unittest.TestCase):\n def test_one(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertNotEqual(result.returncode, 0) + root = self.make_suite() + (root / "harness/tests/common/test_common.py").write_text( + "import unittest\nclass X(unittest.TestCase):\n def test_one(self): self.fail('boom')\n") + result = self.execute(root) + self.assertNotEqual(result.returncode, 0) + self.assertIn("FAILED", result.stderr + result.stdout) + + def test_package_manifest_drift_and_extra_zero_file_fail_closed(self): + cases = ( + ("initializer", "from pathlib import Path\np = Path(__file__).parent / '__init__.py'\np.write_text(p.read_text() + '# drift\\n')\n"), + ("add-source", "from pathlib import Path\nPath(__file__).with_name('test_added.py').write_text('import unittest\\nclass Added(unittest.TestCase):\\n def test_added(self): pass\\n')\n"), + ("remove-source", "from pathlib import Path\n(Path(__file__).parents[1] / 'review' / 'test_review.py').unlink()\n"), + ("extra-zero", "from pathlib import Path\nPath(__file__).with_name('test_empty.py').write_text('import unittest\\n')\n"), + ) + for name, mutation in cases: + with self.subTest(name=name): + root = self.make_suite() + target = root / "harness/tests/common/test_common.py" + target.write_text("import unittest\n" + mutation + + "class X(unittest.TestCase):\n def test_one(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertIn("source/package manifest drift", result.stderr.lower()) + + def test_symlinked_source_and_package_paths_fail_before_import(self): + for kind in ("source", "marker", "role", "nested"): + with self.subTest(kind=kind): + root = self.make_suite() + marker = root / "escaped-import" + outside = root / "outside" + outside.mkdir() + payload = ("from pathlib import Path\n" + + "Path(" + repr(str(marker)) + ").write_text('imported')\n" + + "import unittest\nclass Escape(unittest.TestCase):\n def test_escape(self): pass\n") + if kind == "source": + external = outside / "escape.py" + external.write_text(payload) + (root / "harness/tests/common/test_escape.py").symlink_to(external) + elif kind == "marker": + external = outside / "init.py" + external.write_text(payload) + path = root / "harness/tests/common/__init__.py" + path.unlink() + path.symlink_to(external) + else: + (outside / "__init__.py").write_text(payload) + (outside / "test_escape.py").write_text(payload) + if kind == "role": + path = root / "harness/tests/review" + (path / "__init__.py").unlink() + (path / "test_review.py").unlink() + path.rmdir() + else: + path = root / "harness/tests/common/nested" + path.symlink_to(outside, target_is_directory=True) + result = self.execute(root) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("symlink", result.stderr.lower()) + self.assertFalse(marker.exists(), "escaped code was imported before rejection") + + def test_static_extra_zero_file_fails_with_valid_sibling(self): + root = self.make_suite() + (root / "harness/tests/common/test_empty.py").write_text("# empty candidate\n") + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertIn("zero inventoried tests", result.stderr) + + def test_removed_role_marker_is_manifest_failure(self): + root = self.make_suite() + source = root / "harness/tests/common/test_common.py" + source.write_text("import unittest\nfrom pathlib import Path\n" + "(Path(__file__).parents[1] / 'review' / '__init__.py').unlink()\n" + "class X(unittest.TestCase):\n def test_one(self): pass\n") + result = self.execute(root, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertIn("missing package marker", result.stderr) + + def test_normal_synthetic_success_and_assertion_failure_exit_statuses(self): + root = self.make_suite() + success = self.execute(root) + self.assertEqual(success.returncode, 0, success.stderr) + (root / "harness/tests/common/test_common.py").write_text( + "import unittest\nclass X(unittest.TestCase):\n def test_one(self): self.fail('named assertion')\n") + failure = self.execute(root) + self.assertEqual(failure.returncode, 1) + self.assertIn("named assertion", failure.stderr + failure.stdout) + (root / "harness/tests/common/test_common.py").write_text( + "import unittest\nclass X(unittest.TestCase):\n def test_one(self): raise RuntimeError('named error')\n") + error = self.execute(root) + self.assertEqual(error.returncode, 1) + self.assertIn("named error", error.stderr + error.stdout) diff --git a/harness/tests/common/test_workflow_guard.py b/harness/tests/common/test_workflow_guard.py index 4036482..19b5cc3 100644 --- a/harness/tests/common/test_workflow_guard.py +++ b/harness/tests/common/test_workflow_guard.py @@ -15,7 +15,7 @@ def inventory_digest(ids, statuses): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() -def test_inventory(ids=('case_a',), current=None): +def make_test_inventory(ids=('case_a',), current=None): current = list(ids if current is None else current) ids = list(ids) old_status = {key: {'skip': False, 'expected_failure': False} for key in ids} @@ -157,84 +157,84 @@ class WorkflowGuardTests(unittest.TestCase): self.assertEqual(guard.compare_source(root, out, ['src/a.py'])['status'], 'block') def test_new_regression_tests_do_not_require_deletion_authorization(self): - self.assertEqual(guard.check_tests(test_inventory(current=['case_a', 'case_b']))['status'], 'pass') + self.assertEqual(guard.check_tests(make_test_inventory(current=['case_a', 'case_b']))['status'], 'pass') def test_collected_inventory_requires_provenance(self): - value = test_inventory() + value = make_test_inventory() del value['baseline_collection'] self.assert_block(guard.check_tests, value) def test_collected_inventory_digest_must_match_ids_and_statuses(self): - value = test_inventory() + value = make_test_inventory() value['baseline_collection']['inventory_sha256'] = '0' * 64 self.assert_block(guard.check_tests, value) def test_new_skip_requires_explicit_approval(self): - value = test_inventory() + value = make_test_inventory() value['current_status']['case_a']['skip'] = True value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status']) self.assert_block(guard.check_tests, value) def test_new_expected_failure_requires_explicit_approval(self): - value = test_inventory() + value = make_test_inventory() value['current_status']['case_a']['expected_failure'] = True value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status']) self.assert_block(guard.check_tests, value) def test_rename_requires_real_replacement_and_semantic_evidence(self): - value = test_inventory(current=['case_b']) + value = make_test_inventory(current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': ['ghost'], 'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assert_block(guard.check_tests, value) def test_approved_rename_preserves_mapped_test(self): - value = test_inventory(current=['case_b']) + value = make_test_inventory(current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': ['case_b'], 'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assertEqual(guard.check_tests(value)['status'], 'pass') def test_modified_test_requires_semantic_review(self): - value = test_inventory() + value = make_test_inventory() value['changed_tests'] = [{'test_id': 'case_a', 'baseline_sha256': 'a' * 64, 'current_sha256': 'b' * 64, 'review_evidence': ''}] self.assert_block(guard.check_tests, value) def test_finding_truthy_strings_are_not_red_green_records(self): - value = test_inventory() + value = make_test_inventory() value['findings'] = [{'id': 'finding-a', 'test_id': 'case_a', 'classification': 'assertion', 'red': 'claimed-red', 'green': 'claimed-green'}] self.assert_block(guard.check_tests, value) def test_red_success_exit_cannot_demonstrate_failure(self): - value = test_inventory() + value = make_test_inventory() finding = finding_record() finding['red']['exit_code'] = finding['red']['expected_exit_code'] = 0 value['findings'] = [finding] self.assert_block(guard.check_tests, value) def test_green_failure_exit_cannot_demonstrate_resolution(self): - value = test_inventory() + value = make_test_inventory() finding = finding_record() finding['green']['exit_code'] = finding['green']['expected_exit_code'] = 1 value['findings'] = [finding] self.assert_block(guard.check_tests, value) def test_environment_failure_is_not_valid_red(self): - value = test_inventory() + value = make_test_inventory() finding = finding_record() finding['red']['failure_kind'] = 'environment' value['findings'] = [finding] self.assert_block(guard.check_tests, value) def test_finding_must_name_present_regression_test(self): - value = test_inventory() + value = make_test_inventory() finding = finding_record() finding['test_id'] = 'ghost' value['findings'] = [finding] self.assert_block(guard.check_tests, value) def test_named_red_green_records_with_provenance_pass(self): - value = test_inventory() + value = make_test_inventory() value['findings'] = [finding_record()] self.assertEqual(guard.check_tests(value)['status'], 'pass') @@ -342,31 +342,31 @@ class WorkflowGuardTests(unittest.TestCase): self.assertEqual(json.loads(result.stdout)['status'], 'block') def test_rename_empty_replacement_cannot_stand_in_for_deletion(self): - value = test_inventory(current=['case_b']) + value = make_test_inventory(current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'rename', 'replacement_ids': [], 'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assert_block(guard.check_tests, value) def test_unclassified_empty_replacement_cannot_stand_in_for_deletion(self): - value = test_inventory(current=['case_b']) + value = make_test_inventory(current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'replacement_ids': [], 'reason': 'rename', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assert_block(guard.check_tests, value) def test_explicit_approved_deletion_is_allowed(self): - value = test_inventory(ids=['case_a', 'case_b'], current=['case_b']) + value = make_test_inventory(ids=['case_a', 'case_b'], current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'delete', 'replacement_ids': [], 'reason': 'approved obsolete case deletion', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assertEqual(guard.check_tests(value)['status'], 'pass') def test_status_approval_cannot_authorize_removal(self): - value = test_inventory(current=['case_b']) + value = make_test_inventory(current=['case_b']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'status', 'replacement_ids': ['case_b'], 'reason': 'approved status update', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assert_block(guard.check_tests, value) def test_explicit_status_approval_maps_same_current_id(self): - value = test_inventory() + value = make_test_inventory() value['current_status']['case_a']['skip'] = True value['current_collection']['inventory_sha256'] = inventory_digest(value['current_ids'], value['current_status']) value['authorized_changes'] = [{'old_id': 'case_a', 'kind': 'status', 'replacement_ids': ['case_a'], @@ -374,7 +374,7 @@ class WorkflowGuardTests(unittest.TestCase): self.assertEqual(guard.check_tests(value)['status'], 'pass') def test_unhashable_authorization_id_fails_closed_api(self): - value = test_inventory() + value = make_test_inventory() value['authorized_changes'] = [{'old_id': ['case_a'], 'replacement_ids': [], 'reason': 'fixture', 'approval': 'fixture-approval', 'evidence': 'fixture-review'}] self.assert_block(guard.check_tests, value) @@ -408,10 +408,10 @@ class WorkflowGuardTests(unittest.TestCase): def test_declared_evidence_shape_does_not_authenticate_records(self): # Deliberately synthetic fixture claims: the main agent must check actual evidence separately. - self.assertEqual(guard.check_tests(test_inventory())['status'], 'pass') + self.assertEqual(guard.check_tests(make_test_inventory())['status'], 'pass') def test_unknown_inventory_fields_are_rejected(self): - value = test_inventory() + value = make_test_inventory() value['pretend_verified'] = True self.assert_block(guard.check_tests, value) diff --git a/harness/tests/impl/__init__.py b/harness/tests/impl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/harness/tests/review/__init__.py b/harness/tests/review/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/harness/tests/run.py b/harness/tests/run.py new file mode 100644 index 0000000..436f541 --- /dev/null +++ b/harness/tests/run.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Canonical, source-bound unittest harness.""" +import argparse +import ast +import hashlib +import json +import os +import sys +import unittest +from pathlib import Path + +ROLES = ("common", "spec", "impl", "review") +SOURCE_ROOT = Path(__file__).resolve().parent +REPO_ROOT = SOURCE_ROOT.parents[1] + +class CanonicalError(Exception): + pass + +def fail(message): + raise CanonicalError("canonical harness: " + message) + +def rel(path): + return path.relative_to(REPO_ROOT).as_posix() + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + +def _reject_source_tree_symlinks(): + """Reject links before any source bytes are read or imported.""" + def visit(directory): + try: + entries = list(os.scandir(directory)) + except OSError as exc: + fail(f"cannot inspect {rel(Path(directory))}: {exc}") + return + for entry in entries: + path = Path(entry.path) + if entry.is_symlink(): + fail(f"symlinked source-tree path: {rel(path)}") + if entry.is_dir(follow_symlinks=False): + visit(path) + visit(SOURCE_ROOT) + +def module_name(path): + return ".".join(path.relative_to(REPO_ROOT).with_suffix("").parts) + +def _package_manifest(): + """Return raw-byte fingerprints for all source and package markers.""" + _reject_source_tree_symlinks() + files = {} + if not SOURCE_ROOT.is_dir(): + fail("missing source root") + for directory in [SOURCE_ROOT, *(SOURCE_ROOT / role for role in ROLES)]: + marker = directory / "__init__.py" + if not marker.is_file(): + fail(f"missing package marker: {rel(marker)}") + for path in SOURCE_ROOT.rglob("*"): + if path.is_file() and (path.name == "__init__.py" or path.name.startswith("test") and path.suffix == ".py"): + files[rel(path)] = path.read_bytes() + return files + +def _reject_package_load_tests(manifest=None): + manifest = _package_manifest() if manifest is None else manifest + for name in manifest: + if not name.endswith("__init__.py"): + continue + path = REPO_ROOT / name + try: + tree = ast.parse(path.read_bytes(), filename=str(path)) + except (OSError, SyntaxError) as exc: + fail(f"cannot parse {rel(path)}: {exc}") + if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_tests" for n in tree.body): + fail(f"unsupported dynamic load_tests declaration in {rel(path)}") + +def package_dirs(manifest=None): + _reject_package_load_tests(manifest) + for path in SOURCE_ROOT.rglob("test*.py"): + current = path.parent + while current != SOURCE_ROOT.parent: + if current in (SOURCE_ROOT, *(SOURCE_ROOT / role for role in ROLES)): + current = current.parent + continue + marker = current / "__init__.py" + if not marker.is_file(): + fail(f"nested test package missing marker: {rel(marker)}") + current = current.parent + +def source_files(manifest=None): + package_dirs(manifest) + files = sorted(SOURCE_ROOT.rglob("test*.py")) + role_roots = {SOURCE_ROOT / role for role in ROLES} + unexpected = [p for p in files if not any(p.is_relative_to(d) for d in role_roots)] + if unexpected: + fail("unexpected test file outside role dirs: " + ", ".join(rel(p) for p in unexpected)) + by_role = {role: sorted((SOURCE_ROOT / role).rglob("test*.py")) for role in ROLES} + for role, paths in by_role.items(): + if not paths: + fail(f"required role has no test files: {role}") + return by_role + +def aliases(tree): + names = {"TestCase", "unittest.TestCase"} + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module == "unittest": + for item in node.names: + if item.name == "TestCase": + names.add(item.asname or item.name) + elif isinstance(node, ast.Import): + for item in node.names: + if item.name == "unittest": + names.add((item.asname or item.name) + ".TestCase") + return names + +def inventory_file(path, raw_bytes=None): + try: + raw_bytes = path.read_bytes() if raw_bytes is None else raw_bytes + tree = ast.parse(raw_bytes, filename=str(path)) + except (OSError, SyntaxError) as exc: + fail(f"cannot parse {rel(path)}: {exc}") + def conditional_declarations(nodes, context): + controls = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.With, ast.AsyncWith, ast.Match) + for node in nodes: + if isinstance(node, controls): + stack = list(ast.iter_child_nodes(node)) + while stack: + child = stack.pop() + if isinstance(child, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if child.name.startswith("test") or isinstance(child, ast.ClassDef): + fail(f"unsupported conditional test declaration in {rel(path)}: {context}.{child.name}") + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + stack.extend(ast.iter_child_nodes(child)) + conditional_declarations(tree.body, "module") + if any(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_tests" for n in tree.body): + fail(f"unsupported dynamic load_tests declaration in {rel(path)}") + testcase_names = aliases(tree) + classes = {} + for node in tree.body: + if isinstance(node, ast.ClassDef): + if node.name in classes: + fail(f"duplicate test class declaration in {rel(path)}: {node.name}") + classes[node.name] = node + conditional_declarations(node.body, node.name) + child_names = [c.name for c in node.body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef))] + if len(child_names) != len(set(child_names)): + fail(f"duplicate declaration in {rel(path)}: {node.name}") + state = {} + memo = {} + def base_name(base): + if isinstance(base, ast.Name): return base.id + if isinstance(base, ast.Attribute) and isinstance(base.value, ast.Name): return base.value.id + "." + base.attr + fail(f"unsupported test class base in {rel(path)}: {ast.unparse(base)}") + def is_case(name): + if name in memo: return memo[name] + if state.get(name) == "active": + fail(f"cyclic test class declaration in {rel(path)}: {name}") + node = classes.get(name) + if node is None: + result = name in testcase_names + if not result and name not in testcase_names: + fail(f"unknown test class base in {rel(path)}: {name}") + return result + state[name] = "active" + result = False + for base in node.bases: # eager: never short-circuit validation + result = is_case(base_name(base)) or result + state.pop(name, None) + memo[name] = result + return result + def local_methods(name, stack=()): + if name in stack: + fail(f"cyclic test inheritance in {rel(path)}: {' -> '.join(stack + (name,))}") + node = classes[name] + found = {c.name for c in node.body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef)) and c.name.startswith("test")} + for base in node.bases: + b = base_name(base) + if b in classes: + is_case(b) + found |= local_methods(b, stack + (name,)) + return found + methods = {} + free = [] + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test"): + free.append(node.name) + for name in classes: + case = is_case(name) + own = {c.name for c in classes[name].body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef)) and c.name.startswith("test")} + if case: + methods[name] = local_methods(name) + if free: + fail(f"unsupported free-function test in {rel(path)}: {free[0]}") + ids = {f"{module_name(path)}.{name}.{method}" for name, ms in methods.items() for method in ms} + if not ids: + fail(f"file has zero inventoried tests: {rel(path)}") + return ids + +def inventory(files, manifest=None): + ids, role_ids, hashes = set(), {}, {} + manifest = _package_manifest() if manifest is None else manifest + for role, paths in files.items(): + role_ids[role] = set() + for path in paths: + raw_bytes = manifest.get(rel(path)) + if raw_bytes is None: + fail(f"source missing from package manifest: {rel(path)}") + hashes[rel(path)] = hashlib.sha256(raw_bytes).hexdigest() + found = inventory_file(path, raw_bytes) + overlap = ids & found + if overlap: fail("duplicate test IDs: " + ", ".join(sorted(overlap))) + ids |= found; role_ids[role] |= found + if not role_ids[role]: fail(f"role has zero inventoried tests: {role}") + return ids, role_ids, hashes + +def flatten(suite): + for item in suite: + if isinstance(item, unittest.TestSuite): yield from flatten(item) + else: yield item + +def collect(files, loader=None): + loader = loader or unittest.defaultTestLoader + suite = loader.discover(str(SOURCE_ROOT), pattern="test*.py", top_level_dir=str(REPO_ROOT)) + flat = list(flatten(suite)) + failed = [test for test in flat if isinstance(test, unittest.loader._FailedTest)] + if failed: fail("import/loader failure: " + "; ".join(str(test) for test in failed[:20])) + raw = [test.id() for test in flat] + seen = set(); duplicates = [] + for test_id in raw: + if test_id in seen and test_id not in duplicates: duplicates.append(test_id) + seen.add(test_id) + if duplicates: fail("duplicate loader test IDs: " + ", ".join(duplicates[:20])) + return set(raw), suite + +def validate(): + before_manifest = _package_manifest() + files = source_files(before_manifest) + expected, role_ids, before = inventory(files, before_manifest) + actual, suite = collect(files) + after_manifest = _package_manifest() + if before_manifest != after_manifest: fail("source/package manifest drift detected during collection") + if expected != actual: + fail(f"inventory/loader mismatch: missing={sorted(expected-actual)!r} extra={sorted(actual-expected)!r}") + return expected, role_ids, before, suite + +def main(argv=None): + parser = argparse.ArgumentParser(add_help=True); parser.add_argument("--collect-only", action="store_true") + args = parser.parse_args(argv); sys.path.insert(0, str(REPO_ROOT)) + try: ids, role_ids, hashes, suite = validate() + except CanonicalError as exc: print(str(exc), file=sys.stderr); return 2 + if args.collect_only: + print(json.dumps({"collection_only": True, "role_counts": {r: len(role_ids[r]) for r in ROLES}, "ids": sorted(ids), "source_sha256": hashes}, sort_keys=True)); return 0 + result = unittest.TextTestRunner(verbosity=2).run(suite); return 0 if result.wasSuccessful() else 1 + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/harness/tests/spec/__init__.py b/harness/tests/spec/__init__.py new file mode 100644 index 0000000..e69de29