genichin/gitea-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(harness): reorganize regression suites and preserve contract coverage

Browse Source
This commit is contained in:
gitea-spec-agent
2026-10-16 07:24:49 +00:00
parent b6f87db623
commit 33709ab873
108 changed files with 5869 additions and 5328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/spec/support/test_codec.py
+223
View File
@@ -0,0 +1,223 @@
import copy
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from harness.tests._support.paths import ROOT
from harness.tests._support.spec import MODPATH, mod, publication_fixture, result, verify_fixture
class CodecTests(unittest.TestCase):
def test_validate_and_render_roundtrip(self):
r = result()
self.assertEqual(mod.validate_result(r, {"src/a.py", "tests/test_a.py"}), r)
body = mod.render_review(r, {"src/a.py", "tests/test_a.py"}, declaration={"mode": "none"})
parsed = mod.parse_review(body, {"src/a.py", "tests/test_a.py"})
self.assertEqual(parsed, r)
self.assertEqual(mod.render_review(parsed, {"src/a.py", "tests/test_a.py"}, declaration={"mode": "none"}), body)
def test_validation_rejects_bool_path_traversal_bad_verdict(self):
r = result()
r["target"]["issue"] = True
with self.assertRaises(mod.ValidationError):
mod.validate_result(r, {"src/a.py"})
r = result(); r["criteria"][0]["implementation_paths"] = ["../x"]
with self.assertRaises(mod.ValidationError):
mod.validate_result(r, {"../x"})
r = result("request_changes")
with self.assertRaises(mod.ValidationError):
mod.validate_result(r, {"src/a.py", "tests/test_a.py"})
def test_collect_pages_requires_identity_and_reads_all_pages(self):
pages = [[{"id": 1}, {"id": 2}], [{"id": 3}]]
def fetch(n):
return {"items": pages[n - 1], "total_count": 3, "has_more": n == 1}
self.assertEqual(mod.collect_pages(fetch, "id", page_size=2), pages[0] + pages[1])
def bad(n):
return {"items": [{"id": 1}], "total_count": 2, "has_more": True}
with self.assertRaises(mod.ValidationError):
mod.collect_pages(bad, "id", max_pages=1)
def test_verify_unchanged(self):
self.assertTrue(mod.verify_unchanged({"a": [1]}, {"a": [1]}))
with self.assertRaises(mod.ValidationError):
mod.verify_unchanged({"a": 1}, {"a": 2})
def test_cli_check_pages_and_unchanged(self):
with tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch") as td:
td = Path(td); env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
pages = td / "pages.json"; pages.write_text(json.dumps([
{"items": [{"id": 1}], "total_count": 1, "has_more": False}],), encoding="utf-8")
base = [sys.executable, str(MODPATH)]
p = subprocess.run(base + ["check-pages", "--pages", str(pages), "--key", "id"], capture_output=True, text=True, env=env)
self.assertEqual(p.returncode, 0, p.stderr)
b = td / "before.json"; a = td / "after.json"
b.write_text('{"x":1}', encoding="utf-8"); a.write_text('{"x":1}', encoding="utf-8")
p = subprocess.run(base + ["check-unchanged", "--before", str(b), "--after", str(a)], capture_output=True, text=True, env=env)
self.assertEqual(p.returncode, 0, p.stderr); self.assertIn('"unchanged": true', p.stdout)
def test_cli_validate_render_parse(self):
with tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch") as td:
td = Path(td); rf = td / "r.json"; af = td / "a.json"; bf = td / "b.md"
rf.write_text(json.dumps(result(), ensure_ascii=False), encoding="utf-8")
af.write_text(json.dumps(["src/a.py", "tests/test_a.py"]), encoding="utf-8")
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
base = [sys.executable, str(MODPATH)]
p = subprocess.run(base + ["validate-result", "--result", str(rf), "--allowed-paths", str(af)],
text=True, capture_output=True, env=env)
self.assertEqual(p.returncode, 0, p.stderr)
p = subprocess.run(base + ["render", "--no-delegated-evidence", "--result", str(rf), "--allowed-paths", str(af)],
text=True, capture_output=True, env=env)
self.assertEqual(p.returncode, 0, p.stderr); bf.write_text(p.stdout, encoding="utf-8")
p = subprocess.run(base + ["parse", "--body", str(bf), "--allowed-paths", str(af)],
text=True, capture_output=True, env=env)
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(json.loads(p.stdout), result())
def test_result_codec_preserves_existing_native_format(self):
r = result()
paths = {"src/a.py", "tests/test_a.py"}
body = mod.render_review(r, paths, declaration={"mode": "none"})
self.assertIn(" - 구현 근거: src/a.py; 증거\n", body)
self.assertIn("모든 기준을 충족했다.\n\n<!--", body)
self.assertEqual(mod.parse_review(body, paths), r)
unordered = json.loads(json.dumps(r, sort_keys=True))
self.assertEqual(mod.render_review(unordered, paths, declaration={"mode": "none"}), body)
def test_strict_codec_rejects_unrenderable_and_sensitive_human_values(self):
paths = {"src/a.py", "tests/test_a.py"}
for field, value in (("verdict", []), ("conclusion", "trailing "),
("conclusion", "line\u2028break"), ("conclusion", "ftp://example.test")):
r = result(); r[field] = value
with self.subTest(field=field, value=value), self.assertRaises(mod.ValidationError):
mod.validate_result(r, paths)
for field in ("criterion", "contract", "evidence", "reason"):
r = result(); r["criteria"][0][field] = "secretvalue"
with self.subTest(field=field), self.assertRaises(mod.ValidationError):
mod.validate_result(r, paths, sensitive_values=("secretvalue",))
r = result(); r["verdict"] = "blocked_spec"; r["scope"]["spec_conflicts"] = ["secretvalue"]
with self.assertRaises(mod.ValidationError):
mod.validate_result(r, paths, sensitive_values=("secretvalue",))
with self.assertRaises(mod.ValidationError):
mod.loads_json('{"n":1e999}')
with self.assertRaises(mod.ValidationError):
mod.validate_result(result(), "src/a.py")
def test_parser_rejects_forged_marker_and_body_drift(self):
paths = {"src/a.py", "tests/test_a.py"}
body = mod.render_review(result(), paths, declaration={"mode": "none"})
for bad in (body.replace("issue=3", "issue=4"), body.replace("증거", "다른 증거"),
body.replace("\n", "\r\n"), body + "\n", body.replace("correlation=", "other=")):
with self.subTest(body=bad[-100:]), self.assertRaises(mod.ValidationError):
mod.parse_review(bad, paths)
def test_codec_roundtrips_delimiter_bearing_paths_and_prose(self):
r = result(); r["criteria"][0]["implementation_paths"] = ["src/a, b.py", "src/c; d.py"]
r["criteria"][0]["evidence"] = "확인; 모두 — 일치"
r["verdict"] = "blocked_spec"; r["scope"]["spec_conflicts"] = ["하나, 둘", "세 가지"]
paths = {"src/a, b.py", "src/c; d.py", "tests/test_a.py"}
self.assertEqual(mod.parse_review(mod.render_review(r, paths, declaration={"mode": "none"}), paths), r)
def test_read_only_replay_uses_recursive_exact_json_types(self):
with self.assertRaises(mod.ValidationError):
mod.verify_unchanged({"nested": [True]}, {"nested": [1]})
def test_pagination_is_one_based_and_rejects_short_continuations(self):
calls = []
pages = [{"items": [{"id": 1}, {"id": 2}], "total_count": 3, "has_more": True},
{"items": [{"id": 3}], "total_count": 3, "has_more": False}]
def fetch(page):
calls.append(page)
return pages[page - 1]
self.assertEqual(mod.collect_pages(fetch, "id", page_size=2), [{"id": 1}, {"id": 2}, {"id": 3}])
self.assertEqual(calls, [1, 2])
with self.assertRaises(mod.ValidationError):
mod.collect_pages(lambda _: {"items": [{"id": 1}], "total_count": 3, "has_more": True}, "id", page_size=2)
def test_pagination_validates_all_declared_metadata_and_identity_types(self):
bad_pages = [
{"items": [{"id": 1}], "total_count": 0, "has_more": True},
{"items": [{"id": 1}, {"id": 2}], "total_count": 2, "has_more": True},
{"items": [{"id": []}], "has_more": False},
{"items": [{"id": 1.5}], "has_more": False},
{"items": [{"id": 0}], "has_more": False},
{"items": [{"id": True}], "has_more": False},
{"items": [{"id": 1}, {"id": 1}], "has_more": False},
{"items": None, "has_more": False},
{"items": [], "has_more": True},
]
for page in bad_pages:
with self.subTest(page=bad_pages.index(page)), self.assertRaises(mod.ValidationError):
mod.collect_pages(lambda _: page, "id", page_size=2, max_pages=2)
pages = [{"items": [{"id": 1}], "total_count": 2, "has_more": True},
{"items": [{"id": 2}], "total_count": 3, "has_more": False}]
with self.assertRaises(mod.ValidationError):
mod.collect_pages(lambda i: pages[i - 1], "id", page_size=1)
self.assertEqual(mod.collect_pages(lambda _: {"items": None, "total_count": 0, "has_more": False}, "id"), [])
def test_cli_errors_are_sanitized_without_tracebacks_or_echoed_values(self):
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
base = [sys.executable, str(MODPATH)]
for args in (["unknown-secret-value"], ["check-cache", "--issue", "secret-value"],
["check-unchanged", "--before", "secret-value", "--after", "missing"]):
p = subprocess.run(base + args, capture_output=True, text=True, env=env)
self.assertNotEqual(p.returncode, 0)
self.assertNotIn("secret-value", p.stderr + p.stdout)
self.assertNotIn("Traceback", p.stderr + p.stdout)
self.assertEqual(json.loads(p.stderr), {"error": "invalid input"})
with tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch") as td:
path = Path(td) / "pages.json"
path.write_text(json.dumps([{ "items": [{"id": 1}], "has_more": True, "total_count": 2 }]))
p = subprocess.run(base + ["check-pages", "--pages", str(path), "--key", "id"],
capture_output=True, text=True, env=env)
self.assertNotEqual(p.returncode, 0)
self.assertNotIn("Traceback", p.stderr)
self.assertEqual(json.loads(p.stderr), {"error": "invalid input"})
def test_target_identifiers_preserve_dots_but_reject_marker_breaking_whitespace(self):
r = result(); paths = {"src/a.py", "tests/test_a.py"}
r["target"].update(owner="acme.name", repository="repo.name")
body = mod.render_review(r, paths, declaration={"mode": "none"})
self.assertEqual(mod.parse_review(body, paths), r)
r["target"]["owner"] = "acme name"
with self.assertRaises(mod.ValidationError):
mod.validate_result(r, paths)
def test_rendered_paths_cannot_inject_markdown_html_or_mentions(self):
r = result(); path = "src/<script>@user`[link].py"
r["criteria"][0]["implementation_paths"] = [path]
paths = {path, "tests/test_a.py"}
body = mod.render_review(r, paths, declaration={"mode": "none"})
self.assertNotIn("<script>", body)
self.assertNotIn("@user", body)
self.assertNotIn("`[link]", body)
self.assertEqual(mod.parse_review(body, paths), r)
def test_strict_types_limits_and_duplicate_nested_keys(self):
paths = {"src/a.py", "tests/test_a.py"}
for bad_json in ('{"a":1,"a":2}', '{"a":NaN}',
'{"outer":{"x":1,"x":2}}', '{"x":Infinity}', '{"x":-Infinity}'):
with self.subTest(bad_json=bad_json), self.assertRaises(mod.ValidationError):
mod.loads_json(bad_json)
cases = []
r = result(); r["target"]["pull"] = False; cases.append(r)
r = result(); r["criteria"].append(copy.deepcopy(r["criteria"][0])); cases.append(r)
r = result(); r["criteria"][0]["contract"] = "a" * 4001; cases.append(r)
r = result(); r["criteria"][0]["criterion"] = "a" * 201; cases.append(r)
r = result(); r["criteria"][0]["result"] = {}; cases.append(r)
r = result(); r["criteria"][0]["test_paths"] *= 2; cases.append(r)
r = result(); r["criteria"][0]["test_paths"] = ["/absolute"]; cases.append(r)
r = result(); r["criteria"][0]["test_paths"] = ["tests/../file.py"]; cases.append(r)
r = result(); r["extra"] = None; cases.append(r)
r = result(); del r["scope"]; cases.append(r)
for value in cases:
with self.subTest(case=cases.index(value)), self.assertRaises(mod.ValidationError):
mod.validate_result(value, paths)
if __name__ == "__main__":
unittest.main()