test(harness): reorganize regression suites and preserve contract coverage
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
||||
SCRATCH = Path("/opt/data/cache/scratch/evidence-deadline-regressions")
|
||||
|
||||
|
||||
def git(cwd, *args):
|
||||
return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
class EvidenceCommandDeadlineTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(dir="/opt/data/cache/scratch")
|
||||
root = Path(self.tmp.name)
|
||||
self.contract = root / "harness/operations/repository-cache.md"
|
||||
self.contract.parent.mkdir(parents=True)
|
||||
self.contract.write_text("contract\n")
|
||||
self.repo = root / "repositories/o/p/worktrees/issue-1/checkout"
|
||||
self.artifacts = self.repo.parent / "verification/run-a"
|
||||
self.repo.mkdir(parents=True)
|
||||
self.artifacts.mkdir(parents=True)
|
||||
for p in (root / "repositories", root / "repositories/o", root / "repositories/o/p",
|
||||
root / "repositories/o/p/worktrees", root / "repositories/o/p/worktrees/issue-1",
|
||||
self.repo, self.artifacts.parent, self.artifacts):
|
||||
if p.exists(): p.chmod(0o700)
|
||||
git(self.repo, "init", "-q")
|
||||
git(self.repo, "config", "user.email", "test@example.invalid")
|
||||
git(self.repo, "config", "user.name", "Test")
|
||||
(self.repo / "input.txt").write_text("input\n")
|
||||
git(self.repo, "add", "input.txt")
|
||||
git(self.repo, "commit", "-qm", "initial")
|
||||
self.known_pid_files = set()
|
||||
self.outer_processes = set()
|
||||
|
||||
def tearDown(self):
|
||||
for pid_file in self.known_pid_files:
|
||||
try:
|
||||
pid = int(pid_file.read_text().strip())
|
||||
except (FileNotFoundError, ValueError):
|
||||
continue
|
||||
self._kill_pid_group(pid)
|
||||
for proc in self.outer_processes:
|
||||
if proc.poll() is None:
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
proc.wait(timeout=1)
|
||||
self.tmp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def _kill_pid_group(pid):
|
||||
try:
|
||||
pgid = os.getpgid(pid)
|
||||
if pgid == os.getpgrp():
|
||||
raise AssertionError("refusing cleanup of test runner group")
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
time.sleep(0.05)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
try:
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
def invoke(self, plan, output="manifest.json", op="run", manifest=None, env=None):
|
||||
if op == "run":
|
||||
args = ["run", "--contract-path", str(self.contract), "--repo-root", str(self.repo),
|
||||
"--artifact-root", str(self.artifacts), "--plan", str(plan.name), "--output", output]
|
||||
elif op == "validate":
|
||||
args = ["validate", "--contract-path", str(self.contract), "--repo-root", str(self.repo),
|
||||
"--artifact-root", str(self.artifacts), "--manifest", str(manifest or output)]
|
||||
elif op == "validate-archived-v2":
|
||||
args = ["validate-archived-v2", "--contract-path", str(self.contract), "--repo-root", str(self.repo),
|
||||
"--artifact-root", str(self.artifacts), "--manifest", str(manifest or output)]
|
||||
else:
|
||||
raise AssertionError(op)
|
||||
proc = subprocess.Popen([sys.executable, str(SCRIPT), *args], cwd=self.repo,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, start_new_session=True)
|
||||
self.outer_processes.add(proc)
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=8)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._kill_pid_group(proc.pid)
|
||||
stdout, stderr = proc.communicate(timeout=1)
|
||||
stderr += "\nOUTER_TIMEOUT\n"
|
||||
return subprocess.CompletedProcess(proc.args, proc.returncode, stdout, stderr)
|
||||
|
||||
def make_plan(self, argv, timeout: float | None = 1.5, max_output_bytes=65536,
|
||||
expected_exit_code=0, extra=None, commands=None):
|
||||
plan = self.artifacts / "plan.json"
|
||||
if commands is None:
|
||||
commands = [{"id": "probe", "argv": argv, "timeout_seconds": timeout,
|
||||
"max_output_bytes": max_output_bytes, "expected_exit_code": expected_exit_code,
|
||||
"claims": ["probe"]}]
|
||||
data = {"hash_paths": ["input.txt"], "commands": commands}
|
||||
if extra:
|
||||
data.update(extra)
|
||||
plan.write_text(json.dumps(data))
|
||||
plan.chmod(0o600)
|
||||
return plan
|
||||
|
||||
def pid_file(self, name="child.pid"):
|
||||
path = Path(self.tmp.name) / name
|
||||
self.known_pid_files.add(path)
|
||||
return path
|
||||
|
||||
def assert_no_live_pid(self, pid_file):
|
||||
self.assertTrue(pid_file.is_file(), "child must actually have executed")
|
||||
pid = int(pid_file.read_text().strip())
|
||||
proc = Path("/proc") / str(pid)
|
||||
if not proc.exists():
|
||||
return
|
||||
state = (proc / "stat").read_text().split()[2]
|
||||
self.assertIn(state, {"Z"}, f"descendant still live: pid={pid} state={state}")
|
||||
|
||||
def test_leader_exits_zero_descendant_retains_stdout_is_killed(self):
|
||||
pid = self.pid_file()
|
||||
code = ("import os,signal,time; p=os.fork(); "
|
||||
f"(open({str(pid)!r},'w').write(str(os.getpid())) if p==0 else os._exit(0)); "
|
||||
"signal.signal(signal.SIGTERM,signal.SIG_IGN); time.sleep(30)")
|
||||
result = self.invoke(self.make_plan([sys.executable, "-c", code]))
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "timeout")
|
||||
self.assertTrue(record["termination"]["kill_sent"])
|
||||
self.assertTrue(record["termination"]["group_cleanup_completed"])
|
||||
self.assert_no_live_pid(pid)
|
||||
|
||||
def test_leader_dies_on_term_descendant_ignores_term_and_requires_kill(self):
|
||||
pid = self.pid_file()
|
||||
code = ("import os,signal,time; p=os.fork(); "
|
||||
f"(signal.signal(signal.SIGTERM,signal.SIG_IGN), open({str(pid)!r},'w').write(str(os.getpid())), time.sleep(30)) if p==0 else (signal.signal(signal.SIGTERM,signal.SIG_DFL), time.sleep(30))")
|
||||
result = self.invoke(self.make_plan([sys.executable, "-c", code]))
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "timeout")
|
||||
self.assertTrue(record["termination"]["kill_sent"])
|
||||
self.assertTrue(record["termination"]["group_cleanup_completed"])
|
||||
self.assert_no_live_pid(pid)
|
||||
|
||||
def test_normal_leader_exit_with_live_descendant_is_not_success(self):
|
||||
pid = self.pid_file()
|
||||
code = ("import os,time\np=os.fork()\nif p==0:\n"
|
||||
f" open({str(pid)!r},'w').write(str(os.getpid()))\n"
|
||||
" os.close(1)\n os.close(2)\n time.sleep(30)\nelse:\n os._exit(0)")
|
||||
result = self.invoke(self.make_plan([sys.executable, "-c", code]))
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assert_no_live_pid(pid)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "descendant_cleanup")
|
||||
self.assertTrue(record["termination"]["group_cleanup_completed"])
|
||||
self.assertEqual(self.invoke(None, op="validate").returncode, 0)
|
||||
|
||||
def test_stdout_eof_while_leader_runs_is_bounded(self):
|
||||
code = "import os,time; os.close(1); os.close(2); time.sleep(30)"
|
||||
started = time.monotonic()
|
||||
result = self.invoke(self.make_plan([sys.executable, "-c", code]))
|
||||
self.assertLess(time.monotonic() - started, 6)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "timeout")
|
||||
|
||||
def test_timeout_expected_signal_exit_cannot_count_as_success(self):
|
||||
code = "import time; time.sleep(30)"
|
||||
plan = self.make_plan([sys.executable, "-c", code], expected_exit_code=-15)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assertEqual(json.loads((self.artifacts / "manifest.json").read_text())["result"], "failure")
|
||||
|
||||
def test_infrastructure_failure_stops_second_command_and_failure_validates(self):
|
||||
marker = self.repo / "second-ran.txt"
|
||||
commands = [
|
||||
{"id": "first", "argv": [sys.executable, "-c", "import time; time.sleep(30)"], "timeout_seconds": 0.2, "claims": ["first"]},
|
||||
{"id": "second", "argv": [sys.executable, "-c", f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')"], "timeout_seconds": 1, "claims": ["second"]},
|
||||
]
|
||||
plan = self.make_plan([], commands=commands)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
data = json.loads((self.artifacts / "manifest.json").read_text())
|
||||
self.assertEqual(len(data["commands"]), 1)
|
||||
self.assertFalse(marker.exists())
|
||||
validated = self.invoke(plan, op="validate", manifest="manifest.json")
|
||||
self.assertEqual(validated.returncode, 0, validated.stderr)
|
||||
self.assertEqual(data["result"], "failure")
|
||||
|
||||
def test_output_limit_retains_safe_bounded_failure(self):
|
||||
plan = self.make_plan([sys.executable, "-c", "print('safe-' * 10000)"], max_output_bytes=17)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "output_limit")
|
||||
self.assertLessEqual(len(record["output"].encode()), 17)
|
||||
self.assertTrue(record["output"].startswith("safe-"))
|
||||
|
||||
def test_invalid_utf8_output_is_bounded(self):
|
||||
plan = self.make_plan([sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'\\xff' * 1000)"], timeout=2, max_output_bytes=7)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "output_limit")
|
||||
self.assertLessEqual(len(record["output"].encode()), 7)
|
||||
|
||||
def test_secret_on_timeout_is_rejected_without_manifest_or_leak(self):
|
||||
secret = "SYNTHETIC_TIMEOUT_SECRET_7f3"
|
||||
env = os.environ.copy(); env["SYNTHETIC_SECRET"] = secret
|
||||
plan = self.make_plan([sys.executable, "-c", "import os,time; print(os.environ['SYNTHETIC_SECRET'],flush=True); time.sleep(30)"], extra={"secret_env_names": ["SYNTHETIC_SECRET"]})
|
||||
result = self.invoke(plan, env=env)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertFalse((self.artifacts / "manifest.json").exists())
|
||||
self.assertNotIn(secret, result.stdout + result.stderr)
|
||||
|
||||
def test_secret_on_overflow_is_rejected_without_manifest_or_leak(self):
|
||||
secret = "SYNTHETIC_OVERFLOW_SECRET_9c2"
|
||||
env = os.environ.copy(); env["SYNTHETIC_SECRET"] = secret
|
||||
plan = self.make_plan([sys.executable, "-c", "import os; print(os.environ['SYNTHETIC_SECRET'] + 'x'*1000)"], max_output_bytes=16, extra={"secret_env_names": ["SYNTHETIC_SECRET"]})
|
||||
result = self.invoke(plan, env=env)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertFalse((self.artifacts / "manifest.json").exists())
|
||||
self.assertNotIn(secret, result.stdout + result.stderr)
|
||||
|
||||
def test_timeout_bounds_are_rejected_before_later_command_execution(self):
|
||||
invalids = [None, 0, -1, False, "0.2", float("nan"), float("inf"), 3601, 10**500]
|
||||
for index, invalid in enumerate(invalids):
|
||||
with self.subTest(timeout=invalid):
|
||||
marker = self.repo / f"later-ran-{index}.txt"
|
||||
first = {"id": "bad", "argv": ["true"], "claims": ["bad"]}
|
||||
if invalid is not None:
|
||||
first["timeout_seconds"] = invalid
|
||||
commands = [
|
||||
first,
|
||||
{"id": "later", "argv": [sys.executable, "-c", f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')"], "timeout_seconds": 1, "claims": ["later"]},
|
||||
]
|
||||
result = self.invoke(self.make_plan([], commands=commands), output=f"invalid-{index}.json")
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def test_archived_v2_shape_validates_read_only_but_fresh_v3_rejects(self):
|
||||
plan = self.make_plan(["true"], timeout=1)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
manifest = self.artifacts / "manifest.json"
|
||||
obj = json.loads(manifest.read_text())
|
||||
archived_plan = json.loads(plan.read_text())
|
||||
archived_plan["commands"][0].pop("timeout_seconds")
|
||||
plan.write_text(json.dumps(archived_plan)); plan.chmod(0o600)
|
||||
obj["schema"] = "evidence-manifest/v2"
|
||||
obj["plan"]["sha256"] = hashlib.sha256(plan.read_bytes()).hexdigest()
|
||||
for key in ("timeout_seconds", "execution_status", "termination"):
|
||||
obj["commands"][0].pop(key, None)
|
||||
obj["claims"]["probe"] = obj["claims"]["probe"][:6]
|
||||
manifest.write_text(json.dumps(obj, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
before = manifest.read_bytes()
|
||||
fresh = self.invoke(plan, op="validate", manifest="manifest.json")
|
||||
self.assertEqual(fresh.returncode, 2)
|
||||
archived = self.invoke(plan, op="validate-archived-v2", manifest="manifest.json")
|
||||
self.assertEqual(archived.returncode, 0, archived.stderr)
|
||||
self.assertEqual(manifest.read_bytes(), before)
|
||||
|
||||
def test_tampered_completed_status_or_termination_fails_validation(self):
|
||||
plan = self.make_plan(["true"], timeout=1)
|
||||
self.assertEqual(self.invoke(plan).returncode, 0)
|
||||
manifest = self.artifacts / "manifest.json"
|
||||
original = json.loads(manifest.read_text())
|
||||
for mutation in (lambda obj: obj["commands"][0].__setitem__("execution_status", "timeout"),
|
||||
lambda obj: obj["commands"][0]["termination"].__setitem__("direct_child_reaped", False),
|
||||
lambda obj: obj["commands"][0]["termination"].__setitem__("group_cleanup_completed", False),
|
||||
lambda obj: obj["commands"][0]["termination"].__setitem__("term_sent", True),
|
||||
lambda obj: obj["commands"][0]["termination"].__setitem__("kill_sent", True)):
|
||||
manifest.write_text(json.dumps(original, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
obj = json.loads(manifest.read_text()); mutation(obj)
|
||||
manifest.write_text(json.dumps(obj, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
self.assertEqual(self.invoke(plan, op="validate", manifest="manifest.json").returncode, 2)
|
||||
|
||||
def test_secret_prefix_after_safe_text_on_overflow_is_rejected(self):
|
||||
secret = "SYNTHETIC_BOUNDARY_SECRET_9c2"
|
||||
env = os.environ.copy(); env["SYNTHETIC_SECRET"] = secret
|
||||
plan = self.make_plan([sys.executable, "-c", "import os; print('safe-banner:' + os.environ['SYNTHETIC_SECRET'] + 'x'*1000)"], max_output_bytes=20, extra={"secret_env_names": ["SYNTHETIC_SECRET"]})
|
||||
result = self.invoke(plan, env=env)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertFalse((self.artifacts / "manifest.json").exists())
|
||||
self.assertNotIn(secret[:8], result.stdout + result.stderr)
|
||||
|
||||
def test_megabyte_invalid_utf8_processing_remains_bounded(self):
|
||||
plan = self.make_plan([sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'\\xff' * (2**21))"], timeout=2, max_output_bytes=2**20)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "output_limit")
|
||||
self.assertLessEqual(len(record["output"].encode()), 2**20)
|
||||
|
||||
def test_silent_command_timeout_is_bounded_and_persists_failure(self):
|
||||
plan = self.make_plan([sys.executable, "-c", "import time; time.sleep(30)"])
|
||||
started = time.monotonic()
|
||||
result = self.invoke(plan)
|
||||
elapsed = time.monotonic() - started
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assertLess(elapsed, 6.0)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["execution_status"], "timeout")
|
||||
self.assertTrue(record["termination"]["direct_child_reaped"])
|
||||
self.assertLessEqual(len(record["output"].encode()), record["max_output_bytes"])
|
||||
|
||||
def test_partial_output_survives_timeout_and_is_utf8_bounded(self):
|
||||
plan = self.make_plan([sys.executable, "-c", "import sys,time; sys.stdout.write('prefix\\n'); sys.stdout.flush(); time.sleep(30)"], timeout=1.5, max_output_bytes=64)
|
||||
result = self.invoke(plan)
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
record = json.loads((self.artifacts / "manifest.json").read_text())["commands"][0]
|
||||
self.assertEqual(record["output"], "prefix\n")
|
||||
self.assertLessEqual(len(record["output"].encode("utf-8")), 64)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,245 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
||||
|
||||
|
||||
def run_git(cwd, *args):
|
||||
return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=True)
|
||||
|
||||
class ExternalEvidenceTests(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR', '/opt/data/cache/scratch'))
|
||||
self.root = Path(self.tmp.name)
|
||||
self.contract = self.root / 'harness/operations/repository-cache.md'
|
||||
self.contract.parent.mkdir(parents=True)
|
||||
self.contract.write_text('# Repository cache 운영 지침\n', encoding='utf-8')
|
||||
self.repo = self.root / 'repositories/owner/project/worktrees/issue-1/checkout'
|
||||
self.artifacts = self.repo.parent / 'verification/run-a'
|
||||
self.repo.mkdir(parents=True)
|
||||
self.artifacts.mkdir(parents=True)
|
||||
for path in (self.root / 'repositories', *self.repo.parents):
|
||||
if path == self.root:
|
||||
break
|
||||
path.chmod(448)
|
||||
self.repo.chmod(448)
|
||||
self.artifacts.parent.chmod(448)
|
||||
self.artifacts.chmod(448)
|
||||
run_git(self.repo, 'init', '-q')
|
||||
run_git(self.repo, 'config', 'user.email', 'test@example.invalid')
|
||||
run_git(self.repo, 'config', 'user.name', 'Test')
|
||||
(self.repo / 'input.txt').write_text('hello\n')
|
||||
run_git(self.repo, 'add', 'input.txt')
|
||||
run_git(self.repo, 'commit', '-qm', 'initial')
|
||||
self.plan = self.artifacts / 'evidence-plan.json'
|
||||
self.plan.write_text(json.dumps({'hash_paths': ['input.txt'], 'commands': [{'id': 'check', 'argv': ['git', 'show', 'HEAD:input.txt'], 'timeout_seconds': 5, 'claims': ['output']}]}))
|
||||
self.plan.chmod(384)
|
||||
self.manifest = self.artifacts / 'evidence-manifest.json'
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def invoke(self, op, *args):
|
||||
return subprocess.run([sys.executable, str(SCRIPT), op, '--contract-path', str(self.contract), '--repo-root', str(self.repo), '--artifact-root', str(self.artifacts), *map(str, args)], text=True, capture_output=True)
|
||||
|
||||
def test_ignored_source_input_is_rejected_for_run_and_validate(self):
|
||||
(self.repo / '.gitignore').write_text('ignored.txt\n')
|
||||
run_git(self.repo, 'add', '.gitignore')
|
||||
run_git(self.repo, 'commit', '-qm', 'ignore fixture')
|
||||
ignored = self.repo / 'ignored.txt'
|
||||
ignored.write_text('unbound input\n')
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn('not clean', result.stderr)
|
||||
self.assertFalse(self.manifest.exists())
|
||||
ignored.unlink()
|
||||
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
||||
prior = self.manifest.read_bytes()
|
||||
ignored.write_text('unbound input again\n')
|
||||
result = self.invoke('validate', '--manifest', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn('not clean', result.stderr)
|
||||
self.assertEqual(self.manifest.read_bytes(), prior)
|
||||
|
||||
def test_canonical_control_directory_modes_match_installed_harness(self):
|
||||
self.root.chmod(504)
|
||||
self.contract.parent.parent.chmod(511)
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|
||||
self.assertEqual(self.root.stat().st_mode & 511, 504)
|
||||
self.assertEqual(self.contract.parent.parent.stat().st_mode & 511, 511)
|
||||
|
||||
def test_same_filename_in_source_and_artifacts_is_not_a_collision(self):
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', 'input.txt')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual((self.repo / 'input.txt').read_text(), 'hello\n')
|
||||
self.assertEqual(self.invoke('validate', '--manifest', 'input.txt').returncode, 0)
|
||||
|
||||
def test_real_linked_detached_worktree_preserves_git_registration(self):
|
||||
seed = self.root / 'seed'
|
||||
self.repo.rename(seed)
|
||||
bare = self.repo.parents[2] / 'repository.git'
|
||||
subprocess.run(['git', 'clone', '--bare', str(seed), str(bare)], check=True, capture_output=True)
|
||||
subprocess.run(['git', '--git-dir', str(bare), 'worktree', 'add', '--detach', str(self.repo), 'HEAD'], check=True, capture_output=True)
|
||||
subprocess.run(['git', '--git-dir', str(bare), 'worktree', 'lock', str(self.repo)], check=True, capture_output=True)
|
||||
before = run_git(self.repo, 'worktree', 'list', '--porcelain').stdout
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|
||||
self.assertEqual(run_git(self.repo, 'worktree', 'list', '--porcelain').stdout, before)
|
||||
self.assertNotEqual(run_git(self.repo, 'rev-parse', '--git-common-dir').stdout.strip(), '.git')
|
||||
self.assertEqual(run_git(self.repo, 'status', '--porcelain').stdout, '')
|
||||
|
||||
def test_canonical_root_identity_and_traversal_denied_before_execution(self):
|
||||
targets = (self.repo / 'docs/verification/run-a', self.root / 'scratch/run-a', self.repo.parent.parent / 'issue-2/verification/run-a', self.artifacts.parent / '../run-a')
|
||||
for target in targets:
|
||||
with self.subTest(target=str(target)):
|
||||
result = self.invoke('run', '--artifact-root', target, '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertFalse(self.manifest.exists())
|
||||
for filename in ('../escape.json', '/absolute.json', 'a/../escape.json', './plan.json', 'plan.json/'):
|
||||
with self.subTest(filename=filename):
|
||||
result = self.invoke('run', '--plan', filename, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
result = self.invoke('run', '--contract-path', self.root / 'wrong.md', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
|
||||
def test_artifact_file_and_directory_symlinks_denied(self):
|
||||
alias = self.artifacts / 'alias.json'
|
||||
alias.symlink_to(self.plan.name)
|
||||
result = self.invoke('run', '--plan', alias.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn('unsafe path', result.stderr)
|
||||
outside = self.root / 'outside'
|
||||
self.artifacts.rename(outside)
|
||||
self.artifacts.symlink_to(outside, target_is_directory=True)
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn('symlink', result.stderr)
|
||||
self.assertFalse((outside / self.manifest.name).exists())
|
||||
|
||||
def test_private_permissions_fail_closed_without_repair(self):
|
||||
self.plan.chmod(420)
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertEqual(self.plan.stat().st_mode & 511, 420)
|
||||
self.plan.chmod(384)
|
||||
self.artifacts.chmod(493)
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertEqual(self.artifacts.stat().st_mode & 511, 493)
|
||||
self.assertFalse(self.manifest.exists())
|
||||
|
||||
def test_wrong_uid_is_denied(self):
|
||||
import importlib.util
|
||||
from unittest.mock import patch
|
||||
spec = importlib.util.spec_from_file_location('evidence_manifest_uid_test', SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
with patch.object(module.os, 'getuid', return_value=self.artifacts.stat().st_uid + 1):
|
||||
with self.assertRaisesRegex(ValueError, 'ownership'):
|
||||
module.safe_entry(self.artifacts, directory=True, mode=448)
|
||||
|
||||
def test_group_writable_cache_parent_requires_exclusive_role_membership(self):
|
||||
import importlib.util
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
spec = importlib.util.spec_from_file_location('evidence_manifest_group_test', SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
parent = self.root / 'repositories'
|
||||
parent.chmod(509)
|
||||
uid = parent.stat().st_uid
|
||||
gid = parent.stat().st_gid
|
||||
role = SimpleNamespace(pw_name='role', pw_uid=uid, pw_gid=gid)
|
||||
foreign = SimpleNamespace(pw_name='foreign', pw_uid=uid + 1, pw_gid=gid)
|
||||
with patch.object(module.grp, 'getgrgid', return_value=SimpleNamespace(gr_mem=[])), patch.object(module.pwd, 'getpwall', return_value=[role]), patch.object(module.pwd, 'getpwnam', return_value=role):
|
||||
module.safe_repositories(parent)
|
||||
with patch.object(module.grp, 'getgrgid', return_value=SimpleNamespace(gr_mem=['foreign'])), patch.object(module.pwd, 'getpwall', return_value=[role, foreign]), patch.object(module.pwd, 'getpwnam', side_effect=lambda name: role if name == 'role' else foreign):
|
||||
with self.assertRaisesRegex(ValueError, 'exclusive'):
|
||||
module.safe_repositories(parent)
|
||||
with patch.object(module.grp, 'getgrgid', side_effect=KeyError('unresolved')):
|
||||
with self.assertRaisesRegex(ValueError, 'unverifiable'):
|
||||
module.safe_repositories(parent)
|
||||
self.assertEqual(parent.stat().st_mode & 511, 509)
|
||||
|
||||
def test_no_clobber_rejects_rerun_before_command_execution(self):
|
||||
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
||||
prior = self.manifest.read_bytes()
|
||||
obj = json.loads(self.plan.read_text())
|
||||
marker = self.root / 'rerun-executed'
|
||||
obj['commands'][0]['argv'] = [sys.executable, '-c', 'from pathlib import Path; Path(' + repr(str(marker)) + ").write_text('ran')"]
|
||||
self.plan.write_text(json.dumps(obj))
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn('already exists', result.stderr)
|
||||
self.assertEqual(self.manifest.read_bytes(), prior)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def test_v2_rejects_forged_issue_binding_and_v1_normal_validation(self):
|
||||
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
||||
original = json.loads(self.manifest.read_text())
|
||||
for replacement in (2, True, '1'):
|
||||
obj = json.loads(json.dumps(original))
|
||||
obj['artifact']['issue'] = replacement
|
||||
self.manifest.write_text(json.dumps(obj, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 2)
|
||||
original['schema'] = 'evidence-manifest/v1'
|
||||
self.manifest.write_text(json.dumps(original, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 2)
|
||||
|
||||
def test_artifact_boundary_mutation_by_command_rejected_before_write(self):
|
||||
obj = json.loads(self.plan.read_text())
|
||||
obj['commands'][0]['argv'] = [sys.executable, '-c', 'from pathlib import Path; Path(' + repr(str(self.artifacts)) + ').chmod(0o755)']
|
||||
self.plan.write_text(json.dumps(obj))
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn('permission', result.stderr)
|
||||
self.assertFalse(self.manifest.exists())
|
||||
|
||||
def test_v2_stale_head_rejected_even_when_bound_file_is_unchanged(self):
|
||||
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
||||
(self.repo / 'other.txt').write_text('new source revision\n')
|
||||
run_git(self.repo, 'add', 'other.txt')
|
||||
run_git(self.repo, 'commit', '-qm', 'advance head')
|
||||
result = self.invoke('validate', '--manifest', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn('source revision is not current HEAD', result.stderr)
|
||||
|
||||
def test_v2_rejects_source_symlink_alias(self):
|
||||
(self.repo / 'alias.txt').symlink_to('input.txt')
|
||||
run_git(self.repo, 'add', 'alias.txt')
|
||||
run_git(self.repo, 'commit', '-qm', 'source symlink fixture')
|
||||
obj = json.loads(self.plan.read_text())
|
||||
obj['hash_paths'] = ['alias.txt']
|
||||
self.plan.write_text(json.dumps(obj))
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn('symlink', result.stderr)
|
||||
self.assertFalse(self.manifest.exists())
|
||||
|
||||
def test_external_artifacts_leave_checkout_and_head_unchanged(self):
|
||||
head = run_git(self.repo, 'rev-parse', 'HEAD').stdout.strip()
|
||||
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
data = json.loads(self.manifest.read_text())
|
||||
self.assertEqual(data['schema'], 'evidence-manifest/v3')
|
||||
self.assertEqual(data['source_revision'], head)
|
||||
self.assertEqual(data['artifact']['run_id'], 'run-a')
|
||||
self.assertEqual(data['artifact']['issue'], 1)
|
||||
self.assertEqual(run_git(self.repo, 'status', '--porcelain').stdout, '')
|
||||
self.assertEqual(run_git(self.repo, 'rev-parse', 'HEAD').stdout.strip(), head)
|
||||
self.assertEqual(self.manifest.stat().st_mode & 511, 384)
|
||||
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|
||||
@@ -0,0 +1,206 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
from harness.tests._support.evidence import EvidenceManifestFixture
|
||||
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
||||
|
||||
|
||||
def run_git(cwd, *args):
|
||||
return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=True)
|
||||
|
||||
class EvidenceManifestTests(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.fixture = EvidenceManifestFixture().setup()
|
||||
self.addCleanup(self.fixture.cleanup)
|
||||
self.tmp = self.fixture.tmp
|
||||
self.root = self.fixture.root
|
||||
self.contract = self.fixture.contract
|
||||
self.repo = self.fixture.repo
|
||||
self.artifacts = self.fixture.artifacts
|
||||
|
||||
def invoke(self, *args, **kwargs):
|
||||
return self.fixture.invoke(*args, **kwargs)
|
||||
|
||||
def plan(self, *args, **kwargs):
|
||||
return self.fixture.plan(*args, **kwargs)
|
||||
|
||||
def run_manifest(self, *args, **kwargs):
|
||||
return self.fixture.run_manifest(*args, **kwargs)
|
||||
|
||||
def test_run_captures_success_claim_and_hash(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
manifest = json.loads(out.read_text())
|
||||
self.assertEqual(manifest['commands'][0]['argv'], ['git', 'show', 'HEAD:input.txt'])
|
||||
self.assertEqual(manifest['commands'][0]['exit_code'], 0)
|
||||
self.assertEqual(manifest['commands'][0]['claim_ids'], ['output'])
|
||||
self.assertEqual(manifest['claims']['output'], ['/commands/0/argv', '/commands/0/exit_code', '/commands/0/expected_exit_code', '/commands/0/output', '/commands/0/started_at_utc', '/commands/0/finished_at_utc', '/commands/0/timeout_seconds', '/commands/0/execution_status', '/commands/0/termination'])
|
||||
self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(b'hello\n').hexdigest())
|
||||
self.assertEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', 'manifest.json').returncode, 0)
|
||||
|
||||
def test_unexpected_exit_is_recorded_and_nonzero(self):
|
||||
result, out = self.run_manifest(self.plan([{'argv': [sys.executable, '-c', "print('no'); raise SystemExit(3)"], 'expected_exit_code': 0, 'claims': ['result']}]))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(json.loads(out.read_text())['commands'][0]['exit_code'], 3)
|
||||
|
||||
def test_failed_run_preserves_prior_manifest(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
prior = out.read_bytes()
|
||||
bad = self.plan([{'argv': [sys.executable, '-c', "print('Authorization: Bearer ***')"]}])
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', bad, '--output', out)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(out.read_bytes(), prior)
|
||||
|
||||
def test_command_environment_is_allowlisted_and_output_is_bounded(self):
|
||||
env = os.environ.copy()
|
||||
env['UNLISTED_SECRET'] = 'must-not-reach-child'
|
||||
command = [{'argv': [sys.executable, '-c', "import os; print(os.getenv('UNLISTED_SECRET'))"], 'claims': ['environment']}]
|
||||
plan = self.plan(command)
|
||||
out = self.artifacts / 'manifest.json'
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', plan.name, '--output', out.name, env=env)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(out.read_text())['commands'][0]['output'], 'None\n')
|
||||
env['LEAK'] = 'SECRET_VALUE'
|
||||
plan = self.plan(command, command_env_allowlist=['LEAK'])
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', plan.name, '--output', out.name, env=env)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn('SECRET_VALUE', out.read_text())
|
||||
prior = out.read_bytes()
|
||||
plan = self.plan([{'argv': [sys.executable, '-c', "print('x' * 100000)"], 'max_output_bytes': 16, 'claims': ['bounded']}])
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', plan.name, '--output', 'overflow.json')
|
||||
self.assertEqual(result.returncode, 1)
|
||||
overflow = json.loads((self.artifacts / 'overflow.json').read_text())
|
||||
self.assertEqual(overflow['result'], 'failure')
|
||||
self.assertEqual(overflow['commands'][0]['execution_status'], 'output_limit')
|
||||
self.assertLessEqual(len(overflow['commands'][0]['output'].encode()), 16)
|
||||
self.assertEqual(out.read_bytes(), prior)
|
||||
|
||||
def test_output_collision_and_password_argument_are_rejected(self):
|
||||
plan = self.plan()
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', plan.name, '--output', '../checkout/input.txt')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual((self.repo / 'input.txt').read_text(), 'hello\n')
|
||||
result, out = self.run_manifest(self.plan([{'argv': [sys.executable, '-c', "from pathlib import Path; Path('ran.txt').write_text('ran')", '--password', 'visible-value'], 'claims': ['password-leak']}]))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
self.assertFalse((self.repo / 'ran.txt').exists())
|
||||
|
||||
def test_argv_only_and_traversal_rejected(self):
|
||||
result, out = self.run_manifest(self.plan([{'argv': ['echo', 'shell=true']}]))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
result, out = self.run_manifest(self.plan(hash_paths=['../outside']))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
result, out = self.run_manifest(self.plan(hash_paths=['nested/../input.txt']))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
|
||||
def test_empty_plan_and_malformed_command_record_are_rejected(self):
|
||||
result, out = self.run_manifest(self.plan([]))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
data = json.loads(out.read_text())
|
||||
data['commands'][0]['duration_seconds'] = -1
|
||||
out.write_text(json.dumps(data, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
|
||||
def test_secret_or_header_leak_fails_closed(self):
|
||||
env = os.environ.copy()
|
||||
env['TEST_SECRET'] = 'never-write-this'
|
||||
p = self.plan([{'argv': [sys.executable, '-c', "import os; print(os.environ['TEST_SECRET'])"]}], secret_env_names=['TEST_SECRET'])
|
||||
result = self.invoke('run', '--repo-root', self.repo, '--plan', 'plan.json', '--output', 'manifest.json', env=env)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse((self.artifacts / 'manifest.json').exists())
|
||||
|
||||
def test_source_drift_hash_mismatch_rejected(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0)
|
||||
(self.repo / 'input.txt').write_text('changed\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
|
||||
def test_command_source_mutation_is_rejected_before_manifest_write(self):
|
||||
command = [{'argv': [sys.executable, '-c', "from pathlib import Path; Path('input.txt').write_text('changed\\n')"], 'claims': ['mutation']}]
|
||||
result, out = self.run_manifest(self.plan(command))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
|
||||
def test_command_commit_drift_is_rejected_before_manifest_write(self):
|
||||
script = "from pathlib import Path; import subprocess; Path('other.txt').write_text('committed\\n'); subprocess.run(['git','add','other.txt'],check=True); subprocess.run(['git','commit','-qm','drift'],check=True)"
|
||||
result, out = self.run_manifest(self.plan([{'argv': [sys.executable, '-c', script], 'claims': ['commit-drift']}]))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(out.exists())
|
||||
|
||||
def test_binary_source_hash_is_byte_accurate(self):
|
||||
payload = b'\xff\x00\x80binary\n'
|
||||
(self.repo / 'binary.bin').write_bytes(payload)
|
||||
run_git(self.repo, 'add', 'binary.bin')
|
||||
run_git(self.repo, 'commit', '-qm', 'binary')
|
||||
result, out = self.run_manifest(self.plan(hash_paths=['binary.bin']))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(out.read_text())['files'][0]['sha256'], hashlib.sha256(payload).hexdigest())
|
||||
|
||||
def test_invalid_claim_pointer_rejected(self):
|
||||
result, out = self.run_manifest(self.plan([{'argv': ['true'], 'claims': ['bad']}]))
|
||||
self.assertEqual(result.returncode, 0)
|
||||
data = json.loads(out.read_text())
|
||||
data['claims']['bad'] = '/missing'
|
||||
out.write_text(json.dumps(data, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
|
||||
def test_v2_validate_rejects_forged_plan_exclusion(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
(self.artifacts / 'plan.json').unlink()
|
||||
(self.repo / 'dirty.txt').write_text('dirty\n')
|
||||
data = json.loads(out.read_text())
|
||||
data['excluded_paths'] = ['dirty.txt', 'manifest.json']
|
||||
out.write_text(json.dumps(data, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
|
||||
def test_validate_rejects_empty_or_shell_argv(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
data = json.loads(out.read_text())
|
||||
data['commands'][0]['argv'] = []
|
||||
out.write_text(json.dumps(data, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
data['commands'][0]['argv'] = ['bash', '-c', 'true']
|
||||
out.write_text(json.dumps(data, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
self.assertNotEqual(self.invoke('validate', '--repo-root', self.repo, '--manifest', out.name).returncode, 0)
|
||||
|
||||
def test_archived_v1_descendant_validates_read_only(self):
|
||||
result, out = self.run_manifest()
|
||||
self.assertEqual(result.returncode, 0)
|
||||
obj = json.loads(out.read_text())
|
||||
obj['schema'] = 'evidence-manifest/v1'
|
||||
del obj['artifact']
|
||||
obj['excluded_paths'] = ['plan.json', 'manifest.json']
|
||||
archived_plan = json.loads((self.artifacts / 'plan.json').read_text())
|
||||
archived_plan['commands'][0].pop('timeout_seconds', None)
|
||||
(self.repo / 'plan.json').write_text(json.dumps(archived_plan))
|
||||
obj['plan']['sha256'] = hashlib.sha256((self.repo / 'plan.json').read_bytes()).hexdigest()
|
||||
for key in ('timeout_seconds', 'execution_status', 'termination'):
|
||||
obj['commands'][0].pop(key, None)
|
||||
obj['claims'] = {key: pointers[:6] for key, pointers in obj['claims'].items()}
|
||||
archived = self.repo / 'manifest.json'
|
||||
archived.write_text(json.dumps(obj, sort_keys=True, separators=(',', ':'), ensure_ascii=False) + '\n')
|
||||
run_git(self.repo, 'add', 'plan.json', 'manifest.json')
|
||||
run_git(self.repo, 'commit', '-qm', 'historical evidence fixture')
|
||||
before = archived.read_bytes()
|
||||
archived_result = self.invoke('validate-legacy', '--repo-root', self.repo, '--manifest', 'manifest.json')
|
||||
self.assertEqual(archived_result.returncode, 0, archived_result.stderr)
|
||||
self.assertEqual(archived.read_bytes(), before)
|
||||
self.assertEqual(run_git(self.repo, 'status', '--porcelain').stdout, '')
|
||||
Reference in New Issue
Block a user