|
|
|
@@ -1,7 +1,7 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Create and validate bounded, fail-closed evidence manifests."""
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
import argparse, datetime as dt, grp, hashlib, json, math, os, platform, pwd, re, shutil, stat, subprocess, sys, tempfile, time
|
|
|
|
|
import argparse, datetime as dt, grp, hashlib, json, math, os, platform, pwd, re, selectors, shutil, signal, stat, subprocess, sys, tempfile, time
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
HEX40 = re.compile(r"^[0-9a-f]{40}$")
|
|
|
|
@@ -18,6 +18,7 @@ SECRET_PATTERNS = [
|
|
|
|
|
]
|
|
|
|
|
SAFE_CHILD_ENV = ("PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "VIRTUAL_ENV", "UV_CACHE_DIR", "PYTHONDONTWRITEBYTECODE")
|
|
|
|
|
SHELL_NAMES = {"sh", "bash", "dash", "zsh", "ksh", "fish", "cmd", "cmd.exe", "powershell", "powershell.exe", "pwsh", "pwsh.exe"}
|
|
|
|
|
MAX_OUTPUT_BYTES = 1 << 20
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def die(msg):
|
|
|
|
@@ -140,6 +141,35 @@ def source_file(root, value):
|
|
|
|
|
def canonical(obj): return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
|
|
|
|
|
def text_bytes(b): return b.decode("utf-8", "replace")
|
|
|
|
|
|
|
|
|
|
def bounded_text(b, limit):
|
|
|
|
|
# Linear, including all-invalid UTF-8. Repeated slicing/reencoding would
|
|
|
|
|
# turn a 1 MiB invalid-byte stream into unbounded quadratic postprocessing.
|
|
|
|
|
return text_bytes(b).encode("utf-8")[:limit].decode("utf-8", "ignore")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sensitive_boundary(text, secrets):
|
|
|
|
|
"""Reject a captured suffix that is a known secret prefix (linear KMP)."""
|
|
|
|
|
for secret in secrets:
|
|
|
|
|
if not secret or not text:
|
|
|
|
|
continue
|
|
|
|
|
prefix = [0] * len(secret)
|
|
|
|
|
for i in range(1, len(secret)):
|
|
|
|
|
j = prefix[i-1]
|
|
|
|
|
while j and secret[i] != secret[j]:
|
|
|
|
|
j = prefix[j-1]
|
|
|
|
|
if secret[i] == secret[j]:
|
|
|
|
|
j += 1
|
|
|
|
|
prefix[i] = j
|
|
|
|
|
matched = 0
|
|
|
|
|
for char in text[-len(secret):]:
|
|
|
|
|
while matched and (matched == len(secret) or secret[matched] != char):
|
|
|
|
|
matched = prefix[matched-1]
|
|
|
|
|
if matched < len(secret) and secret[matched] == char:
|
|
|
|
|
matched += 1
|
|
|
|
|
if matched:
|
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def scan_sensitive(text, secrets, forbidden):
|
|
|
|
|
for s in secrets:
|
|
|
|
@@ -161,24 +191,143 @@ def validate_argv(argv):
|
|
|
|
|
if executable in SHELL_NAMES or any(x.lower() in ("shell", "shell=true") for x in argv): raise ValueError("shell commands are forbidden")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_bounded(argv, root, env, max_bytes):
|
|
|
|
|
proc=subprocess.Popen(argv,cwd=root,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=False,env=env,bufsize=0)
|
|
|
|
|
stream=proc.stdout
|
|
|
|
|
if stream is None: raise ValueError("command output pipe unavailable")
|
|
|
|
|
data=bytearray()
|
|
|
|
|
def valid_timeout(value):
|
|
|
|
|
# Range comparisons reject NaN/Infinity without converting huge integers.
|
|
|
|
|
return isinstance(value, (int, float)) and not isinstance(value, bool) and 0 < value <= 3600
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _group_has_live_members(pgid):
|
|
|
|
|
"""Linux-only liveness: orphan zombies are not running descendants.
|
|
|
|
|
|
|
|
|
|
This is cleanup, not a sandbox: a process that changes session/group escapes.
|
|
|
|
|
Failure to inspect an extant group is a cleanup failure, never success.
|
|
|
|
|
"""
|
|
|
|
|
try:
|
|
|
|
|
while True:
|
|
|
|
|
chunk=stream.read(min(65536,max_bytes-len(data)+1))
|
|
|
|
|
if not chunk: break
|
|
|
|
|
data.extend(chunk)
|
|
|
|
|
if len(data)>max_bytes:
|
|
|
|
|
proc.kill(); proc.wait()
|
|
|
|
|
raise ValueError("command output exceeds max_output_bytes")
|
|
|
|
|
return proc.wait(),bytes(data)
|
|
|
|
|
os.killpg(pgid, 0)
|
|
|
|
|
except ProcessLookupError:
|
|
|
|
|
return False
|
|
|
|
|
for entry in Path("/proc").iterdir():
|
|
|
|
|
if not entry.name.isdecimal():
|
|
|
|
|
continue
|
|
|
|
|
try:
|
|
|
|
|
fields = (entry / "stat").read_text(encoding="utf-8", errors="replace").rsplit(")", 1)[1].split()
|
|
|
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
|
|
|
continue
|
|
|
|
|
if int(fields[2]) == pgid and fields[0] not in ("Z", "X"):
|
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _cleanup_process_group(proc):
|
|
|
|
|
record = {"term_sent": False, "kill_sent": False,
|
|
|
|
|
"direct_child_reaped": False, "group_cleanup_completed": False}
|
|
|
|
|
try:
|
|
|
|
|
if _group_has_live_members(proc.pid):
|
|
|
|
|
try:
|
|
|
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
|
|
|
record["term_sent"] = True
|
|
|
|
|
except ProcessLookupError:
|
|
|
|
|
pass
|
|
|
|
|
grace = time.monotonic() + 0.25
|
|
|
|
|
while _group_has_live_members(proc.pid) and time.monotonic() < grace:
|
|
|
|
|
time.sleep(min(0.01, max(0, grace - time.monotonic())))
|
|
|
|
|
if _group_has_live_members(proc.pid):
|
|
|
|
|
try:
|
|
|
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
|
|
|
record["kill_sent"] = True
|
|
|
|
|
except ProcessLookupError:
|
|
|
|
|
pass
|
|
|
|
|
reap_deadline = time.monotonic() + 0.5
|
|
|
|
|
while _group_has_live_members(proc.pid) and time.monotonic() < reap_deadline:
|
|
|
|
|
time.sleep(min(0.01, max(0, reap_deadline - time.monotonic())))
|
|
|
|
|
record["group_cleanup_completed"] = not _group_has_live_members(proc.pid)
|
|
|
|
|
proc.wait(timeout=max(0, reap_deadline - time.monotonic()))
|
|
|
|
|
record["direct_child_reaped"] = True
|
|
|
|
|
if not record["group_cleanup_completed"]:
|
|
|
|
|
return record, ValueError("process group cleanup incomplete")
|
|
|
|
|
return record, None
|
|
|
|
|
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError) as exc:
|
|
|
|
|
# Even inspection failure must attempt group termination, including when
|
|
|
|
|
# the leader already exited. Never use leader.poll() to skip signals.
|
|
|
|
|
try:
|
|
|
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
|
|
|
record["kill_sent"] = True
|
|
|
|
|
except OSError:
|
|
|
|
|
pass
|
|
|
|
|
try:
|
|
|
|
|
proc.wait(timeout=0.1)
|
|
|
|
|
record["direct_child_reaped"] = True
|
|
|
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
|
|
|
pass
|
|
|
|
|
return record, exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_bounded(argv, root, env, max_bytes, timeout_seconds):
|
|
|
|
|
validate_argv(argv)
|
|
|
|
|
if not valid_timeout(timeout_seconds):
|
|
|
|
|
raise ValueError("invalid timeout_seconds")
|
|
|
|
|
if type(max_bytes) is not int or not 0 <= max_bytes <= MAX_OUTPUT_BYTES:
|
|
|
|
|
raise ValueError("invalid command bounds")
|
|
|
|
|
deadline = time.monotonic() + timeout_seconds
|
|
|
|
|
proc = subprocess.Popen(argv, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
|
|
|
stdin=subprocess.DEVNULL, shell=False, env=env, bufsize=0,
|
|
|
|
|
start_new_session=True)
|
|
|
|
|
stream = proc.stdout
|
|
|
|
|
sel = None
|
|
|
|
|
data = bytearray()
|
|
|
|
|
status = "completed"
|
|
|
|
|
try:
|
|
|
|
|
if stream is None:
|
|
|
|
|
raise ValueError("command output pipe unavailable")
|
|
|
|
|
os.set_blocking(stream.fileno(), False)
|
|
|
|
|
sel = selectors.DefaultSelector()
|
|
|
|
|
sel.register(stream, selectors.EVENT_READ)
|
|
|
|
|
pipe_open = True
|
|
|
|
|
while pipe_open:
|
|
|
|
|
remaining = deadline - time.monotonic()
|
|
|
|
|
if remaining <= 0:
|
|
|
|
|
status = "timeout"
|
|
|
|
|
break
|
|
|
|
|
ready = sel.select(remaining)
|
|
|
|
|
if not ready:
|
|
|
|
|
status = "timeout"
|
|
|
|
|
break
|
|
|
|
|
try:
|
|
|
|
|
chunk = os.read(stream.fileno(), min(65536, max_bytes - len(data) + 1))
|
|
|
|
|
except BlockingIOError:
|
|
|
|
|
continue
|
|
|
|
|
if not chunk:
|
|
|
|
|
sel.unregister(stream)
|
|
|
|
|
pipe_open = False
|
|
|
|
|
break
|
|
|
|
|
take = min(len(chunk), max_bytes - len(data))
|
|
|
|
|
data.extend(chunk[:take])
|
|
|
|
|
if len(chunk) > take:
|
|
|
|
|
status = "output_limit"
|
|
|
|
|
break
|
|
|
|
|
if status == "completed":
|
|
|
|
|
try:
|
|
|
|
|
proc.wait(timeout=max(0, deadline - time.monotonic()))
|
|
|
|
|
except subprocess.TimeoutExpired:
|
|
|
|
|
status = "timeout"
|
|
|
|
|
# Cleanup all same-group descendants even after normal leader exit.
|
|
|
|
|
termination, cleanup_error = _cleanup_process_group(proc)
|
|
|
|
|
if cleanup_error is not None:
|
|
|
|
|
status = "cleanup_failed"
|
|
|
|
|
elif status == "completed" and (termination["term_sent"] or termination["kill_sent"]):
|
|
|
|
|
# A zero-exit leader with unfinished descendants is not a completed
|
|
|
|
|
# verification command merely because cleanup successfully killed them.
|
|
|
|
|
status = "descendant_cleanup"
|
|
|
|
|
return {"status": status, "output": bytes(data), "exit_code": proc.returncode,
|
|
|
|
|
"termination": termination}
|
|
|
|
|
except BaseException:
|
|
|
|
|
_cleanup_process_group(proc)
|
|
|
|
|
raise
|
|
|
|
|
finally:
|
|
|
|
|
stream.close()
|
|
|
|
|
if proc.poll() is None:
|
|
|
|
|
proc.kill(); proc.wait()
|
|
|
|
|
if sel is not None:
|
|
|
|
|
sel.close()
|
|
|
|
|
if stream is not None:
|
|
|
|
|
stream.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def status_paths(root, *, include_ignored=True):
|
|
|
|
@@ -199,7 +348,7 @@ def check_clean(root, allowed, *, include_ignored=True):
|
|
|
|
|
if bad: raise ValueError("source tree is not clean: " + ", ".join(bad))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def plan_checks(plan, root):
|
|
|
|
|
def plan_checks(plan, root, *, require_timeout=True):
|
|
|
|
|
if not isinstance(plan, dict): raise ValueError("plan must be an object")
|
|
|
|
|
for key in ("hash_paths", "commands", "limitations", "secret_env_names", "forbidden_regexes", "command_env_allowlist"):
|
|
|
|
|
if key in plan and not isinstance(plan[key], list): raise ValueError(f"{key} must be a list")
|
|
|
|
@@ -221,14 +370,16 @@ def plan_checks(plan, root):
|
|
|
|
|
if not isinstance(spec, dict): raise ValueError("command must be object")
|
|
|
|
|
argv=spec.get("argv"); validate_argv(argv)
|
|
|
|
|
expected=spec.get("expected_exit_code",0); maxb=spec.get("max_output_bytes",65536)
|
|
|
|
|
if not isinstance(expected,int) or isinstance(expected,bool) or not isinstance(maxb,int) or maxb < 0: raise ValueError("invalid command bounds")
|
|
|
|
|
timeout=spec.get("timeout_seconds")
|
|
|
|
|
if require_timeout and not valid_timeout(timeout): raise ValueError("invalid timeout_seconds")
|
|
|
|
|
if not isinstance(expected,int) or isinstance(expected,bool) or not isinstance(maxb,int) or isinstance(maxb,bool) or maxb < 0 or (require_timeout and maxb > MAX_OUTPUT_BYTES): raise ValueError("invalid command bounds")
|
|
|
|
|
claims=spec.get("claims",[])
|
|
|
|
|
if not isinstance(claims,list) or not claims or any(not isinstance(x,str) or not ID.fullmatch(x) for x in claims): raise ValueError("claims must be a nonempty list of valid IDs")
|
|
|
|
|
if any(x in seen for x in claims): raise ValueError("duplicate claim ID")
|
|
|
|
|
command_id=spec.get("id",f"command-{index}"); purpose=spec.get("purpose","")
|
|
|
|
|
if not isinstance(command_id,str) or not ID.fullmatch(command_id) or command_id in command_ids: raise ValueError("invalid or duplicate command ID")
|
|
|
|
|
if not isinstance(purpose,str): raise ValueError("command purpose must be a string")
|
|
|
|
|
command_ids.add(command_id); seen.update(claims); commands.append((spec,argv,expected,maxb,claims))
|
|
|
|
|
command_ids.add(command_id); seen.update(claims); commands.append((spec,argv,expected,maxb,claims,timeout))
|
|
|
|
|
return secrets, forbidden, requested, child_names, commands
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -261,7 +412,7 @@ def run(args):
|
|
|
|
|
if not out.parent.is_dir(): raise ValueError("output parent must already exist")
|
|
|
|
|
plan_bytes=plan_path.read_bytes(); plan_sha256=hashlib.sha256(plan_bytes).hexdigest()
|
|
|
|
|
plan=json.loads(plan_bytes.decode("utf-8")); secrets,forbidden,env_keys,child_names,commandspec=plan_checks(plan,root)
|
|
|
|
|
for _,argv,_,_,_ in commandspec:
|
|
|
|
|
for _,argv,_,_,_,_ in commandspec:
|
|
|
|
|
violation=scan_sensitive("\n".join(argv),secrets,forbidden)
|
|
|
|
|
if violation: raise ValueError("sensitive command rejected before execution: "+violation)
|
|
|
|
|
check_clean(root,set())
|
|
|
|
@@ -276,12 +427,16 @@ def run(args):
|
|
|
|
|
files.append({"path":rel,"sha256":sha,"blob_oid":oid})
|
|
|
|
|
if plan_rel==out_rel: raise ValueError("plan/output paths must not overlap")
|
|
|
|
|
child_env={name:os.environ[name] for name in child_names if name in os.environ}
|
|
|
|
|
result_commands=[]; claim_map={}
|
|
|
|
|
for i,(spec,argv,expected,maxb,claim_ids) in enumerate(commandspec):
|
|
|
|
|
started=utc_now(); t=time.monotonic(); returncode,raw=run_bounded(argv,root,child_env,maxb); finished=utc_now()
|
|
|
|
|
output=text_bytes(raw); violation=scan_sensitive("\n".join(argv)+"\n"+output,secrets,forbidden)
|
|
|
|
|
result_commands=[]; claim_map={}; infrastructure_failure=False
|
|
|
|
|
for i,(spec,argv,expected,maxb,claim_ids,timeout) in enumerate(commandspec):
|
|
|
|
|
started=utc_now(); t=time.monotonic(); outcome=run_bounded(argv,root,child_env,maxb,timeout); finished=utc_now()
|
|
|
|
|
raw_output=outcome["output"]; full_output=text_bytes(raw_output)
|
|
|
|
|
violation=scan_sensitive("\n".join(argv)+"\n"+full_output,secrets,forbidden)
|
|
|
|
|
if not violation and outcome["status"] != "completed" and sensitive_boundary(full_output, secrets):
|
|
|
|
|
violation = "secret boundary"
|
|
|
|
|
if violation: raise ValueError("sensitive evidence rejected: "+violation)
|
|
|
|
|
c={"id":spec.get("id",f"command-{i}"),"purpose":spec.get("purpose",""),"argv":argv,"exit_code":returncode,"expected_exit_code":expected,"max_output_bytes":maxb,"output":output,"started_at_utc":started,"finished_at_utc":finished,"duration_seconds":time.monotonic()-t,"claim_ids":claim_ids}
|
|
|
|
|
output=bounded_text(raw_output,maxb)
|
|
|
|
|
c={"id":spec.get("id",f"command-{i}"),"purpose":spec.get("purpose",""),"argv":argv,"exit_code":outcome["exit_code"],"expected_exit_code":expected,"max_output_bytes":maxb,"timeout_seconds":timeout,"execution_status":outcome["status"],"termination":outcome["termination"],"output":output,"started_at_utc":started,"finished_at_utc":finished,"duration_seconds":time.monotonic()-t,"claim_ids":claim_ids}
|
|
|
|
|
result_commands.append(c)
|
|
|
|
|
for cid in claim_ids:
|
|
|
|
|
claim_map[cid]=[
|
|
|
|
@@ -291,7 +446,13 @@ def run(args):
|
|
|
|
|
f"/commands/{i}/output",
|
|
|
|
|
f"/commands/{i}/started_at_utc",
|
|
|
|
|
f"/commands/{i}/finished_at_utc",
|
|
|
|
|
f"/commands/{i}/timeout_seconds",
|
|
|
|
|
f"/commands/{i}/execution_status",
|
|
|
|
|
f"/commands/{i}/termination",
|
|
|
|
|
]
|
|
|
|
|
if outcome["status"] != "completed" or not outcome["termination"]["direct_child_reaped"] or not outcome["termination"]["group_cleanup_completed"]:
|
|
|
|
|
infrastructure_failure=True
|
|
|
|
|
break
|
|
|
|
|
env: dict[str, str | None]={k:None for k in env_keys}
|
|
|
|
|
if "os" in env: env["os"]=platform.system()
|
|
|
|
|
if "kernel" in env: env["kernel"]=platform.release()
|
|
|
|
@@ -312,8 +473,8 @@ def run(args):
|
|
|
|
|
path,_=source_file(root,record["path"])
|
|
|
|
|
if hashlib.sha256(path.read_bytes()).hexdigest()!=record["sha256"]:
|
|
|
|
|
raise ValueError(f"source changed during verification: {record['path']}")
|
|
|
|
|
success=all(c["exit_code"]==c["expected_exit_code"] for c in result_commands)
|
|
|
|
|
manifest={"schema":"evidence-manifest/v2","source_revision":source,"source_tree_clean":True,"artifact":identity,"plan":{"path":plan_rel,"sha256":plan_sha256},"generated_at":utc_now(),"environment":env,"commands":result_commands,"files":files,"claims":claim_map,"redaction":{"checked":True,"violations":[]},"limitations":plan.get("limitations",[]),"result":"success" if success else "failure"}
|
|
|
|
|
success=not infrastructure_failure and len(result_commands)==len(commandspec) and all(c["execution_status"]=="completed" and c["exit_code"]==c["expected_exit_code"] for c in result_commands)
|
|
|
|
|
manifest={"schema":"evidence-manifest/v3","source_revision":source,"source_tree_clean":True,"artifact":identity,"plan":{"path":plan_rel,"sha256":plan_sha256},"generated_at":utc_now(),"environment":env,"commands":result_commands,"files":files,"claims":claim_map,"redaction":{"checked":True,"violations":[]},"limitations":plan.get("limitations",[]),"result":"success" if success else "failure"}
|
|
|
|
|
atomic_write(out,canonical(manifest))
|
|
|
|
|
return 0 if success else 1
|
|
|
|
|
|
|
|
|
@@ -331,6 +492,7 @@ def pointer(obj,ptr):
|
|
|
|
|
|
|
|
|
|
def validate(args):
|
|
|
|
|
legacy = args.op == "validate-legacy"
|
|
|
|
|
archived = args.op == "validate-archived-v2"
|
|
|
|
|
identity = None
|
|
|
|
|
if legacy:
|
|
|
|
|
root=Path(args.repo_root).resolve(); artifact=root; mp,mp_rel=inside(root,args.manifest)
|
|
|
|
@@ -339,7 +501,8 @@ def validate(args):
|
|
|
|
|
raw=mp.read_text(encoding="utf-8"); obj=json.loads(raw)
|
|
|
|
|
if canonical(obj)!=raw: raise ValueError("manifest is not canonical JSON")
|
|
|
|
|
required={"schema","source_revision","source_tree_clean","plan","generated_at","commands","files","claims","redaction","environment","limitations","result"} | ({"excluded_paths"} if legacy else {"artifact"})
|
|
|
|
|
if set(obj)!=required or obj["schema"]!=("evidence-manifest/v1" if legacy else "evidence-manifest/v2") or obj["result"] not in ("success","failure"): raise ValueError("invalid manifest structure")
|
|
|
|
|
expected_schema = "evidence-manifest/v1" if legacy else ("evidence-manifest/v2" if archived else "evidence-manifest/v3")
|
|
|
|
|
if set(obj)!=required or obj["schema"]!=expected_schema or obj["result"] not in ("success","failure"): raise ValueError("invalid manifest structure")
|
|
|
|
|
if obj["source_tree_clean"] is not True: raise ValueError("source tree was not clean")
|
|
|
|
|
if not isinstance(obj["generated_at"],str) or not ISO.fullmatch(obj["generated_at"]): raise ValueError("invalid generated timestamp")
|
|
|
|
|
if legacy:
|
|
|
|
@@ -357,7 +520,8 @@ def validate(args):
|
|
|
|
|
if plan_path.suffix.lower()!=".json" or not plan_path.is_file() or plan_path.is_symlink(): raise ValueError("bound plan is unavailable")
|
|
|
|
|
plan_bytes=plan_path.read_bytes()
|
|
|
|
|
if hashlib.sha256(plan_bytes).hexdigest()!=obj["plan"]["sha256"]: raise ValueError("plan hash mismatch")
|
|
|
|
|
plan=json.loads(plan_bytes.decode("utf-8")); plan_secrets,plan_forbidden,requested_env,_,planned_commands=plan_checks(plan,root)
|
|
|
|
|
archive_shape = legacy or archived
|
|
|
|
|
plan=json.loads(plan_bytes.decode("utf-8")); plan_secrets,plan_forbidden,requested_env,_,planned_commands=plan_checks(plan,root,require_timeout=not archive_shape)
|
|
|
|
|
check_clean(root,set(obj["excluded_paths"]) if legacy else set(),include_ignored=not legacy)
|
|
|
|
|
src=obj["source_revision"]
|
|
|
|
|
if not isinstance(src,str) or not HEX40.fullmatch(src): raise ValueError("invalid source revision")
|
|
|
|
@@ -371,7 +535,7 @@ def validate(args):
|
|
|
|
|
if obj["redaction"]!={"checked":True,"violations":[]}: raise ValueError("redaction failed")
|
|
|
|
|
if not isinstance(obj["claims"],dict) or any(
|
|
|
|
|
not isinstance(k,str) or not ID.fullmatch(k) or not isinstance(v,list)
|
|
|
|
|
or len(v)!=6 or any(not isinstance(ptr,str) for ptr in v)
|
|
|
|
|
or len(v)!=(6 if archive_shape else 9) or any(not isinstance(ptr,str) for ptr in v)
|
|
|
|
|
for k,v in obj["claims"].items()
|
|
|
|
|
): raise ValueError("invalid claims")
|
|
|
|
|
if not isinstance(obj["files"],list) or not obj["files"]: raise ValueError("files must be nonempty")
|
|
|
|
@@ -386,20 +550,48 @@ def validate(args):
|
|
|
|
|
oid,sha=tracked_blob(root,src,rel)
|
|
|
|
|
if oid!=f["blob_oid"] or sha!=f["sha256"] or hashlib.sha256(path.read_bytes()).hexdigest()!=sha: raise ValueError("file hash mismatch")
|
|
|
|
|
if legacy and any(f["path"] in set(obj["excluded_paths"]) for f in obj["files"]): raise ValueError("evidence paths overlap bound source paths")
|
|
|
|
|
if not isinstance(obj["commands"],list) or not obj["commands"] or len(obj["commands"])!=len(planned_commands): raise ValueError("commands do not match plan")
|
|
|
|
|
if not isinstance(obj["commands"],list) or not obj["commands"] or len(obj["commands"]) > len(planned_commands): raise ValueError("commands do not match plan")
|
|
|
|
|
if archive_shape and len(obj["commands"]) != len(planned_commands): raise ValueError("commands do not match plan")
|
|
|
|
|
if not archive_shape and len(obj["commands"]) < len(planned_commands):
|
|
|
|
|
final = obj["commands"][-1]
|
|
|
|
|
if obj["result"] != "failure" or not isinstance(final, dict) or final.get("execution_status") not in ("timeout", "output_limit", "cleanup_failed", "descendant_cleanup"):
|
|
|
|
|
raise ValueError("invalid interrupted command prefix")
|
|
|
|
|
command_ids=set()
|
|
|
|
|
for i,c in enumerate(obj["commands"]):
|
|
|
|
|
spec,planned_argv,planned_exit,planned_max,planned_claims=planned_commands[i]
|
|
|
|
|
command_keys={"id","purpose","argv","exit_code","expected_exit_code","max_output_bytes","output","started_at_utc","finished_at_utc","duration_seconds","claim_ids"}
|
|
|
|
|
if archive_shape:
|
|
|
|
|
spec,planned_argv,planned_exit,planned_max,planned_claims,_=planned_commands[i]
|
|
|
|
|
planned_timeout=None
|
|
|
|
|
command_keys={"id","purpose","argv","exit_code","expected_exit_code","max_output_bytes","output","started_at_utc","finished_at_utc","duration_seconds","claim_ids"}
|
|
|
|
|
else:
|
|
|
|
|
spec,planned_argv,planned_exit,planned_max,planned_claims,planned_timeout=planned_commands[i]
|
|
|
|
|
command_keys={"id","purpose","argv","exit_code","expected_exit_code","max_output_bytes","timeout_seconds","execution_status","termination","output","started_at_utc","finished_at_utc","duration_seconds","claim_ids"}
|
|
|
|
|
if not isinstance(c,dict) or set(c)!=command_keys: raise ValueError("invalid command structure")
|
|
|
|
|
if not isinstance(c["id"],str) or not ID.fullmatch(c["id"]) or c["id"] in command_ids or not isinstance(c["purpose"],str): raise ValueError("invalid command identity")
|
|
|
|
|
command_ids.add(c["id"])
|
|
|
|
|
validate_argv(c["argv"])
|
|
|
|
|
if not isinstance(c["output"],str): raise ValueError("invalid command types")
|
|
|
|
|
if any(not isinstance(c[k],int) or isinstance(c[k],bool) for k in ("exit_code","expected_exit_code","max_output_bytes")) or c["max_output_bytes"]<0 or len(c["output"].encode())>c["max_output_bytes"]: raise ValueError("invalid command bounds")
|
|
|
|
|
if (not isinstance(c["exit_code"],int) or isinstance(c["exit_code"],bool)) and not (not archive_shape and c["exit_code"] is None): raise ValueError("invalid command bounds")
|
|
|
|
|
if not isinstance(c["expected_exit_code"],int) or isinstance(c["expected_exit_code"],bool) or not isinstance(c["max_output_bytes"],int) or isinstance(c["max_output_bytes"],bool) or c["max_output_bytes"]<0 or (not archive_shape and c["max_output_bytes"]>MAX_OUTPUT_BYTES) or len(c["output"].encode())>c["max_output_bytes"]: raise ValueError("invalid command bounds")
|
|
|
|
|
if not archive_shape:
|
|
|
|
|
if not valid_timeout(c["timeout_seconds"]): raise ValueError("invalid timeout_seconds")
|
|
|
|
|
if c["execution_status"] not in ("completed","timeout","output_limit","cleanup_failed","descendant_cleanup"): raise ValueError("invalid execution status")
|
|
|
|
|
if not archive_shape and (not isinstance(c["termination"],dict) or set(c["termination"])!={"term_sent","kill_sent","direct_child_reaped","group_cleanup_completed"} or any(type(c["termination"][k]) is not bool for k in c["termination"])): raise ValueError("invalid termination record")
|
|
|
|
|
if not archive_shape:
|
|
|
|
|
signalled = c["termination"]["term_sent"] or c["termination"]["kill_sent"]
|
|
|
|
|
if c["execution_status"] == "completed" and signalled:
|
|
|
|
|
raise ValueError("completed command has termination signal")
|
|
|
|
|
if c["execution_status"] == "descendant_cleanup" and not signalled:
|
|
|
|
|
raise ValueError("descendant cleanup missing termination signal")
|
|
|
|
|
settled = c["termination"]["direct_child_reaped"] and c["termination"]["group_cleanup_completed"]
|
|
|
|
|
if c["execution_status"] == "cleanup_failed":
|
|
|
|
|
if settled: raise ValueError("cleanup failure inconsistent")
|
|
|
|
|
elif not settled or c["exit_code"] is None:
|
|
|
|
|
raise ValueError("execution cleanup incomplete")
|
|
|
|
|
if c["execution_status"] != "completed" and i != len(obj["commands"])-1:
|
|
|
|
|
raise ValueError("command executed after infrastructure failure")
|
|
|
|
|
if not isinstance(c["duration_seconds"],(int,float)) or isinstance(c["duration_seconds"],bool) or not math.isfinite(c["duration_seconds"]) or c["duration_seconds"]<0: raise ValueError("invalid command duration")
|
|
|
|
|
if not isinstance(c["claim_ids"],list) or any(not isinstance(x,str) or not ID.fullmatch(x) for x in c["claim_ids"]): raise ValueError("invalid command fields")
|
|
|
|
|
if c["id"]!=spec.get("id",f"command-{i}") or c["purpose"]!=spec.get("purpose","") or c["argv"]!=planned_argv or c["expected_exit_code"]!=planned_exit or c["max_output_bytes"]!=planned_max or c["claim_ids"]!=planned_claims: raise ValueError("command record does not match plan")
|
|
|
|
|
if c["id"]!=spec.get("id",f"command-{i}") or c["purpose"]!=spec.get("purpose","") or c["argv"]!=planned_argv or c["expected_exit_code"]!=planned_exit or c["max_output_bytes"]!=planned_max or (not archive_shape and c["timeout_seconds"]!=planned_timeout) or c["claim_ids"]!=planned_claims: raise ValueError("command record does not match plan")
|
|
|
|
|
if not ISO.fullmatch(c["started_at_utc"]) or not ISO.fullmatch(c["finished_at_utc"]): raise ValueError("invalid timestamp")
|
|
|
|
|
for cid in c["claim_ids"]:
|
|
|
|
|
if cid in claim_seen or cid not in obj["claims"]: raise ValueError("duplicate or missing claim")
|
|
|
|
@@ -413,12 +605,14 @@ def validate(args):
|
|
|
|
|
f"/commands/{i}/started_at_utc",
|
|
|
|
|
f"/commands/{i}/finished_at_utc",
|
|
|
|
|
]
|
|
|
|
|
if not archive_shape:
|
|
|
|
|
expected_pointers += [f"/commands/{i}/timeout_seconds", f"/commands/{i}/execution_status", f"/commands/{i}/termination"]
|
|
|
|
|
if pointers!=expected_pointers: raise ValueError("claim pointers are not canonical")
|
|
|
|
|
for ptr in pointers: pointer(obj,ptr)
|
|
|
|
|
violation=scan_sensitive("\n".join(c["argv"])+"\n"+c["output"],plan_secrets,plan_forbidden)
|
|
|
|
|
if violation: raise ValueError("redaction violation")
|
|
|
|
|
if claim_seen != set(obj["claims"]): raise ValueError("unreferenced claim")
|
|
|
|
|
expected="success" if all(c["exit_code"]==c["expected_exit_code"] for c in obj["commands"]) else "failure"
|
|
|
|
|
expected="success" if (all(c["exit_code"]==c["expected_exit_code"] for c in obj["commands"]) if archive_shape else all(c["execution_status"]=="completed" and c["termination"]["direct_child_reaped"] and c["termination"]["group_cleanup_completed"] and c["exit_code"]==c["expected_exit_code"] for c in obj["commands"])) and len(obj["commands"])==len(planned_commands) else "failure"
|
|
|
|
|
if obj["result"]!=expected: raise ValueError("result inconsistent")
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
@@ -433,6 +627,9 @@ def main():
|
|
|
|
|
parser.add_argument("--artifact-root",required=True)
|
|
|
|
|
old=sub.add_parser("validate-legacy",help="read-only archived v1 compatibility; not a new publication gate")
|
|
|
|
|
old.add_argument("--repo-root",required=True); old.add_argument("--manifest",required=True)
|
|
|
|
|
archived=sub.add_parser("validate-archived-v2",help="read-only archived v2 compatibility; not a new publication gate")
|
|
|
|
|
for parser in (archived,):
|
|
|
|
|
parser.add_argument("--contract-path",required=True); parser.add_argument("--repo-root",required=True); parser.add_argument("--artifact-root",required=True); parser.add_argument("--manifest",required=True)
|
|
|
|
|
a=p.parse_args()
|
|
|
|
|
try: return run(a) if a.op=="run" else validate(a)
|
|
|
|
|
except (OSError, ValueError, TypeError, KeyError, AttributeError, UnicodeError, json.JSONDecodeError, re.error, subprocess.SubprocessError) as e: return die(e)
|
|
|
|
|
