#!/usr/bin/env python3 """Create and validate bounded, fail-closed evidence manifests.""" from __future__ import annotations import argparse, datetime as dt, grp, hashlib, json, math, os, platform, pwd, re, shutil, stat, subprocess, sys, tempfile, time from pathlib import Path HEX40 = re.compile(r"^[0-9a-f]{40}$") HEX64 = re.compile(r"^[0-9a-f]{64}$") ID = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,127}$") ISO = re.compile(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z$") SAFE_ENV = ("os", "kernel", "architecture", "python", "uv") SECRET_PATTERNS = [ re.compile(r"(?:cookie|set-cookie|authorization|proxy-authorization|x-api-key|api-key|x-auth-token)\s*:", re.I), re.compile(r"\bbearer\s+[^\s]+", re.I), re.compile(r"-----BEGIN [^-]*PRIVATE KEY-----", re.I), re.compile(r"\b(?:password|passwd|secret|token)\s*[:=]", re.I), re.compile(r"(?:^|\s)--?(?:password|passwd|secret|token)(?:\s+|=)\S+", re.I), ] SAFE_CHILD_ENV = ("PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "VIRTUAL_ENV", "UV_CACHE_DIR", "PYTHONDONTWRITEBYTECODE") SHELL_NAMES = {"sh", "bash", "dash", "zsh", "ksh", "fish", "cmd", "cmd.exe", "powershell", "powershell.exe", "pwsh", "pwsh.exe"} def die(msg): print(str(msg), file=sys.stderr); return 2 def git(root: Path, *args, check=True): p = subprocess.run(["git", f"-c", f"safe.directory={root}", *args], cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if check and p.returncode: raise ValueError(p.stderr.decode("utf-8", "replace").strip() or "git failed") return p def inside(root, value, *, allow_absolute=False): if not isinstance(value, str) or not value or (Path(value).is_absolute() and not allow_absolute): raise ValueError("paths must be nonempty repository-relative paths") p = Path(value) if ".." in p.parts: raise ValueError("path traversal is forbidden") if p.parts and p.parts[0] == ".git": raise ValueError(".git paths are forbidden") candidate = (root / p).resolve() try: candidate.relative_to(root.resolve()) except ValueError: raise ValueError(f"path outside repository: {value}") if (root / p).is_symlink() and not candidate.is_relative_to(root.resolve()): raise ValueError("symlink escapes repository") return candidate, candidate.relative_to(root.resolve()).as_posix() def safe_entry(path, *, directory=False, mode=None, owned=True, writable_mask=0o022): info = path.lstat() if stat.S_ISLNK(info.st_mode) or not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)): raise ValueError("unsafe path entry") if owned and info.st_uid != os.getuid(): raise ValueError("path ownership mismatch") actual = stat.S_IMODE(info.st_mode) if mode is not None and actual != mode: raise ValueError("path permission mismatch") if directory and actual & writable_mask: raise ValueError("writable directory boundary") def safe_repositories(path): safe_entry(path, directory=True, writable_mask=0o002) info=path.lstat() if info.st_mode & 0o020: try: group=grp.getgrgid(info.st_gid) members=set(group.gr_mem) | {entry.pw_name for entry in pwd.getpwall() if entry.pw_gid==info.st_gid} member_uids={pwd.getpwnam(name).pw_uid for name in members} except KeyError: raise ValueError("unverifiable repositories group") if member_uids!={os.getuid()}: raise ValueError("repositories group is not exclusive to executing role") def lexical_absolute(value): path = Path(value) if not path.is_absolute() or any(x in ("", ".", "..") for x in str(value).split("/")[1:]): raise ValueError("canonical absolute path required") for ancestor in reversed((path, *path.parents)): if ancestor.is_symlink(): raise ValueError("symlink path boundary") return path def artifact_context(args): contract = lexical_absolute(args.contract_path) if contract.name != "repository-cache.md" or contract.parent.name != "operations" or contract.parent.parent.name != "harness": raise ValueError("invalid canonical contract path") harness_root = contract.parents[2] # Read-only harness code parents are not private runtime directories. # Their type/symlink gates and the trusted NFS root follow the contract. safe_entry(harness_root, directory=True, owned=False, writable_mask=0o002) for directory in (contract.parent.parent, contract.parent): safe_entry(directory, directory=True, owned=False, writable_mask=0) safe_entry(contract, owned=False) repo = lexical_absolute(args.repo_root) artifact = lexical_absolute(args.artifact_root) relative = repo.relative_to(harness_root).parts if len(relative) != 6 or relative[0] != "repositories" or relative[3] != "worktrees" or relative[5] != "checkout": raise ValueError("repository is not canonical Issue checkout") owner, repository, issue_name = relative[1], relative[2], relative[4] identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,254}") if not identifier.fullmatch(owner) or not identifier.fullmatch(repository) or not re.fullmatch(r"issue-[1-9][0-9]*", issue_name): raise ValueError("invalid Issue identity") if artifact.parent != repo.parent / "verification" or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", artifact.name): raise ValueError("artifact root is not canonical verification run") safe_repositories(harness_root / "repositories") for directory in (repo.parents[3], repo.parents[2], repo.parents[1], repo.parent, artifact.parent, artifact): safe_entry(directory, directory=True, mode=0o700) safe_entry(repo, directory=True) identity = {"owner": owner, "repository": repository, "issue": int(issue_name[6:]), "run_id": artifact.name} return repo, artifact, identity def artifact_file(root, value, *, must_exist=True): if not isinstance(value, str) or not value or any(ord(c) < 32 or ord(c) == 127 for c in value) or "\\" in value: raise ValueError("invalid artifact path") if Path(value).is_absolute() or any(part in ("", ".", "..") for part in value.split("/")): raise ValueError("artifact path must be run-relative") path = root / value for parent in reversed(path.parents): if parent == root or parent.is_relative_to(root): safe_entry(parent, directory=True, mode=0o700) if must_exist: safe_entry(path, mode=0o600) elif path.exists() or path.is_symlink(): raise ValueError("completed artifact already exists") return path, value def utc_now(): return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") def source_file(root, value): if not isinstance(value, str) or not value or Path(value).is_absolute() or "\\" in value or any(ord(c) < 32 or ord(c) == 127 for c in value): raise ValueError("invalid source path") if any(part in ("", ".", "..", ".git") for part in value.split("/")): raise ValueError("source path traversal or metadata path") lexical_absolute(str(root) + "/" + value) path, rel = inside(root, value) safe_entry(path) return path, rel def canonical(obj): return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" def text_bytes(b): return b.decode("utf-8", "replace") def scan_sensitive(text, secrets, forbidden): for s in secrets: if s and s in text: return "secret value" for pattern in SECRET_PATTERNS + forbidden: if pattern.search(text): return f"forbidden pattern: {pattern.pattern}" return None def validate_regexes(values): if not isinstance(values, list) or any(not isinstance(x, str) for x in values): raise ValueError("forbidden_regexes must be strings") try: return [re.compile(x) for x in values] except re.error as e: raise ValueError(f"invalid forbidden regex: {e}") def validate_argv(argv): if not isinstance(argv,list) or not argv or any(not isinstance(x,str) for x in argv): raise ValueError("commands require nonempty argv string arrays") executable=Path(argv[0]).name.lower() if executable in SHELL_NAMES or any(x.lower() in ("shell", "shell=true") for x in argv): raise ValueError("shell commands are forbidden") def run_bounded(argv, root, env, max_bytes): proc=subprocess.Popen(argv,cwd=root,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=False,env=env,bufsize=0) stream=proc.stdout if stream is None: raise ValueError("command output pipe unavailable") data=bytearray() try: while True: chunk=stream.read(min(65536,max_bytes-len(data)+1)) if not chunk: break data.extend(chunk) if len(data)>max_bytes: proc.kill(); proc.wait() raise ValueError("command output exceeds max_output_bytes") return proc.wait(),bytes(data) finally: stream.close() if proc.poll() is None: proc.kill(); proc.wait() def status_paths(root, *, include_ignored=True): flags=("--ignored=matching", "--untracked-files=all") if include_ignored else () raw = git(root, "status", "--porcelain=v1", "-z", *flags).stdout out=[] for entry in raw.split(b"\0"): if not entry: continue path = entry[3:] if len(entry) >= 3 else b"" if entry[:2] in (b"R ", b" R", b"R?"): path = path.split(b"\0",1)[-1] out.append(path.decode("utf-8", "surrogateescape")) return out def check_clean(root, allowed, *, include_ignored=True): bad = [p for p in status_paths(root, include_ignored=include_ignored) if p not in allowed] if bad: raise ValueError("source tree is not clean: " + ", ".join(bad)) def plan_checks(plan, root): if not isinstance(plan, dict): raise ValueError("plan must be an object") for key in ("hash_paths", "commands", "limitations", "secret_env_names", "forbidden_regexes", "command_env_allowlist"): if key in plan and not isinstance(plan[key], list): raise ValueError(f"{key} must be a list") names = plan.get("secret_env_names", []) if any(not isinstance(n, str) for n in names): raise ValueError("secret_env_names must be strings") secrets = [os.environ.get(n, "") for n in names if os.environ.get(n, "")] forbidden = validate_regexes(plan.get("forbidden_regexes", [])) requested = plan.get("environment", list(SAFE_ENV[:4])) if not isinstance(requested, list) or len(set(requested)) != len(requested) or any(x not in SAFE_ENV for x in requested): raise ValueError("invalid environment allowlist") child_names=plan.get("command_env_allowlist",list(SAFE_CHILD_ENV)) if any(not isinstance(x,str) or not x or x not in SAFE_CHILD_ENV for x in child_names): raise ValueError("command_env_allowlist may contain only safe operational names") child_names=list(dict.fromkeys([*child_names,*names])) hash_paths=plan.get("hash_paths",[]) if not hash_paths or any(not isinstance(x,str) or not x for x in hash_paths): raise ValueError("hash_paths must be a nonempty string list") command_plans=plan.get("commands",[]) if not command_plans: raise ValueError("commands must be nonempty") seen=set(); command_ids=set(); commands=[] for index,spec in enumerate(command_plans): if not isinstance(spec, dict): raise ValueError("command must be object") argv=spec.get("argv"); validate_argv(argv) expected=spec.get("expected_exit_code",0); maxb=spec.get("max_output_bytes",65536) if not isinstance(expected,int) or isinstance(expected,bool) or not isinstance(maxb,int) or maxb < 0: raise ValueError("invalid command bounds") claims=spec.get("claims",[]) if not isinstance(claims,list) or not claims or any(not isinstance(x,str) or not ID.fullmatch(x) for x in claims): raise ValueError("claims must be a nonempty list of valid IDs") if any(x in seen for x in claims): raise ValueError("duplicate claim ID") command_id=spec.get("id",f"command-{index}"); purpose=spec.get("purpose","") if not isinstance(command_id,str) or not ID.fullmatch(command_id) or command_id in command_ids: raise ValueError("invalid or duplicate command ID") if not isinstance(purpose,str): raise ValueError("command purpose must be a string") command_ids.add(command_id); seen.update(claims); commands.append((spec,argv,expected,maxb,claims)) return secrets, forbidden, requested, child_names, commands def tracked_blob(root, source, rel): info=git(root,"ls-tree",source,"--",rel).stdout.decode("utf-8","replace").strip().split() if len(info)<3 or info[1] not in ("blob",) or info[0] not in ("100644","100755"): raise ValueError(f"hash path is not tracked regular file: {rel}") oid=info[2]; blob=git(root,"cat-file","blob",oid).stdout return oid, hashlib.sha256(blob).hexdigest() def atomic_write(path, data): if not path.parent.is_dir(): raise ValueError("output parent must already exist") fd,tmp=tempfile.mkstemp(prefix="."+path.name+".", dir=path.parent) try: with os.fdopen(fd,"w",encoding="utf-8",newline="") as f: f.write(data); f.flush(); os.fsync(f.fileno()) os.link(tmp,path) os.unlink(tmp) dir_fd = os.open(path.parent, os.O_RDONLY) try: os.fsync(dir_fd) finally: os.close(dir_fd) finally: try: os.unlink(tmp) except FileNotFoundError: pass def run(args): root,artifact,identity=artifact_context(args); plan_path,plan_rel=artifact_file(artifact,args.plan); out,out_rel=artifact_file(artifact,args.output,must_exist=False) if plan_path.suffix.lower()!=".json": raise ValueError("plan must be a JSON file") if not out.parent.is_dir(): raise ValueError("output parent must already exist") plan_bytes=plan_path.read_bytes(); plan_sha256=hashlib.sha256(plan_bytes).hexdigest() plan=json.loads(plan_bytes.decode("utf-8")); secrets,forbidden,env_keys,child_names,commandspec=plan_checks(plan,root) for _,argv,_,_,_ in commandspec: violation=scan_sensitive("\n".join(argv),secrets,forbidden) if violation: raise ValueError("sensitive command rejected before execution: "+violation) check_clean(root,set()) source=git(root,"rev-parse","HEAD").stdout.decode().strip() if not HEX40.fullmatch(source): raise ValueError("source revision is not full SHA") files=[] for item in plan.get("hash_paths",[]): path,rel=source_file(root,item) if not path.is_file() or path.is_symlink(): raise ValueError(f"hash path is not a regular file: {item}") oid,sha=tracked_blob(root,source,rel) if hashlib.sha256(path.read_bytes()).hexdigest()!=sha: raise ValueError(f"source hash mismatch: {rel}") files.append({"path":rel,"sha256":sha,"blob_oid":oid}) if plan_rel==out_rel: raise ValueError("plan/output paths must not overlap") child_env={name:os.environ[name] for name in child_names if name in os.environ} result_commands=[]; claim_map={} for i,(spec,argv,expected,maxb,claim_ids) in enumerate(commandspec): started=utc_now(); t=time.monotonic(); returncode,raw=run_bounded(argv,root,child_env,maxb); finished=utc_now() output=text_bytes(raw); violation=scan_sensitive("\n".join(argv)+"\n"+output,secrets,forbidden) if violation: raise ValueError("sensitive evidence rejected: "+violation) c={"id":spec.get("id",f"command-{i}"),"purpose":spec.get("purpose",""),"argv":argv,"exit_code":returncode,"expected_exit_code":expected,"max_output_bytes":maxb,"output":output,"started_at_utc":started,"finished_at_utc":finished,"duration_seconds":time.monotonic()-t,"claim_ids":claim_ids} result_commands.append(c) for cid in claim_ids: claim_map[cid]=[ f"/commands/{i}/argv", f"/commands/{i}/exit_code", f"/commands/{i}/expected_exit_code", f"/commands/{i}/output", f"/commands/{i}/started_at_utc", f"/commands/{i}/finished_at_utc", ] env: dict[str, str | None]={k:None for k in env_keys} if "os" in env: env["os"]=platform.system() if "kernel" in env: env["kernel"]=platform.release() if "architecture" in env: env["architecture"]=platform.machine() if "python" in env: env["python"]=platform.python_version() if "uv" in env: p=subprocess.run(["uv","--version"],stdout=subprocess.PIPE,stderr=subprocess.PIPE) if shutil.which("uv") else None env["uv"]=text_bytes((p.stdout+p.stderr).strip()) if p and p.returncode==0 else None # Commands are evidence producers, not trusted source mutators. Recheck the # tree and every bound blob before persisting their claims. if artifact_context(args)!=(root,artifact,identity): raise ValueError("artifact boundary changed during verification") artifact_file(artifact,args.plan) artifact_file(artifact,args.output,must_exist=False) check_clean(root,set()) if git(root,"rev-parse","HEAD").stdout.decode().strip()!=source: raise ValueError("HEAD changed during verification") if not plan_path.is_file() or hashlib.sha256(plan_path.read_bytes()).hexdigest()!=plan_sha256: raise ValueError("plan changed during verification") for record in files: path,_=source_file(root,record["path"]) if hashlib.sha256(path.read_bytes()).hexdigest()!=record["sha256"]: raise ValueError(f"source changed during verification: {record['path']}") success=all(c["exit_code"]==c["expected_exit_code"] for c in result_commands) manifest={"schema":"evidence-manifest/v2","source_revision":source,"source_tree_clean":True,"artifact":identity,"plan":{"path":plan_rel,"sha256":plan_sha256},"generated_at":utc_now(),"environment":env,"commands":result_commands,"files":files,"claims":claim_map,"redaction":{"checked":True,"violations":[]},"limitations":plan.get("limitations",[]),"result":"success" if success else "failure"} atomic_write(out,canonical(manifest)) return 0 if success else 1 def pointer(obj,ptr): if not isinstance(ptr,str) or not ptr.startswith("/"): raise ValueError("invalid claim pointer") cur=obj for part in ptr[1:].split("/"): part=part.replace("~1","/").replace("~0","~") if isinstance(cur,list) and part.isdigit() and int(part)c["max_output_bytes"]: raise ValueError("invalid command bounds") if not isinstance(c["duration_seconds"],(int,float)) or isinstance(c["duration_seconds"],bool) or not math.isfinite(c["duration_seconds"]) or c["duration_seconds"]<0: raise ValueError("invalid command duration") if not isinstance(c["claim_ids"],list) or any(not isinstance(x,str) or not ID.fullmatch(x) for x in c["claim_ids"]): raise ValueError("invalid command fields") if c["id"]!=spec.get("id",f"command-{i}") or c["purpose"]!=spec.get("purpose","") or c["argv"]!=planned_argv or c["expected_exit_code"]!=planned_exit or c["max_output_bytes"]!=planned_max or c["claim_ids"]!=planned_claims: raise ValueError("command record does not match plan") if not ISO.fullmatch(c["started_at_utc"]) or not ISO.fullmatch(c["finished_at_utc"]): raise ValueError("invalid timestamp") for cid in c["claim_ids"]: if cid in claim_seen or cid not in obj["claims"]: raise ValueError("duplicate or missing claim") claim_seen.add(cid) pointers=obj["claims"][cid] expected_pointers=[ f"/commands/{i}/argv", f"/commands/{i}/exit_code", f"/commands/{i}/expected_exit_code", f"/commands/{i}/output", f"/commands/{i}/started_at_utc", f"/commands/{i}/finished_at_utc", ] if pointers!=expected_pointers: raise ValueError("claim pointers are not canonical") for ptr in pointers: pointer(obj,ptr) violation=scan_sensitive("\n".join(c["argv"])+"\n"+c["output"],plan_secrets,plan_forbidden) if violation: raise ValueError("redaction violation") if claim_seen != set(obj["claims"]): raise ValueError("unreferenced claim") expected="success" if all(c["exit_code"]==c["expected_exit_code"] for c in obj["commands"]) else "failure" if obj["result"]!=expected: raise ValueError("result inconsistent") return 0 def main(): p=argparse.ArgumentParser(); sub=p.add_subparsers(dest="op",required=True) r=sub.add_parser("run"); r.add_argument("--plan",required=True); r.add_argument("--output",required=True) v=sub.add_parser("validate"); v.add_argument("--manifest",required=True) for parser in (r,v): parser.add_argument("--contract-path",required=True) parser.add_argument("--repo-root",required=True) parser.add_argument("--artifact-root",required=True) old=sub.add_parser("validate-legacy",help="read-only archived v1 compatibility; not a new publication gate") old.add_argument("--repo-root",required=True); old.add_argument("--manifest",required=True) a=p.parse_args() try: return run(a) if a.op=="run" else validate(a) except (OSError, ValueError, TypeError, KeyError, AttributeError, UnicodeError, json.JSONDecodeError, re.error, subprocess.SubprocessError) as e: return die(e) if __name__=="__main__": sys.exit(main())