fix(ops): make release provenance gate fail closed

docs/operations.md

@@ -45,30 +45,33 @@ pkill -TERM -f trex-core-daemon
 배포, 복구, 수동 stop/start/recreate 전에 아래 gate를 실행해 **검증 가능한 release record**를 남긴다.
 
 ```bash
-# PRE (대상 인스턴스 1개만 지정)
+# PRE (대상 인스턴스 1개만 지정; POST도 같은 operation-id로 별도 실행)
 python scripts/release_provenance_gate.py \
   --actor "<operator>" \
   --reason "<ticket-or-incident>" \
   --instance "<instance_key>" \
+  --operation-id "<deployment-or-incident-id>" \
+  --phase "pre" \
   --command-class "manual-stop" \
-  --expected-oci-revision "<running-image-oci-revision>" \
-  --image-digest "<running-image-digest>" \
+  --expected-oci-revision "<40-character-running-image-oci-revision>" \
+  --image-digest "sha256:<64-hex-running-image-digest>" \
   --runtime-file "trex_core/runtime/daemon.py" \
   --runtime-file "trex_core/runtime/cli.py" \
-  --capture-command "pending_slots=.venv/bin/trex-core slots --status pending --json" \
-  --capture-command "reconciliation_slots=.venv/bin/trex-core slots --status all --json" \
+  --capture-command "pending_slots=.venv/bin/trex-core slots --status pending --instance <instance_key> --json" \
+  --capture-command "reconciliation_slots=.venv/bin/trex-core slots --status all --instance <instance_key> --json" \
   --capture-command "queue_consumers=rabbitmqctl list_consumers"
 ```
 
-- gate는 `git status --porcelain` clean 여부를 검사한다. dirty checkout이면 종료 코드 `2`로 중단하고 변경 파일을 `artifacts/validation/release_provenance_isolation/`으로 복사해 격리한다.
-- `checkout.head_revision`과 `--expected-oci-revision` 일치 여부를 검사해 audit checkout HEAD가 실행 이미지 OCI revision과 같은지 검증한다.
-- `--image-digest`와 `--runtime-file` 검증 결과(sha256 + revision 일치 여부)가 JSON record에 저장된다.
-- actor/timestamp/instance/command-class/reason이 JSON record에 함께 저장되어 stop/start/recreate 감사 추적이 가능하다.
-- PRE/POST 모두 실행해 재기동 전후의 `pending_slots`, `reconciliation_slots`, `queue_consumers`, broker-side-effect 로그(`order_submit/fill/reject/cancel`)를 자동 캡처한다.
-- broker-side-effect 로그 입력은 기본 `/tmp/trex-core.log`를 사용하며, 로그 파일 위치가 다르면 `--broker-log-path <path>`로 명시한다.
-- 이 스크립트는 read-only capture 전용이다. 주문 제출·취소, DB 복구, 자동 sync/verify live 실행을 암묵적으로 수행하지 않는다.
+- gate는 audit checkout이 detached HEAD인지 확인하고 NUL 구분 `git status --porcelain=v1`로 clean 여부를 검사한다. branch checkout 또는 dirty checkout이면 운영 상태 조회 명령도 실행하지 않고 종료 코드 `2`로 중단하며, dirty 변경 파일은 audit checkout 밖의 `<checkout>-release-provenance-isolation/`에 작업별 디렉터리로 복사한다. 우회 옵션은 없다.
+- `checkout.head_revision`과 필수 `--expected-oci-revision` 일치 여부를 검사해 audit checkout HEAD가 실행 이미지 OCI revision과 같은지 검증한다.
+- 필수 `--image-digest` 형식과 하나 이상의 `--runtime-file` 검증 결과(sha256 + revision 일치 여부)가 JSON record에 저장된다.
+- actor/timestamp/instance/operation-id/phase/command-class/reason이 JSON record에 함께 저장되어 PRE/POST 및 stop/start/recreate 감사 추적이 가능하다. actor/reason에 credential 형태의 key/value가 포함되면 기록 전에 거부한다.
+- PRE/POST 모두 실행해 재기동 전후의 `pending_slots`, `reconciliation_slots`, `queue_consumers`, broker-side-effect 로그(`order_submit/fill/reject/cancel`)를 자동 캡처한다. 필수 조회 또는 로그 캡처가 누락·실패하면 gate는 실패한다.
+- capture command는 위의 인스턴스 지정 `trex-core slots` 두 형식과 `rabbitmqctl list_consumers`만 허용한다. 주문 제출·취소, DB 복구, 자동 sync/verify live 실행은 거부한다.
+- broker-side-effect 로그 입력은 기본 `/tmp/trex-core.log`를 사용하며, 로그 파일 위치가 다르면 `--broker-log-path <path>`로 명시한다. record에는 원문 대신 파일 sha256·스캔 행 수와 event 종류·행 번호·행 sha256만 남긴다. 파일을 정상 스캔했지만 일치 이벤트가 없으면 `captured_empty`, 파일 누락·읽기 실패는 각각 `unavailable`·`error`로 기록하며 후자는 gate를 차단한다.
+- record와 격리 디렉터리는 audit checkout 밖에 둔다. 기본 record 위치는 `<checkout>-release-provenance/`이며, 충돌 없는 파일명을 exclusive-create하고 read-only mode로 전환해 기존 record를 덮어쓰지 않는다.
 
-재기동 후에는 동일 명령(예: `--command-class "manual-start"`)으로 POST record를 추가 저장한다.
+재기동 후에는 같은 `--operation-id`, `--phase "post"`, 해당 command class(예: `manual-start`)로 POST record를 추가 저장한다.
 
 ---
 

scripts/release_provenance_gate.py

@@ -1,25 +1,48 @@
-"""Collect release/restart provenance and enforce audit-checkout clean gate."""
+#!/usr/bin/env python3
+"""Collect release/restart provenance and enforce audit-checkout gates."""
 
 from __future__ import annotations
 
 import argparse
 import json
+import re
 import shlex
+import sys
+import uuid
 from datetime import UTC, datetime
 from pathlib import Path
 
+if __package__ in {None, ""}:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from trex_core.adapters.persistence.ops_checks import (
     collect_release_provenance,
     isolate_dirty_checkout_files,
 )
 
-_DEFAULT_OUTPUT_DIR = Path("artifacts/validation/release_provenance")
-_DEFAULT_ISOLATE_DIR = Path("artifacts/validation/release_provenance_isolation")
+_COMMAND_CLASSES = ("deploy", "recovery", "manual-stop", "manual-start", "recreate")
+_REQUIRED_CAPTURE_KEYS = frozenset(
+    {"pending_slots", "reconciliation_slots", "queue_consumers"}
+)
+_SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
 
 
-def _record_path(output_dir: Path, *, instance: str, command_class: str) -> Path:
-    stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ")
-    return output_dir / f"{stamp}--{instance}--{command_class}.json"
+def _record_path(
+    output_dir: Path,
+    *,
+    operation_id: str,
+    phase: str,
+    instance: str,
+    command_class: str,
+) -> Path:
+    for value in (operation_id, phase, instance, command_class):
+        if not _SAFE_ID.fullmatch(value):
+            raise ValueError("record filename components must be safe identifiers")
+    stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%S.%fZ")
+    nonce = uuid.uuid4().hex[:12]
+    return output_dir / (
+        f"{stamp}--{operation_id}--{phase}--{instance}--{command_class}--{nonce}.json"
+    )
 
 
 def _parse_capture_command(raw: str) -> tuple[str, list[str]]:
@@ -33,67 +56,119 @@ def _parse_capture_command(raw: str) -> tuple[str, list[str]]:
     return key, parts
 
 
+def _validate_capture_command(key: str, command: list[str], *, instance: str) -> None:
+    executable = Path(command[0]).name
+    args = command[1:]
+    expected = {
+        "pending_slots": ["slots", "--status", "pending", "--instance", instance, "--json"],
+        "reconciliation_slots": ["slots", "--status", "all", "--instance", instance, "--json"],
+        "queue_consumers": ["list_consumers"],
+    }
+    if key not in expected:
+        raise ValueError(f"unsupported capture key: {key}")
+    if key == "queue_consumers":
+        valid = executable == "rabbitmqctl" and args == expected[key]
+    else:
+        valid = executable == "trex-core" and args == expected[key]
+    if not valid:
+        raise ValueError(f"capture command is not the approved read-only form for {key}")
+
+
+def _outside_checkout(path: Path, checkout: Path, *, label: str) -> Path:
+    resolved = path.resolve()
+    try:
+        resolved.relative_to(checkout.resolve())
+    except ValueError:
+        return resolved
+    raise ValueError(f"{label} must be outside the audit checkout")
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--actor", required=True)
     parser.add_argument("--reason", required=True)
     parser.add_argument("--instance", required=True)
-    parser.add_argument("--command-class", required=True)
+    parser.add_argument("--operation-id", required=True)
+    parser.add_argument("--phase", required=True, choices=("pre", "post"))
+    parser.add_argument("--command-class", required=True, choices=_COMMAND_CLASSES)
     parser.add_argument("--checkout-dir", default=".")
-    parser.add_argument("--expected-oci-revision")
-    parser.add_argument("--image-digest")
-    parser.add_argument("--runtime-file", action="append", default=[])
+    parser.add_argument("--expected-oci-revision", required=True)
+    parser.add_argument("--image-digest", required=True)
+    parser.add_argument("--runtime-file", action="append", required=True)
     parser.add_argument("--broker-log-path", default="/tmp/trex-core.log")
-    parser.add_argument("--capture-command", action="append", default=[])
-    parser.add_argument("--output-dir", default=str(_DEFAULT_OUTPUT_DIR))
-    parser.add_argument("--isolate-dir", default=str(_DEFAULT_ISOLATE_DIR))
-    parser.add_argument("--allow-dirty", action="store_true")
+    parser.add_argument("--capture-command", action="append", required=True)
+    parser.add_argument("--output-dir")
+    parser.add_argument("--isolate-dir")
     args = parser.parse_args(argv)
 
-    capture_commands = {}
-    for raw in args.capture_command:
-        key, command = _parse_capture_command(raw)
-        capture_commands[key] = command
+    try:
+        capture_commands: dict[str, list[str]] = {}
+        for raw in args.capture_command:
+            key, command = _parse_capture_command(raw)
+            if key in capture_commands:
+                raise ValueError(f"duplicate capture key: {key}")
+            _validate_capture_command(key, command, instance=args.instance)
+            capture_commands[key] = command
+        if set(capture_commands) != _REQUIRED_CAPTURE_KEYS:
+            missing = sorted(_REQUIRED_CAPTURE_KEYS - set(capture_commands))
+            extra = sorted(set(capture_commands) - _REQUIRED_CAPTURE_KEYS)
+            raise ValueError(f"capture command keys mismatch: missing={missing}, extra={extra}")
 
-    report = collect_release_provenance(
-        actor=args.actor,
-        reason=args.reason,
-        instance=args.instance,
-        command_class=args.command_class,
-        checkout_dir=args.checkout_dir,
-        expected_oci_revision=args.expected_oci_revision,
-        image_digest=args.image_digest,
-        runtime_files=args.runtime_file,
-        capture_commands=capture_commands,
-        broker_log_path=args.broker_log_path,
-    )
-
-    output_dir = Path(args.output_dir).resolve()
-    output_dir.mkdir(parents=True, exist_ok=True)
-    output_path = _record_path(output_dir, instance=args.instance, command_class=args.command_class)
-
-    dirty_checkout = bool(report["checkout"]["dirty_paths"])
-    runtime_ok = all(
-        item.get("matches_expected_revision", False) for item in report["runtime_file_verification"]
-    )
-    oci_ok = bool(report["artifact_identity"]["oci_revision_matches_head"])
-    allow_dirty_passed = bool(args.allow_dirty and runtime_ok and oci_ok)
-    report["gate"]["allow_dirty"] = bool(args.allow_dirty)
-    report["gate"]["allow_dirty_passed"] = allow_dirty_passed
-
-    if dirty_checkout and not args.allow_dirty:
-        report["dirty_checkout_isolation"] = isolate_dirty_checkout_files(
-            checkout_dir=args.checkout_dir,
-            isolate_dir=args.isolate_dir,
+        checkout = Path(args.checkout_dir).resolve()
+        output_default = checkout.parent / f"{checkout.name}-release-provenance"
+        isolate_default = checkout.parent / f"{checkout.name}-release-provenance-isolation"
+        output_dir = _outside_checkout(
+            Path(args.output_dir) if args.output_dir else output_default,
+            checkout,
+            label="output directory",
+        )
+        isolate_dir = _outside_checkout(
+            Path(args.isolate_dir) if args.isolate_dir else isolate_default,
+            checkout,
+            label="isolation directory",
         )
 
-    output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+        report = collect_release_provenance(
+            actor=args.actor,
+            reason=args.reason,
+            instance=args.instance,
+            command_class=args.command_class,
+            operation_id=args.operation_id,
+            phase=args.phase,
+            checkout_dir=checkout,
+            expected_oci_revision=args.expected_oci_revision,
+            image_digest=args.image_digest,
+            runtime_files=args.runtime_file,
+            capture_commands=capture_commands,
+            broker_log_path=args.broker_log_path,
+        )
+    except (RuntimeError, ValueError) as exc:
+        parser.error(str(exc))
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = _record_path(
+        output_dir,
+        operation_id=args.operation_id,
+        phase=args.phase,
+        instance=args.instance,
+        command_class=args.command_class,
+    )
+
+    dirty_checkout = bool(report["checkout"]["dirty_paths"])
+    if dirty_checkout:
+        report["dirty_checkout_isolation"] = isolate_dirty_checkout_files(
+            checkout_dir=checkout,
+            isolate_dir=isolate_dir,
+        )
+
+    # Exclusive create prevents a provenance record from being overwritten.
+    with output_path.open("x", encoding="utf-8") as fp:
+        fp.write(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
+    output_path.chmod(0o444)
     print(str(output_path))
-    if dirty_checkout and not args.allow_dirty:
+    if dirty_checkout:
         return 2
-    if report["gate"]["passed"] or allow_dirty_passed:
-        return 0
-    return 1
+    return 0 if report["gate"]["passed"] else 1
 
 
 if __name__ == "__main__":

tests/adapters/persistence/test_ops_checks.py

@@ -106,27 +106,71 @@ def _init_git_repo(tmp_path: Path) -> tuple[Path, str]:
     subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True, capture_output=True)
     subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, capture_output=True)
     head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
+    subprocess.run(["git", "checkout", "--detach", head], cwd=repo, check=True, capture_output=True)
     return repo, head
 
 
+def _provenance_kwargs(repo: Path, head: str) -> dict:
+    broker_log = repo.parent / "broker.log"
+    broker_log.write_text("event=order_submit token=must-not-leak\n", encoding="utf-8")
+    bindir = repo.parent / "bin"
+    bindir.mkdir(exist_ok=True)
+    trex = bindir / "trex-core"
+    rabbit = bindir / "rabbitmqctl"
+    trex.write_text("#!/bin/sh\nprintf '{}\\n'\n", encoding="utf-8")
+    rabbit.write_text("#!/bin/sh\nprintf 'consumer\\n'\n", encoding="utf-8")
+    trex.chmod(0o755)
+    rabbit.chmod(0o755)
+    instance = "inst-a"
+    return {
+        "actor": "operator-a",
+        "reason": "deploy",
+        "instance": instance,
+        "command_class": "manual-start",
+        "operation_id": "op-1",
+        "phase": "pre",
+        "checkout_dir": repo,
+        "expected_oci_revision": head,
+        "image_digest": "sha256:" + ("a" * 64),
+        "runtime_files": ["tracked.txt"],
+        "capture_commands": {
+            "pending_slots": [
+                str(trex), "slots", "--status", "pending", "--instance", instance, "--json"
+            ],
+            "reconciliation_slots": [
+                str(trex), "slots", "--status", "all", "--instance", instance, "--json"
+            ],
+            "queue_consumers": [str(rabbit), "list_consumers"],
+        },
+        "broker_log_path": broker_log,
+    }
+
+
 def test_collect_release_provenance_passes_on_clean_checkout(tmp_path):
     repo, head = _init_git_repo(tmp_path)
-    report = collect_release_provenance(
-        actor="operator-a",
-        reason="deploy",
-        instance="inst-a",
-        command_class="manual-start",
-        checkout_dir=repo,
-        expected_oci_revision=head,
-        image_digest="sha256:" + ("a" * 64),
-        runtime_files=["tracked.txt"],
-        capture_commands={},
-        broker_log_path=None,
-    )
+    report = collect_release_provenance(**_provenance_kwargs(repo, head))
     assert report["checkout"]["git_clean"] is True
+    assert report["checkout"]["detached_head"] is True
     assert report["artifact_identity"]["oci_revision_matches_head"] is True
     assert report["runtime_file_verification"][0]["matches_expected_revision"] is True
     assert report["gate"]["passed"] is True
+    assert report["operation_id"] == "op-1"
+    assert report["phase"] == "pre"
+    rendered = str(report)
+    assert "must-not-leak" not in rendered
+    assert report["state_capture"]["broker_side_effect_logs"]["matches"][0]["event"] == "order_submit"
+
+
+def test_collect_release_provenance_blocks_attached_checkout(tmp_path):
+    repo, head = _init_git_repo(tmp_path)
+    subprocess.run(["git", "switch", "-c", "audit-attached"], cwd=repo, check=True, capture_output=True)
+
+    report = collect_release_provenance(**_provenance_kwargs(repo, head))
+
+    assert report["checkout"]["git_clean"] is True
+    assert report["checkout"]["detached_head"] is False
+    assert report["state_capture"]["command_outputs"] == {}
+    assert report["gate"]["passed"] is False
 
 
 def test_collect_release_provenance_blocks_on_revision_or_runtime_mismatch(tmp_path):
@@ -134,17 +178,8 @@ def test_collect_release_provenance_blocks_on_revision_or_runtime_mismatch(tmp_p
     (repo / "tracked.txt").write_text("v2\n", encoding="utf-8")
     subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True, capture_output=True)
     subprocess.run(["git", "commit", "-m", "v2"], cwd=repo, check=True, capture_output=True)
-    report = collect_release_provenance(
-        actor="operator-a",
-        reason="deploy",
-        instance="inst-a",
-        command_class="manual-start",
-        checkout_dir=repo,
-        expected_oci_revision=old_head,
-        runtime_files=["tracked.txt"],
-        capture_commands={},
-        broker_log_path=None,
-    )
+    kwargs = _provenance_kwargs(repo, old_head)
+    report = collect_release_provenance(**kwargs)
     assert report["checkout"]["git_clean"] is True
     assert report["artifact_identity"]["oci_revision_matches_head"] is False
     assert report["runtime_file_verification"][0]["matches_expected_revision"] is False
@@ -159,24 +194,18 @@ def test_isolate_dirty_checkout_files_copies_dirty_and_untracked(tmp_path):
     manifest = isolate_dirty_checkout_files(checkout_dir=repo, isolate_dir=isolated)
     assert set(manifest["dirty_paths"]) == {"tracked.txt", "new.txt"}
     assert set(manifest["copied_paths"]) == {"tracked.txt", "new.txt"}
-    assert (isolated / "tracked.txt").read_text(encoding="utf-8") == "dirty\n"
-    assert (isolated / "new.txt").read_text(encoding="utf-8") == "new\n"
-    assert (isolated / "manifest.json").is_file()
+    snapshot = Path(manifest["isolate_dir"])
+    assert snapshot.parent == isolated
+    assert (snapshot / "tracked.txt").read_text(encoding="utf-8") == "dirty\n"
+    assert (snapshot / "new.txt").read_text(encoding="utf-8") == "new\n"
+    assert (snapshot / "manifest.json").is_file()
 
 
 def test_collect_release_provenance_rejects_runtime_path_escape(tmp_path):
     repo, head = _init_git_repo(tmp_path)
-    report = collect_release_provenance(
-        actor="operator-a",
-        reason="deploy",
-        instance="inst-a",
-        command_class="manual-start",
-        checkout_dir=repo,
-        expected_oci_revision=head,
-        runtime_files=["../outside.txt"],
-        capture_commands={},
-        broker_log_path=None,
-    )
+    kwargs = _provenance_kwargs(repo, head)
+    kwargs["runtime_files"] = ["../outside.txt"]
+    report = collect_release_provenance(**kwargs)
     runtime = report["runtime_file_verification"][0]
     assert runtime["path_within_checkout"] is False
     assert runtime["matches_expected_revision"] is False
@@ -186,13 +215,93 @@ def test_collect_release_provenance_rejects_runtime_path_escape(tmp_path):
 def test_collect_release_provenance_reports_git_failure_context(tmp_path):
     not_repo = tmp_path / "not-repo"
     not_repo.mkdir()
+    trex = tmp_path / "trex-core"
+    rabbit = tmp_path / "rabbitmqctl"
     with pytest.raises(RuntimeError, match="git command failed in checkout="):
         collect_release_provenance(
             actor="operator-a",
             reason="deploy",
             instance="inst-a",
             command_class="manual-start",
+            operation_id="op-1",
+            phase="pre",
             checkout_dir=not_repo,
-            capture_commands={},
-            broker_log_path=None,
+            expected_oci_revision="0" * 40,
+            image_digest="sha256:" + ("a" * 64),
+            runtime_files=["tracked.txt"],
+            capture_commands={
+                "pending_slots": [
+                    str(trex), "slots", "--status", "pending", "--instance", "inst-a", "--json"
+                ],
+                "reconciliation_slots": [
+                    str(trex), "slots", "--status", "all", "--instance", "inst-a", "--json"
+                ],
+                "queue_consumers": [str(rabbit), "list_consumers"],
+            },
         )
+
+
+def test_dirty_checkout_blocks_before_capture_commands_run(tmp_path):
+    repo, head = _init_git_repo(tmp_path)
+    marker = tmp_path / "capture-ran"
+    (repo / "tracked.txt").write_text("dirty\n", encoding="utf-8")
+    kwargs = _provenance_kwargs(repo, head)
+    for command in kwargs["capture_commands"].values():
+        executable = Path(command[0])
+        executable.write_text(f"#!/bin/sh\ntouch {marker}\n", encoding="utf-8")
+        executable.chmod(0o755)
+
+    report = collect_release_provenance(**kwargs)
+
+    assert report["gate"]["passed"] is False
+    assert report["gate"]["deployment_blocked"] is True
+    assert report["state_capture"]["command_outputs"] == {}
+    assert not marker.exists()
+
+
+def test_capture_failure_or_missing_broker_log_blocks_gate(tmp_path):
+    repo, head = _init_git_repo(tmp_path)
+    kwargs = _provenance_kwargs(repo, head)
+    trex = Path(kwargs["capture_commands"]["pending_slots"][0])
+    trex.write_text("#!/bin/sh\nexit 3\n", encoding="utf-8")
+    trex.chmod(0o755)
+    kwargs["broker_log_path"] = tmp_path / "missing.log"
+
+    report = collect_release_provenance(**kwargs)
+
+    assert report["gate"]["passed"] is False
+    assert report["gate"]["state_captures_succeeded"] is False
+    assert report["gate"]["broker_log_captured"] is False
+
+
+def test_empty_broker_log_is_explicitly_captured_as_no_matching_events(tmp_path):
+    repo, head = _init_git_repo(tmp_path)
+    kwargs = _provenance_kwargs(repo, head)
+    Path(kwargs["broker_log_path"]).write_bytes(b"")
+
+    report = collect_release_provenance(**kwargs)
+
+    broker = report["state_capture"]["broker_side_effect_logs"]
+    assert broker["capture_status"] == "captured_empty"
+    assert broker["file_sha256"] == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    assert broker["matches"] == []
+    assert report["gate"]["passed"] is True
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("expected_oci_revision", None),
+        ("image_digest", "not-a-digest"),
+        ("runtime_files", []),
+        ("capture_commands", {}),
+        ("instance", "../escape"),
+        ("reason", "password=must-not-be-recorded"),
+    ],
+)
+def test_required_provenance_inputs_fail_closed(tmp_path, field, value):
+    repo, head = _init_git_repo(tmp_path)
+    kwargs = _provenance_kwargs(repo, head)
+    kwargs[field] = value
+    with pytest.raises((TypeError, ValueError)):
+        collect_release_provenance(**kwargs)

tests/scripts/test_release_provenance_gate.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from scripts import release_provenance_gate
+
+
+def test_documented_python_invocation_loads_project_without_pythonpath():
+    script = Path(release_provenance_gate.__file__).resolve()
+    env = os.environ.copy()
+    env.pop("PYTHONPATH", None)
+
+    completed = subprocess.run(
+        [sys.executable, str(script), "--help"],
+        cwd=script.parent,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert completed.returncode == 0, completed.stderr
+    assert "--expected-oci-revision" in completed.stdout
+    assert script.stat().st_mode & 0o111 != 0
+
+
+def _init_repo(tmp_path: Path) -> tuple[Path, str]:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True)
+    subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=repo, check=True)
+    subprocess.run(["git", "config", "user.name", "Test"], cwd=repo, check=True)
+    (repo / "runtime.py").write_text("print('ok')\n", encoding="utf-8")
+    subprocess.run(["git", "add", "runtime.py"], cwd=repo, check=True)
+    subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, capture_output=True)
+    head = subprocess.run(
+        ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True
+    ).stdout.strip()
+    subprocess.run(["git", "checkout", "--detach", head], cwd=repo, check=True, capture_output=True)
+    return repo, head
+
+
+def _fake_read_only_commands(tmp_path: Path) -> tuple[Path, Path]:
+    bindir = tmp_path / "bin"
+    bindir.mkdir(parents=True)
+    trex = bindir / "trex-core"
+    rabbit = bindir / "rabbitmqctl"
+    trex.write_text("#!/bin/sh\nprintf '{}\\n'\n", encoding="utf-8")
+    rabbit.write_text("#!/bin/sh\nprintf 'consumer\\n'\n", encoding="utf-8")
+    trex.chmod(0o755)
+    rabbit.chmod(0o755)
+    return trex, rabbit
+
+
+def _argv(tmp_path: Path, repo: Path, head: str) -> list[str]:
+    trex, rabbit = _fake_read_only_commands(tmp_path)
+    broker_log = tmp_path / "broker.log"
+    broker_log.write_text("event=order_submit account=must-not-leak\n", encoding="utf-8")
+    return [
+        "--actor", "operator-a",
+        "--reason", "deploy release",
+        "--instance", "inst-a",
+        "--operation-id", "op-123",
+        "--phase", "pre",
+        "--command-class", "deploy",
+        "--checkout-dir", str(repo),
+        "--expected-oci-revision", head,
+        "--image-digest", "sha256:" + ("a" * 64),
+        "--runtime-file", "runtime.py",
+        "--broker-log-path", str(broker_log),
+        "--capture-command", f"pending_slots={trex} slots --status pending --instance inst-a --json",
+        "--capture-command", f"reconciliation_slots={trex} slots --status all --instance inst-a --json",
+        "--capture-command", f"queue_consumers={rabbit} list_consumers",
+    ]
+
+
+def test_cli_writes_unique_record_outside_checkout_and_keeps_checkout_clean(tmp_path, capsys):
+    repo, head = _init_repo(tmp_path)
+    argv = _argv(tmp_path, repo, head)
+
+    assert release_provenance_gate.main(argv) == 0
+    first = Path(capsys.readouterr().out.strip())
+    assert release_provenance_gate.main(argv) == 0
+    second = Path(capsys.readouterr().out.strip())
+
+    assert first != second
+    assert first.parent == tmp_path / "repo-release-provenance"
+    assert first.stat().st_mode & 0o222 == 0
+    assert second.stat().st_mode & 0o222 == 0
+    payload = json.loads(first.read_text(encoding="utf-8"))
+    assert payload["gate"]["passed"] is True
+    assert "must-not-leak" not in first.read_text(encoding="utf-8")
+    status = subprocess.run(
+        ["git", "status", "--porcelain"], cwd=repo, check=True, capture_output=True, text=True
+    ).stdout
+    assert status == ""
+
+
+def test_cli_dirty_checkout_is_fail_closed_and_isolated(tmp_path, capsys):
+    repo, head = _init_repo(tmp_path)
+    argv = _argv(tmp_path, repo, head)
+    (repo / "runtime.py").write_text("dirty\n", encoding="utf-8")
+
+    assert release_provenance_gate.main(argv) == 2
+    output = Path(capsys.readouterr().out.strip())
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert output.stat().st_mode & 0o222 == 0
+    assert payload["gate"]["passed"] is False
+    assert payload["gate"]["deployment_blocked"] is True
+    assert payload["state_capture"]["command_outputs"] == {}
+    snapshot = Path(payload["dirty_checkout_isolation"]["isolate_dir"])
+    assert snapshot.parent == tmp_path / "repo-release-provenance-isolation"
+    assert (snapshot / "runtime.py").read_text() == "dirty\n"
+
+
+def test_cli_rejects_unsafe_command_and_checkout_local_output(tmp_path):
+    repo, head = _init_repo(tmp_path)
+    argv = _argv(tmp_path, repo, head)
+    unsafe = argv.index(next(x for x in argv if x.startswith("pending_slots=")))
+    argv[unsafe] = "pending_slots=trex-core cancel slot-1 --yes"
+    with pytest.raises(SystemExit):
+        release_provenance_gate.main(argv)
+
+    argv = _argv(tmp_path / "other", repo, head)
+    argv.extend(["--output-dir", str(repo / "artifacts")])
+    with pytest.raises(SystemExit):
+        release_provenance_gate.main(argv)

trex_core/adapters/persistence/ops_checks.py

@@ -13,8 +13,10 @@ from __future__ import annotations
 
 import hashlib
 import json
+import re
 import shutil
 import subprocess
+import uuid
 from collections import deque
 from datetime import UTC, date, datetime
 from pathlib import Path
@@ -56,6 +58,22 @@ def find_ledger_gap_slots(
         return [dict(zip(_COLS, row)) for row in cur.fetchall()]
 
 
+_SENSITIVE_KEY = re.compile(
+    r"(?:account|acct|token|credential|password|passwd|secret|api[_-]?key|dsn)",
+    re.IGNORECASE,
+)
+_SENSITIVE_VALUE = re.compile(
+    r"(?i)\b(account|acct|token|credential|password|passwd|secret|api[_-]?key|dsn)"
+    r"\s*[:=]\s*([^\s,;]+)"
+)
+_FULL_GIT_REVISION = re.compile(r"[0-9a-f]{40}")
+_IMAGE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
+_SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
+_REQUIRED_CAPTURE_KEYS = frozenset(
+    {"pending_slots", "reconciliation_slots", "queue_consumers"}
+)
+
+
 def _run_git(checkout_dir: Path, *args: str) -> str:
     command = ["git", *args]
     try:
@@ -71,24 +89,91 @@ def _run_git(checkout_dir: Path, *args: str) -> str:
         ) from exc
 
 
-def _porcelain_paths(status_output: str) -> list[str]:
+def _git_status(checkout_dir: Path) -> tuple[list[dict[str, str]], list[str]]:
+    try:
+        raw = subprocess.check_output(
+            ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"],
+            cwd=str(checkout_dir),
+        )
+    except subprocess.CalledProcessError as exc:
+        raise RuntimeError(f"git status failed in checkout={checkout_dir}") from exc
+    fields = raw.decode("utf-8", errors="surrogateescape").split("\0")
+    entries: list[dict[str, str]] = []
     paths: list[str] = []
-    for raw in status_output.splitlines():
-        line = raw.rstrip()
-        if len(line) < 3:
+    index = 0
+    while index < len(fields):
+        field = fields[index]
+        index += 1
+        if not field:
             continue
-        if line.startswith("?? ") or (len(line) >= 4 and line[2] == " "):
-            body = line[3:]
-        elif len(line) >= 3 and line[1] == " ":
-            body = line[2:]
-        else:
-            body = line[3:] if len(line) > 3 else ""
-        if " -> " in body:
-            body = body.split(" -> ", 1)[1]
-        body = body.strip()
-        if body:
-            paths.append(body)
-    return paths
+        if len(field) < 4:
+            raise RuntimeError("unexpected git status porcelain record")
+        code, path = field[:2], field[3:]
+        entry = {"code": code, "path": path}
+        if "R" in code or "C" in code:
+            if index >= len(fields) or not fields[index]:
+                raise RuntimeError("unexpected git rename/copy porcelain record")
+            entry["original_path"] = fields[index]
+            index += 1
+        entries.append(entry)
+        paths.append(path)
+    return entries, paths
+
+
+def _git_head_is_detached(checkout_dir: Path) -> bool:
+    proc = subprocess.run(
+        ["git", "symbolic-ref", "-q", "HEAD"],
+        cwd=checkout_dir,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if proc.returncode == 0:
+        return False
+    if proc.returncode == 1:
+        return True
+    detail = _sanitize_text(proc.stderr.strip()) or f"exit={proc.returncode}"
+    raise RuntimeError(f"git symbolic-ref failed in checkout={checkout_dir}: {detail}")
+
+
+def _sanitize_text(value: str) -> str:
+    return _SENSITIVE_VALUE.sub(lambda match: f"{match.group(1)}=[REDACTED]", value)
+
+
+def _sanitize_json(value: object) -> object:
+    if isinstance(value, dict):
+        return {
+            str(key): "[REDACTED]" if _SENSITIVE_KEY.search(str(key)) else _sanitize_json(item)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_sanitize_json(item) for item in value]
+    if isinstance(value, str):
+        return _sanitize_text(value)
+    return value
+
+
+def _validate_capture_commands(
+    capture_commands: dict[str, list[str]], *, instance: str
+) -> None:
+    if set(capture_commands) != _REQUIRED_CAPTURE_KEYS:
+        raise ValueError("all required state capture commands must be supplied exactly once")
+    expected = {
+        "pending_slots": ["slots", "--status", "pending", "--instance", instance, "--json"],
+        "reconciliation_slots": ["slots", "--status", "all", "--instance", instance, "--json"],
+        "queue_consumers": ["list_consumers"],
+    }
+    for key, command in capture_commands.items():
+        if not command:
+            raise ValueError(f"capture command is empty for {key}")
+        executable = Path(command[0]).name
+        valid = (
+            executable == "rabbitmqctl" and command[1:] == expected[key]
+            if key == "queue_consumers"
+            else executable == "trex-core" and command[1:] == expected[key]
+        )
+        if not valid:
+            raise ValueError(f"capture command is not the approved read-only form for {key}")
 
 
 def _git_blob_bytes(checkout_dir: Path, revision: str, relpath: str) -> bytes | None:
@@ -125,26 +210,83 @@ def _capture_command(command: list[str], checkout_dir: Path) -> dict:
             check=False,
             timeout=20,
         )
-        return {"command": command, "returncode": done.returncode, "stdout": done.stdout, "stderr": done.stderr}
+        try:
+            stdout: object = _sanitize_json(json.loads(done.stdout))
+            stdout_format = "json"
+        except json.JSONDecodeError:
+            stdout = _sanitize_text(done.stdout)
+            stdout_format = "text"
+        return {
+            "command": command,
+            "returncode": done.returncode,
+            "stdout": stdout,
+            "stdout_format": stdout_format,
+            "stderr": _sanitize_text(done.stderr),
+        }
     except (OSError, subprocess.SubprocessError) as exc:  # pragma: no cover
-        return {"command": command, "returncode": None, "stdout": "", "stderr": str(exc)}
+        return {
+            "command": command,
+            "returncode": None,
+            "stdout": "",
+            "stdout_format": "text",
+            "stderr": type(exc).__name__,
+        }
 
 
 def _capture_broker_side_effect_logs(log_path: Path) -> dict:
     if not log_path.exists():
-        return {"path": str(log_path), "exists": False, "matches": []}
-    matches: deque[str] = deque(maxlen=200)
-    with log_path.open("r", encoding="utf-8", errors="replace") as fp:
-        for line in fp:
-            lowered = line.lower()
-            if (
-                "event=order_submit" in lowered
-                or "event=order_fill" in lowered
-                or "event=order_reject" in lowered
-                or "cancel" in lowered
-            ):
-                matches.append(line.rstrip("\n"))
-    return {"path": str(log_path), "exists": True, "matches": list(matches)}
+        return {
+            "path": str(log_path),
+            "exists": False,
+            "capture_status": "unavailable",
+            "matches": [],
+        }
+    matches: deque[dict[str, object]] = deque(maxlen=200)
+    file_hash = hashlib.sha256()
+    scanned_lines = 0
+    try:
+        with log_path.open("rb") as fp:
+            for line_number, raw_line in enumerate(fp, start=1):
+                scanned_lines = line_number
+                file_hash.update(raw_line)
+                lowered = raw_line.decode("utf-8", errors="replace").lower()
+                event = next(
+                    (
+                        name
+                        for marker, name in (
+                            ("event=order_submit", "order_submit"),
+                            ("event=order_fill", "order_fill"),
+                            ("event=order_reject", "order_reject"),
+                            ("event=order_cancel", "order_cancel"),
+                        )
+                        if marker in lowered
+                    ),
+                    None,
+                )
+                if event is not None:
+                    matches.append(
+                        {
+                            "event": event,
+                            "line_number": line_number,
+                            "line_sha256": _sha256_bytes(raw_line.rstrip(b"\n")),
+                        }
+                    )
+    except OSError as exc:
+        return {
+            "path": str(log_path),
+            "exists": True,
+            "capture_status": "error",
+            "error_type": type(exc).__name__,
+            "matches": [],
+        }
+    return {
+        "path": str(log_path),
+        "exists": True,
+        "capture_status": "captured" if matches else "captured_empty",
+        "file_sha256": file_hash.hexdigest(),
+        "scanned_lines": scanned_lines,
+        "matches": list(matches),
+    }
 
 
 def collect_release_provenance(
@@ -153,28 +295,45 @@ def collect_release_provenance(
     reason: str,
     instance: str,
     command_class: str,
+    operation_id: str,
+    phase: str,
+    expected_oci_revision: str,
+    image_digest: str,
+    runtime_files: list[str],
+    capture_commands: dict[str, list[str]],
     checkout_dir: str | Path = ".",
-    expected_oci_revision: str | None = None,
-    image_digest: str | None = None,
-    runtime_files: list[str] | None = None,
-    capture_commands: dict[str, list[str]] | None = None,
     broker_log_path: str | Path | None = "/tmp/trex-core.log",
 ) -> dict:
-    """Collect immutable release/restart provenance with fail-closed gates."""
+    """Collect release/restart provenance with mandatory fail-closed evidence."""
+    if not actor.strip() or not reason.strip():
+        raise ValueError("actor and reason must be non-empty")
+    if _sanitize_text(actor) != actor or _sanitize_text(reason) != reason:
+        raise ValueError("actor and reason must not contain sensitive key/value material")
+    if not _SAFE_ID.fullmatch(instance) or not _SAFE_ID.fullmatch(operation_id):
+        raise ValueError("instance and operation_id must be safe identifiers")
+    if phase not in {"pre", "post"}:
+        raise ValueError("phase must be pre or post")
+    if not _FULL_GIT_REVISION.fullmatch(expected_oci_revision):
+        raise ValueError("expected_oci_revision must be a full lowercase git SHA")
+    if not _IMAGE_DIGEST.fullmatch(image_digest):
+        raise ValueError("image_digest must be sha256:<64 lowercase hex>")
+    if not runtime_files:
+        raise ValueError("at least one runtime file is required")
+    _validate_capture_commands(capture_commands, instance=instance)
+
     checkout = Path(checkout_dir).resolve()
     head = _run_git(checkout, "rev-parse", "HEAD")
-    status = _run_git(checkout, "status", "--porcelain")
-    dirty_paths = _porcelain_paths(status)
+    detached_head = _git_head_is_detached(checkout)
+    status_entries, dirty_paths = _git_status(checkout)
     git_clean = len(dirty_paths) == 0
-    ref_revision = expected_oci_revision or head
     runtime_reports = []
-    for relpath in runtime_files or []:
+    for relpath in runtime_files:
         abs_path = _resolve_under(checkout, relpath)
         path_valid = abs_path is not None
         exists = bool(path_valid and abs_path.is_file())
         current_bytes = abs_path.read_bytes() if exists else b""
         current_sha = _sha256_bytes(current_bytes) if exists else None
-        expected_bytes = _git_blob_bytes(checkout, ref_revision, relpath)
+        expected_bytes = _git_blob_bytes(checkout, expected_oci_revision, relpath)
         expected_sha = _sha256_bytes(expected_bytes) if expected_bytes is not None else None
         runtime_reports.append(
             {
@@ -182,29 +341,55 @@ def collect_release_provenance(
                 "path_within_checkout": path_valid,
                 "exists": exists,
                 "current_sha256": current_sha,
-                "expected_revision": ref_revision,
+                "expected_revision": expected_oci_revision,
                 "expected_sha256": expected_sha,
                 "matches_expected_revision": bool(
                     path_valid and exists and expected_bytes is not None and current_bytes == expected_bytes
                 ),
             }
         )
-    command_outputs = {}
-    for key, cmd in (capture_commands or {}).items():
-        command_outputs[key] = _capture_command(cmd, checkout)
-    oci_revision_matches_head = expected_oci_revision is None or expected_oci_revision == head
+
+    # A dirty audit checkout blocks before any operator command is executed.
+    command_outputs: dict[str, dict] = {}
+    broker_logs = None
+    if detached_head and git_clean:
+        for key, cmd in capture_commands.items():
+            command_outputs[key] = _capture_command(cmd, checkout)
+        broker_logs = (
+            _capture_broker_side_effect_logs(Path(broker_log_path))
+            if broker_log_path
+            else None
+        )
+    oci_revision_matches_head = expected_oci_revision == head
     runtime_ok = all(item["matches_expected_revision"] for item in runtime_reports)
-    gate_passed = git_clean and oci_revision_matches_head and runtime_ok
-    payload = {
+    commands_ok = set(command_outputs) == _REQUIRED_CAPTURE_KEYS and all(
+        item.get("returncode") == 0 for item in command_outputs.values()
+    )
+    broker_logs_ok = bool(
+        broker_logs
+        and broker_logs.get("capture_status") in {"captured", "captured_empty"}
+    )
+    gate_passed = (
+        detached_head
+        and git_clean
+        and oci_revision_matches_head
+        and runtime_ok
+        and commands_ok
+        and broker_logs_ok
+    )
+    return {
         "timestamp_utc": datetime.now(tz=UTC).isoformat(),
         "actor": actor,
         "reason": reason,
         "instance": instance,
         "command_class": command_class,
+        "operation_id": operation_id,
+        "phase": phase,
         "checkout": {
             "path": str(checkout),
             "head_revision": head,
-            "git_status_porcelain": status.splitlines(),
+            "detached_head": detached_head,
+            "git_status_porcelain": status_entries,
             "git_clean": git_clean,
             "dirty_paths": dirty_paths,
         },
@@ -216,18 +401,16 @@ def collect_release_provenance(
         "runtime_file_verification": runtime_reports,
         "state_capture": {
             "command_outputs": command_outputs,
-            "broker_side_effect_logs": (
-                _capture_broker_side_effect_logs(Path(broker_log_path))
-                if broker_log_path
-                else None
-            ),
+            "broker_side_effect_logs": broker_logs,
         },
         "gate": {
             "passed": gate_passed,
             "deployment_blocked": not gate_passed,
+            "runtime_files_verified": runtime_ok,
+            "state_captures_succeeded": commands_ok,
+            "broker_log_captured": broker_logs_ok,
         },
     }
-    return payload
 
 
 def isolate_dirty_checkout_files(
@@ -237,9 +420,10 @@ def isolate_dirty_checkout_files(
 ) -> dict:
     """Copy dirty/untracked checkout files to an isolation directory."""
     checkout = Path(checkout_dir).resolve()
-    dst = Path(isolate_dir).resolve()
-    status = _run_git(checkout, "status", "--porcelain")
-    dirty_paths = _porcelain_paths(status)
+    dst_root = Path(isolate_dir).resolve()
+    stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%S.%fZ")
+    dst = dst_root / f"{stamp}--{uuid.uuid4().hex[:12]}"
+    _, dirty_paths = _git_status(checkout)
     copied: list[str] = []
     for rel in dirty_paths:
         src = _resolve_under(checkout, rel)
@@ -261,8 +445,6 @@ def isolate_dirty_checkout_files(
         "copied_paths": copied,
     }
     dst.mkdir(parents=True, exist_ok=True)
-    (dst / "manifest.json").write_text(
-        json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
-        encoding="utf-8",
-    )
+    with (dst / "manifest.json").open("x", encoding="utf-8") as fp:
+        fp.write(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
     return manifest