feat(ops): add release provenance gate and audit checkout isolation

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

trex_core/adapters/persistence/ops_checks.py

@@ -11,8 +11,13 @@ change; the transition log is durable so this is a permanent detector.
 """
 from __future__ import annotations
 
-from datetime import date
-from typing import Any, Optional
+import hashlib
+import json
+import shutil
+import subprocess
+from datetime import UTC, date, datetime
+from pathlib import Path
+from typing import Any
 
 from trex_core.adapters.persistence.postgresql import _require_psycopg
 
@@ -37,7 +42,7 @@ _COLS = ("slot_id", "symbol", "portfolio_id", "broker_order_id", "sold_at")
 def find_ledger_gap_slots(
     dsn: str,
     *,
-    trading_date: Optional[date] = None,
+    trading_date: date | None = None,
     psycopg_module: Any = None,
 ) -> list[dict]:
     """Return sell fills with no corresponding ledger_outbox row (FR-5).
@@ -45,7 +50,192 @@ def find_ledger_gap_slots(
     Pass `trading_date` to scope to one (UTC) day of sell fills; omit for all.
     """
     psycopg = _require_psycopg(psycopg_module)
-    with psycopg.connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(_LEDGER_GAP_SQL, {"td": trading_date})
-            return [dict(zip(_COLS, row)) for row in cur.fetchall()]
+    with psycopg.connect(dsn) as conn, conn.cursor() as cur:
+        cur.execute(_LEDGER_GAP_SQL, {"td": trading_date})
+        return [dict(zip(_COLS, row)) for row in cur.fetchall()]
+
+
+def _run_git(checkout_dir: Path, *args: str) -> str:
+    return subprocess.check_output(
+        ["git", *args],
+        cwd=str(checkout_dir),
+        text=True,
+    ).strip()
+
+
+def _porcelain_paths(status_output: str) -> list[str]:
+    paths: list[str] = []
+    for raw in status_output.splitlines():
+        line = raw.rstrip()
+        if len(line) < 3:
+            continue
+        if line.startswith("?? ") or (len(line) >= 4 and line[2] == " "):
+            body = line[3:]
+        elif len(line) >= 3 and line[1] == " ":
+            body = line[2:]
+        else:
+            body = line[3:] if len(line) > 3 else ""
+        if " -> " in body:
+            body = body.split(" -> ", 1)[1]
+        body = body.strip()
+        if body:
+            paths.append(body)
+    return paths
+
+
+def _git_blob_bytes(checkout_dir: Path, revision: str, relpath: str) -> bytes | None:
+    try:
+        return subprocess.check_output(
+            ["git", "show", f"{revision}:{relpath}"],
+            cwd=str(checkout_dir),
+        )
+    except subprocess.CalledProcessError:
+        return None
+
+
+def _sha256_bytes(payload: bytes) -> str:
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _capture_command(command: list[str], checkout_dir: Path) -> dict:
+    try:
+        done = subprocess.run(
+            command,
+            cwd=str(checkout_dir),
+            text=True,
+            capture_output=True,
+            check=False,
+            timeout=20,
+        )
+        return {"command": command, "returncode": done.returncode, "stdout": done.stdout, "stderr": done.stderr}
+    except (OSError, subprocess.SubprocessError) as exc:  # pragma: no cover
+        return {"command": command, "returncode": None, "stdout": "", "stderr": str(exc)}
+
+
+def _capture_broker_side_effect_logs(log_path: Path) -> dict:
+    if not log_path.exists():
+        return {"path": str(log_path), "exists": False, "matches": []}
+    matches: list[str] = []
+    for line in log_path.read_text(encoding="utf-8", errors="replace").splitlines():
+        lowered = line.lower()
+        if (
+            "event=order_submit" in lowered
+            or "event=order_fill" in lowered
+            or "event=order_reject" in lowered
+            or "cancel" in lowered
+        ):
+            matches.append(line)
+    return {"path": str(log_path), "exists": True, "matches": matches[-200:]}
+
+
+def collect_release_provenance(
+    *,
+    actor: str,
+    reason: str,
+    instance: str,
+    command_class: str,
+    checkout_dir: str | Path = ".",
+    expected_oci_revision: str | None = None,
+    image_digest: str | None = None,
+    runtime_files: list[str] | None = None,
+    capture_commands: dict[str, list[str]] | None = None,
+    broker_log_path: str | Path | None = "/tmp/trex-core.log",
+) -> dict:
+    """Collect immutable release/restart provenance with fail-closed gates."""
+    checkout = Path(checkout_dir).resolve()
+    head = _run_git(checkout, "rev-parse", "HEAD")
+    status = _run_git(checkout, "status", "--porcelain")
+    dirty_paths = _porcelain_paths(status)
+    git_clean = len(dirty_paths) == 0
+    ref_revision = expected_oci_revision or head
+    runtime_reports = []
+    for relpath in runtime_files or []:
+        abs_path = (checkout / relpath).resolve()
+        exists = abs_path.is_file()
+        current_bytes = abs_path.read_bytes() if exists else b""
+        current_sha = _sha256_bytes(current_bytes) if exists else None
+        expected_bytes = _git_blob_bytes(checkout, ref_revision, relpath)
+        expected_sha = _sha256_bytes(expected_bytes) if expected_bytes is not None else None
+        runtime_reports.append(
+            {
+                "path": relpath,
+                "exists": exists,
+                "current_sha256": current_sha,
+                "expected_revision": ref_revision,
+                "expected_sha256": expected_sha,
+                "matches_expected_revision": bool(exists and expected_bytes is not None and current_bytes == expected_bytes),
+            }
+        )
+    command_outputs = {}
+    for key, cmd in (capture_commands or {}).items():
+        command_outputs[key] = _capture_command(cmd, checkout)
+    oci_revision_matches_head = expected_oci_revision is None or expected_oci_revision == head
+    runtime_ok = all(item["matches_expected_revision"] for item in runtime_reports)
+    gate_passed = git_clean and oci_revision_matches_head and runtime_ok
+    payload = {
+        "timestamp_utc": datetime.now(tz=UTC).isoformat(),
+        "actor": actor,
+        "reason": reason,
+        "instance": instance,
+        "command_class": command_class,
+        "checkout": {
+            "path": str(checkout),
+            "head_revision": head,
+            "git_status_porcelain": status.splitlines(),
+            "git_clean": git_clean,
+            "dirty_paths": dirty_paths,
+        },
+        "artifact_identity": {
+            "expected_oci_revision": expected_oci_revision,
+            "oci_revision_matches_head": oci_revision_matches_head,
+            "image_digest": image_digest,
+        },
+        "runtime_file_verification": runtime_reports,
+        "state_capture": {
+            "command_outputs": command_outputs,
+            "broker_side_effect_logs": (
+                _capture_broker_side_effect_logs(Path(broker_log_path))
+                if broker_log_path
+                else None
+            ),
+        },
+        "gate": {
+            "passed": gate_passed,
+            "deployment_blocked": not gate_passed,
+        },
+    }
+    return payload
+
+
+def isolate_dirty_checkout_files(
+    *,
+    checkout_dir: str | Path,
+    isolate_dir: str | Path,
+) -> dict:
+    """Copy dirty/untracked checkout files to an isolation directory."""
+    checkout = Path(checkout_dir).resolve()
+    dst = Path(isolate_dir).resolve()
+    status = _run_git(checkout, "status", "--porcelain")
+    dirty_paths = _porcelain_paths(status)
+    copied: list[str] = []
+    for rel in dirty_paths:
+        src = (checkout / rel).resolve()
+        if not src.exists() or src.is_dir():
+            continue
+        target = dst / rel
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, target)
+        copied.append(rel)
+    manifest = {
+        "timestamp_utc": datetime.now(tz=UTC).isoformat(),
+        "checkout_dir": str(checkout),
+        "isolate_dir": str(dst),
+        "dirty_paths": dirty_paths,
+        "copied_paths": copied,
+    }
+    dst.mkdir(parents=True, exist_ok=True)
+    (dst / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return manifest