trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ops): make release provenance gate fail closed

Browse Source
This commit is contained in:
Hermes Agent
2026-09-13 16:34:44 +09:00
parent 8378e03dcc
commit 7dc2f65455
5 changed files with 669 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/adapters/persistence/ops_checks.py
+244 -62
View File
@@ -13,8 +13,10 @@ from __future__ import annotations
import hashlib
import json
import re
import shutil
import subprocess
import uuid
from collections import deque
from datetime import UTC, date, datetime
from pathlib import Path
@@ -56,6 +58,22 @@ def find_ledger_gap_slots(
return [dict(zip(_COLS, row)) for row in cur.fetchall()]
_SENSITIVE_KEY = re.compile(
r"(?:account|acct|token|credential|password|passwd|secret|api[_-]?key|dsn)",
re.IGNORECASE,
)
_SENSITIVE_VALUE = re.compile(
r"(?i)\b(account|acct|token|credential|password|passwd|secret|api[_-]?key|dsn)"
r"\s*[:=]\s*([^\s,;]+)"
)
_FULL_GIT_REVISION = re.compile(r"[0-9a-f]{40}")
_IMAGE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
_SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
_REQUIRED_CAPTURE_KEYS = frozenset(
{"pending_slots", "reconciliation_slots", "queue_consumers"}
)
def _run_git(checkout_dir: Path, *args: str) -> str:
command = ["git", *args]
try:
@@ -71,24 +89,91 @@ def _run_git(checkout_dir: Path, *args: str) -> str:
) from exc
def _porcelain_paths(status_output: str) -> list[str]:
def _git_status(checkout_dir: Path) -> tuple[list[dict[str, str]], list[str]]:
try:
raw = subprocess.check_output(
["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"],
cwd=str(checkout_dir),
)
except subprocess.CalledProcessError as exc:
raise RuntimeError(f"git status failed in checkout={checkout_dir}") from exc
fields = raw.decode("utf-8", errors="surrogateescape").split("\0")
entries: list[dict[str, str]] = []
paths: list[str] = []
for raw in status_output.splitlines():
line = raw.rstrip()
if len(line) < 3:
index = 0
while index < len(fields):
field = fields[index]
index += 1
if not field:
continue
if line.startswith("?? ") or (len(line) >= 4 and line[2] == " "):
body = line[3:]
elif len(line) >= 3 and line[1] == " ":
body = line[2:]
else:
body = line[3:] if len(line) > 3 else ""
if " -> " in body:
body = body.split(" -> ", 1)[1]
body = body.strip()
if body:
paths.append(body)
return paths
if len(field) < 4:
raise RuntimeError("unexpected git status porcelain record")
code, path = field[:2], field[3:]
entry = {"code": code, "path": path}
if "R" in code or "C" in code:
if index >= len(fields) or not fields[index]:
raise RuntimeError("unexpected git rename/copy porcelain record")
entry["original_path"] = fields[index]
index += 1
entries.append(entry)
paths.append(path)
return entries, paths
def _git_head_is_detached(checkout_dir: Path) -> bool:
proc = subprocess.run(
["git", "symbolic-ref", "-q", "HEAD"],
cwd=checkout_dir,
capture_output=True,
text=True,
check=False,
)
if proc.returncode == 0:
return False
if proc.returncode == 1:
return True
detail = _sanitize_text(proc.stderr.strip()) or f"exit={proc.returncode}"
raise RuntimeError(f"git symbolic-ref failed in checkout={checkout_dir}: {detail}")
def _sanitize_text(value: str) -> str:
return _SENSITIVE_VALUE.sub(lambda match: f"{match.group(1)}=[REDACTED]", value)
def _sanitize_json(value: object) -> object:
if isinstance(value, dict):
return {
str(key): "[REDACTED]" if _SENSITIVE_KEY.search(str(key)) else _sanitize_json(item)
for key, item in value.items()
}
if isinstance(value, list):
return [_sanitize_json(item) for item in value]
if isinstance(value, str):
return _sanitize_text(value)
return value
def _validate_capture_commands(
capture_commands: dict[str, list[str]], *, instance: str
) -> None:
if set(capture_commands) != _REQUIRED_CAPTURE_KEYS:
raise ValueError("all required state capture commands must be supplied exactly once")
expected = {
"pending_slots": ["slots", "--status", "pending", "--instance", instance, "--json"],
"reconciliation_slots": ["slots", "--status", "all", "--instance", instance, "--json"],
"queue_consumers": ["list_consumers"],
}
for key, command in capture_commands.items():
if not command:
raise ValueError(f"capture command is empty for {key}")
executable = Path(command[0]).name
valid = (
executable == "rabbitmqctl" and command[1:] == expected[key]
if key == "queue_consumers"
else executable == "trex-core" and command[1:] == expected[key]
)
if not valid:
raise ValueError(f"capture command is not the approved read-only form for {key}")
def _git_blob_bytes(checkout_dir: Path, revision: str, relpath: str) -> bytes | None:
@@ -125,26 +210,83 @@ def _capture_command(command: list[str], checkout_dir: Path) -> dict:
check=False,
timeout=20,
)
return {"command": command, "returncode": done.returncode, "stdout": done.stdout, "stderr": done.stderr}
try:
stdout: object = _sanitize_json(json.loads(done.stdout))
stdout_format = "json"
except json.JSONDecodeError:
stdout = _sanitize_text(done.stdout)
stdout_format = "text"
return {
"command": command,
"returncode": done.returncode,
"stdout": stdout,
"stdout_format": stdout_format,
"stderr": _sanitize_text(done.stderr),
}
except (OSError, subprocess.SubprocessError) as exc: # pragma: no cover
return {"command": command, "returncode": None, "stdout": "", "stderr": str(exc)}
return {
"command": command,
"returncode": None,
"stdout": "",
"stdout_format": "text",
"stderr": type(exc).__name__,
}
def _capture_broker_side_effect_logs(log_path: Path) -> dict:
if not log_path.exists():
return {"path": str(log_path), "exists": False, "matches": []}
matches: deque[str] = deque(maxlen=200)
with log_path.open("r", encoding="utf-8", errors="replace") as fp:
for line in fp:
lowered = line.lower()
if (
"event=order_submit" in lowered
or "event=order_fill" in lowered
or "event=order_reject" in lowered
or "cancel" in lowered
):
matches.append(line.rstrip("\n"))
return {"path": str(log_path), "exists": True, "matches": list(matches)}
return {
"path": str(log_path),
"exists": False,
"capture_status": "unavailable",
"matches": [],
}
matches: deque[dict[str, object]] = deque(maxlen=200)
file_hash = hashlib.sha256()
scanned_lines = 0
try:
with log_path.open("rb") as fp:
for line_number, raw_line in enumerate(fp, start=1):
scanned_lines = line_number
file_hash.update(raw_line)
lowered = raw_line.decode("utf-8", errors="replace").lower()
event = next(
(
name
for marker, name in (
("event=order_submit", "order_submit"),
("event=order_fill", "order_fill"),
("event=order_reject", "order_reject"),
("event=order_cancel", "order_cancel"),
)
if marker in lowered
),
None,
)
if event is not None:
matches.append(
{
"event": event,
"line_number": line_number,
"line_sha256": _sha256_bytes(raw_line.rstrip(b"\n")),
}
)
except OSError as exc:
return {
"path": str(log_path),
"exists": True,
"capture_status": "error",
"error_type": type(exc).__name__,
"matches": [],
}
return {
"path": str(log_path),
"exists": True,
"capture_status": "captured" if matches else "captured_empty",
"file_sha256": file_hash.hexdigest(),
"scanned_lines": scanned_lines,
"matches": list(matches),
}
def collect_release_provenance(
@@ -153,28 +295,45 @@ def collect_release_provenance(
reason: str,
instance: str,
command_class: str,
operation_id: str,
phase: str,
expected_oci_revision: str,
image_digest: str,
runtime_files: list[str],
capture_commands: dict[str, list[str]],
checkout_dir: str | Path = ".",
expected_oci_revision: str | None = None,
image_digest: str | None = None,
runtime_files: list[str] | None = None,
capture_commands: dict[str, list[str]] | None = None,
broker_log_path: str | Path | None = "/tmp/trex-core.log",
) -> dict:
"""Collect immutable release/restart provenance with fail-closed gates."""
"""Collect release/restart provenance with mandatory fail-closed evidence."""
if not actor.strip() or not reason.strip():
raise ValueError("actor and reason must be non-empty")
if _sanitize_text(actor) != actor or _sanitize_text(reason) != reason:
raise ValueError("actor and reason must not contain sensitive key/value material")
if not _SAFE_ID.fullmatch(instance) or not _SAFE_ID.fullmatch(operation_id):
raise ValueError("instance and operation_id must be safe identifiers")
if phase not in {"pre", "post"}:
raise ValueError("phase must be pre or post")
if not _FULL_GIT_REVISION.fullmatch(expected_oci_revision):
raise ValueError("expected_oci_revision must be a full lowercase git SHA")
if not _IMAGE_DIGEST.fullmatch(image_digest):
raise ValueError("image_digest must be sha256:<64 lowercase hex>")
if not runtime_files:
raise ValueError("at least one runtime file is required")
_validate_capture_commands(capture_commands, instance=instance)
checkout = Path(checkout_dir).resolve()
head = _run_git(checkout, "rev-parse", "HEAD")
status = _run_git(checkout, "status", "--porcelain")
dirty_paths = _porcelain_paths(status)
detached_head = _git_head_is_detached(checkout)
status_entries, dirty_paths = _git_status(checkout)
git_clean = len(dirty_paths) == 0
ref_revision = expected_oci_revision or head
runtime_reports = []
for relpath in runtime_files or []:
for relpath in runtime_files:
abs_path = _resolve_under(checkout, relpath)
path_valid = abs_path is not None
exists = bool(path_valid and abs_path.is_file())
current_bytes = abs_path.read_bytes() if exists else b""
current_sha = _sha256_bytes(current_bytes) if exists else None
expected_bytes = _git_blob_bytes(checkout, ref_revision, relpath)
expected_bytes = _git_blob_bytes(checkout, expected_oci_revision, relpath)
expected_sha = _sha256_bytes(expected_bytes) if expected_bytes is not None else None
runtime_reports.append(
{
@@ -182,29 +341,55 @@ def collect_release_provenance(
"path_within_checkout": path_valid,
"exists": exists,
"current_sha256": current_sha,
"expected_revision": ref_revision,
"expected_revision": expected_oci_revision,
"expected_sha256": expected_sha,
"matches_expected_revision": bool(
path_valid and exists and expected_bytes is not None and current_bytes == expected_bytes
),
}
)
command_outputs = {}
for key, cmd in (capture_commands or {}).items():
command_outputs[key] = _capture_command(cmd, checkout)
oci_revision_matches_head = expected_oci_revision is None or expected_oci_revision == head
# A dirty audit checkout blocks before any operator command is executed.
command_outputs: dict[str, dict] = {}
broker_logs = None
if detached_head and git_clean:
for key, cmd in capture_commands.items():
command_outputs[key] = _capture_command(cmd, checkout)
broker_logs = (
_capture_broker_side_effect_logs(Path(broker_log_path))
if broker_log_path
else None
)
oci_revision_matches_head = expected_oci_revision == head
runtime_ok = all(item["matches_expected_revision"] for item in runtime_reports)
gate_passed = git_clean and oci_revision_matches_head and runtime_ok
payload = {
commands_ok = set(command_outputs) == _REQUIRED_CAPTURE_KEYS and all(
item.get("returncode") == 0 for item in command_outputs.values()
)
broker_logs_ok = bool(
broker_logs
and broker_logs.get("capture_status") in {"captured", "captured_empty"}
)
gate_passed = (
detached_head
and git_clean
and oci_revision_matches_head
and runtime_ok
and commands_ok
and broker_logs_ok
)
return {
"timestamp_utc": datetime.now(tz=UTC).isoformat(),
"actor": actor,
"reason": reason,
"instance": instance,
"command_class": command_class,
"operation_id": operation_id,
"phase": phase,
"checkout": {
"path": str(checkout),
"head_revision": head,
"git_status_porcelain": status.splitlines(),
"detached_head": detached_head,
"git_status_porcelain": status_entries,
"git_clean": git_clean,
"dirty_paths": dirty_paths,
},
@@ -216,18 +401,16 @@ def collect_release_provenance(
"runtime_file_verification": runtime_reports,
"state_capture": {
"command_outputs": command_outputs,
"broker_side_effect_logs": (
_capture_broker_side_effect_logs(Path(broker_log_path))
if broker_log_path
else None
),
"broker_side_effect_logs": broker_logs,
},
"gate": {
"passed": gate_passed,
"deployment_blocked": not gate_passed,
"runtime_files_verified": runtime_ok,
"state_captures_succeeded": commands_ok,
"broker_log_captured": broker_logs_ok,
},
}
return payload
def isolate_dirty_checkout_files(
@@ -237,9 +420,10 @@ def isolate_dirty_checkout_files(
) -> dict:
"""Copy dirty/untracked checkout files to an isolation directory."""
checkout = Path(checkout_dir).resolve()
dst = Path(isolate_dir).resolve()
status = _run_git(checkout, "status", "--porcelain")
dirty_paths = _porcelain_paths(status)
dst_root = Path(isolate_dir).resolve()
stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%S.%fZ")
dst = dst_root / f"{stamp}--{uuid.uuid4().hex[:12]}"
_, dirty_paths = _git_status(checkout)
copied: list[str] = []
for rel in dirty_paths:
src = _resolve_under(checkout, rel)
@@ -261,8 +445,6 @@ def isolate_dirty_checkout_files(
"copied_paths": copied,
}
dst.mkdir(parents=True, exist_ok=True)
(dst / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
with (dst / "manifest.json").open("x", encoding="utf-8") as fp:
fp.write(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
return manifest