retire GHCR image publication
This commit is contained in:
Hermes Agent
@@ -1,515 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic verification and publication helpers for the release image."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Protocol, Sequence
|
||||
|
||||
IMAGE = "ghcr.io/fourblocks2020/trex-core"
|
||||
CANARY = b"TREX_SECRET_CANARY_DO_NOT_SHIP"
|
||||
COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
DIGEST_RE = re.compile(r"(?:^|\s)digest:\s*(sha256:[0-9a-f]{64})(?:\s|$)")
|
||||
IMAGE_ID_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
VERSION_TAG_RE = re.compile(r"^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$")
|
||||
DEFAULT_MIGRATION_DIR = (
|
||||
Path(__file__).resolve().parents[1] / "trex_core" / "adapters" / "persistence" / "sql"
|
||||
)
|
||||
MAX_ARCHIVE_DEPTH = 8
|
||||
MAX_ARCHIVE_MEMBERS = 200_000
|
||||
MAX_ARCHIVE_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024
|
||||
MAX_ARCHIVE_MEMBER_BYTES = 512 * 1024 * 1024
|
||||
MAX_ARCHIVE_INPUT_BYTES = 1024 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ScanBudget:
|
||||
max_depth: int = MAX_ARCHIVE_DEPTH
|
||||
max_members: int = MAX_ARCHIVE_MEMBERS
|
||||
max_extracted_bytes: int = MAX_ARCHIVE_EXTRACTED_BYTES
|
||||
max_member_bytes: int = MAX_ARCHIVE_MEMBER_BYTES
|
||||
max_input_bytes: int = MAX_ARCHIVE_INPUT_BYTES
|
||||
members: int = 0
|
||||
extracted_bytes: int = 0
|
||||
|
||||
def check_depth(self, depth: int) -> None:
|
||||
if depth > self.max_depth:
|
||||
_scan_limit_exceeded()
|
||||
|
||||
def add_member(self, size: int) -> None:
|
||||
self.members += 1
|
||||
if self.members > self.max_members:
|
||||
_scan_limit_exceeded()
|
||||
if size < 0 or size > self.max_member_bytes:
|
||||
_scan_limit_exceeded()
|
||||
self.extracted_bytes += size
|
||||
if self.extracted_bytes > self.max_extracted_bytes:
|
||||
_scan_limit_exceeded()
|
||||
|
||||
|
||||
def _scan_limit_exceeded() -> None:
|
||||
raise ValueError("archive scan resource limit exceeded")
|
||||
|
||||
|
||||
class Runner(Protocol):
|
||||
def run(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
input_text: str | None = None,
|
||||
timeout: int | None = None,
|
||||
) -> Result: ...
|
||||
|
||||
|
||||
class SubprocessRunner:
|
||||
def run(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
input_text: str | None = None,
|
||||
timeout: int | None = None,
|
||||
) -> Result:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
input=input_text,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return Result(completed.stdout, completed.stderr)
|
||||
|
||||
|
||||
def validate_commit(commit: str) -> str:
|
||||
if not COMMIT_RE.fullmatch(commit):
|
||||
raise ValueError("source commit must be 40 lowercase hexadecimal characters")
|
||||
return commit
|
||||
|
||||
|
||||
def validate_publication_trigger(event_name: str, ref: str) -> str:
|
||||
if event_name != "push":
|
||||
raise ValueError("publication event must be a push")
|
||||
if not VERSION_TAG_RE.fullmatch(ref):
|
||||
raise ValueError("publication ref must be a strict vX.Y.Z tag")
|
||||
return ref.removeprefix("refs/tags/")
|
||||
|
||||
|
||||
def verify_checkout(commit: str, runner: Runner) -> str:
|
||||
expected = validate_commit(commit)
|
||||
actual = runner.run(["git", "rev-parse", "HEAD"]).stdout.strip()
|
||||
if actual != expected:
|
||||
raise ValueError("checked out HEAD does not match source commit")
|
||||
return actual
|
||||
|
||||
|
||||
def release_tags(commit: str, event_name: str, ref: str) -> list[str]:
|
||||
validate_commit(commit)
|
||||
release = validate_publication_trigger(event_name, ref)
|
||||
return [f"{IMAGE}:{commit[:12]}", f"{IMAGE}:{release}"]
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_checksum(archive: Path, checksum: Path) -> None:
|
||||
checksum.write_text(f"{sha256_file(archive)} {archive.name}\n", encoding="ascii")
|
||||
|
||||
|
||||
def verify_checksum(archive: Path, checksum: Path) -> None:
|
||||
if not archive.is_file() or not checksum.is_file():
|
||||
raise ValueError("image archive or checksum evidence is missing")
|
||||
fields = checksum.read_text(encoding="ascii").strip().split()
|
||||
if len(fields) != 2 or fields[1] != archive.name or not re.fullmatch(r"[0-9a-f]{64}", fields[0]):
|
||||
raise ValueError("checksum evidence has an invalid format")
|
||||
if sha256_file(archive) != fields[0]:
|
||||
raise ValueError("image archive checksum mismatch")
|
||||
|
||||
|
||||
def validate_image_id(image_id: str) -> str:
|
||||
if not IMAGE_ID_RE.fullmatch(image_id):
|
||||
raise ValueError("image ID must be sha256 followed by 64 lowercase hexadecimal characters")
|
||||
return image_id
|
||||
|
||||
|
||||
def inspect_image_id(image: str, runner: Runner) -> str:
|
||||
result = runner.run(["docker", "image", "inspect", "--format", "{{.Id}}", image])
|
||||
return validate_image_id(result.stdout.strip())
|
||||
|
||||
|
||||
def write_image_id(image: str, output: Path, runner: Runner) -> str:
|
||||
image_id = inspect_image_id(image, runner)
|
||||
output.write_text(f"{image_id}\n", encoding="ascii")
|
||||
return image_id
|
||||
|
||||
|
||||
def read_image_id(path: Path) -> str:
|
||||
text = path.read_text(encoding="ascii")
|
||||
if not re.fullmatch(r"sha256:[0-9a-f]{64}\n?", text):
|
||||
raise ValueError("tested image ID evidence has an invalid format")
|
||||
return text.rstrip("\n")
|
||||
|
||||
|
||||
def verify_image_id(image: str, expected_path: Path, runner: Runner) -> str:
|
||||
expected = read_image_id(expected_path)
|
||||
actual = inspect_image_id(image, runner)
|
||||
if actual != expected:
|
||||
raise ValueError("loaded image ID does not match tested image ID")
|
||||
return actual
|
||||
|
||||
|
||||
SMOKE_PROBES: tuple[tuple[str, ...], ...] = (
|
||||
("python", "-c", "import trex_core"),
|
||||
("trex-core-daemon", "--help"),
|
||||
("trex-core", "--help"),
|
||||
("trex-core-reset", "--help"),
|
||||
)
|
||||
|
||||
|
||||
def canonical_migration_names(migration_dir: Path = DEFAULT_MIGRATION_DIR) -> tuple[str, ...]:
|
||||
names = tuple(sorted(path.name for path in migration_dir.glob("*.sql") if path.is_file()))
|
||||
if not names:
|
||||
raise ValueError("canonical migration inventory is empty")
|
||||
return names
|
||||
|
||||
|
||||
def migration_inventory_code(expected: tuple[str, ...]) -> str:
|
||||
return (
|
||||
"from trex_core.adapters.persistence.migrate import migration_files;"
|
||||
f"expected={expected!r};"
|
||||
"actual=tuple(path.name for path in migration_files());"
|
||||
"assert expected and actual == expected, 'installed migration inventory mismatch'"
|
||||
)
|
||||
|
||||
|
||||
def _ensure_result_clean(result: Result) -> None:
|
||||
if CANARY in (result.stdout + result.stderr).encode():
|
||||
raise ValueError("canary marker found on a scanned surface")
|
||||
|
||||
|
||||
def smoke_image(
|
||||
image: str,
|
||||
runner: Runner,
|
||||
timeout: int = 30,
|
||||
migration_dir: Path = DEFAULT_MIGRATION_DIR,
|
||||
) -> None:
|
||||
expected_migrations = canonical_migration_names(migration_dir)
|
||||
probes = (
|
||||
*SMOKE_PROBES,
|
||||
("python", "-I", "-c", migration_inventory_code(expected_migrations)),
|
||||
)
|
||||
for index, probe in enumerate(probes):
|
||||
name = f"trex-core-smoke-{os.getpid()}-{index}"
|
||||
command = [
|
||||
"docker",
|
||||
"run",
|
||||
"--name",
|
||||
name,
|
||||
"--network",
|
||||
"none",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"64",
|
||||
"--tmpfs",
|
||||
"/tmp:rw,noexec,nosuid,size=16m",
|
||||
"--entrypoint",
|
||||
probe[0],
|
||||
image,
|
||||
*probe[1:],
|
||||
]
|
||||
failure: BaseException | None = None
|
||||
try:
|
||||
_ensure_result_clean(runner.run(command, timeout=timeout))
|
||||
except BaseException as error:
|
||||
failure = error
|
||||
try:
|
||||
_ensure_result_clean(
|
||||
runner.run(["docker", "rm", "--force", name], timeout=timeout)
|
||||
)
|
||||
except BaseException as cleanup_error:
|
||||
if failure is None:
|
||||
raise RuntimeError("smoke container cleanup failed") from cleanup_error
|
||||
if failure is not None:
|
||||
raise RuntimeError(f"smoke probe {index + 1} failed") from failure
|
||||
|
||||
|
||||
def _scan_stream(path: Path) -> None:
|
||||
with path.open("rb") as stream:
|
||||
overlap = b""
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
data = overlap + chunk
|
||||
if CANARY in data:
|
||||
raise ValueError("canary marker found on a scanned surface")
|
||||
overlap = data[-(len(CANARY) - 1):]
|
||||
|
||||
|
||||
def _scan_metadata(member: tarfile.TarInfo) -> None:
|
||||
values = [member.name, member.linkname, member.uname, member.gname]
|
||||
for key, value in member.pax_headers.items():
|
||||
values.extend((key, value))
|
||||
if any(CANARY in str(value).encode("utf-8", errors="replace") for value in values):
|
||||
raise ValueError("canary marker found on a scanned surface")
|
||||
|
||||
|
||||
def _scan_tar(path: Path, budget: _ScanBudget | None = None, depth: int = 0) -> None:
|
||||
budget = budget or _ScanBudget()
|
||||
if path.stat().st_size > (budget.max_input_bytes if depth == 0 else budget.max_member_bytes):
|
||||
_scan_limit_exceeded()
|
||||
_scan_stream(path)
|
||||
try:
|
||||
archive = tarfile.open(path, mode="r:*")
|
||||
except (tarfile.TarError, OSError, EOFError):
|
||||
return
|
||||
with archive:
|
||||
budget.check_depth(depth)
|
||||
for member in archive:
|
||||
budget.add_member(member.size if member.isfile() else 0)
|
||||
_scan_metadata(member)
|
||||
if not member.isfile():
|
||||
continue
|
||||
extracted = archive.extractfile(member)
|
||||
if extracted is None:
|
||||
continue
|
||||
with tempfile.NamedTemporaryFile() as nested:
|
||||
overlap = b""
|
||||
copied = 0
|
||||
while chunk := extracted.read(1024 * 1024):
|
||||
copied += len(chunk)
|
||||
if copied > member.size or copied > budget.max_member_bytes:
|
||||
_scan_limit_exceeded()
|
||||
data = overlap + chunk
|
||||
if CANARY in data:
|
||||
raise ValueError("canary marker found on a scanned surface")
|
||||
nested.write(chunk)
|
||||
overlap = data[-(len(CANARY) - 1):]
|
||||
nested.flush()
|
||||
_scan_tar(Path(nested.name), budget, depth + 1)
|
||||
|
||||
|
||||
def scan_image(image: str, saved_archive: Path, runner: Runner, timeout: int = 60) -> None:
|
||||
for command in (
|
||||
["docker", "history", "--no-trunc", image],
|
||||
["docker", "image", "inspect", image],
|
||||
):
|
||||
_ensure_result_clean(runner.run(command, timeout=timeout))
|
||||
|
||||
container = runner.run(["docker", "create", "--network", "none", image], timeout=timeout)
|
||||
container_id = container.stdout.strip()
|
||||
if not container_id:
|
||||
raise RuntimeError("docker create returned no container id")
|
||||
try:
|
||||
_ensure_result_clean(container)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
filesystem = Path(directory) / "filesystem.tar"
|
||||
_ensure_result_clean(
|
||||
runner.run(
|
||||
["docker", "export", "--output", str(filesystem), container_id],
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
_scan_tar(filesystem)
|
||||
finally:
|
||||
try:
|
||||
_ensure_result_clean(
|
||||
runner.run(["docker", "rm", "--force", container_id], timeout=timeout)
|
||||
)
|
||||
except BaseException as error:
|
||||
raise RuntimeError("canary scan container cleanup failed") from error
|
||||
_scan_tar(saved_archive)
|
||||
|
||||
|
||||
def assert_clean_output(paths: Sequence[Path]) -> None:
|
||||
for path in paths:
|
||||
_scan_stream(path)
|
||||
|
||||
|
||||
def _push_digest(output: str) -> str:
|
||||
digests = set(DIGEST_RE.findall(output))
|
||||
if len(digests) != 1:
|
||||
raise ValueError("registry push did not return exactly one valid digest")
|
||||
return digests.pop()
|
||||
|
||||
|
||||
def _append(path: Path | None, text: str) -> None:
|
||||
if path is None:
|
||||
return
|
||||
with path.open("a", encoding="utf-8") as destination:
|
||||
destination.write(text)
|
||||
|
||||
|
||||
def publish_image(
|
||||
*,
|
||||
archive: Path,
|
||||
checksum: Path,
|
||||
image_id: Path,
|
||||
local_image: str,
|
||||
commit: str,
|
||||
event_name: str,
|
||||
ref: str,
|
||||
actor: str,
|
||||
token: str,
|
||||
runner: Runner,
|
||||
output_path: Path | None = None,
|
||||
summary_path: Path | None = None,
|
||||
) -> str:
|
||||
tags = release_tags(commit, event_name, ref)
|
||||
version_tag = validate_publication_trigger(event_name, ref)
|
||||
verify_checksum(archive, checksum)
|
||||
expected_image_id = read_image_id(image_id)
|
||||
loaded = runner.run(["docker", "load", "--input", str(archive)])
|
||||
load_lines = {line.strip() for line in (loaded.stdout + "\n" + loaded.stderr).splitlines()}
|
||||
if f"Loaded image: {local_image}" not in load_lines:
|
||||
raise ValueError("docker load did not report the expected local image")
|
||||
actual_image_id = inspect_image_id(local_image, runner)
|
||||
if actual_image_id != expected_image_id:
|
||||
raise ValueError("loaded image ID does not match tested image ID")
|
||||
revision = runner.run(
|
||||
[
|
||||
"docker",
|
||||
"image",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{ index .Config.Labels \"org.opencontainers.image.revision\" }}",
|
||||
local_image,
|
||||
]
|
||||
).stdout.strip()
|
||||
if revision != commit:
|
||||
raise ValueError("loaded image revision does not match source commit")
|
||||
|
||||
if not actor or not token:
|
||||
raise ValueError("registry credentials are missing")
|
||||
for tag in tags:
|
||||
runner.run(["docker", "tag", local_image, tag])
|
||||
runner.run(
|
||||
["docker", "login", "ghcr.io", "--username", actor, "--password-stdin"],
|
||||
input_text=token,
|
||||
)
|
||||
digests = []
|
||||
for tag in tags:
|
||||
result = runner.run(["docker", "push", tag])
|
||||
digests.append(_push_digest(result.stdout + "\n" + result.stderr))
|
||||
if len(set(digests)) != 1:
|
||||
raise ValueError("published tags returned different registry digests")
|
||||
|
||||
digest = digests[0]
|
||||
reference = f"{IMAGE}@{digest}"
|
||||
_append(
|
||||
output_path,
|
||||
f"image={reference}\ndigest={digest}\nsource_commit={commit}\nversion_tag={version_tag}\n",
|
||||
)
|
||||
_append(
|
||||
summary_path,
|
||||
"## Published trex-core image\n\n"
|
||||
f"- Image: `{reference}`\n"
|
||||
f"- Digest: `{digest}`\n"
|
||||
f"- Source commit: `{commit}`\n"
|
||||
f"- Version tag: `{version_tag}`\n",
|
||||
)
|
||||
return digest
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
smoke = commands.add_parser("smoke")
|
||||
smoke.add_argument("--image", required=True)
|
||||
smoke.add_argument("--timeout", type=int, default=30)
|
||||
|
||||
verify_checkout_parser = commands.add_parser("verify-checkout")
|
||||
verify_checkout_parser.add_argument("--commit", required=True)
|
||||
|
||||
scan = commands.add_parser("scan")
|
||||
scan.add_argument("--image", required=True)
|
||||
scan.add_argument("--archive", type=Path, required=True)
|
||||
|
||||
checksum = commands.add_parser("checksum")
|
||||
checksum.add_argument("--archive", type=Path, required=True)
|
||||
checksum.add_argument("--output", type=Path, required=True)
|
||||
|
||||
record_id = commands.add_parser("record-image-id")
|
||||
record_id.add_argument("--image", required=True)
|
||||
record_id.add_argument("--output", type=Path, required=True)
|
||||
|
||||
verify_id = commands.add_parser("verify-image-id")
|
||||
verify_id.add_argument("--image", required=True)
|
||||
verify_id.add_argument("--expected", type=Path, required=True)
|
||||
|
||||
clean = commands.add_parser("assert-clean-output")
|
||||
clean.add_argument("paths", nargs="+", type=Path)
|
||||
|
||||
publish = commands.add_parser("publish")
|
||||
publish.add_argument("--archive", type=Path, required=True)
|
||||
publish.add_argument("--checksum", type=Path, required=True)
|
||||
publish.add_argument("--image-id", type=Path, required=True)
|
||||
publish.add_argument("--local-image", required=True)
|
||||
publish.add_argument("--commit", required=True)
|
||||
publish.add_argument("--event-name", required=True)
|
||||
publish.add_argument("--ref", required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
runner = SubprocessRunner()
|
||||
if args.command == "smoke":
|
||||
smoke_image(args.image, runner, args.timeout)
|
||||
elif args.command == "verify-checkout":
|
||||
verify_checkout(args.commit, runner)
|
||||
elif args.command == "scan":
|
||||
scan_image(args.image, args.archive, runner)
|
||||
elif args.command == "checksum":
|
||||
write_checksum(args.archive, args.output)
|
||||
elif args.command == "record-image-id":
|
||||
write_image_id(args.image, args.output, runner)
|
||||
elif args.command == "verify-image-id":
|
||||
verify_image_id(args.image, args.expected, runner)
|
||||
elif args.command == "assert-clean-output":
|
||||
assert_clean_output(args.paths)
|
||||
elif args.command == "publish":
|
||||
publish_image(
|
||||
archive=args.archive,
|
||||
checksum=args.checksum,
|
||||
image_id=args.image_id,
|
||||
local_image=args.local_image,
|
||||
commit=args.commit,
|
||||
event_name=args.event_name,
|
||||
ref=args.ref,
|
||||
actor=os.environ.get("GITHUB_ACTOR", ""),
|
||||
token=os.environ.get("REGISTRY_TOKEN", ""),
|
||||
runner=runner,
|
||||
output_path=Path(path) if (path := os.environ.get("GITHUB_OUTPUT")) else None,
|
||||
summary_path=Path(path) if (path := os.environ.get("GITHUB_STEP_SUMMARY")) else None,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:
|
||||
print(f"image release failed: {error}", file=sys.stderr)
|
||||
raise SystemExit(1) from error
|
||||
Reference in New Issue
Block a user