diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index fe8fc48..a9c1728 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -28,13 +28,13 @@ jobs: assert re.fullmatch(r"[0-9a-f]{40}", os.environ["RELEASE_SHA"]) PY - name: Check out exact tagged source commit - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ github.sha }} fetch-depth: 1 - run: python scripts/image_release.py validate-checkout --actual "$(git rev-parse HEAD)" --expected "${GITHUB_SHA}" - - run: python scripts/image_release.py validate-event --event "${GITHUB_EVENT_NAME}" --ref "${GITHUB_REF}" --commit "${GITHUB_SHA}" - - uses: actions/setup-python@v5 + - run: python scripts/image_release.py validate-event --event "${GITHUB_EVENT_NAME}" --ref "${GITHUB_REF}" --commit "${GITHUB_SHA}" --project pyproject.toml + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" - run: python -m pip install "uv==0.11.14" @@ -47,7 +47,7 @@ jobs: - name: Create excluded synthetic canary run: printf '%s\n' 'PORTFOLIO_HUB_SECRET_CANARY_DO_NOT_SHIP' > .env.ci-canary - name: Build exact source image once - run: docker build --provenance=false --build-arg "SOURCE_COMMIT=${GITHUB_SHA}" --tag "portfolio-hub-ci:${GITHUB_SHA}" . + run: docker build --platform linux/amd64 --provenance=false --build-arg "SOURCE_COMMIT=${GITHUB_SHA}" --build-arg "RELEASE_VERSION=${GITHUB_REF_NAME}" --tag "portfolio-hub-ci:${GITHUB_SHA}" . - name: Smoke installed payload, health, migration and isolated PostgreSQL run: python scripts/image_release.py smoke --image "portfolio-hub-ci:${GITHUB_SHA}" --inventory migration-inventory.txt --timeout 180 - name: Record tested image ID @@ -63,7 +63,7 @@ jobs: python scripts/image_release.py assert-clean-output canary-scan.stdout canary-scan.stderr exit "${scan_status}" - run: python scripts/image_release.py checksum --archive portfolio-hub-image.tar --output portfolio-hub-image.tar.sha256 - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: tested-image-${{ github.run_id }}-${{ github.run_attempt }} if-no-files-found: error @@ -83,7 +83,7 @@ jobs: contents: read packages: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: tested-image-${{ github.run_id }}-${{ github.run_attempt }} path: tested-image diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 16e9db0..6c1ff3a 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -14,8 +14,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" - run: python -m pip install "uv==0.11.14" @@ -25,3 +25,14 @@ jobs: run: uv run ruff check --select F,I app/db/migrations.py scripts/image_release.py tests/release - name: Validate container contracts without registry writes run: uv run pytest -q tests/release + - name: Build and smoke exact pull-request or main source image + run: | + uv run python -c 'from app.db.migrations import migration_inventory; print("\n".join(migration_inventory()))' > migration-inventory.txt + printf '%s\n' 'PORTFOLIO_HUB_SECRET_CANARY_DO_NOT_SHIP' > .env.ci-canary + RELEASE_VERSION="v$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" + docker build --platform linux/amd64 --provenance=false \ + --build-arg "SOURCE_COMMIT=${GITHUB_SHA}" --build-arg "RELEASE_VERSION=${RELEASE_VERSION}" \ + --tag "portfolio-hub-ci:${GITHUB_SHA}" . + uv run python scripts/image_release.py smoke --image "portfolio-hub-ci:${GITHUB_SHA}" --inventory migration-inventory.txt --timeout 180 + docker save --output portfolio-hub-image.tar "portfolio-hub-ci:${GITHUB_SHA}" + uv run python scripts/image_release.py scan --image "portfolio-hub-ci:${GITHUB_SHA}" --archive portfolio-hub-image.tar diff --git a/Dockerfile b/Dockerfile index a060bbd..fce6721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,28 @@ -# syntax=docker/dockerfile:1 -FROM ghcr.io/astral-sh/uv:0.11.14 AS uv -FROM python:3.11-slim AS build +# syntax=docker/dockerfile:1@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32 +FROM ghcr.io/astral-sh/uv:0.11.14@sha256:1025398289b62de8269e70c45b91ffa37c373f38118d7da036fb8bb8efc85d97 AS uv +FROM python:3.11-slim@sha256:a630a63cdb314e2d138a2fca3e375e319e8568346ffafac5b980f888630ac4f1 AS build ARG SOURCE_COMMIT +ARG RELEASE_VERSION WORKDIR /opt/portfolio-hub RUN test "${#SOURCE_COMMIT}" -eq 40 && case "${SOURCE_COMMIT}" in *[!0-9a-f]*) false;; *) true;; esac \ - || { echo "SOURCE_COMMIT must be a 40-character lowercase commit" >&2; exit 1; } + && printf '%s' "${RELEASE_VERSION}" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' \ + || { echo "SOURCE_COMMIT and strict RELEASE_VERSION=vX.Y.Z are required" >&2; exit 1; } COPY --from=uv /uv /usr/local/bin/uv COPY pyproject.toml uv.lock README.md ./ COPY app/ ./app/ RUN uv sync --locked --no-dev --no-editable -FROM python:3.11-slim AS runtime +FROM python:3.11-slim@sha256:a630a63cdb314e2d138a2fca3e375e319e8568346ffafac5b980f888630ac4f1 AS runtime ARG SOURCE_COMMIT +ARG RELEASE_VERSION LABEL org.opencontainers.image.source="https://github.com/fourblocks2020/portfolio-hub" \ - org.opencontainers.image.revision="${SOURCE_COMMIT}" + org.opencontainers.image.revision="${SOURCE_COMMIT}" \ + org.opencontainers.image.version="${RELEASE_VERSION}" RUN test "${#SOURCE_COMMIT}" -eq 40 && case "${SOURCE_COMMIT}" in *[!0-9a-f]*) false;; *) true;; esac \ + && printf '%s' "${RELEASE_VERSION}" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' \ && groupadd --gid 10001 portfoliohub \ && useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin portfoliohub \ - || { echo "SOURCE_COMMIT must be a 40-character lowercase commit" >&2; exit 1; } + || { echo "SOURCE_COMMIT and strict RELEASE_VERSION=vX.Y.Z are required" >&2; exit 1; } WORKDIR /opt/portfolio-hub COPY --from=build /opt/portfolio-hub/.venv ./.venv ENV PATH="/opt/portfolio-hub/.venv/bin:${PATH}" PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 diff --git a/app/cli/commands/health.py b/app/cli/commands/health.py index 2536386..5c43390 100644 --- a/app/cli/commands/health.py +++ b/app/cli/commands/health.py @@ -5,9 +5,13 @@ from __future__ import annotations import typer from app.cli.core.utils import print_json -from app.db.migrations import bootstrap_or_upgrade_to_head +from app.db.migrations import ( + bootstrap_or_upgrade_to_head, + current_revision, + head_revision, +) from app.repositories.factory import build_portfolio_db_config -from app.repositories.postgres import PostgresPortfolioRepository, create_database_if_missing, database_exists +from app.repositories.postgres import create_database_if_missing, database_exists def initialize_database(*, settings, maintenance_db: str = "postgres") -> dict[str, object]: @@ -90,6 +94,15 @@ def register_health_commands(app: typer.Typer) -> None: } if hasattr(repository, "ping"): repository.ping() + if app_context.settings.db_enabled: + database_revision = current_revision(app_context.settings) + expected_revision = head_revision(app_context.settings) + health["database_revision"] = database_revision + health["expected_revision"] = expected_revision + if database_revision != expected_revision: + raise RuntimeError( + f"database revision {database_revision!r} is not packaged head {expected_revision!r}" + ) except Exception as exc: app_context.console.print(f"health-check failed: {exc}") raise typer.Exit(code=1) from exc diff --git a/app/core/config.py b/app/core/config.py index f96cf0e..9fdbefb 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -18,7 +18,7 @@ if env_file.exists(): class Settings(BaseSettings): app_name: str = "PortfolioHub" - app_version: str = "0.1.0" + app_version: str = "0.1.1" environment: str = "development" allowed_origins_raw: str = Field(default="http://localhost:3000", validation_alias="PORTFOLIOHUB_ALLOWED_ORIGINS") bootstrap_username: str = Field(default="", validation_alias="PORTFOLIOHUB_BOOTSTRAP_USERNAME") diff --git a/app/db/migrations.py b/app/db/migrations.py index 88033de..fb6ba2b 100644 --- a/app/db/migrations.py +++ b/app/db/migrations.py @@ -7,6 +7,7 @@ from importlib import resources import psycopg from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory from app.core.config import Settings @@ -61,15 +62,19 @@ def bootstrap_or_upgrade_to_head(settings: Settings) -> str: def current_revision(settings: Settings) -> str | None: - output: list[str] = [] + """Read the database revision without applying or stamping migrations.""" + with _connect(settings) as conn, conn.cursor() as cur: + cur.execute("SELECT version_num FROM alembic_version") + row = cur.fetchone() + return str(row[0]) if row else None - def capture(message: str, *_args, **_kwargs) -> None: - if message.strip(): - output.append(message.strip()) - config = get_alembic_config(settings) - command.current(config, verbose=False) - return output[-1] if output else None +def head_revision(settings: Settings) -> str: + """Return the single packaged Alembic head without touching the database.""" + heads = ScriptDirectory.from_config(get_alembic_config(settings)).get_heads() + if len(heads) != 1: + raise RuntimeError(f"expected one Alembic head, found {len(heads)}") + return heads[0] def _connect(settings: Settings) -> psycopg.Connection: diff --git a/docs/container-operations.md b/docs/container-operations.md index cfe89c9..45f1684 100644 --- a/docs/container-operations.md +++ b/docs/container-operations.md @@ -1,56 +1,116 @@ -# PortfolioHub container publication and operator handoff +# PortfolioHub v0.1.1 container contract ## Publication is not deployment -Only a requester-authorized strict `vX.Y.Z` tag push may publish. The workflow builds once and gives the same tested image exactly two selectors: +Only a requester-authorized push of the strict tag `v0.1.1` can publish this revision. The workflow checks that the tag equals `[project].version`, checks out `github.sha`, runs the complete Python suite, builds one `linux/amd64` image from `uv.lock`, and tests those exact bytes before publication. It gives the same image exactly two selectors: - `ghcr.io/fourblocks2020/portfolio-hub:` -- `ghcr.io/fourblocks2020/portfolio-hub:vX.Y.Z` +- `ghcr.io/fourblocks2020/portfolio-hub:v0.1.1` -It never publishes `latest`. The registry digest is the immutable artifact identity: +It never publishes `latest`. Both selectors must report the same registry digest; deployment and rollback must use the immutable identity: ```text ghcr.io/fourblocks2020/portfolio-hub@sha256:<64-lowercase-hex-digest> ``` -Publication does **not** deploy, place, replace, start, stop, or restart a container in any environment. This repository change adds no production Compose, Kubernetes, Helm, systemd, Dockge, or other deployment configuration. An authorized deployment system and operator must consume the digest separately. +OCI labels are fail-closed build inputs: `org.opencontainers.image.source`, the full 40-character `org.opencontainers.image.revision`, and strict `org.opencontainers.image.version=v0.1.1`. Publication does **not** deploy, replace, start, stop, or restart a production service. This repository provides no production Compose, Kubernetes, Helm, systemd, or Dockge configuration. -## Operator handoff contract +The Dockerfile frontend, Python base image, uv image, and GitHub Actions are digest/commit pinned. The workflow intentionally uses `--provenance=false` to preserve the tested `docker save` archive and build-once/no-rebuild identity. Registry-attached provenance, SBOM, signing, and attestations are a nonblocking v0.1.1 supply-chain gap; they must not be added by rebuilding a different image in the publish job. -A successful publication hands the operator or deployment system only these verified inputs: +## Image runtime -- exact `vX.Y.Z` source tag; -- full source commit and OCI `org.opencontainers.image.revision`; -- immutable registry digest; -- matching commit-prefix and version selectors; -- successful tested-image workflow evidence. +- platform: `linux/amd64`; +- process: exec-form `uvicorn app.main:app --host 0.0.0.0 --port 8000` as PID 1; +- network: HTTP on fixed container port `8000` (configure only the host/service mapping, not the container port); +- identity: numeric UID/GID `10001:10001`; +- filesystem: no application data path is written; the root filesystem may be read-only; +- temporary space: a bounded `/tmp` tmpfs is safe but not required by normal application behavior; +- shutdown: send `SIGTERM`, allow at least 10 seconds for Uvicorn graceful shutdown, then escalate only under the deployment platform's policy; +- restart: application replicas may use the platform's normal failure restart policy; the one-shot migration must use restart `no` and must never loop after failure. -Before any separately authorized use, verify that the digest and OCI revision identify the intended source. The publication workflow itself does not claim runtime deployment or production readiness. +Recommended runtime restrictions are a read-only root filesystem, all capabilities dropped, `no-new-privileges`, a bounded PID limit, and no Docker socket or host filesystem mounts. No build argument, label, layer, or image environment contains an application secret. -## Required pre-start migration contract +## Environment contract -Database migration is a separate, explicit pre-start operation using the exact image digest and runtime-injected production settings: +Values are read at process start; changing any value requires a container restart. Empty optional values mean the shown default/disabled behavior. -1. execute `pfhub migrate-db` as a one-shot operation; -2. stop on nonzero migration result; -3. start the application only after migration succeeds; -4. evaluate HTTP liveness and database readiness separately. +### Production database and secrets -The image's default application command does not run migrations. Do not run migration implicitly from every application replica. Production migration execution remains outside this implementation and requires separate authorization. +| Variable | Production contract | Default | Secret | Effect | +| --- | --- | --- | --- | --- | +| `PORTFOLIOHUB_DB_ENABLED` | required; `true` for persistent production | `false` | no | Selects PostgreSQL instead of in-memory repository | +| `TIMESCALEDB_HOST` | required when DB enabled | `localhost` | no | PostgreSQL host | +| `TIMESCALEDB_PORT` | optional | `5432` | no | PostgreSQL port | +| `TIMESCALEDB_NAME` | required when DB enabled | `portfoliohub` | no | database name | +| `TIMESCALEDB_USER` | required when DB enabled | `portfoliohub` | usually sensitive | database user | +| `TIMESCALEDB_PASSWORD` | required by production DB policy | empty | **yes** | database password | +| `TIMESCALEDB_SSLMODE` | optional; set explicitly in production | `prefer` | no | psycopg SSL mode | +| `TIMESCALEDB_CONNECT_TIMEOUT_SEC` | optional | `10` | no | connection timeout | +| `TIMESCALEDB_APPLICATION_NAME` | optional | `portfoliohub` | no | PostgreSQL application name | +| `CREDENTIAL_ENCRYPTION_KEY` | required when DB enabled | empty | **yes** | Fernet key for stored broker credentials; startup fails closed if absent | -## Runtime security inputs +### Authentication, bootstrap, and HTTP behavior -The image contract provides: +| Variable | Required/default | Secret | Effect | +| --- | --- | --- | --- | +| `PORTFOLIOHUB_ALLOWED_ORIGINS` | optional; `http://localhost:3000` | no | comma-separated CORS origins; set the trex-web origin in production | +| `PORTFOLIOHUB_ROOT_PATH` | optional; empty | no | reverse-proxy ASGI root path; does not rewrite the actual route paths | +| `PORTFOLIOHUB_READ_TOKENS` | optional; empty | **yes** | comma-separated legacy read service tokens | +| `PORTFOLIOHUB_TRADE_WRITE_TOKENS` | optional; empty | **yes** | comma-separated legacy ledger-write service tokens | +| `PORTFOLIOHUB_ADMIN_TOKENS` | optional; empty | **yes** | comma-separated legacy admin service tokens | +| `PORTFOLIOHUB_SESSION_TOKEN_TTL_HOURS` | optional; `12` | no | login session lifetime | +| `PORTFOLIOHUB_CLIENT_TOKEN_DEFAULT_TTL_DAYS` | optional; `365` | no | retained client-token default | +| `PORTFOLIOHUB_BOT_TOKEN_DEFAULT_TTL_DAYS` | optional; `365` | no | bot-token default | +| `PORTFOLIOHUB_BOOTSTRAP_USERNAME` | optional; empty | no | with password, creates the first admin only when no users exist | +| `PORTFOLIOHUB_BOOTSTRAP_EMAIL` | optional; empty | personal data | bootstrap user email | +| `PORTFOLIOHUB_BOOTSTRAP_PASSWORD` | optional; empty | **yes** | bootstrap password; inject temporarily and remove after first successful start | +| `PORTFOLIOHUB_ENVIRONMENT` | optional; `development` | no | reported environment metadata | +| `PORTFOLIOHUB_APP_NAME` | optional; `PortfolioHub` | no | application/OpenAPI display name | +| `PORTFOLIOHUB_APP_VERSION` | do not override for v0.1.1 | `0.1.1` | no | health/OpenAPI version metadata | -- default UID/GID `10001:10001`; -- explicit command `uvicorn app.main:app --host 0.0.0.0 --port 8000`; -- no application-required writable data path; -- migration resources installed inside the Python package. +`REDIS_HOST`, `REDIS_PORT`, `REDIS_DB`, `REDIS_PASSWORD`, `LOKI_HOST`, `LOKI_PORT`, and `LOKI_ENABLED` are still parsed for source compatibility but the inspected v0.1.1 application does not open Redis or Loki connections. They are not production dependencies of this image. -A deployment system should enforce a read-only root filesystem, dropped capabilities, no-new-privileges, and only a bounded temporary filesystem when needed. PostgreSQL credentials, token configuration, and `CREDENTIAL_ENCRYPTION_KEY` are runtime secrets; they must never be baked into the image or publication artifact. Secret injection and writable-mount configuration belong to the separately authorized deployment environment, not this repository change. +### Legacy and sibling mapping -## Image and database rollback boundary +The existing database variable names remain `TIMESCALEDB_*`; v0.1.1 deliberately does not invent a `DATABASE_URL` alias. Existing `PORTFOLIOHUB_{READ,TRADE_WRITE,ADMIN}_TOKENS` Bearer-token behavior remains compatible. `trex-core`'s `TREX_PORTFOLIO_HUB_BASE_URL` and TOML `portfolio_hub_token`, and `trex-web`'s server-only `PORTFOLIO_HUB_API_BASE_URL`, are **caller-side** settings: point them to `http://portfolio-hub:8000` (or the equivalent service URL) and do not inject them into this container. -Rollback identity is a previously verified immutable image digest whose OCI revision is known. This document defines that identity boundary only; it does not add or execute rollback automation. +## Liveness and readiness -Image rollback is **not** database rollback. Selecting older application bytes does not reverse Alembic migrations, schema or data changes, tokens, or external side effects. Database restore, downgrade, or forward-fix requires a separately reviewed and authorized operational procedure. +- **Liveness:** unauthenticated `GET /health` must return HTTP 200 with `status=ok`. It is intentionally process-only and does not query PostgreSQL. `db_enabled` only reports configuration; it is not readiness. +- **Readiness:** run `pfhub --format json health-check` with the same runtime environment. With DB enabled it performs a repository ping, reads `alembic_version` without migration, compares it with the single packaged Alembic head, and exits nonzero on connectivity, missing schema, multiple heads, or revision mismatch. +- **API identity:** `GET /openapi.json` is stable for smoke verification. Health version is `0.1.1`. + +Do not route traffic from a static `/health` success alone. Readiness checks never apply, stamp, or repair migrations. + +## One-shot migration and 0028 -> 0029 policy + +The installed image contains exactly **29** migration files and one Alembic head, `20260814_0029`. The application default command does not invoke Alembic. Migration is a separate pre-start operation using the exact same digest and database environment: + +1. back up the database and record the current immutable image digest and `alembic_version`; +2. require the normal upgrade origin `20260715_0028`; for an older revision, review and apply the full ordered chain first; +3. stop writes or otherwise establish an operator-approved maintenance boundary; +4. execute `pfhub migrate-db` once with restart disabled; +5. stop rollout on any nonzero result and preserve sanitized logs; +6. run `pfhub --format json health-check` and require both `database_revision` and `expected_revision` to equal `20260814_0029`; +7. start application replicas and verify `/health`, `/openapi.json`, authenticated caller paths, and strategy data; +8. a repeated `pfhub migrate-db` is expected to be idempotent and remain at `0029`, but it is verification—not an application-start hook. + +Migration `0029` is additive/idempotent around the strategy catalog: it preserves existing strategy IDs and portfolio references, reuses the canonical `target_value`/`{}` strategy, and backfills catalog data. Before migration, inspect duplicate or invalid strategy references and take a backup. After migration, verify strategy listing and portfolio active-strategy fields before resuming writes. + +A pre-Alembic legacy schema is detected only when all three tables `strategy_sets`, `portfolios`, and `portfolio_ledger` exist; the CLI stamps `0027` and then runs real upgrades `0028` and `0029`. This compatibility path must be used only after backup and schema review. A partially matching schema is not auto-stamped and must fail for operator investigation. + +Image rollback is **not** database rollback. Selecting older application bytes does not reverse schema/data changes, tokens, or side effects. Database restore, downgrade, or forward-fix requires separate review and authorization; no automatic downgrade is provided. + +## API verification and known caller blockers + +v0.1.1 preserves Bearer authentication and the caller route surface observed in sibling source. The release gate runs the complete portfolio-hub source test suite (including auth success/failure, strategy catalog/data, portfolio and ledger APIs), then verifies the built image's health/OpenAPI surface, exact 29-file migration inventory, isolated PostgreSQL migration twice, readiness at `0029`, persistent-volume restart, non-root/read-only operation, secret canary absence, and graceful SIGTERM. Publication consumes the tested archive without rebuilding it. + +Route presence is **not** a positive cross-repository caller-compatibility claim. Direct source comparison found the following unresolved consumers, which the requester explicitly separated as BLOCKED follow-up rather than adding legacy shims to portfolio-hub: + +- trex-web omits required `strategy_id` when creating a portfolio, so that mutation currently returns `422`; +- trex-web can omit required `account_ref` when creating or rotating credentials, so those requests currently return `422`; +- trex-web sends daemon activation state through the general PATCH route, which can return `200` without changing activation; it must use the activate/deactivate routes; +- portfolio-hub returns singular `broker` from `/v1/user_info`, while trex-web types `brokers[]`; +- portfolio-hub represents duplicate ledger success as `status="duplicate"`, while trex-core currently checks `duplicate=true`, so the core result classification is inconsistent. + +Do not claim trex-core/trex-web compatibility or switch production callers until those consumer changes and cross-repository tests are completed. diff --git a/pyproject.toml b/pyproject.toml index 7e31f10..02e107c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "portfoliohub" -version = "0.1.0" +version = "0.1.1" description = "PortfolioHub API for portfolio management and ledger access" readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/image_release.py b/scripts/image_release.py index 614da29..22f16ba 100755 --- a/scripts/image_release.py +++ b/scripts/image_release.py @@ -13,6 +13,7 @@ import sys import tarfile import tempfile import time +import tomllib from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @@ -24,6 +25,8 @@ COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") VERSION_REF_RE = re.compile(r"^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") IMAGE_ID_RE = re.compile(r"^sha256:[0-9a-f]{64}$") DIGEST_RE = re.compile(r"(?:^|\s)digest:\s*(sha256:[0-9a-f]{64})(?:\s|$)") +EXPECTED_MIGRATION_COUNT = 29 +EXPECTED_ALEMBIC_HEAD = "20260814_0029" @dataclass(frozen=True) class Result: @@ -52,11 +55,24 @@ def validate_checkout_identity(actual: str, expected: str) -> str: raise ValueError("checked-out HEAD does not match the tagged source commit") return actual -def validate_release_event(event_name: str, ref: str, commit: str) -> str: +def project_release_version(project: Path) -> str: + value = tomllib.loads(project.read_text(encoding="utf-8"))["project"]["version"] + version = f"v{value}" + if not VERSION_REF_RE.fullmatch(f"refs/tags/{version}"): + raise ValueError("project version must be strict semantic version X.Y.Z") + return version + + +def validate_release_event( + event_name: str, ref: str, commit: str, expected_version: str | None = None +) -> str: validate_commit(commit) if event_name != "push" or not VERSION_REF_RE.fullmatch(ref): raise ValueError("publication requires a push of a strict vX.Y.Z tag") - return ref.removeprefix("refs/tags/") + version = ref.removeprefix("refs/tags/") + if expected_version is not None and version != expected_version: + raise ValueError("release tag does not match the project version") + return version def release_tags(event_name: str, ref: str, commit: str) -> tuple[str, str]: version = validate_release_event(event_name, ref, commit) @@ -92,6 +108,10 @@ def _clean(result: Result) -> Result: def inspect_value(image: str, template: str, runner: Runner) -> str: return _clean(runner.run(["docker", "image", "inspect", "--format", template, image])).stdout.strip() + +def inspect_container_value(container: str, template: str, runner: Runner) -> str: + return _clean(runner.run(["docker", "container", "inspect", "--format", template, container])).stdout.strip() + def inspect_image_id(image: str, runner: Runner) -> str: value = inspect_value(image, "{{.Id}}", runner) if not IMAGE_ID_RE.fullmatch(value): @@ -101,6 +121,10 @@ def inspect_image_id(image: str, runner: Runner) -> str: def inspect_revision(image: str, runner: Runner) -> str: return inspect_value(image, '{{ index .Config.Labels "org.opencontainers.image.revision" }}', runner) + +def inspect_version(image: str, runner: Runner) -> str: + return inspect_value(image, '{{ index .Config.Labels "org.opencontainers.image.version" }}', runner) + def write_image_id(image: str, output: Path, runner: Runner) -> str: image_id = inspect_image_id(image, runner) output.write_text(image_id + "\n", encoding="ascii") @@ -144,6 +168,10 @@ def smoke_installed_image(image: str, expected_inventory: tuple[str, ...], runne timeout: int = 30) -> None: if not expected_inventory: raise ValueError("expected migration inventory must be non-empty") + if len(expected_inventory) != EXPECTED_MIGRATION_COUNT: + raise ValueError(f"expected exactly {EXPECTED_MIGRATION_COUNT} migrations") + if not expected_inventory[-1].startswith(EXPECTED_ALEMBIC_HEAD): + raise ValueError(f"migration inventory must end at {EXPECTED_ALEMBIC_HEAD}") inventory_code = ( "import json; from app.db.migrations import migration_inventory; " f"expected={json.dumps(list(expected_inventory))}; actual=list(migration_inventory()); " @@ -173,10 +201,11 @@ def _wait(command: list[str], runner: Runner, deadline: float, description: str) raise RuntimeError(f"timed out waiting for {description}") from last def startup_smoke(image: str, runner: Runner, timeout: int = 120) -> dict: - """Run DB-disabled health, then isolated PostgreSQL migration, ping, and health.""" + """Run isolated health/OpenAPI, repeated migration, readiness, persistence and SIGTERM.""" suffix = f"{os.getpid()}-{time.time_ns()}" plain = f"portfolio-hub-plain-{suffix}"; db = f"portfolio-hub-db-{suffix}" - migrate = f"portfolio-hub-migrate-{suffix}"; app = f"portfolio-hub-app-{suffix}" + migrate = f"portfolio-hub-migrate-{suffix}"; migrate_again = f"portfolio-hub-migrate-again-{suffix}" + app = f"portfolio-hub-app-{suffix}"; app_again = f"portfolio-hub-app-again-{suffix}" network = f"portfolio-hub-smoke-{suffix}"; volume = f"portfolio-hub-db-{suffix}" names: list[str] = [] deadline = time.monotonic() + timeout @@ -185,8 +214,10 @@ def startup_smoke(image: str, runner: Runner, timeout: int = 120) -> dict: "-e", "TIMESCALEDB_PORT=5432", "-e", "TIMESCALEDB_NAME=smoke", "-e", "TIMESCALEDB_USER=smoke", "-e", "TIMESCALEDB_PASSWORD=synthetic-db-only", "-e", "TIMESCALEDB_SSLMODE=disable", "-e", "CREDENTIAL_ENCRYPTION_KEY=bnVsbC1vbmx5LXN5bnRoZXRpYy1rZXktMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwPQ=="] - http_probe = "import json,urllib.request; d=json.load(urllib.request.urlopen('http://TARGET:8000/health')); assert d['status']=='ok'; print(json.dumps(d,sort_keys=True))" + http_probe = "import json,urllib.request; d=json.load(urllib.request.urlopen('http://TARGET:8000/health')); assert d['status']=='ok'; o=json.load(urllib.request.urlopen('http://TARGET:8000/openapi.json')); required={'/auth/login','/v1/user_info','/v1/portfolios','/v1/strategies','/v1/portfolios/{portfolio_id}/ledger/trades'}; assert required <= set(o['paths']), required-set(o['paths']); print(json.dumps(d,sort_keys=True))" try: + architecture = inspect_value(image, "{{.Architecture}}", runner) + if architecture != "amd64": raise RuntimeError(f"image architecture must be amd64, got {architecture!r}") runner.run(["docker", "network", "create", "--internal", network], timeout=10) runner.run(["docker", "volume", "create", volume], timeout=10) names.append(plain) @@ -208,10 +239,19 @@ def startup_smoke(image: str, runner: Runner, timeout: int = 120) -> dict: names.append(migrate) runner.run(["docker", "run", "--name", migrate, "--network", network, *db_env, "--entrypoint", "pfhub", image, "migrate-db"], timeout=60) + names.append(migrate_again) + runner.run(["docker", "run", "--name", migrate_again, "--network", network, *db_env, + "--entrypoint", "pfhub", image, "migrate-db"], timeout=60) check = "import psycopg; c=psycopg.connect(host='postgres',dbname='smoke',user='smoke',password='synthetic-db-only'); r=c.execute(\"select version_num from alembic_version\").fetchone(); assert r and r[0]; assert c.execute('select 1').fetchone()==(1,); print(r[0])" head = _clean(runner.run(["docker", "run", "--rm", "--network", network, "--entrypoint", "python", image, "-c", check], timeout=30)).stdout.strip() - if not head: raise RuntimeError("Alembic head/DB ping returned no revision") + if head != EXPECTED_ALEMBIC_HEAD: + raise RuntimeError(f"Alembic head must be {EXPECTED_ALEMBIC_HEAD}, got {head!r}") + readiness = _clean(runner.run(["docker", "run", "--rm", "--network", network, *db_env, + "--entrypoint", "pfhub", image, "--format", "json", "health-check"], timeout=30)) + readiness_payload = json.loads(readiness.stdout) + if readiness_payload.get("database_revision") != EXPECTED_ALEMBIC_HEAD: + raise RuntimeError("readiness did not report the packaged Alembic head") names.append(app) runner.run(["docker", "run", "-d", "--name", app, "--network", network, "--network-alias", "app", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", @@ -221,9 +261,24 @@ def startup_smoke(image: str, runner: Runner, timeout: int = 120) -> dict: app_payload = json.loads(app_result.stdout) if not isinstance(app_payload, dict) or app_payload.get("db_enabled") is not True: raise RuntimeError("DB-enabled health payload has the wrong shape") + _clean(runner.run(["docker", "stop", "--time", "10", app], timeout=20)) + exit_code = inspect_container_value(app, "{{.State.ExitCode}}", runner) + if exit_code != "0": raise RuntimeError(f"SIGTERM shutdown exit code was {exit_code!r}") + names.append(app_again) + runner.run(["docker", "run", "-d", "--name", app_again, "--network", network, + "--network-alias", "app-again", "--read-only", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", + *db_env, image], timeout=20) + repeated_result = _wait(["docker", "run", "--rm", "--network", network, "--entrypoint", "python", + image, "-c", http_probe.replace("TARGET", "app-again")], runner, deadline, + "restarted DB-enabled /health and OpenAPI") + if json.loads(repeated_result.stdout).get("db_enabled") is not True: + raise RuntimeError("persistent restart health payload has the wrong shape") user = inspect_value(image, "{{.Config.User}}", runner) if user not in {"10001", "10001:10001"}: raise RuntimeError("image default user must be 10001") - result = {"plain_health": "ok", "db_health": "ok", "db_ping": "ok", "alembic_head": head} + result = {"architecture": architecture, "plain_health": "ok", "db_health": "ok", + "db_readiness": "ok", "db_ping": "ok", "alembic_head": head, + "repeated_migration": "ok", "persistent_restart": "ok", "sigterm": "graceful"} except BaseException as error: failure = error try: _cleanup(names, network, volume, runner, 15) except BaseException as error: @@ -304,6 +359,9 @@ def publish_image(*, archive: Path, checksum: Path, image_id: Path, local_image: actual_id = inspect_image_id(local_image, runner) if actual_id != expected_id: raise ValueError("loaded image ID does not match tested image ID") if inspect_revision(local_image, runner) != commit: raise ValueError("loaded image revision does not match tagged source commit") + version = validate_release_event(event_name, ref, commit) + if inspect_version(local_image, runner) != version: + raise ValueError("loaded image version label does not match release tag") if not actor or not token: raise ValueError("registry credentials are missing") for tag in tags: runner.run(["docker", "tag", local_image, tag]) runner.run(["docker", "login", "ghcr.io", "--username", actor, "--password-stdin"], input_text=token) @@ -320,7 +378,7 @@ def publish_image(*, archive: Path, checksum: Path, image_id: Path, local_image: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(); commands = parser.add_subparsers(dest="command", required=True) - event = commands.add_parser("validate-event"); event.add_argument("--event", required=True); event.add_argument("--ref", required=True); event.add_argument("--commit", required=True) + event = commands.add_parser("validate-event"); event.add_argument("--event", required=True); event.add_argument("--ref", required=True); event.add_argument("--commit", required=True); event.add_argument("--project", type=Path) checkout = commands.add_parser("validate-checkout"); checkout.add_argument("--actual", required=True); checkout.add_argument("--expected", required=True) smoke = commands.add_parser("smoke"); smoke.add_argument("--image", required=True); smoke.add_argument("--inventory", type=Path, required=True); smoke.add_argument("--timeout", type=int, default=120) scan = commands.add_parser("scan"); scan.add_argument("--image", required=True); scan.add_argument("--archive", type=Path, required=True) @@ -334,7 +392,9 @@ def _parser() -> argparse.ArgumentParser: def main(argv: Sequence[str] | None = None) -> int: args = _parser().parse_args(argv); runner = SubprocessRunner() - if args.command == "validate-event": validate_release_event(args.event, args.ref, args.commit) + if args.command == "validate-event": + expected = project_release_version(args.project) if args.project else None + validate_release_event(args.event, args.ref, args.commit, expected) elif args.command == "validate-checkout": validate_checkout_identity(args.actual, args.expected) elif args.command == "smoke": inventory = tuple(line for line in args.inventory.read_text().splitlines() if line) diff --git a/tests/release/test_image_release.py b/tests/release/test_image_release.py index 056426d..f8a6f9f 100644 --- a/tests/release/test_image_release.py +++ b/tests/release/test_image_release.py @@ -24,13 +24,17 @@ class RecordingRunner: if command[:2] == ["docker", "load"]: return image_release.Result(f"Loaded image: {LOCAL}\n") if command[:3] == ["docker", "image", "inspect"] and "{{.Id}}" in command: return image_release.Result(IMAGE_ID + "\n") if command[:3] == ["docker", "image", "inspect"] and "{{.Config.User}}" in command: return image_release.Result("10001:10001\n") + if command[:3] == ["docker", "image", "inspect"] and "{{.Architecture}}" in command: return image_release.Result("amd64\n") + if command[:3] == ["docker", "container", "inspect"] and "{{.State.ExitCode}}" in command: return image_release.Result("0\n") + if command[:3] == ["docker", "image", "inspect"] and any("org.opencontainers.image.version" in part for part in command): return image_release.Result("v1.2.3\n") if command[:3] == ["docker", "image", "inspect"]: return image_release.Result(COMMIT + "\n") if command[:2] == ["docker", "push"]: self.push_count += 1; return image_release.Result(f"digest: {DIGEST} size: 1\n") if command[:2] == ["docker", "create"]: return image_release.Result("created-container\n") if command[:3] == ["docker", "run", "--rm"] and "urllib.request" in command[-1]: enabled = "TARGET:8000" not in command[-1] and "plain" not in command[-1] return image_release.Result('{"status":"ok","db_enabled":' + str(enabled).lower() + '}') - if command[:3] == ["docker", "run", "--rm"] and "alembic_version" in command[-1]: return image_release.Result("20260715_0028\n") + if command[:3] == ["docker", "run", "--rm"] and "alembic_version" in command[-1]: return image_release.Result("20260814_0029\n") + if command[:3] == ["docker", "run", "--rm"] and "health-check" in command: return image_release.Result('{"database_revision":"20260814_0029"}\n') return image_release.Result() def artifacts(tmp_path: Path): @@ -58,11 +62,24 @@ def test_checkout_identity_is_exact(): assert image_release.validate_checkout_identity(COMMIT, COMMIT) == COMMIT with pytest.raises(ValueError): image_release.validate_checkout_identity("d"*40, COMMIT) + +def test_project_version_must_match_release_tag(tmp_path): + project = tmp_path / "pyproject.toml" + project.write_text('[project]\nversion = "0.1.1"\n') + assert image_release.project_release_version(project) == "v0.1.1" + assert image_release.validate_release_event("push", "refs/tags/v0.1.1", COMMIT, "v0.1.1") == "v0.1.1" + with pytest.raises(ValueError, match="project version"): + image_release.validate_release_event("push", "refs/tags/v0.1.2", COMMIT, "v0.1.1") + def test_installed_smoke_has_import_cli_exact_inventory_and_cleanup(): - runner = RecordingRunner(); image_release.smoke_installed_image("candidate", ("001.py",), runner, 17) + inventory = tuple( + [f"202601{i:02d}_{i:04d}_migration.py" for i in range(1, 29)] + + ["20260814_0029_strategy_catalog_v1.py"] + ) + runner = RecordingRunner(); image_release.smoke_installed_image("candidate", inventory, runner, 17) runs = [x[0] for x in runner.commands if x[0][:2] == ["docker","run"]] assert len(runs) == 3 and all("--read-only" in x and "none" in x for x in runs) - assert "migration_inventory" in runs[-1][-1] and "001.py" in runs[-1][-1] + assert "migration_inventory" in runs[-1][-1] and "20260814_0029" in runs[-1][-1] assert len([x for x in runner.commands if x[0][:3] == ["docker","rm","--force"]]) == 3 def _assert_exact_startup_cleanup(commands: list[list[str]]) -> None: diff --git a/tests/release/test_release_contracts.py b/tests/release/test_release_contracts.py index 3708386..d52fd78 100644 --- a/tests/release/test_release_contracts.py +++ b/tests/release/test_release_contracts.py @@ -24,15 +24,44 @@ def test_tag_publication_is_strict_single_build_handoff_without_latest(): assert "ref: ${{ github.sha }}" in text and text.count("docker build ")==1 assert "docker build" not in publish and "packages: write" not in verify and "packages: write" in publish assert "tested-image/scripts/image_release.py publish" in publish and ":latest" not in text + assert "--platform linux/amd64" in verify and '--build-arg "RELEASE_VERSION=${GITHUB_REF_NAME}"' in verify for name in ("portfolio-hub-image.tar","portfolio-hub-image.tar.sha256","portfolio-hub-image.id","migration-inventory.txt"): assert name in text def test_dockerfile_python311_locked_non_root_explicit_uvicorn_and_resources(): text=(REPO/"Dockerfile").read_text() - assert text.count("FROM python:3.11-slim")==2 and "uv sync --locked --no-dev --no-editable" in text + assert text.count("FROM python:3.11-slim@sha256:")==2 and "uv sync --locked --no-dev --no-editable" in text + assert "FROM ghcr.io/astral-sh/uv:0.11.14@sha256:" in text + assert text.startswith("# syntax=docker/dockerfile:1@sha256:") assert "USER 10001:10001" in text and 'CMD ["uvicorn", "app.main:app"' in text assert "COPY app/ ./app/" in text and "COPY ." not in text assert "org.opencontainers.image.source" in text and "org.opencontainers.image.revision" in text + assert "org.opencontainers.image.version" in text and "ARG RELEASE_VERSION" in text + + +def test_workflow_actions_are_commit_pinned_and_verify_version_is_project_derived(): + for workflow in (PUBLISH, VERIFY): + for line in workflow.read_text().splitlines(): + if "uses: actions/" in line: + revision = line.split("@", 1)[1].split()[0] + assert len(revision) == 40 and all(char in "0123456789abcdef" for char in revision) + verify = VERIFY.read_text() + assert "tomllib" in verify and 'RELEASE_VERSION=${RELEASE_VERSION}' in verify + assert 'RELEASE_VERSION=v0.1.1' not in verify + + +def test_release_version_and_migration_contract_are_v011_head_0029(): + project = (REPO / "pyproject.toml").read_text() + config = (REPO / "app/core/config.py").read_text() + migrations = sorted( + path.name + for path in (REPO / "app/db/alembic/versions").glob("*.py") + if path.name != "__init__.py" + ) + assert 'version = "0.1.1"' in project + assert 'app_version: str = "0.1.1"' in config + assert len(migrations) == 29 + assert migrations[-1].startswith("20260814_0029") def test_dockerignore_is_deny_by_default_required_inputs_only(): lines=(REPO/".dockerignore").read_text().splitlines(); assert lines[0]=="**" diff --git a/tests/test_cli.py b/tests/test_cli.py index 69b1cd6..a68f88b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -783,14 +783,29 @@ def test_cli_write_command_can_abort() -> None: assert repo._portfolio is None -def test_cli_health_check_reports_repository() -> None: +def test_cli_health_check_reports_repository(monkeypatch) -> None: repo = FakeRepo(_portfolio()) runner = build_runner(repo) + monkeypatch.setattr(health_commands, "current_revision", lambda _settings: "20260814_0029") + monkeypatch.setattr(health_commands, "head_revision", lambda _settings: "20260814_0029") result = runner.invoke(runner.app, ["health-check"]) assert result.exit_code == 0 assert "repository: FakeRepo" in result.stdout + assert "database_revision: 20260814_0029" in result.stdout + + +def test_cli_health_check_rejects_database_behind_packaged_head(monkeypatch) -> None: + repo = FakeRepo(_portfolio()) + runner = build_runner(repo) + monkeypatch.setattr(health_commands, "current_revision", lambda _settings: "20260715_0028") + monkeypatch.setattr(health_commands, "head_revision", lambda _settings: "20260814_0029") + + result = runner.invoke(runner.app, ["health-check"]) + + assert result.exit_code == 1 + assert "is not packaged head" in result.stdout def test_cli_init_db_reports_created(monkeypatch) -> None: diff --git a/tests/test_strategy_catalog_migration.py b/tests/test_strategy_catalog_migration.py index 542320f..73c4260 100644 --- a/tests/test_strategy_catalog_migration.py +++ b/tests/test_strategy_catalog_migration.py @@ -10,7 +10,7 @@ from unittest.mock import patch _MIGRATION_PATH = ( Path(__file__).parents[1] - / "migrations/versions/20260814_0029_strategy_catalog_v1.py" + / "app/db/alembic/versions/20260814_0029_strategy_catalog_v1.py" ) diff --git a/uv.lock b/uv.lock index bd8f59d..a5a01d4 100644 --- a/uv.lock +++ b/uv.lock @@ -592,7 +592,7 @@ wheels = [ [[package]] name = "portfoliohub" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "alembic" },