fix(migrations): harden BAT-025 PostgreSQL verification
This commit is contained in:
Hermes Agent
@@ -1,133 +1,273 @@
|
||||
"""BAT-025: portfolio_ledger manual metadata migration behavior tests.
|
||||
"""Real PostgreSQL integration tests for BAT-025 / REQ-057.
|
||||
|
||||
Focus areas:
|
||||
- fresh schema: add required columns safely
|
||||
- runtime-precreated schema: source backfill + source constraints 보정
|
||||
- forward-only downgrade policy
|
||||
These tests deliberately require an explicit local maintenance DSN and create a
|
||||
unique throwaway database per test. They reject non-loopback hosts and any
|
||||
non-``postgres`` maintenance database so test execution cannot target a shared
|
||||
or production application database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import unquote, urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
import importlib.util
|
||||
import psycopg
|
||||
import pytest
|
||||
from psycopg import sql
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.db.migrations import (
|
||||
bootstrap_or_upgrade_to_head,
|
||||
get_alembic_config,
|
||||
upgrade_to_head,
|
||||
)
|
||||
from app.repositories.postgres import PortfolioDBConfig, PostgresPortfolioRepository
|
||||
|
||||
_LOCAL_DSN_ENV = "BAT025_LOCAL_POSTGRES_DSN"
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"}
|
||||
_HEAD = "20260715_0028"
|
||||
_PRE_BAT025 = "20260628_0027"
|
||||
|
||||
|
||||
_MIGRATION_PATH = Path(__file__).resolve().parents[1] / "migrations" / "versions" / "20260715_0028_portfolio_ledger_manual_metadata.py"
|
||||
_MIGRATION_NAME = "m_20260715_0028_portfolio_ledger_manual_metadata"
|
||||
_spec = importlib.util.spec_from_file_location(_MIGRATION_NAME, _MIGRATION_PATH)
|
||||
assert _spec is not None and _spec.loader is not None
|
||||
_migration = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_migration)
|
||||
def _maintenance_dsn() -> str:
|
||||
dsn = os.environ.get(_LOCAL_DSN_ENV)
|
||||
if not dsn:
|
||||
pytest.skip(f"{_LOCAL_DSN_ENV} is required for real PostgreSQL integration tests")
|
||||
parsed = urlparse(dsn)
|
||||
if parsed.scheme not in {"postgres", "postgresql"}:
|
||||
pytest.fail("BAT-025 test DSN must use postgres/postgresql")
|
||||
if parsed.hostname not in _LOOPBACK_HOSTS:
|
||||
pytest.fail("BAT-025 tests refuse a non-loopback PostgreSQL host")
|
||||
if parsed.path.lstrip("/") != "postgres":
|
||||
pytest.fail("BAT-025 test DSN must connect to local maintenance database 'postgres'")
|
||||
if not parsed.username:
|
||||
pytest.fail("BAT-025 test DSN must include a local database user")
|
||||
return dsn
|
||||
|
||||
|
||||
class _FakeInspector:
|
||||
def __init__(self, columns: list[str]) -> None:
|
||||
self._columns = columns
|
||||
|
||||
def get_table_names(self) -> list[str]:
|
||||
return ["portfolio_ledger"]
|
||||
|
||||
def get_columns(self, table_name: str) -> list[dict[str, Any]]:
|
||||
assert table_name == "portfolio_ledger"
|
||||
return [{"name": name} for name in self._columns]
|
||||
def _settings_for_database(maintenance_dsn: str, database: str) -> Settings:
|
||||
parsed = urlparse(maintenance_dsn)
|
||||
return Settings(
|
||||
db_enabled=True,
|
||||
timescaledb_host=parsed.hostname,
|
||||
timescaledb_port=parsed.port or 5432,
|
||||
timescaledb_name=database,
|
||||
timescaledb_user=unquote(parsed.username or ""),
|
||||
timescaledb_password=unquote(parsed.password or ""),
|
||||
timescaledb_sslmode="disable",
|
||||
timescaledb_connect_timeout_sec=3,
|
||||
timescaledb_application_name="bat025-real-postgres-test",
|
||||
)
|
||||
|
||||
|
||||
class _MissingLedgerInspector(_FakeInspector):
|
||||
def get_table_names(self) -> list[str]:
|
||||
return ["other_table"]
|
||||
@pytest.fixture()
|
||||
def isolated_settings() -> Settings:
|
||||
maintenance_dsn = _maintenance_dsn()
|
||||
database = f"portfolio_hub_bat025_{uuid4().hex}"
|
||||
with psycopg.connect(maintenance_dsn, autocommit=True) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
|
||||
try:
|
||||
yield _settings_for_database(maintenance_dsn, database)
|
||||
finally:
|
||||
with psycopg.connect(maintenance_dsn, autocommit=True) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE datname = %s AND pid <> pg_backend_pid()",
|
||||
(database,),
|
||||
)
|
||||
cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(database)))
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self, scalar_value: Any = None) -> None:
|
||||
self._scalar_value = scalar_value
|
||||
|
||||
def scalar(self) -> Any:
|
||||
return self._scalar_value
|
||||
def _repo(settings: Settings) -> PostgresPortfolioRepository:
|
||||
return PostgresPortfolioRepository(
|
||||
PortfolioDBConfig(
|
||||
host=settings.active_db_host,
|
||||
port=settings.active_db_port,
|
||||
dbname=settings.active_db_name,
|
||||
user=settings.active_db_user,
|
||||
password=settings.active_db_password,
|
||||
sslmode=settings.active_db_sslmode,
|
||||
connect_timeout_sec=settings.active_db_connect_timeout_sec,
|
||||
application_name=settings.active_db_application_name,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _FakeBind:
|
||||
def __init__(self) -> None:
|
||||
self.queries: list[str] = []
|
||||
|
||||
def execute(self, statement: object, *args: object, **kwargs: object) -> _FakeResult:
|
||||
self.queries.append(str(statement).strip())
|
||||
return _FakeResult(0)
|
||||
def _current_revision(settings: Settings) -> str:
|
||||
with psycopg.connect(
|
||||
host=settings.active_db_host,
|
||||
port=settings.active_db_port,
|
||||
dbname=settings.active_db_name,
|
||||
user=settings.active_db_user,
|
||||
password=settings.active_db_password,
|
||||
sslmode=settings.active_db_sslmode,
|
||||
) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT version_num FROM alembic_version")
|
||||
return str(cur.fetchone()[0])
|
||||
|
||||
|
||||
def _sql_contains(queries: list[str], needle: str) -> bool:
|
||||
return any(needle in q for q in queries)
|
||||
def _metadata_columns(settings: Settings) -> dict[str, tuple[str, str, str | None]]:
|
||||
with psycopg.connect(
|
||||
host=settings.active_db_host,
|
||||
port=settings.active_db_port,
|
||||
dbname=settings.active_db_name,
|
||||
user=settings.active_db_user,
|
||||
password=settings.active_db_password,
|
||||
sslmode=settings.active_db_sslmode,
|
||||
) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT column_name, data_type, is_nullable, column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'portfolio_ledger'
|
||||
AND column_name IN ('source', 'target_ledger_entry_id',
|
||||
'confirmation_evidence', 'actor_user_id')
|
||||
"""
|
||||
)
|
||||
return {row[0]: (row[1], row[2], row[3]) for row in cur.fetchall()}
|
||||
|
||||
|
||||
def test_upgrade_adds_all_columns_on_fresh_portfolio_ledger() -> None:
|
||||
fake_bind = _FakeBind()
|
||||
fake_inspector = _FakeInspector(columns=[])
|
||||
|
||||
with (
|
||||
patch.object(_migration.op, "get_bind", return_value=fake_bind),
|
||||
patch.object(_migration, "inspect", return_value=fake_inspector),
|
||||
patch.object(_migration.op, "add_column") as add_column,
|
||||
patch.object(_migration.op, "alter_column") as alter_column,
|
||||
):
|
||||
_migration.upgrade()
|
||||
|
||||
assert add_column.call_count == 4
|
||||
called = [call.args[1].name for call in add_column.call_args_list]
|
||||
assert called == [
|
||||
"source",
|
||||
"target_ledger_entry_id",
|
||||
"confirmation_evidence",
|
||||
"actor_user_id",
|
||||
]
|
||||
alter_column.assert_not_called()
|
||||
def _seed_portfolio(repo: PostgresPortfolioRepository, suffix: str) -> str:
|
||||
portfolio_id = f"bat025-{suffix}"
|
||||
user_id = f"bat025-user-{suffix}"
|
||||
symbol = f"BAT025-{suffix}"
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO users (id, username) VALUES (%s, %s)",
|
||||
(user_id, user_id),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO assets (symbol, markets) VALUES (%s, ARRAY[]::text[]) RETURNING asset_id",
|
||||
(symbol,),
|
||||
)
|
||||
asset_id = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO portfolios (
|
||||
portfolio_id, name, owner_user_id, cash_balance, asset_quantity, asset_price,
|
||||
profit_buffer, asset_id, broker, status
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(portfolio_id, portfolio_id, user_id, 1_000_000, 0, 0, 0, asset_id, "bat025", "enabled"),
|
||||
)
|
||||
return portfolio_id
|
||||
|
||||
|
||||
def test_upgrade_preserves_data_and_enforces_source_when_precreated() -> None:
|
||||
fake_bind = _FakeBind()
|
||||
fake_inspector = _FakeInspector(columns=[
|
||||
"source",
|
||||
"target_ledger_entry_id",
|
||||
"confirmation_evidence",
|
||||
"actor_user_id",
|
||||
])
|
||||
def test_upgrade_from_pre_0028_preserves_business_data_and_backfills_source(
|
||||
isolated_settings: Settings,
|
||||
) -> None:
|
||||
"""A real 0027 -> head upgrade preserves a representative ledger row."""
|
||||
from alembic import command
|
||||
|
||||
with (
|
||||
patch.object(_migration.op, "get_bind", return_value=fake_bind),
|
||||
patch.object(_migration, "inspect", return_value=fake_inspector),
|
||||
patch.object(_migration.op, "add_column") as add_column,
|
||||
patch.object(_migration.op, "alter_column") as alter_column,
|
||||
):
|
||||
_migration.upgrade()
|
||||
command.upgrade(get_alembic_config(isolated_settings), _PRE_BAT025)
|
||||
repo = _repo(isolated_settings)
|
||||
portfolio_id = _seed_portfolio(repo, "preserve")
|
||||
recorded_at = datetime(2026, 7, 15, tzinfo=timezone.utc)
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO portfolio_ledger (
|
||||
portfolio_id, recorded_at, side, price, quantity, fee_amount, tax_amount,
|
||||
cash_balance_after, asset_quantity_after, total_value_after,
|
||||
profit_buffer_after, strategy_set_id, note, executed_at
|
||||
) VALUES (%s, %s, 'buy', 123.45, 7, 1, 0, 998_000, 7, 998_864.15, 0,
|
||||
'business-strategy', 'preserve-me', %s)
|
||||
""",
|
||||
(portfolio_id, recorded_at, recorded_at),
|
||||
)
|
||||
cur.execute("SELECT COUNT(*) FROM portfolio_ledger")
|
||||
before_count = cur.fetchone()[0]
|
||||
|
||||
add_column.assert_not_called()
|
||||
alter_column.assert_called_once()
|
||||
alter_kwargs = alter_column.call_args.kwargs
|
||||
assert str(alter_kwargs["existing_type"]).upper().startswith("TEXT")
|
||||
assert alter_kwargs["server_default"].text == "'broker'"
|
||||
assert alter_kwargs["nullable"] is False
|
||||
assert _sql_contains(fake_bind.queries, "UPDATE portfolio_ledger SET source = 'broker' WHERE source IS NULL")
|
||||
upgrade_to_head(isolated_settings)
|
||||
|
||||
assert _current_revision(isolated_settings) == _HEAD
|
||||
assert _metadata_columns(isolated_settings) == {
|
||||
"source": ("text", "NO", "'broker'::text"),
|
||||
"target_ledger_entry_id": ("bigint", "YES", None),
|
||||
"confirmation_evidence": ("text", "YES", None),
|
||||
"actor_user_id": ("text", "YES", None),
|
||||
}
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT COUNT(*), side, price::text, quantity::text, note, source "
|
||||
"FROM portfolio_ledger GROUP BY side, price, quantity, note, source"
|
||||
)
|
||||
count, side, price, quantity, note, source = cur.fetchone()
|
||||
assert count == before_count
|
||||
assert (side, price, quantity, note, source) == ("buy", "123.45000000", "7.00000000", "preserve-me", "broker")
|
||||
repo.close()
|
||||
|
||||
|
||||
def test_upgrade_noop_when_portfolio_ledger_missing() -> None:
|
||||
fake_bind = _FakeBind()
|
||||
fake_inspector = _MissingLedgerInspector(columns=[])
|
||||
def test_runtime_precreated_schema_with_null_source_converges_without_duplicate_ddl(
|
||||
isolated_settings: Settings,
|
||||
) -> None:
|
||||
"""ensure_schema-first DB stamps 0027 then truly executes 0028 to head."""
|
||||
repo = _repo(isolated_settings)
|
||||
repo.ensure_schema()
|
||||
portfolio_id = _seed_portfolio(repo, "runtime")
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute("ALTER TABLE portfolio_ledger ALTER COLUMN source DROP NOT NULL")
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO portfolio_ledger (
|
||||
portfolio_id, side, price, quantity, fee_amount, tax_amount,
|
||||
cash_balance_after, asset_quantity_after, total_value_after,
|
||||
profit_buffer_after, strategy_set_id, note, source
|
||||
) VALUES (%s, 'buy', 10, 1, 0, 0, 990, 1, 1000, 0, 'runtime', 'null-source', NULL)
|
||||
""",
|
||||
(portfolio_id,),
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(_migration.op, "get_bind", return_value=fake_bind),
|
||||
patch.object(_migration, "inspect", return_value=fake_inspector),
|
||||
patch.object(_migration.op, "add_column") as add_column,
|
||||
patch.object(_migration.op, "alter_column") as alter_column,
|
||||
):
|
||||
_migration.upgrade()
|
||||
|
||||
add_column.assert_not_called()
|
||||
alter_column.assert_not_called()
|
||||
assert fake_bind.queries == []
|
||||
assert bootstrap_or_upgrade_to_head(isolated_settings) == "head"
|
||||
assert _current_revision(isolated_settings) == _HEAD
|
||||
assert _metadata_columns(isolated_settings)["source"] == ("text", "NO", "'broker'::text")
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute("SELECT source FROM portfolio_ledger WHERE portfolio_id = %s", (portfolio_id,))
|
||||
assert cur.fetchone()[0] == "broker"
|
||||
repo.close()
|
||||
|
||||
|
||||
def test_downgrade_is_forward_only_noop() -> None:
|
||||
with patch.object(_migration.op, "get_bind", return_value=MagicMock()):
|
||||
result = _migration.downgrade()
|
||||
assert result is None
|
||||
def test_broker_and_manual_ledger_writes_regress_after_real_upgrade(
|
||||
isolated_settings: Settings,
|
||||
) -> None:
|
||||
"""Synthetic broker and manual writes use the migrated columns on real PostgreSQL."""
|
||||
upgrade_to_head(isolated_settings)
|
||||
repo = _repo(isolated_settings)
|
||||
portfolio_id = _seed_portfolio(repo, "writes")
|
||||
|
||||
broker, created = repo.insert_ledger_trade(
|
||||
portfolio_id=portfolio_id,
|
||||
order_id="bat025-order",
|
||||
execution_id="bat025-execution",
|
||||
side="buy",
|
||||
quantity=2,
|
||||
price=100,
|
||||
executed_at=datetime.now(timezone.utc),
|
||||
)
|
||||
manual = repo.record_manual_ledger(
|
||||
portfolio_id,
|
||||
action="INVEST",
|
||||
amount=50,
|
||||
actor_user_id="bat025-user",
|
||||
confirmation_evidence="synthetic-local-fixture",
|
||||
)
|
||||
|
||||
assert created is True
|
||||
assert broker.source == "broker"
|
||||
assert manual.source == "manual"
|
||||
assert manual.actor_user_id == "bat025-user"
|
||||
assert manual.confirmation_evidence == "synthetic-local-fixture"
|
||||
with repo._connect().cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT source, actor_user_id, confirmation_evidence FROM portfolio_ledger "
|
||||
"WHERE ledger_entry_id = %s",
|
||||
(manual.ledger_entry_id,),
|
||||
)
|
||||
assert cur.fetchone() == ("manual", "bat025-user", "synthetic-local-fixture")
|
||||
repo.close()
|
||||
|
||||
Reference in New Issue
Block a user