trex/portfolio-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve ledger venue provenance (#39)

Browse Source 
* feat: preserve ledger venue provenance

* fix: satisfy ledger test lint ratchet

* fix: advance image migration inventory
This commit is contained in:
genichin
2026-09-19 19:30:32 +09:00
committed by GitHub
parent 567d662913
commit 996127f215
23 changed files with 768 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/release/test_image_release.py
+8 -4
View File
@@ -33,8 +33,8 @@ class RecordingRunner:
if command[:3] == ["docker", "run", "--rm"] and "urllib.request" in command[-1]:
enabled = "TARGET:8000" not in command[-1] and "plain" not in command[-1]
return image_release.Result('{"status":"ok","db_enabled":' + str(enabled).lower() + '}')
if command[:3] == ["docker", "run", "--rm"] and "alembic_version" in command[-1]: return image_release.Result("20260913_0031\n")
if command[:3] == ["docker", "run", "--rm"] and "health-check" in command: return image_release.Result('{"database_revision":"20260913_0031"}\n')
if command[:3] == ["docker", "run", "--rm"] and "alembic_version" in command[-1]: return image_release.Result("20260919_0032\n")
if command[:3] == ["docker", "run", "--rm"] and "health-check" in command: return image_release.Result('{"database_revision":"20260919_0032"}\n')
return image_release.Result()
def artifacts(tmp_path: Path):
@@ -74,12 +74,16 @@ def test_project_version_must_match_release_tag(tmp_path):
def test_installed_smoke_has_import_cli_exact_inventory_and_cleanup():
inventory = tuple(
[f"202601{i:02d}_{i:04d}_migration.py" for i in range(1, 30)]
+ ["20260820_0030_portfolio_ledger_keyset_index.py", "20260913_0031_corporate_actions.py"]
+ [
"20260820_0030_portfolio_ledger_keyset_index.py",
"20260913_0031_corporate_actions.py",
"20260919_0032_ledger_venue_provenance.py",
]
)
runner = RecordingRunner(); image_release.smoke_installed_image("candidate", inventory, runner, 17)
runs = [x[0] for x in runner.commands if x[0][:2] == ["docker","run"]]
assert len(runs) == 3 and all("--read-only" in x and "none" in x for x in runs)
assert "migration_inventory" in runs[-1][-1] and "20260913_0031" in runs[-1][-1]
assert "migration_inventory" in runs[-1][-1] and "20260919_0032" in runs[-1][-1]
assert len([x for x in runner.commands if x[0][:3] == ["docker","rm","--force"]]) == 3
def _assert_exact_startup_cleanup(commands: list[list[str]]) -> None:
tests/release/test_release_contracts.py
+3 -3
View File
@@ -54,7 +54,7 @@ def test_workflow_actions_are_commit_pinned_and_verify_version_is_project_derive
assert 'RELEASE_VERSION=v0.1.3' not in verify
def test_release_version_and_migration_contract_has_current_head_0031():
def test_release_version_and_migration_contract_has_current_head_0032():
project = (REPO / "pyproject.toml").read_text()
config = (REPO / "app/core/config.py").read_text()
migrations = sorted(
@@ -64,8 +64,8 @@ def test_release_version_and_migration_contract_has_current_head_0031():
)
assert 'version = "0.3.4"' in project
assert 'app_version: str = "0.3.4"' in config
assert len(migrations) == 31
assert migrations[-1].startswith("20260913_0031_")
assert len(migrations) == 32
assert migrations[-1].startswith("20260919_0032_")
def test_dockerignore_is_deny_by_default_required_inputs_only():
lines=(REPO/".dockerignore").read_text().splitlines(); assert lines[0]=="**"
tests/test_bat025_portfolio_ledger_manual_metadata_migration.py
+89 -4
View File
@@ -34,7 +34,7 @@ from app.repositories.postgres import PortfolioDBConfig, PostgresPortfolioReposi
_LOCAL_DSN_ENV = "BAT025_LOCAL_POSTGRES_DSN"
_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"}
_HEAD = "20260715_0028"
_HEAD = "20260919_0032"
_PRE_BAT025 = "20260628_0027"
@@ -138,6 +138,26 @@ def _metadata_columns(settings: Settings) -> dict[str, tuple[str, str, str | Non
return {row[0]: (row[1], row[2], row[3]) for row in cur.fetchall()}
def _venue_columns(settings: Settings) -> dict[str, tuple[str, str, str | None]]:
with psycopg.connect(
host=settings.active_db_host,
port=settings.active_db_port,
dbname=settings.active_db_name,
user=settings.active_db_user,
password=settings.active_db_password,
sslmode=settings.active_db_sslmode,
) as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT column_name, data_type, is_nullable, column_default
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'portfolio_ledger'
AND column_name IN ('execution_venue', 'order_venue')
"""
)
return {row[0]: (row[1], row[2], row[3]) for row in cur.fetchall()}
def _seed_portfolio(repo: PostgresPortfolioRepository, suffix: str) -> str:
portfolio_id = f"bat025-{suffix}"
user_id = f"bat025-user-{suffix}"
@@ -215,14 +235,79 @@ def test_upgrade_from_pre_0028_preserves_business_data_and_backfills_source(
"confirmation_evidence": ("text", "YES", None),
"actor_user_id": ("text", "YES", None),
}
assert _venue_columns(isolated_settings) == {
"execution_venue": ("text", "YES", None),
"order_venue": ("text", "YES", None),
}
with repo._connect().cursor() as cur:
cur.execute(
"SELECT COUNT(*), side, price::text, quantity::text, note, source "
"FROM portfolio_ledger GROUP BY side, price, quantity, note, source"
"SELECT COUNT(*), side, price::text, quantity::text, note, source, "
"execution_venue, order_venue FROM portfolio_ledger "
"GROUP BY side, price, quantity, note, source, execution_venue, order_venue"
)
count, side, price, quantity, note, source = cur.fetchone()
row = cur.fetchone()
assert row is not None
count, side, price, quantity, note, source, execution_venue, order_venue = row
assert count == before_count
assert (side, price, quantity, note, source) == ("buy", "123.45000000", "7.00000000", "preserve-me", "broker")
assert (execution_venue, order_venue) == (None, None)
with pytest.raises(psycopg.errors.CheckViolation), repo._connect().cursor() as cur:
cur.execute(
"UPDATE portfolio_ledger SET execution_venue = 'SOR' WHERE portfolio_id = %s",
(portfolio_id,),
)
upgrade_to_head(isolated_settings)
assert _current_revision(isolated_settings) == _HEAD
repo.close()
def test_venue_migration_downgrade_removes_only_new_columns(
isolated_settings: Settings,
) -> None:
from alembic import command
upgrade_to_head(isolated_settings)
repo = _repo(isolated_settings)
portfolio_id = _seed_portfolio(repo, "venue-downgrade")
recorded_at = datetime(2026, 9, 19, tzinfo=UTC)
with repo._connect().cursor() as cur:
cur.execute(
"""
INSERT INTO portfolio_ledger (
portfolio_id, recorded_at, side, price, quantity,
cash_balance_after, asset_quantity_after, total_value_after,
strategy_set_id, note, executed_at,
execution_venue, order_venue
) VALUES (%s, %s, 'buy', 100, 1, 900, 1, 1000,
'downgrade', 'preserve-me', %s, 'NXT', 'SOR')
""",
(portfolio_id, recorded_at, recorded_at),
)
command.downgrade(get_alembic_config(isolated_settings), "20260913_0031")
assert _current_revision(isolated_settings) == "20260913_0031"
assert _venue_columns(isolated_settings) == {}
with repo._connect().cursor() as cur:
cur.execute(
"SELECT COUNT(*), MIN(note) FROM portfolio_ledger WHERE portfolio_id = %s",
(portfolio_id,),
)
assert cur.fetchone() == (1, "preserve-me")
upgrade_to_head(isolated_settings)
assert _venue_columns(isolated_settings) == {
"execution_venue": ("text", "YES", None),
"order_venue": ("text", "YES", None),
}
with repo._connect().cursor() as cur:
cur.execute(
"SELECT execution_venue, order_venue FROM portfolio_ledger WHERE portfolio_id = %s",
(portfolio_id,),
)
assert cur.fetchone() == (None, None)
repo.close()
tests/test_daemon_assignment.py
+2
View File
@@ -230,6 +230,8 @@ def test_ledger_trade_201_when_daemon_matches(monkeypatch) -> None:
trade.price = 50000.0
trade.fee_amount = 0.0
trade.tax_amount = 0.0
trade.execution_venue = None
trade.order_venue = None
trade.executed_at = datetime(2026, 5, 13, 1, 0, 0, tzinfo=timezone.utc)
trade.recorded_at = datetime(2026, 5, 13, 1, 0, 0, tzinfo=timezone.utc)
mock_repo, client = _setup_ledger_mocks(monkeypatch, principal, portfolio, trade)
tests/test_ledger_consolidation_integration.py
+179
View File
@@ -12,10 +12,13 @@ from __future__ import annotations
import os
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from threading import Barrier
import pytest
from app.repositories.portfolio_repository import LedgerTradeConflictError
from app.repositories.postgres import PortfolioDBConfig, PostgresPortfolioRepository
@@ -133,6 +136,182 @@ def test_duplicate_broker_write_is_idempotent(repo, seeded_portfolio):
assert cur.fetchone()[0] == 1
def test_duplicate_venue_enrichment_is_metadata_only(repo, seeded_portfolio):
args = {
"portfolio_id": seeded_portfolio,
"order_id": "ord-venue-enrich",
"execution_id": "exec-venue-enrich",
"side": "buy",
"quantity": 5.0,
"price": 10_000.0,
"executed_at": _now(),
}
_first, created = repo.insert_ledger_trade(**args)
assert created is True
before = repo.get(seeded_portfolio)
enriched, created = repo.insert_ledger_trade(
**args,
execution_venue="NXT",
order_venue="SOR",
)
assert created is False
assert enriched.execution_venue == "NXT"
assert enriched.order_venue == "SOR"
after = repo.get(seeded_portfolio)
assert after.cash_balance == before.cash_balance
assert after.asset_quantity == before.asset_quantity
with repo._connect().cursor() as cur:
cur.execute(
"""
SELECT COUNT(*), MIN(execution_venue), MIN(order_venue)
FROM portfolio_ledger
WHERE portfolio_id = %s AND order_id = %s AND execution_id = %s
""",
(seeded_portfolio, args["order_id"], args["execution_id"]),
)
assert cur.fetchone() == (1, "NXT", "SOR")
def test_duplicate_venue_conflict_is_atomic(repo, seeded_portfolio):
args = {
"portfolio_id": seeded_portfolio,
"order_id": "ord-venue-conflict",
"execution_id": "exec-venue-conflict",
"side": "buy",
"quantity": 1.0,
"price": 10_000.0,
"executed_at": _now(),
}
repo.insert_ledger_trade(**args, execution_venue="KRX")
before = repo.get(seeded_portfolio)
with pytest.raises(LedgerTradeConflictError):
repo.insert_ledger_trade(
**args,
execution_venue="NXT",
order_venue="SOR",
)
after = repo.get(seeded_portfolio)
assert after.cash_balance == before.cash_balance
assert after.asset_quantity == before.asset_quantity
with repo._connect().cursor() as cur:
cur.execute(
"""
SELECT execution_venue, order_venue
FROM portfolio_ledger
WHERE portfolio_id = %s AND order_id = %s AND execution_id = %s
""",
(seeded_portfolio, args["order_id"], args["execution_id"]),
)
assert cur.fetchone() == ("KRX", None)
def test_duplicate_null_venues_do_not_erase_existing_values(repo, seeded_portfolio):
args = {
"portfolio_id": seeded_portfolio,
"order_id": "ord-venue-preserve",
"execution_id": "exec-venue-preserve",
"side": "buy",
"quantity": 1.0,
"price": 10_000.0,
"executed_at": _now(),
}
repo.insert_ledger_trade(
**args,
execution_venue="KRX",
order_venue="SOR",
)
duplicate, created = repo.insert_ledger_trade(**args)
assert created is False
assert duplicate.execution_venue == "KRX"
assert duplicate.order_venue == "SOR"
def test_duplicate_economic_conflict_is_rejected(repo, seeded_portfolio):
args = {
"portfolio_id": seeded_portfolio,
"order_id": "ord-economic-conflict",
"execution_id": "exec-economic-conflict",
"side": "buy",
"quantity": 1.0,
"price": 10_000.0,
"executed_at": _now(),
}
repo.insert_ledger_trade(**args)
with pytest.raises(LedgerTradeConflictError):
repo.insert_ledger_trade(**{**args, "quantity": 2.0}, order_venue="SOR")
with repo._connect().cursor() as cur:
cur.execute(
"""
SELECT quantity::float8, order_venue
FROM portfolio_ledger
WHERE portfolio_id = %s AND order_id = %s AND execution_id = %s
""",
(seeded_portfolio, args["order_id"], args["execution_id"]),
)
assert cur.fetchone() == (1.0, None)
def test_concurrent_conflicting_venue_enrichment_has_no_last_write_wins(
repo, seeded_portfolio
):
args = {
"portfolio_id": seeded_portfolio,
"order_id": "ord-venue-concurrent",
"execution_id": "exec-venue-concurrent",
"side": "buy",
"quantity": 1.0,
"price": 10_000.0,
"executed_at": _now(),
}
repo.insert_ledger_trade(**args)
barrier = Barrier(2)
def enrich(execution_venue: str):
worker_repo = PostgresPortfolioRepository(repo._config)
barrier.wait()
try:
entry, created = worker_repo.insert_ledger_trade(
portfolio_id=seeded_portfolio,
order_id="ord-venue-concurrent",
execution_id="exec-venue-concurrent",
side="buy",
quantity=1.0,
price=10_000.0,
executed_at=args["executed_at"],
execution_venue=execution_venue,
)
return ("ok", created, entry.execution_venue)
except LedgerTradeConflictError:
return ("conflict", False, None)
finally:
worker_repo.close()
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(enrich, ("KRX", "NXT")))
assert sorted(result[0] for result in results) == ["conflict", "ok"]
winner = next(result[2] for result in results if result[0] == "ok")
assert winner in {"KRX", "NXT"}
with repo._connect().cursor() as cur:
cur.execute(
"""
SELECT COUNT(*), MIN(execution_venue), MIN(order_venue)
FROM portfolio_ledger
WHERE portfolio_id = %s AND order_id = %s AND execution_id = %s
""",
(seeded_portfolio, args["order_id"], args["execution_id"]),
)
assert cur.fetchone() == (1, winner, None)
def test_negative_balance_allowed(repo, seeded_portfolio):
"""REQ-037 AC-3 + OQ-3: oversell produces negative asset_quantity without rejection."""
entry, created = repo.insert_ledger_trade(
tests/test_v1_portfolios_ledger.py
+91
View File
@@ -147,6 +147,97 @@ def test_ledger_trade_duplicate_same_payload() -> None:
assert resp.json()["status"] == "duplicate"
def test_ledger_trade_round_trips_and_forwards_venue_provenance() -> None:
settings = _make_settings()
repo = _make_service_repo(
trade=_make_trade(execution_venue="NXT", order_venue="SOR")
)
with _with_repo(repo):
client = _make_client(settings)
resp = client.post(
_NEW_PATH,
json=_trade_payload(execution_venue="NXT", order_venue="SOR"),
headers={"Authorization": "Bearer svc-token"},
)
assert resp.status_code == 201
assert resp.json()["execution_venue"] == "NXT"
assert resp.json()["order_venue"] == "SOR"
assert repo.insert_ledger_trade.call_args.kwargs["execution_venue"] == "NXT"
assert repo.insert_ledger_trade.call_args.kwargs["order_venue"] == "SOR"
def test_ledger_trade_allows_unknown_actual_venue_with_sor_route() -> None:
settings = _make_settings()
repo = _make_service_repo(trade=_make_trade(execution_venue=None, order_venue="SOR"))
with _with_repo(repo):
client = _make_client(settings)
resp = client.post(
_NEW_PATH,
json=_trade_payload(execution_venue=None, order_venue="SOR"),
headers={"Authorization": "Bearer svc-token"},
)
assert resp.status_code == 201
assert resp.json()["execution_venue"] is None
assert resp.json()["order_venue"] == "SOR"
def test_ledger_trade_rejects_sor_as_execution_venue() -> None:
settings = _make_settings()
repo = _make_service_repo()
with _with_repo(repo):
client = _make_client(settings)
resp = client.post(
_NEW_PATH,
json=_trade_payload(execution_venue="SOR", order_venue="SOR"),
headers={"Authorization": "Bearer svc-token"},
)
assert resp.status_code == 422
repo.insert_ledger_trade.assert_not_called()
def test_ledger_trade_rejects_unknown_request_fields() -> None:
settings = _make_settings()
repo = _make_service_repo()
with _with_repo(repo):
client = _make_client(settings)
resp = client.post(
_NEW_PATH,
json=_trade_payload(broker_ordered_at="2026-05-01T09:59:59+09:00"),
headers={"Authorization": "Bearer svc-token"},
)
assert resp.status_code == 422
repo.insert_ledger_trade.assert_not_called()
def test_ledger_trade_openapi_exposes_nullable_venue_enums() -> None:
settings = _make_settings()
repo = _make_service_repo()
with _with_repo(repo):
client = _make_client(settings)
response = client.get("/openapi.json")
assert response.status_code == 200
properties = response.json()["components"]["schemas"]["LedgerTradeRequest"][
"properties"
]
execution_options = properties["execution_venue"]["anyOf"]
order_options = properties["order_venue"]["anyOf"]
assert {tuple(option.get("enum", ())) for option in execution_options} == {
(),
("KRX", "NXT"),
}
assert {tuple(option.get("enum", ())) for option in order_options} == {
(),
("KRX", "NXT", "SOR"),
}
assert {option.get("type") for option in execution_options} == {"string", "null"}
assert {option.get("type") for option in order_options} == {"string", "null"}
# ---------------------------------------------------------------------------
# AC-3: same key + different payload → 409
# ---------------------------------------------------------------------------
tests/test_v1_portfolios_manual_ledger.py
+5
View File
@@ -1030,6 +1030,8 @@ def test_unified_ledger_includes_corporate_action_fields() -> None:
"evidence_ref": "notice://ca-7",
"actor_user_id": _USER_ID,
"recorded_at": "2026-09-02T00:00:00Z",
"execution_venue": None,
"order_venue": None,
}
@@ -1094,6 +1096,7 @@ def test_unified_ledger_paginates_mixed_sources_and_hides_side() -> None:
LedgerEntry(
ledger_entry_id=99, portfolio_id=_PORTFOLIO_ID, side="buy", quantity=2, price=10, source="broker",
recorded_at=datetime.now(UTC), cash_balance_after=900, asset_quantity_after=12,
execution_venue="NXT", order_venue="SOR",
)
)
first = client.get(f"/v1/portfolios/{_PORTFOLIO_ID}/ledger?limit=1", headers=_auth())
@@ -1101,6 +1104,8 @@ def test_unified_ledger_paginates_mixed_sources_and_hides_side() -> None:
first_item = first.json()["items"][0]
assert first_item["source"] == "broker"
assert first_item["action"] == "BUY"
assert first_item["execution_venue"] == "NXT"
assert first_item["order_venue"] == "SOR"
assert "side" not in first_item
assert first.json()["next_cursor"]
second = client.get(