trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #90 from fourblocks2020/copilot/bug-ledger-shared-outbox-consumption

Browse Source 
Isolate ledger outbox delivery by instance and add lease-based claims
This commit is contained in:
genichin
2026-09-13 10:48:57 +09:00
committed by GitHub
parent 60a835fde1 762a953c95
commit 3cb0fdc303
18 changed files with 817 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/adapters/persistence/test_migration_and_drift.py
+70 -44
View File
@@ -6,9 +6,10 @@ iteration: EXPECTED_SLOT_FIELDS (code contract) vs daily_slot actual columns
"""
from __future__ import annotations
from datetime import UTC, datetime
import psycopg
import pytest
from datetime import datetime, timezone
from trex_core.adapters.persistence.migrate import apply_migrations, migration_files
from trex_core.adapters.persistence.postgresql import (
@@ -33,24 +34,22 @@ _EXPECTED_TABLES = {
def _columns(dsn: str, table: str) -> set[str]:
with psycopg.connect(dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = %s",
(table,),
)
return {r[0] for r in cur.fetchall()}
with psycopg.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = %s",
(table,),
)
return {r[0] for r in cur.fetchall()}
def test_migrations_create_expected_tables(pg_clean):
assert migration_files(), "no migration files found"
with psycopg.connect(pg_clean) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT table_name FROM information_schema.tables WHERE table_schema='public'"
)
tables = {r[0] for r in cur.fetchall()}
with psycopg.connect(pg_clean) as conn, conn.cursor() as cur:
cur.execute(
"SELECT table_name FROM information_schema.tables WHERE table_schema='public'"
)
tables = {r[0] for r in cur.fetchall()}
assert _EXPECTED_TABLES <= tables
@@ -61,20 +60,19 @@ def test_daily_slot_columns_match_expected_contract(pg_clean):
def test_strategy_state_schema_matches_instance_map_contract(pg_clean):
with psycopg.connect(pg_clean) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT column_name, data_type, udt_name "
"FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'"
)
columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()}
cur.execute(
"SELECT a.attname FROM pg_index i "
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) "
"WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary"
)
primary_key = {row[0] for row in cur.fetchall()}
with psycopg.connect(pg_clean) as conn, conn.cursor() as cur:
cur.execute(
"SELECT column_name, data_type, udt_name "
"FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'"
)
columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()}
cur.execute(
"SELECT a.attname FROM pg_index i "
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) "
"WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary"
)
primary_key = {row[0] for row in cur.fetchall()}
assert columns == {"slot_id": ("uuid", "uuid"), "data": ("jsonb", "jsonb")}
assert primary_key == {"slot_id"}
@@ -111,17 +109,16 @@ def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean):
apply_migrations(pg_clean)
assert repo.fetch_slot(slot.slot_id).venue == "SOR"
with psycopg.connect(pg_clean) as conn:
with conn.cursor() as cur:
with pytest.raises(psycopg.errors.CheckViolation):
cur.execute(
"UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s",
(slot.slot_id,),
)
with psycopg.connect(pg_clean) as conn, conn.cursor() as cur:
with pytest.raises(psycopg.errors.CheckViolation):
cur.execute(
"UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s",
(slot.slot_id,),
)
def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="test-instance")
normal = LedgerTrade(
asset_id="PF-NORMAL",
order_id="O-NORMAL",
@@ -129,7 +126,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
side="buy",
qty=1,
price=1000,
executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc),
executed_at=datetime(2026, 8, 19, tzinfo=UTC),
venue="NXT",
order_venue="KRX",
)
@@ -140,7 +137,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
side="buy",
qty=1,
price=1000,
executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc),
executed_at=datetime(2026, 8, 19, tzinfo=UTC),
venue="KRX",
order_venue="SOR",
)
@@ -172,10 +169,39 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
assert legacy_row.trade.venue is None
assert legacy_row.trade.order_venue == "SOR"
with psycopg.connect(pg_clean) as conn, conn.cursor() as cur:
with pytest.raises(psycopg.errors.CheckViolation):
cur.execute(
"UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s",
(normal.correlation_id,),
)
def test_migration_029_fail_closes_unowned_legacy_outbox_rows(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
trade = LedgerTrade(
asset_id="PF-LEGACY",
order_id="O-LEGACY",
execution_id="E-LEGACY",
side="buy",
qty=1,
price=1000,
executed_at=datetime(2026, 9, 12, tzinfo=UTC),
)
outbox.enqueue(trade)
with psycopg.connect(pg_clean) as conn:
with conn.cursor() as cur:
with pytest.raises(psycopg.errors.CheckViolation):
cur.execute(
"UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s",
(normal.correlation_id,),
)
cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP NOT NULL")
cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP DEFAULT")
cur.execute(
"UPDATE ledger_outbox SET instance_key=NULL WHERE correlation_id=%s",
(trade.correlation_id,),
)
conn.commit()
apply_migrations(pg_clean)
stored = outbox.get(trade.correlation_id)
assert stored is not None
assert stored.instance_key == "__unowned__"
assert outbox.list_pending() == []
tests/adapters/persistence/test_postgresql_ledger_outbox.py
+203 -16
View File
@@ -4,43 +4,53 @@ Skipped without TREX_POSTGRESQL_TEST_DSN (same gate as other PG suites).
"""
from __future__ import annotations
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
import pytest
from trex_core.adapters.ledger.in_memory import InMemoryLedger
from trex_core.adapters.persistence.postgresql import PostgreSQLLedgerOutbox
from trex_core.application.ledger_outbox import (
LedgerWriter,
OutboxClaimLostError,
OutboxOutcome,
WorkItemStatus,
)
from trex_core.application.ledger_port import LedgerTrade
from trex_core.application.ledger_port import (
LedgerRetryableError,
LedgerTrade,
LedgerWriteOutcome,
LedgerWriteResult,
)
def _trade(execution_id="paper:1:10") -> LedgerTrade:
return LedgerTrade(
asset_id="PF-1", order_id="ORD-1", execution_id=execution_id,
side="buy", qty=10, price=1000.0, fee=1.5, tax=0.0,
executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc),
executed_at=datetime(2026, 6, 4, tzinfo=UTC),
venue="NXT", order_venue="SOR",
)
def test_enqueue_persists_and_is_idempotent(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
a = outbox.enqueue(_trade())
b = outbox.enqueue(_trade())
assert a.correlation_id == b.correlation_id
pending = outbox.list_pending()
assert len(pending) == 1
assert pending[0].status is WorkItemStatus.PENDING
assert pending[0].instance_key == "inst-a"
assert pending[0].trade.venue == "NXT"
assert pending[0].trade.order_venue == "SOR"
def test_writer_completes_persisted_item(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
ledger = InMemoryLedger()
writer = LedgerWriter(outbox, ledger)
writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
outbox.enqueue(_trade(execution_id="pg-1"))
stats = writer.run_once()
assert stats.completed == 1 and ledger.count() == 1
@@ -50,9 +60,9 @@ def test_writer_completes_persisted_item(pg_clean):
def test_writer_marks_conflict_blocked_on_pg(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
ledger = InMemoryLedger(conflict_executions={"pg-x"})
writer = LedgerWriter(outbox, ledger)
writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
outbox.enqueue(_trade(execution_id="pg-x"))
writer.run_once()
row = outbox.get("PF-1:ORD-1:pg-x")
@@ -61,10 +71,10 @@ def test_writer_marks_conflict_blocked_on_pg(pg_clean):
def test_writer_retries_on_pg(pg_clean):
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
ledger = InMemoryLedger(fail_n_then_succeed=1)
# dcy-015: base 0 => next_retry_at == now, so the row is immediately due again.
writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0)
writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0, instance_key="inst-a")
outbox.enqueue(_trade(execution_id="pg-r"))
writer.run_once()
row1 = outbox.get("PF-1:ORD-1:pg-r")
@@ -79,9 +89,9 @@ def test_writer_retries_on_pg(pg_clean):
def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
"""dcy-015 / REQ-019: 404/permanent → terminal delivery_failed persisted
(migration 007 status enum + last_outcome). No retry."""
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
ledger = InMemoryLedger(permanent_executions={"pg-perm"})
writer = LedgerWriter(outbox, ledger)
writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
outbox.enqueue(_trade(execution_id="pg-perm"))
s = writer.run_once()
assert s.delivery_failed == 1
@@ -94,22 +104,199 @@ def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
def test_pending_cash_delta_on_pg(pg_clean):
"""dcy-018 / REQ-029: estimated cash delta of pending rows summed by asset (SQL)."""
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
outbox.enqueue(LedgerTrade(asset_id="PF-1", order_id="O-1", execution_id="cd-buy",
side="buy", qty=10, price=1000.0, fee=1.0, tax=0.0,
executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc)))
executed_at=datetime(2026, 6, 4, tzinfo=UTC)))
deltas = outbox.pending_cash_delta()
assert deltas["PF-1"] == -(10 * 1000.0 + 1.0) # buy => cash down by notional+fee
def test_backoff_row_skipped_by_pending_scan_on_pg(pg_clean):
"""dcy-015 / REQ-019: a future next_retry_at excludes the row from list_pending."""
outbox = PostgreSQLLedgerOutbox(pg_clean)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
ledger = InMemoryLedger(fail_n_then_succeed=99)
# large base => next_retry_at well in the future
writer = LedgerWriter(outbox, ledger, retry_base_sec=600.0, max_attempts=99)
writer = LedgerWriter(
outbox, ledger, retry_base_sec=600.0, max_attempts=99, instance_key="inst-a"
)
outbox.enqueue(_trade(execution_id="pg-bo"))
writer.run_once() # -> retryable, next_retry_at ~ now + 600s
row = outbox.get("PF-1:ORD-1:pg-bo")
assert row.status is WorkItemStatus.PENDING and row.next_retry_at is not None
assert outbox.list_pending() == [] # still backing off -> skipped
def test_pending_claims_are_isolated_per_instance(pg_clean):
ledger = InMemoryLedger()
outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b")
writer_a = LedgerWriter(outbox_a, ledger, instance_key="inst-a")
writer_b = LedgerWriter(outbox_b, ledger, instance_key="inst-b")
row_a = outbox_a.enqueue(_trade(execution_id="pg-a"))
row_b = outbox_b.enqueue(_trade(execution_id="pg-b"))
stats_a = writer_a.run_once()
assert stats_a.completed == 1 and ledger.count() == 1
assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.COMPLETED
assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.PENDING
stats_b = writer_b.run_once()
assert stats_b.completed == 1 and ledger.count() == 2
assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.COMPLETED
def test_claim_expiry_allows_safe_recovery_after_worker_crash(pg_clean):
now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
outbox_a = PostgreSQLLedgerOutbox(
pg_clean,
instance_key="inst-a",
claim_ttl_sec=5.0,
worker_id="worker-a",
clock=lambda: now["t"],
)
outbox_b = PostgreSQLLedgerOutbox(
pg_clean,
instance_key="inst-a",
claim_ttl_sec=5.0,
worker_id="worker-b",
clock=lambda: now["t"],
)
row = outbox_a.enqueue(_trade(execution_id="pg-crash"))
claimed_a = outbox_a.claim_pending()
assert [item.correlation_id for item in claimed_a] == [row.correlation_id]
assert outbox_b.claim_pending() == []
now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
claimed_b = outbox_b.claim_pending()
assert [item.correlation_id for item in claimed_b] == [row.correlation_id]
def test_stale_claim_cannot_overwrite_new_owner_outcome(pg_clean):
now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
outbox_a = PostgreSQLLedgerOutbox(
pg_clean,
instance_key="inst-a",
claim_ttl_sec=5.0,
worker_id="worker-a",
clock=lambda: now["t"],
)
outbox_b = PostgreSQLLedgerOutbox(
pg_clean,
instance_key="inst-a",
claim_ttl_sec=5.0,
worker_id="worker-b",
clock=lambda: now["t"],
)
row = outbox_a.enqueue(_trade(execution_id="pg-stale"))
[claim_a] = outbox_a.claim_pending()
now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
[claim_b] = outbox_b.claim_pending()
outbox_b.mark_outcome(
row.correlation_id,
OutboxOutcome.COMPLETED,
expected_claim_expires_at=claim_b.claim_expires_at,
)
with pytest.raises(OutboxClaimLostError):
outbox_a.mark_outcome(
row.correlation_id,
OutboxOutcome.RETRYABLE_FAILURE,
error="stale owner",
next_retry_at=now["t"],
expected_claim_expires_at=claim_a.claim_expires_at,
)
stored = outbox_a.get(row.correlation_id)
assert stored is not None
assert stored.status is WorkItemStatus.COMPLETED
assert stored.last_outcome is OutboxOutcome.COMPLETED
@pytest.mark.parametrize("instance_key", ["", " "])
def test_postgresql_outbox_rejects_empty_instance_before_connect(pg_clean, instance_key):
with pytest.raises(ValueError, match="non-empty"):
PostgreSQLLedgerOutbox(pg_clean, instance_key=instance_key)
def test_original_owner_can_finalize_expired_unreclaimed_claim(pg_clean):
now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
outbox = PostgreSQLLedgerOutbox(
pg_clean,
instance_key="inst-a",
claim_ttl_sec=5.0,
worker_id="worker-a",
clock=lambda: now["t"],
)
row = outbox.enqueue(_trade(execution_id="pg-expired"))
[claim] = outbox.claim_pending()
now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
stored = outbox.mark_outcome(
row.correlation_id,
OutboxOutcome.COMPLETED,
expected_claim_expires_at=claim.claim_expires_at,
)
assert stored.status is WorkItemStatus.COMPLETED
assert stored.last_outcome is OutboxOutcome.COMPLETED
def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean):
class _ForbiddenLedger:
def __init__(self):
self.calls: list[str] = []
def record_trade(self, trade):
self.calls.append(trade.execution_id)
raise LedgerRetryableError("portfolio-hub 403 Forbidden")
outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b")
writer_b = LedgerWriter(
outbox_b, _ForbiddenLedger(), retry_base_sec=0.0, instance_key="inst-b"
)
row_a = outbox_a.enqueue(_trade(execution_id="pg-403-a"))
row_b = outbox_b.enqueue(_trade(execution_id="pg-403-b"))
stats_b = writer_b.run_once()
assert stats_b.retryable == 1
assert outbox_b.get(row_b.correlation_id).last_outcome is OutboxOutcome.RETRYABLE_FAILURE
assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.PENDING
def test_concurrent_workers_claim_each_row_at_most_once(pg_clean):
outbox_a = PostgreSQLLedgerOutbox(
pg_clean, instance_key="inst-a", worker_id="worker-a"
)
outbox_b = PostgreSQLLedgerOutbox(
pg_clean, instance_key="inst-a", worker_id="worker-b"
)
row = outbox_a.enqueue(_trade(execution_id="pg-concurrent"))
with ThreadPoolExecutor(max_workers=2) as pool:
claims = list(pool.map(lambda box: box.claim_pending(), (outbox_a, outbox_b)))
claimed_ids = [item.correlation_id for batch in claims for item in batch]
assert claimed_ids == [row.correlation_id]
def test_writer_marks_duplicate_completed_from_adapter_response_on_pg(pg_clean):
class _DuplicateLedger:
def record_trade(self, _trade):
return LedgerWriteOutcome(LedgerWriteResult.DUPLICATE_OK)
outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
writer = LedgerWriter(outbox, _DuplicateLedger(), instance_key="inst-a")
row = outbox.enqueue(_trade(execution_id="pg-dup"))
outcome = writer.process_correlation_id(row.correlation_id)
assert outcome is OutboxOutcome.DUPLICATE_COMPLETED
stored = outbox.get(row.correlation_id)
assert stored is not None
assert stored.status is WorkItemStatus.COMPLETED
assert stored.last_outcome is OutboxOutcome.DUPLICATE_COMPLETED