trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: isolate ledger outbox delivery by instance

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-12 13:24:58 +00:00
committed by GitHub
co-authored by genichin
parent 304a06f97c
commit 09b1b4f4c3
9 changed files with 429 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/adapters/persistence/postgresql.py
+79 -12
View File
@@ -13,9 +13,10 @@ domain layer (NFR-1).
from __future__ import annotations
import json
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Any, Iterator, Optional
from typing import Any, Optional
from uuid import UUID, uuid4
from trex_core.application.ledger_outbox import (
@@ -26,9 +27,9 @@ from trex_core.application.ledger_outbox import (
from trex_core.application.ledger_port import LedgerTrade
from trex_core.application.ports import IntentRecord
from trex_core.application.strategy_state import (
_STATE_CAP_BYTES,
SetResult,
StrategyStateEntry,
_STATE_CAP_BYTES,
)
from trex_core.domain.slot import Slot, SlotNotFoundError, SlotState, SlotTransitionRecord
@@ -565,16 +566,32 @@ class _PgIntentRecord:
_OUTBOX_COLUMNS = (
"correlation_id, asset_id, order_id, execution_id, side, qty, price, "
"fee, tax, executed_at, status, attempts, last_outcome, last_error, "
"created_at, updated_at, next_retry_at, venue, order_venue"
"created_at, updated_at, next_retry_at, venue, order_venue, instance_key"
)
class PostgreSQLLedgerOutbox:
"""ledger_outbox-backed LedgerOutbox. correlation_id PK is the dedup gate."""
def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None:
def __init__(
self,
dsn: str,
*,
instance_key: str = "default",
claim_ttl_sec: float = 60.0,
worker_id: str | None = None,
clock=None,
psycopg_module: Any = None,
) -> None:
self._psycopg = _require_psycopg(psycopg_module)
self._dsn = dsn
self._instance_key = instance_key
self._claim_ttl_sec = claim_ttl_sec
self._worker_id = worker_id or f"ledger-writer-{uuid4().hex}"
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
def _now(self) -> datetime:
return self._clock()
def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem:
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
@@ -582,8 +599,8 @@ class PostgreSQLLedgerOutbox:
cur.execute(
"INSERT INTO ledger_outbox "
"(correlation_id, asset_id, order_id, execution_id, side, "
" qty, price, fee, tax, executed_at, venue, order_venue) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
" qty, price, fee, tax, executed_at, venue, order_venue, instance_key) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
"ON CONFLICT (correlation_id) DO NOTHING",
(
trade.correlation_id,
@@ -598,6 +615,7 @@ class PostgreSQLLedgerOutbox:
trade.executed_at,
trade.venue,
trade.order_venue,
self._instance_key,
),
)
conn.commit()
@@ -611,12 +629,53 @@ class PostgreSQLLedgerOutbox:
cur.execute(
f"SELECT {_OUTBOX_COLUMNS} FROM ledger_outbox "
"WHERE status = 'pending' "
"AND instance_key = %s "
"AND (next_retry_at IS NULL OR next_retry_at <= now()) "
"ORDER BY created_at LIMIT %s",
(limit,),
"AND (claim_expires_at IS NULL OR claim_expires_at <= now()) "
"ORDER BY created_at, correlation_id LIMIT %s",
(self._instance_key, limit),
)
return [self._row_to_item(r) for r in cur.fetchall()]
def claim_pending(
self,
limit: int = 32,
*,
correlation_id: str | None = None,
) -> list[LedgerWorkItem]:
now = self._now()
lease_until = now + timedelta(seconds=self._claim_ttl_sec)
correlation_sql = ""
params: list[Any] = [self._instance_key]
if correlation_id is not None:
correlation_sql = "AND correlation_id = %s "
params.append(correlation_id)
params.extend([limit, self._worker_id, lease_until, now])
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
with conn.cursor() as cur:
cur.execute(
"WITH candidates AS ("
"SELECT correlation_id FROM ledger_outbox "
"WHERE status = 'pending' "
"AND instance_key = %s "
"AND (next_retry_at IS NULL OR next_retry_at <= now()) "
"AND (claim_expires_at IS NULL OR claim_expires_at <= now()) "
f"{correlation_sql}"
"ORDER BY created_at, correlation_id "
"LIMIT %s "
"FOR UPDATE SKIP LOCKED"
") "
"UPDATE ledger_outbox o "
"SET claim_owner = %s, claim_expires_at = %s, updated_at = %s "
"FROM candidates "
"WHERE o.correlation_id = candidates.correlation_id "
f"RETURNING {_OUTBOX_COLUMNS}",
params,
)
rows = [self._row_to_item(r) for r in cur.fetchall()]
conn.commit()
return rows
def pending_cash_delta(self) -> dict[str, float]:
# dcy-018 / REQ-029 (OQ-4): estimated cash delta of all pending rows by
# asset_id. SUM in SQL: sell → +(qty*price - fee - tax), buy → negative.
@@ -627,7 +686,10 @@ class PostgreSQLLedgerOutbox:
"SUM(CASE WHEN side='sell' "
" THEN (qty*price - fee - tax) "
" ELSE -(qty*price + fee + tax) END) "
"FROM ledger_outbox WHERE status='pending' GROUP BY asset_id"
"FROM ledger_outbox "
"WHERE status='pending' AND instance_key=%s "
"GROUP BY asset_id",
(self._instance_key,),
)
return {r[0]: float(r[1]) for r in cur.fetchall() if r[1] is not None}
@@ -671,16 +733,20 @@ class PostgreSQLLedgerOutbox:
nra = next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None
cur.execute(
"UPDATE ledger_outbox SET status=%s, attempts=attempts+%s, "
"last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s "
"WHERE correlation_id=%s",
"last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s, "
"claim_owner=NULL, claim_expires_at=NULL "
"WHERE correlation_id=%s AND instance_key=%s "
"AND (claim_owner=%s OR claim_owner IS NULL)",
(
status_value,
attempts_delta,
outcome.value,
error,
nra,
datetime.now(tz=timezone.utc),
self._now(),
correlation_id,
self._instance_key,
self._worker_id,
),
)
conn.commit()
@@ -714,6 +780,7 @@ class PostgreSQLLedgerOutbox:
created_at=row[14],
updated_at=row[15],
next_retry_at=row[16],
instance_key=row[19],
)
trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql
+22
View File
@@ -0,0 +1,22 @@
-- GitHub #84: persist outbox owner instance and add claim/lease fields so
-- shared-DB daemon writers only deliver their own rows and can safely recover
-- abandoned work after lease expiry. Existing rows remain fail-closed when the
-- owner instance cannot be inferred.
ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS instance_key TEXT;
ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS claim_owner TEXT;
ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS claim_expires_at TIMESTAMPTZ;
UPDATE ledger_outbox
SET instance_key='__unowned__'
WHERE instance_key IS NULL;
ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET DEFAULT '__unowned__';
ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET NOT NULL;
CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_pending
ON ledger_outbox (instance_key, status, created_at)
WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_claim_expires
ON ledger_outbox (instance_key, claim_expires_at)
WHERE status = 'pending';