trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: isolate ledger outbox delivery by instance

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-12 13:24:58 +00:00
committed by GitHub
co-authored by genichin
parent 304a06f97c
commit 09b1b4f4c3
9 changed files with 429 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/application/ledger_outbox.py
+103 -5
View File
@@ -104,6 +104,7 @@ class LedgerWorkItem:
next_retry_at: Optional[datetime] = None
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
instance_key: Optional[str] = None
class LedgerOutbox(Protocol):
@@ -118,6 +119,15 @@ class LedgerOutbox(Protocol):
"""Return pending work items in FIFO order, up to `limit`."""
...
def claim_pending(
self,
limit: int = 32,
*,
correlation_id: str | None = None,
) -> list[LedgerWorkItem]:
"""Atomically claim pending items for delivery."""
...
def pending_cash_delta(self) -> dict[str, float]:
"""Sum the estimated cash delta of ALL not-yet-settled (pending) rows,
grouped by asset_id (dcy-018 / REQ-029, OQ-4 estimate overlay).
@@ -191,13 +201,27 @@ class LedgerWriter:
self._alarm_emitter = alarm_emitter
self._ledger_skip_store = ledger_skip_store
self._instance_key = instance_key
self.counters: dict[str, int] = {}
def _count(self, key: str) -> None:
self.counters[key] = self.counters.get(key, 0) + 1
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
def run_once(self, limit: int = 32) -> WriterStats:
stats = WriterStats()
for item in self._outbox.list_pending(limit):
items = self._outbox.claim_pending(limit)
claim_result = "claimed" if items else "empty"
self._count(f"claim:{claim_result}")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d",
self._instance_key,
claim_result,
len(items),
limit,
)
for item in items:
outcome = self._process(item)
if outcome is OutboxOutcome.COMPLETED:
stats.completed += 1
@@ -216,6 +240,32 @@ class LedgerWriter:
item = self._outbox.get(correlation_id)
if item is None:
return None
if (
self._instance_key is not None
and item.instance_key is not None
and item.instance_key != self._instance_key
):
self._count("claim:foreign_instance")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=foreign_instance "
"correlation_id=%s owner_instance_key=%s",
self._instance_key,
correlation_id,
item.instance_key,
)
return None
if item.status is WorkItemStatus.PENDING:
claimed = self._outbox.claim_pending(1, correlation_id=correlation_id)
if not claimed:
self._count("claim:unavailable")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=unavailable "
"correlation_id=%s",
self._instance_key,
correlation_id,
)
return None
item = claimed[0]
return self._process(item)
def _backoff_delay(self, attempts: int) -> float:
@@ -229,6 +279,7 @@ class LedgerWriter:
existing = self._outbox.get(item.correlation_id)
if existing is not None and existing.status is WorkItemStatus.COMPLETED:
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED)
self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED)
return OutboxOutcome.DUPLICATE_COMPLETED
try:
outcome = self._ledger.record_trade(item.trade)
@@ -236,6 +287,7 @@ class LedgerWriter:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)
)
self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc))
# dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
@@ -252,8 +304,10 @@ class LedgerWriter:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.PERMANENT_FAILURE, error=str(exc)
)
self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE, error=str(exc))
_log.warning(
"ledger delivery_failed (permanent): correlation_id=%s reason=%s",
"ledger delivery_failed (permanent): instance_key=%s correlation_id=%s reason=%s",
item.instance_key or self._instance_key,
item.correlation_id, exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry.
@@ -273,10 +327,14 @@ class LedgerWriter:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.DEAD_LETTERED, error=str(exc)
)
self._log_outcome(item, OutboxOutcome.DEAD_LETTERED, error=str(exc))
_log.warning(
"ledger delivery_failed (dead-letter, attempts=%d): "
"correlation_id=%s reason=%s",
new_attempts, item.correlation_id, exc,
"instance_key=%s correlation_id=%s reason=%s",
new_attempts,
item.instance_key or self._instance_key,
item.correlation_id,
exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries.
if self._alarm_emitter is not None:
@@ -296,6 +354,7 @@ class LedgerWriter:
item.correlation_id, OutboxOutcome.RETRYABLE_FAILURE,
error=str(exc), next_retry_at=next_retry_at,
)
self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE, error=str(exc))
return OutboxOutcome.RETRYABLE_FAILURE
# dcy-018 / REQ-019/030 (option D): write back portfolio-hub authoritative
# cash/quantity to the snapshot store on new+duplicate (OQ-2 b). *_after
@@ -308,11 +367,32 @@ class LedgerWriter:
if outcome.result is LedgerWriteResult.DUPLICATE_OK:
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED)
self._resolve_ledger_skip(item.trade)
self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED)
return OutboxOutcome.DUPLICATE_COMPLETED
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.COMPLETED)
self._resolve_ledger_skip(item.trade)
self._log_outcome(item, OutboxOutcome.COMPLETED)
return OutboxOutcome.COMPLETED
def _log_outcome(
self,
item: LedgerWorkItem,
outcome: OutboxOutcome,
*,
error: str | None = None,
) -> None:
self._count(f"outcome:{outcome.value}")
msg = "ledger outbox outcome: instance_key=%s correlation_id=%s outcome=%s"
args: tuple[object, ...] = (
item.instance_key or self._instance_key,
item.correlation_id,
outcome.value,
)
if error is not None:
msg += " error=%s"
args = (*args, error)
_log.info(msg, *args)
def _resolve_ledger_skip(self, trade: LedgerTrade) -> None:
if (
self._ledger_skip_store is None
@@ -358,7 +438,12 @@ class LedgerWriter:
class InMemoryLedgerOutbox:
"""Process-local outbox. Thread-safe; idempotent on correlation_id."""
def __init__(self, *, clock: Optional[Callable[[], datetime]] = None) -> None:
def __init__(
self,
*,
clock: Optional[Callable[[], datetime]] = None,
instance_key: str = "default",
) -> None:
import threading
self._rows: dict[str, LedgerWorkItem] = {}
@@ -367,6 +452,7 @@ class InMemoryLedgerOutbox:
# dcy-015: shared clock lets tests align backoff (next_retry_at) with
# list_pending's "now" deterministically.
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
self._instance_key = instance_key
def _now(self) -> datetime:
return self._clock()
@@ -383,6 +469,7 @@ class InMemoryLedgerOutbox:
status=WorkItemStatus.PENDING,
created_at=self._now(),
updated_at=self._now(),
instance_key=self._instance_key,
)
self._rows[cid] = row
self._order.append(cid)
@@ -404,6 +491,17 @@ class InMemoryLedgerOutbox:
break
return out
def claim_pending(
self,
limit: int = 32,
*,
correlation_id: str | None = None,
) -> list[LedgerWorkItem]:
pending = self.list_pending(limit if correlation_id is None else max(limit, 1))
if correlation_id is None:
return pending[:limit]
return [row for row in pending if row.correlation_id == correlation_id][:1]
def pending_cash_delta(self) -> dict[str, float]:
with self._lock:
out: dict[str, float] = {}