Merge pull request #90 from fourblocks2020/copilot/bug-ledger-shared-outbox-consumption

Isolate ledger outbox delivery by instance and add lease-based claims

tests/application/test_execution_cancel.py

@@ -70,7 +70,7 @@ def _wire(broker, *, stale_sec=None):
     svc = SlotService(repo)
     intents = IntentService(svc, repo, InMemoryIntentStore(),
                             resolve_symbol=lambda pid: "005930")
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger()
     execution = ExecutionService(
         svc, repo, broker, ledger_outbox=outbox,

tests/application/test_execution_fill_price_backfill.py

@@ -120,7 +120,7 @@ def _wire(broker):
         resolve_symbol=lambda pid: "005930",
         instance_key=INST,
     )
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     execution = ExecutionService(svc, repo, broker, ledger_outbox=outbox, instance_key=INST)
     repo.reconcile_universe(INST, {"PF-1": "005930"})
     return repo, svc, intents, execution, outbox
@@ -966,7 +966,7 @@ class TestPaperBrokerRegression:
             resolve_symbol=lambda pid: "005930",
             instance_key=INST,
         )
-        outbox2 = InMemoryLedgerOutbox()
+        outbox2 = InMemoryLedgerOutbox(instance_key="test-instance")
         execution2 = ExecutionService(svc2, repo2, paper, ledger_outbox=outbox2, instance_key=INST)
         repo2.reconcile_universe(INST, {"PF-1": "005930"})
 

tests/application/test_execution_ledger.py

@@ -119,7 +119,11 @@ def _wire(broker, *, ledger=None, clock=None, alarm_emitter=None, ledger_skip_st
     svc = SlotService(repo)
     intents = IntentService(svc, repo, InMemoryIntentStore(),
                             resolve_symbol=lambda pid: "005930")
-    outbox = InMemoryLedgerOutbox(clock=clock) if clock else InMemoryLedgerOutbox()
+    outbox = (
+        InMemoryLedgerOutbox(clock=clock, instance_key="default")
+        if clock
+        else InMemoryLedgerOutbox(instance_key="default")
+    )
     execution = ExecutionService(
         svc,
         repo,
@@ -430,7 +434,7 @@ def test_duplicate_execution_id_results_in_zero_duplicate_in_ledger():
     # And even if the writer somehow re-processes (simulated by a fresh outbox
     # row for the same execution_id, e.g. after a manual replay), the ledger
     # adapter dedup'd it server-side and the outcome is duplicate_completed.
-    replay_outbox = InMemoryLedgerOutbox()
+    replay_outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     replay_writer = LedgerWriter(replay_outbox, ledger)
     replay_outbox.enqueue(first.trade)
     stats = replay_writer.run_once()

tests/application/test_ledger_option_d.py

@@ -39,7 +39,7 @@ def _store_with(pid="PF-1", cash=1_000_000.0, qty=0.0):
 # --- write-back (REQ-019/030) ----------------------------------------------
 def test_writeback_updates_snapshot_on_new():
     store = _store_with()
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger(cash_after=300_000.0, quantity_after=10.0)
     writer = LedgerWriter(outbox, ledger, snapshot_store=store)
     outbox.enqueue(_trade())
@@ -50,7 +50,7 @@ def test_writeback_updates_snapshot_on_new():
 
 def test_writeback_applies_on_duplicate_too():
     store = _store_with(cash=1_000_000.0)
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     # execution already recorded once → second write is duplicate, *_after still authoritative
     ledger = InMemoryLedger(cash_after=300_000.0, quantity_after=10.0)
     ledger.record_trade(_trade(execution_id="dup"))   # pre-record → next is duplicate
@@ -62,7 +62,7 @@ def test_writeback_applies_on_duplicate_too():
 
 def test_writeback_skipped_when_after_missing():
     store = _store_with(cash=1_000_000.0)
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger()  # no cash_after/quantity_after → None
     writer = LedgerWriter(outbox, ledger, snapshot_store=store)
     outbox.enqueue(_trade())
@@ -74,7 +74,7 @@ def test_writeback_best_effort_does_not_block_completion():
     class _BoomStore:
         def update_cash_position(self, *a, **k):
             raise RuntimeError("boom")
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger(cash_after=1.0, quantity_after=1.0)
     writer = LedgerWriter(outbox, ledger, snapshot_store=_BoomStore())
     outbox.enqueue(_trade())
@@ -83,7 +83,7 @@ def test_writeback_best_effort_does_not_block_completion():
 
 
 def test_no_writeback_without_store():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger(cash_after=1.0, quantity_after=1.0)
     writer = LedgerWriter(outbox, ledger)  # no snapshot_store
     outbox.enqueue(_trade())
@@ -92,7 +92,7 @@ def test_no_writeback_without_store():
 
 # --- estimate overlay (REQ-029 / OQ-4) — via make_slots_rpc_handler ----------
 def test_pending_cash_delta_sums_by_asset():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     outbox.enqueue(_trade(execution_id="b", side="buy", qty=10, price=1000.0, fee=1.0))
     deltas = outbox.pending_cash_delta()
     assert deltas["PF-1"] == trade_cash_delta(_trade(side="buy", qty=10, price=1000.0, fee=1.0))
@@ -101,7 +101,7 @@ def test_pending_cash_delta_sums_by_asset():
 
 def test_apply_cash_estimate_overlays_pending_delta():
     """Cash-estimated overlay applied through slots handler (REQ-029 OQ-4)."""
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     outbox.enqueue(_trade(execution_id="b", side="buy", qty=10, price=1000.0, fee=1.0))
     store = _store_with(pid="PF-1", cash=1_000_000.0, qty=0.0)
     repo = InMemorySlotRepository()
@@ -117,7 +117,7 @@ def test_apply_cash_estimate_overlays_pending_delta():
 def test_estimate_clears_after_writeback():
     """Once the ledger write completes (no longer pending), the estimate is gone."""
     store = _store_with(cash=1_000_000.0)
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger(cash_after=989_999.0, quantity_after=10.0)
     writer = LedgerWriter(outbox, ledger, snapshot_store=store)
     outbox.enqueue(_trade(side="buy", qty=10, price=1000.0, fee=1.0))

tests/application/test_ledger_outbox.py

@@ -1,16 +1,20 @@
 """LedgerOutbox + LedgerWriter direct unit tests (REQ-019)."""
 from __future__ import annotations
 
+from dataclasses import replace
 from datetime import UTC, datetime, timedelta
 
+import pytest
+
 from trex_core.adapters.ledger.in_memory import InMemoryLedger
 from trex_core.application.ledger_outbox import (
     InMemoryLedgerOutbox,
     LedgerWriter,
+    OutboxClaimLostError,
     OutboxOutcome,
     WorkItemStatus,
 )
-from trex_core.application.ledger_port import LedgerTrade
+from trex_core.application.ledger_port import LedgerPermanentError, LedgerTrade
 
 
 def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade:
@@ -22,7 +26,7 @@ def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade:
 
 
 def test_enqueue_is_idempotent_on_correlation_id():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     a = outbox.enqueue(_trade())
     b = outbox.enqueue(_trade())
     assert a.correlation_id == b.correlation_id
@@ -30,7 +34,7 @@ def test_enqueue_is_idempotent_on_correlation_id():
 
 
 def test_writer_run_once_drains_pending_and_marks_completed():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger()
     writer = LedgerWriter(outbox, ledger)
     outbox.enqueue(_trade(execution_id="e-1"))
@@ -42,7 +46,7 @@ def test_writer_run_once_drains_pending_and_marks_completed():
 
 
 def test_writer_marks_conflict_blocked_and_does_not_retry():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger(conflict_executions={"e-x"})
     writer = LedgerWriter(outbox, ledger)
     outbox.enqueue(_trade(execution_id="e-x"))
@@ -58,7 +62,9 @@ def test_writer_retries_until_success():
     # dcy-015: retryable rows now back off; share an injected clock between the
     # outbox (list_pending "now") and the writer (next_retry_at) to advance past it.
     now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
-    outbox = InMemoryLedgerOutbox(clock=lambda: now["t"])
+    outbox = InMemoryLedgerOutbox(
+        clock=lambda: now["t"], instance_key="test-instance"
+    )
     ledger = InMemoryLedger(fail_n_then_succeed=1)
     writer = LedgerWriter(outbox, ledger, clock=lambda: now["t"])
     outbox.enqueue(_trade(execution_id="e-r"))
@@ -71,15 +77,134 @@ def test_writer_retries_until_success():
 
 
 def test_process_correlation_id_only_drains_requested_row():
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="inst-a")
     ledger = InMemoryLedger()
-    writer = LedgerWriter(outbox, ledger)
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
     first = outbox.enqueue(_trade(execution_id="e-1"))
     second = outbox.enqueue(_trade(execution_id="e-2"))
 
-    outcome = writer.process_correlation_id(first.correlation_id)
+    outcome = writer.process_correlation_id(second.correlation_id)
 
     assert outcome is OutboxOutcome.COMPLETED
     assert ledger.count() == 1
-    assert outbox.get(first.correlation_id).status is WorkItemStatus.COMPLETED
-    assert outbox.get(second.correlation_id).status is WorkItemStatus.PENDING
+    assert writer.counters["claim:claimed"] == 1
+    assert writer.counters["outcome:completed"] == 1
+    assert outbox.get(first.correlation_id).status is WorkItemStatus.PENDING
+    assert outbox.get(second.correlation_id).status is WorkItemStatus.COMPLETED
+
+
+def test_process_correlation_id_skips_foreign_instance_row():
+    outbox = InMemoryLedgerOutbox(instance_key="inst-a")
+    ledger = InMemoryLedger()
+    item = outbox.enqueue(_trade(execution_id="e-foreign"))
+
+    with pytest.raises(ValueError, match="must match"):
+        LedgerWriter(outbox, ledger, instance_key="inst-b")
+    assert ledger.count() == 0
+    assert outbox.get(item.correlation_id).status is WorkItemStatus.PENDING
+
+
+def test_process_correlation_id_rejects_legacy_row_without_instance_owner():
+    class _LegacyOutbox(InMemoryLedgerOutbox):
+        def get(self, correlation_id):
+            item = super().get(correlation_id)
+            return replace(item, instance_key=None) if item is not None else None
+
+    outbox = _LegacyOutbox(instance_key="inst-a")
+    ledger = InMemoryLedger()
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
+    item = outbox.enqueue(_trade())
+
+    assert writer.process_correlation_id(item.correlation_id) is None
+    assert ledger.count() == 0
+
+
+def test_writer_exposes_claim_and_outcome_counters():
+    outbox = InMemoryLedgerOutbox(instance_key="inst-a")
+    ledger = InMemoryLedger()
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
+    outbox.enqueue(_trade(execution_id="e-count"))
+
+    stats = writer.run_once()
+
+    assert stats.completed == 1
+    assert writer.counters["claim:claimed"] == 1
+    assert writer.counters["outcome:completed"] == 1
+
+
+@pytest.mark.parametrize("instance_key", ["", "   "])
+def test_writer_rejects_explicit_empty_instance_key(instance_key):
+    outbox = InMemoryLedgerOutbox(instance_key="inst-a")
+    with pytest.raises(ValueError, match="non-empty"):
+        LedgerWriter(outbox, InMemoryLedger(), instance_key=instance_key)
+
+
+@pytest.mark.parametrize("instance_key", ["", "   "])
+def test_outbox_rejects_empty_instance_key(instance_key):
+    with pytest.raises(ValueError, match="non-empty"):
+        InMemoryLedgerOutbox(instance_key=instance_key)
+
+
+def test_in_memory_claim_is_exclusive_and_stale_lease_is_fenced():
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+    outbox = InMemoryLedgerOutbox(
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        clock=lambda: now["t"],
+    )
+    item = outbox.enqueue(_trade(execution_id="e-lease"))
+
+    [claim_a] = outbox.claim_pending()
+    assert outbox.claim_pending() == []
+    assert outbox.list_pending() == []
+
+    now["t"] += timedelta(seconds=6)
+    [claim_b] = outbox.claim_pending()
+    assert claim_b.claim_expires_at != claim_a.claim_expires_at
+    with pytest.raises(OutboxClaimLostError):
+        outbox.mark_outcome(
+            item.correlation_id,
+            OutboxOutcome.COMPLETED,
+            expected_claim_expires_at=claim_a.claim_expires_at,
+        )
+
+    stored = outbox.mark_outcome(
+        item.correlation_id,
+        OutboxOutcome.COMPLETED,
+        expected_claim_expires_at=claim_b.claim_expires_at,
+    )
+    assert stored.status is WorkItemStatus.COMPLETED
+    assert stored.claim_expires_at is None
+
+
+def test_outcome_logs_do_not_render_exception_text(caplog):
+    sentinel = "TOKEN_SENTINEL_ACCOUNT_DSN_PAYLOAD"
+
+    class _FailingLedger:
+        def record_trade(self, _trade):
+            raise LedgerPermanentError(sentinel)
+
+    outbox = InMemoryLedgerOutbox(instance_key="inst-a")
+    writer = LedgerWriter(outbox, _FailingLedger(), instance_key="inst-a")
+    outbox.enqueue(_trade(execution_id="e-sensitive"))
+
+    with caplog.at_level("INFO"):
+        stats = writer.run_once()
+
+    assert stats.delivery_failed == 1
+    assert sentinel not in caplog.text
+
+
+def test_lost_claim_is_counted_without_false_completion():
+    class _ClaimLosingOutbox(InMemoryLedgerOutbox):
+        def mark_outcome(self, correlation_id, outcome, **kwargs):
+            raise OutboxClaimLostError(correlation_id)
+
+    outbox = _ClaimLosingOutbox(instance_key="inst-a")
+    outbox.enqueue(_trade())
+    writer = LedgerWriter(outbox, InMemoryLedger(), instance_key="inst-a")
+
+    stats = writer.run_once()
+
+    assert stats.claim_lost == 1
+    assert stats.completed == 0

tests/application/test_ledger_outbox_backoff.py

@@ -47,7 +47,7 @@ class _ProgrammableLedger:
 
 
 def _wire(behavior, *, clock, **writer_kw):
-    outbox = InMemoryLedgerOutbox(clock=clock)
+    outbox = InMemoryLedgerOutbox(clock=clock, instance_key="test-instance")
     ledger = _ProgrammableLedger(behavior)
     writer = LedgerWriter(outbox, ledger, clock=clock, rng=lambda: 0.0, **writer_kw)
     return outbox, ledger, writer

tests/application/test_messaging_event_publisher.py

@@ -22,7 +22,7 @@ def _wire(transport):
     svc = SlotService(repo)
     intents = IntentService(svc, repo, InMemoryIntentStore(),
                             resolve_symbol=lambda pid: "005930")
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     ledger = InMemoryLedger()
     writer = LedgerWriter(outbox, ledger)
     repo.reconcile_universe("default", {"PF-1": "SYM"})
@@ -203,7 +203,7 @@ def test_sell_fill_dlq_sentinel_absent_when_strategy_id_set():
     intents = IntentService(svc, repo, InMemoryIntentStore(),
                             resolve_symbol=lambda pid: "005930")
     from trex_core.application.ledger_outbox import InMemoryLedgerOutbox
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(instance_key="test-instance")
     execution = ExecutionService(svc, repo, PaperBroker(fill_after_polls=0),
                                  ledger_outbox=outbox, event_emitter=emitter)
     repo.reconcile_universe("default", {"PF-1": "SYM"})