fix: fence ledger outbox claims by instance

This commit is contained in:
Hermes Agent
2026-09-13 09:41:51 +09:00
parent f801c53e0c
commit 762a953c95
15 changed files with 316 additions and 88 deletions
+106 -10
View File
@@ -1,16 +1,20 @@
"""LedgerOutbox + LedgerWriter direct unit tests (REQ-019)."""
from __future__ import annotations
from dataclasses import replace
from datetime import UTC, datetime, timedelta
import pytest
from trex_core.adapters.ledger.in_memory import InMemoryLedger
from trex_core.application.ledger_outbox import (
InMemoryLedgerOutbox,
LedgerWriter,
OutboxClaimLostError,
OutboxOutcome,
WorkItemStatus,
)
from trex_core.application.ledger_port import LedgerTrade
from trex_core.application.ledger_port import LedgerPermanentError, LedgerTrade
def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade:
@@ -22,7 +26,7 @@ def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade:
def test_enqueue_is_idempotent_on_correlation_id():
outbox = InMemoryLedgerOutbox()
outbox = InMemoryLedgerOutbox(instance_key="test-instance")
a = outbox.enqueue(_trade())
b = outbox.enqueue(_trade())
assert a.correlation_id == b.correlation_id
@@ -30,7 +34,7 @@ def test_enqueue_is_idempotent_on_correlation_id():
def test_writer_run_once_drains_pending_and_marks_completed():
outbox = InMemoryLedgerOutbox()
outbox = InMemoryLedgerOutbox(instance_key="test-instance")
ledger = InMemoryLedger()
writer = LedgerWriter(outbox, ledger)
outbox.enqueue(_trade(execution_id="e-1"))
@@ -42,7 +46,7 @@ def test_writer_run_once_drains_pending_and_marks_completed():
def test_writer_marks_conflict_blocked_and_does_not_retry():
outbox = InMemoryLedgerOutbox()
outbox = InMemoryLedgerOutbox(instance_key="test-instance")
ledger = InMemoryLedger(conflict_executions={"e-x"})
writer = LedgerWriter(outbox, ledger)
outbox.enqueue(_trade(execution_id="e-x"))
@@ -58,7 +62,9 @@ def test_writer_retries_until_success():
# dcy-015: retryable rows now back off; share an injected clock between the
# outbox (list_pending "now") and the writer (next_retry_at) to advance past it.
now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
outbox = InMemoryLedgerOutbox(clock=lambda: now["t"])
outbox = InMemoryLedgerOutbox(
clock=lambda: now["t"], instance_key="test-instance"
)
ledger = InMemoryLedger(fail_n_then_succeed=1)
writer = LedgerWriter(outbox, ledger, clock=lambda: now["t"])
outbox.enqueue(_trade(execution_id="e-r"))
@@ -90,17 +96,29 @@ def test_process_correlation_id_only_drains_requested_row():
def test_process_correlation_id_skips_foreign_instance_row():
outbox = InMemoryLedgerOutbox(instance_key="inst-a")
ledger = InMemoryLedger()
writer = LedgerWriter(outbox, ledger, instance_key="inst-b")
item = outbox.enqueue(_trade(execution_id="e-foreign"))
outcome = writer.process_correlation_id(item.correlation_id)
assert outcome is None
with pytest.raises(ValueError, match="must match"):
LedgerWriter(outbox, ledger, instance_key="inst-b")
assert ledger.count() == 0
assert writer.counters["claim:foreign_instance"] == 1
assert outbox.get(item.correlation_id).status is WorkItemStatus.PENDING
def test_process_correlation_id_rejects_legacy_row_without_instance_owner():
class _LegacyOutbox(InMemoryLedgerOutbox):
def get(self, correlation_id):
item = super().get(correlation_id)
return replace(item, instance_key=None) if item is not None else None
outbox = _LegacyOutbox(instance_key="inst-a")
ledger = InMemoryLedger()
writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
item = outbox.enqueue(_trade())
assert writer.process_correlation_id(item.correlation_id) is None
assert ledger.count() == 0
def test_writer_exposes_claim_and_outcome_counters():
outbox = InMemoryLedgerOutbox(instance_key="inst-a")
ledger = InMemoryLedger()
@@ -112,3 +130,81 @@ def test_writer_exposes_claim_and_outcome_counters():
assert stats.completed == 1
assert writer.counters["claim:claimed"] == 1
assert writer.counters["outcome:completed"] == 1
@pytest.mark.parametrize("instance_key", ["", " "])
def test_writer_rejects_explicit_empty_instance_key(instance_key):
outbox = InMemoryLedgerOutbox(instance_key="inst-a")
with pytest.raises(ValueError, match="non-empty"):
LedgerWriter(outbox, InMemoryLedger(), instance_key=instance_key)
@pytest.mark.parametrize("instance_key", ["", " "])
def test_outbox_rejects_empty_instance_key(instance_key):
with pytest.raises(ValueError, match="non-empty"):
InMemoryLedgerOutbox(instance_key=instance_key)
def test_in_memory_claim_is_exclusive_and_stale_lease_is_fenced():
now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
outbox = InMemoryLedgerOutbox(
instance_key="inst-a",
claim_ttl_sec=5.0,
clock=lambda: now["t"],
)
item = outbox.enqueue(_trade(execution_id="e-lease"))
[claim_a] = outbox.claim_pending()
assert outbox.claim_pending() == []
assert outbox.list_pending() == []
now["t"] += timedelta(seconds=6)
[claim_b] = outbox.claim_pending()
assert claim_b.claim_expires_at != claim_a.claim_expires_at
with pytest.raises(OutboxClaimLostError):
outbox.mark_outcome(
item.correlation_id,
OutboxOutcome.COMPLETED,
expected_claim_expires_at=claim_a.claim_expires_at,
)
stored = outbox.mark_outcome(
item.correlation_id,
OutboxOutcome.COMPLETED,
expected_claim_expires_at=claim_b.claim_expires_at,
)
assert stored.status is WorkItemStatus.COMPLETED
assert stored.claim_expires_at is None
def test_outcome_logs_do_not_render_exception_text(caplog):
sentinel = "TOKEN_SENTINEL_ACCOUNT_DSN_PAYLOAD"
class _FailingLedger:
def record_trade(self, _trade):
raise LedgerPermanentError(sentinel)
outbox = InMemoryLedgerOutbox(instance_key="inst-a")
writer = LedgerWriter(outbox, _FailingLedger(), instance_key="inst-a")
outbox.enqueue(_trade(execution_id="e-sensitive"))
with caplog.at_level("INFO"):
stats = writer.run_once()
assert stats.delivery_failed == 1
assert sentinel not in caplog.text
def test_lost_claim_is_counted_without_false_completion():
class _ClaimLosingOutbox(InMemoryLedgerOutbox):
def mark_outcome(self, correlation_id, outcome, **kwargs):
raise OutboxClaimLostError(correlation_id)
outbox = _ClaimLosingOutbox(instance_key="inst-a")
outbox.enqueue(_trade())
writer = LedgerWriter(outbox, InMemoryLedger(), instance_key="inst-a")
stats = writer.run_once()
assert stats.claim_lost == 1
assert stats.completed == 0