fix(ledger): dcy-015 outbox 영속 오류 분류 + 백오프/dead-letter (BAT-019)

portfolio-hub 404(미등록 포트폴리오)가 retryable 로 분류되어 백오프·상한 없이
초당 영구 재시도되던 버그(issue #13, 실측 930회) 정정.

거버넌스: 전부 change-req (신규 REQ 없음, dcy-015 OQ-3; 백오프 env 는 REQ-019 흡수).
- REQ-017: 404/400/422 → LedgerPermanentError 분류 (409 conflict / 408·429·5xx·
  network retryable 불변). LedgerPort 에 permanent 타입 추가
- REQ-019: 영속 → 터미널 delivery_failed 즉시(재시도 0). retryable 지수 백오프
  +jitter+next_retry_at, max_attempts 초과 시 dead-letter delivery_failed.
  list_pending 이 백오프 행 스킵(head-of-line 보호). 마이그레이션 007(status enum
  +next_retry_at). 백오프 env(base/cap/max-attempts)
- REQ-018: 터미널/dead-letter WARN + WriterStats.delivery_failed 카운터 가시화
- REQ-020: runtime-flows/data-model/interface-contract 정합

단일 터미널 delivery_failed(permanent/dead-letter, last_outcome 로 사유 구분) —
conflict_blocked 패턴과 동형. dead-letter 는 운영자 재처리 가능 상태.

검증: 단위 311 passed/34 skipped, 인프라 게이트(PG+RabbitMQ) 345 passed, ruff clean.
신규 test_ledger_outbox_backoff + adapter/PG 테스트, 기존 retryable 백오프 정합.
BAT-019 release-candidate.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/adapters/ledger/test_portfolio_hub_ledger.py

@@ -10,6 +10,7 @@ import pytest
 from trex_core.adapters.ledger.portfolio_hub import PortfolioHubLedger
 from trex_core.application.ledger_port import (
     LedgerConflictError,
+    LedgerPermanentError,
     LedgerRetryableError,
     LedgerTrade,
 )
@@ -76,3 +77,24 @@ def test_network_error_is_retryable():
     adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
     with pytest.raises(LedgerRetryableError):
         adapter.record_trade(_trade())
+
+
+@pytest.mark.parametrize("status", [404, 400, 422])
+def test_client_errors_are_permanent(status):
+    """dcy-015 / REQ-017: 404/400/422 won't be fixed by retrying → permanent."""
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(status, text='{"error":"portfolio not found"}')
+
+    adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
+    with pytest.raises(LedgerPermanentError):
+        adapter.record_trade(_trade())
+
+
+def test_429_stays_retryable():
+    """dcy-015 / REQ-017: rate-limit is transient → retryable (not permanent)."""
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(429, text="slow down")
+
+    adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
+    with pytest.raises(LedgerRetryableError):
+        adapter.record_trade(_trade())

tests/adapters/persistence/test_postgresql_ledger_outbox.py

@@ -60,7 +60,8 @@ def test_writer_marks_conflict_blocked_on_pg(pg_clean):
 def test_writer_retries_on_pg(pg_clean):
     outbox = PostgreSQLLedgerOutbox(pg_clean)
     ledger = InMemoryLedger(fail_n_then_succeed=1)
-    writer = LedgerWriter(outbox, ledger)
+    # dcy-015: base 0 => next_retry_at == now, so the row is immediately due again.
+    writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0)
     outbox.enqueue(_trade(execution_id="pg-r"))
     writer.run_once()
     row1 = outbox.get("PF-1:ORD-1:pg-r")
@@ -70,3 +71,32 @@ def test_writer_retries_on_pg(pg_clean):
     row2 = outbox.get("PF-1:ORD-1:pg-r")
     assert row2.status is WorkItemStatus.COMPLETED
     assert ledger.count() == 1
+
+
+def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
+    """dcy-015 / REQ-019: 404/permanent → terminal delivery_failed persisted
+    (migration 007 status enum + last_outcome). No retry."""
+    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    ledger = InMemoryLedger(permanent_executions={"pg-perm"})
+    writer = LedgerWriter(outbox, ledger)
+    outbox.enqueue(_trade(execution_id="pg-perm"))
+    s = writer.run_once()
+    assert s.delivery_failed == 1
+    row = outbox.get("PF-1:ORD-1:pg-perm")
+    assert row.status is WorkItemStatus.DELIVERY_FAILED
+    assert row.last_outcome is OutboxOutcome.PERMANENT_FAILURE
+    # terminal — not returned by a subsequent pending scan
+    assert outbox.list_pending() == []
+
+
+def test_backoff_row_skipped_by_pending_scan_on_pg(pg_clean):
+    """dcy-015 / REQ-019: a future next_retry_at excludes the row from list_pending."""
+    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    ledger = InMemoryLedger(fail_n_then_succeed=99)
+    # large base => next_retry_at well in the future
+    writer = LedgerWriter(outbox, ledger, retry_base_sec=600.0, max_attempts=99)
+    outbox.enqueue(_trade(execution_id="pg-bo"))
+    writer.run_once()  # -> retryable, next_retry_at ~ now + 600s
+    row = outbox.get("PF-1:ORD-1:pg-bo")
+    assert row.status is WorkItemStatus.PENDING and row.next_retry_at is not None
+    assert outbox.list_pending() == []  # still backing off -> skipped

tests/application/test_execution_ledger.py

@@ -25,13 +25,16 @@ from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
 DAY = date(2026, 6, 4)
 
 
-def _wire(broker, *, ledger=None):
+def _wire(broker, *, ledger=None, clock=None):
     repo = InMemorySlotRepository()
     svc = SlotService(repo)
     intents = IntentService(svc, repo, InMemoryIntentStore())
-    outbox = InMemoryLedgerOutbox()
+    outbox = InMemoryLedgerOutbox(clock=clock) if clock else InMemoryLedgerOutbox()
     execution = ExecutionService(svc, repo, broker, ledger_outbox=outbox)
-    writer = LedgerWriter(outbox, ledger or InMemoryLedger())
+    writer = (
+        LedgerWriter(outbox, ledger or InMemoryLedger(), clock=clock)
+        if clock else LedgerWriter(outbox, ledger or InMemoryLedger())
+    )
     svc.preallocate_daily_slots(DAY, 5)
     return repo, svc, intents, execution, outbox, writer
 
@@ -134,17 +137,26 @@ def test_duplicate_execution_id_results_in_zero_duplicate_in_ledger():
 
 
 def test_retryable_failure_then_eventually_completes():
-    """REQ-019 AC-2: retry until completed; slot transition unaffected."""
+    """REQ-019 AC-2: retry until completed; slot transition unaffected.
+
+    dcy-015: retryable rows back off, so advance a shared clock past each
+    next_retry_at between writer passes.
+    """
+    from datetime import datetime, timedelta, timezone
+    now = {"t": datetime(2026, 6, 4, tzinfo=timezone.utc)}
     ledger = InMemoryLedger(fail_n_then_succeed=2)
-    repo, svc, intents, execution, outbox, writer = _wire(PaperBroker(fill_after_polls=0),
-                                                          ledger=ledger)
+    repo, svc, intents, execution, outbox, writer = _wire(
+        PaperBroker(fill_after_polls=0), ledger=ledger, clock=lambda: now["t"],
+    )
     sid = _buy(intents).slot_id
     execution.run_once()
     assert repo.fetch_slot(sid).status is SlotState.HOLDING  # slot already transitioned
     stats1 = writer.run_once()
     assert stats1.retryable == 1 and stats1.completed == 0 and ledger.count() == 0
+    now["t"] += timedelta(seconds=600)  # past backoff
     stats2 = writer.run_once()
     assert stats2.retryable == 1 and ledger.count() == 0
+    now["t"] += timedelta(seconds=600)  # past backoff again
     stats3 = writer.run_once()
     assert stats3.completed == 1 and ledger.count() == 1
     # outbox row terminal-completed

tests/application/test_ledger_outbox.py

@@ -1,7 +1,7 @@
 """LedgerOutbox + LedgerWriter direct unit tests (REQ-019)."""
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 from trex_core.adapters.ledger.in_memory import InMemoryLedger
 from trex_core.application.ledger_outbox import (
@@ -55,11 +55,16 @@ def test_writer_marks_conflict_blocked_and_does_not_retry():
 
 
 def test_writer_retries_until_success():
-    outbox = InMemoryLedgerOutbox()
+    # dcy-015: retryable rows now back off; share an injected clock between the
+    # outbox (list_pending "now") and the writer (next_retry_at) to advance past it.
+    now = {"t": datetime(2026, 6, 4, tzinfo=timezone.utc)}
+    outbox = InMemoryLedgerOutbox(clock=lambda: now["t"])
     ledger = InMemoryLedger(fail_n_then_succeed=1)
-    writer = LedgerWriter(outbox, ledger)
+    writer = LedgerWriter(outbox, ledger, clock=lambda: now["t"])
     outbox.enqueue(_trade(execution_id="e-r"))
     s1 = writer.run_once()
     assert s1.retryable == 1 and ledger.count() == 0
+    assert writer.run_once().completed == 0  # still in backoff -> skipped
+    now["t"] += timedelta(seconds=600)       # past the backoff window
     s2 = writer.run_once()
     assert s2.completed == 1 and ledger.count() == 1

tests/application/test_ledger_outbox_backoff.py

@@ -0,0 +1,156 @@
+"""Ledger outbox permanent-error + backoff + dead-letter tests (dcy-015 / issue #13).
+
+A programmable LedgerPort raises chosen errors so we can drive: 404→permanent
+terminal (no retry), retryable→backoff (next_retry_at skip), max_attempts→
+dead-letter terminal, and head-of-line protection.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+from trex_core.application.ledger_outbox import (
+    InMemoryLedgerOutbox,
+    LedgerWriter,
+    OutboxOutcome,
+    WorkItemStatus,
+)
+from trex_core.application.ledger_port import (
+    LedgerPermanentError,
+    LedgerRetryableError,
+    LedgerTrade,
+    LedgerWriteResult,
+)
+
+UTC = timezone.utc
+
+
+def _trade(cid_suffix="1", asset="PF-1") -> LedgerTrade:
+    return LedgerTrade(
+        asset_id=asset, order_id=f"O-{cid_suffix}", execution_id=f"e-{cid_suffix}",
+        side="buy", qty=10, price=1000.0,
+        executed_at=datetime(2026, 6, 4, tzinfo=UTC),
+    )
+
+
+class _ProgrammableLedger:
+    """LedgerPort double: `behavior(trade)` returns a result or raises."""
+
+    def __init__(self, behavior):
+        self.behavior = behavior
+        self.calls = 0
+
+    def record_trade(self, trade) -> LedgerWriteResult:
+        self.calls += 1
+        return self.behavior(trade)
+
+
+def _wire(behavior, *, clock, **writer_kw):
+    outbox = InMemoryLedgerOutbox(clock=clock)
+    ledger = _ProgrammableLedger(behavior)
+    writer = LedgerWriter(outbox, ledger, clock=clock, rng=lambda: 0.0, **writer_kw)
+    return outbox, ledger, writer
+
+
+def test_permanent_error_terminates_immediately_no_retry():
+    """S-1: 404/permanent → delivery_failed terminal, adapter called exactly once."""
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+
+    def behavior(trade):
+        raise LedgerPermanentError("404 portfolio not registered")
+
+    outbox, ledger, writer = _wire(behavior, clock=lambda: now["t"])
+    outbox.enqueue(_trade())
+    s = writer.run_once()
+    assert s.delivery_failed == 1 and s.retryable == 0
+    [item] = outbox.all_items()
+    assert item.status is WorkItemStatus.DELIVERY_FAILED
+    assert item.last_outcome is OutboxOutcome.PERMANENT_FAILURE
+    # no further attempts — terminal, not pending
+    now["t"] += timedelta(seconds=3600)
+    assert writer.run_once().delivery_failed == 0
+    assert ledger.calls == 1  # never retried
+
+
+def test_retryable_backoff_skips_until_next_retry_at():
+    """S-2/S-4: a retryable row backs off and is skipped by list_pending until due."""
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+
+    def behavior(trade):
+        raise LedgerRetryableError("503 upstream")
+
+    outbox, ledger, writer = _wire(
+        behavior, clock=lambda: now["t"], retry_base_sec=10.0, max_attempts=99,
+    )
+    outbox.enqueue(_trade())
+    assert writer.run_once().retryable == 1
+    [item] = outbox.all_items()
+    assert item.status is WorkItemStatus.PENDING and item.next_retry_at is not None
+    # within backoff window -> skipped (no adapter call)
+    calls_before = ledger.calls
+    assert writer.run_once().retryable == 0
+    assert ledger.calls == calls_before
+    # advance past backoff -> retried
+    now["t"] += timedelta(seconds=60)
+    assert writer.run_once().retryable == 1
+    assert ledger.calls == calls_before + 1
+
+
+def test_retryable_dead_letters_after_max_attempts():
+    """S-2: retryable exhausts max_attempts → delivery_failed (dead-letter)."""
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+
+    def behavior(trade):
+        raise LedgerRetryableError("503 upstream")
+
+    outbox, ledger, writer = _wire(
+        behavior, clock=lambda: now["t"], retry_base_sec=1.0, retry_cap_sec=2.0,
+        max_attempts=3,
+    )
+    outbox.enqueue(_trade())
+    # attempt 1, 2 -> retryable; attempt 3 -> dead-letter
+    assert writer.run_once().retryable == 1
+    now["t"] += timedelta(seconds=10)
+    assert writer.run_once().retryable == 1
+    now["t"] += timedelta(seconds=10)
+    s = writer.run_once()
+    assert s.delivery_failed == 1
+    [item] = outbox.all_items()
+    assert item.status is WorkItemStatus.DELIVERY_FAILED
+    assert item.last_outcome is OutboxOutcome.DEAD_LETTERED
+    assert item.attempts == 3
+
+
+def test_permanent_failure_emits_warning(caplog):
+    """S-3: terminal delivery_failed is visible (WARN), not silent."""
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+
+    def behavior(trade):
+        raise LedgerPermanentError("404 not found")
+
+    outbox, ledger, writer = _wire(behavior, clock=lambda: now["t"])
+    outbox.enqueue(_trade())
+    with caplog.at_level(logging.WARNING):
+        writer.run_once()
+    assert any("delivery_failed" in r.message for r in caplog.records)
+
+
+def test_backoff_row_does_not_block_new_rows():
+    """S-4: a backing-off row does not occupy the FIFO batch; new rows proceed."""
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+
+    def behavior(trade):
+        if trade.asset_id == "PF-bad":
+            raise LedgerRetryableError("503")
+        return LedgerWriteResult.NEW
+
+    outbox, ledger, writer = _wire(
+        behavior, clock=lambda: now["t"], retry_base_sec=100.0, max_attempts=99,
+    )
+    outbox.enqueue(_trade(cid_suffix="bad", asset="PF-bad"))
+    writer.run_once()  # PF-bad -> backoff
+    outbox.enqueue(_trade(cid_suffix="good", asset="PF-good"))
+    s = writer.run_once()  # PF-bad skipped (backoff), PF-good completes
+    assert s.completed == 1 and s.retryable == 0
+    good = outbox.get("PF-good:O-good:e-good")
+    assert good.status is WorkItemStatus.COMPLETED