fix(ledger): dcy-015 outbox 영속 오류 분류 + 백오프/dead-letter (BAT-019)
portfolio-hub 404(미등록 포트폴리오)가 retryable 로 분류되어 백오프·상한 없이 초당 영구 재시도되던 버그(issue #13, 실측 930회) 정정. 거버넌스: 전부 change-req (신규 REQ 없음, dcy-015 OQ-3; 백오프 env 는 REQ-019 흡수). - REQ-017: 404/400/422 → LedgerPermanentError 분류 (409 conflict / 408·429·5xx· network retryable 불변). LedgerPort 에 permanent 타입 추가 - REQ-019: 영속 → 터미널 delivery_failed 즉시(재시도 0). retryable 지수 백오프 +jitter+next_retry_at, max_attempts 초과 시 dead-letter delivery_failed. list_pending 이 백오프 행 스킵(head-of-line 보호). 마이그레이션 007(status enum +next_retry_at). 백오프 env(base/cap/max-attempts) - REQ-018: 터미널/dead-letter WARN + WriterStats.delivery_failed 카운터 가시화 - REQ-020: runtime-flows/data-model/interface-contract 정합 단일 터미널 delivery_failed(permanent/dead-letter, last_outcome 로 사유 구분) — conflict_blocked 패턴과 동형. dead-letter 는 운영자 재처리 가능 상태. 검증: 단위 311 passed/34 skipped, 인프라 게이트(PG+RabbitMQ) 345 passed, ruff clean. 신규 test_ledger_outbox_backoff + adapter/PG 테스트, 기존 retryable 백오프 정합. BAT-019 release-candidate. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import pytest
|
||||
from trex_core.adapters.ledger.portfolio_hub import PortfolioHubLedger
|
||||
from trex_core.application.ledger_port import (
|
||||
LedgerConflictError,
|
||||
LedgerPermanentError,
|
||||
LedgerRetryableError,
|
||||
LedgerTrade,
|
||||
)
|
||||
@@ -76,3 +77,24 @@ def test_network_error_is_retryable():
|
||||
adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
|
||||
with pytest.raises(LedgerRetryableError):
|
||||
adapter.record_trade(_trade())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [404, 400, 422])
|
||||
def test_client_errors_are_permanent(status):
|
||||
"""dcy-015 / REQ-017: 404/400/422 won't be fixed by retrying → permanent."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(status, text='{"error":"portfolio not found"}')
|
||||
|
||||
adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
|
||||
with pytest.raises(LedgerPermanentError):
|
||||
adapter.record_trade(_trade())
|
||||
|
||||
|
||||
def test_429_stays_retryable():
|
||||
"""dcy-015 / REQ-017: rate-limit is transient → retryable (not permanent)."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(429, text="slow down")
|
||||
|
||||
adapter = PortfolioHubLedger("https://portfolio-hub.test", client=_client(handler))
|
||||
with pytest.raises(LedgerRetryableError):
|
||||
adapter.record_trade(_trade())
|
||||
|
||||
@@ -60,7 +60,8 @@ def test_writer_marks_conflict_blocked_on_pg(pg_clean):
|
||||
def test_writer_retries_on_pg(pg_clean):
|
||||
outbox = PostgreSQLLedgerOutbox(pg_clean)
|
||||
ledger = InMemoryLedger(fail_n_then_succeed=1)
|
||||
writer = LedgerWriter(outbox, ledger)
|
||||
# dcy-015: base 0 => next_retry_at == now, so the row is immediately due again.
|
||||
writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0)
|
||||
outbox.enqueue(_trade(execution_id="pg-r"))
|
||||
writer.run_once()
|
||||
row1 = outbox.get("PF-1:ORD-1:pg-r")
|
||||
@@ -70,3 +71,32 @@ def test_writer_retries_on_pg(pg_clean):
|
||||
row2 = outbox.get("PF-1:ORD-1:pg-r")
|
||||
assert row2.status is WorkItemStatus.COMPLETED
|
||||
assert ledger.count() == 1
|
||||
|
||||
|
||||
def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
|
||||
"""dcy-015 / REQ-019: 404/permanent → terminal delivery_failed persisted
|
||||
(migration 007 status enum + last_outcome). No retry."""
|
||||
outbox = PostgreSQLLedgerOutbox(pg_clean)
|
||||
ledger = InMemoryLedger(permanent_executions={"pg-perm"})
|
||||
writer = LedgerWriter(outbox, ledger)
|
||||
outbox.enqueue(_trade(execution_id="pg-perm"))
|
||||
s = writer.run_once()
|
||||
assert s.delivery_failed == 1
|
||||
row = outbox.get("PF-1:ORD-1:pg-perm")
|
||||
assert row.status is WorkItemStatus.DELIVERY_FAILED
|
||||
assert row.last_outcome is OutboxOutcome.PERMANENT_FAILURE
|
||||
# terminal — not returned by a subsequent pending scan
|
||||
assert outbox.list_pending() == []
|
||||
|
||||
|
||||
def test_backoff_row_skipped_by_pending_scan_on_pg(pg_clean):
|
||||
"""dcy-015 / REQ-019: a future next_retry_at excludes the row from list_pending."""
|
||||
outbox = PostgreSQLLedgerOutbox(pg_clean)
|
||||
ledger = InMemoryLedger(fail_n_then_succeed=99)
|
||||
# large base => next_retry_at well in the future
|
||||
writer = LedgerWriter(outbox, ledger, retry_base_sec=600.0, max_attempts=99)
|
||||
outbox.enqueue(_trade(execution_id="pg-bo"))
|
||||
writer.run_once() # -> retryable, next_retry_at ~ now + 600s
|
||||
row = outbox.get("PF-1:ORD-1:pg-bo")
|
||||
assert row.status is WorkItemStatus.PENDING and row.next_retry_at is not None
|
||||
assert outbox.list_pending() == [] # still backing off -> skipped
|
||||
|
||||
Reference in New Issue
Block a user