trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ledger): dcy-015 outbox 영속 오류 분류 + 백오프/dead-letter (BAT-019)

Browse Source 
portfolio-hub 404(미등록 포트폴리오)가 retryable 로 분류되어 백오프·상한 없이
초당 영구 재시도되던 버그(issue #13, 실측 930회) 정정.

거버넌스: 전부 change-req (신규 REQ 없음, dcy-015 OQ-3; 백오프 env 는 REQ-019 흡수).
- REQ-017: 404/400/422 → LedgerPermanentError 분류 (409 conflict / 408·429·5xx·
  network retryable 불변). LedgerPort 에 permanent 타입 추가
- REQ-019: 영속 → 터미널 delivery_failed 즉시(재시도 0). retryable 지수 백오프
  +jitter+next_retry_at, max_attempts 초과 시 dead-letter delivery_failed.
  list_pending 이 백오프 행 스킵(head-of-line 보호). 마이그레이션 007(status enum
  +next_retry_at). 백오프 env(base/cap/max-attempts)
- REQ-018: 터미널/dead-letter WARN + WriterStats.delivery_failed 카운터 가시화
- REQ-020: runtime-flows/data-model/interface-contract 정합

단일 터미널 delivery_failed(permanent/dead-letter, last_outcome 로 사유 구분) —
conflict_blocked 패턴과 동형. dead-letter 는 운영자 재처리 가능 상태.

검증: 단위 311 passed/34 skipped, 인프라 게이트(PG+RabbitMQ) 345 passed, ruff clean.
신규 test_ledger_outbox_backoff + adapter/PG 테스트, 기존 retryable 백오프 정합.
BAT-019 release-candidate.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
genichin
2026-06-08 16:12:17 +09:00
co-authored by Claude Opus 4.8
parent 0b7db66463
commit b851b927ee
29 changed files with 1069 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/adapters/ledger/in_memory.py
+9
View File
@@ -5,6 +5,7 @@ import threading
from trex_core.application.ledger_port import (
LedgerConflictError,
LedgerPermanentError,
LedgerRetryableError,
LedgerTrade,
LedgerWriteResult,
@@ -21,6 +22,8 @@ class InMemoryLedger:
then accept.
conflict_executions -- execution_ids that raise LedgerConflictError
(simulates portfolio-hub 409).
permanent_executions -- execution_ids that raise LedgerPermanentError
(simulates portfolio-hub 404/400/422, dcy-015).
"""
def __init__(
@@ -28,12 +31,14 @@ class InMemoryLedger:
*,
fail_n_then_succeed: int = 0,
conflict_executions: set[str] | None = None,
permanent_executions: set[str] | None = None,
) -> None:
self._trades: dict[str, LedgerTrade] = {}
self._order: list[LedgerTrade] = []
self._lock = threading.Lock()
self._fail_remaining = fail_n_then_succeed
self._conflicts = set(conflict_executions or set())
self._permanent = set(permanent_executions or set())
def record_trade(self, trade: LedgerTrade) -> LedgerWriteResult:
with self._lock:
@@ -41,6 +46,10 @@ class InMemoryLedger:
raise LedgerConflictError(
f"ledger conflict for execution_id={trade.execution_id}"
)
if trade.execution_id in self._permanent:
raise LedgerPermanentError(
f"ledger permanent error for execution_id={trade.execution_id}"
)
if self._fail_remaining > 0:
self._fail_remaining -= 1
raise LedgerRetryableError("simulated transient failure")
trex_core/adapters/ledger/portfolio_hub.py
+15 -1
View File
@@ -6,7 +6,9 @@ Live portfolio-hub contract (verified against openapi, REQ-017 CHG-04):
execution_id, order_id
- 2xx (201) => recorded (or duplicate accepted idempotently)
- 409 => LedgerConflictError (operator resolution)
- 4xx (other)/5xx => LedgerRetryableError (outbox retries)
- 404/400/422 => LedgerPermanentError (dcy-015/REQ-017 — retry can't fix:
portfolio not registered / malformed trade)
- 4xx (other)/5xx => LedgerRetryableError (outbox backoff retries)
- httpx error => LedgerRetryableError
`asset_id` on LedgerTrade carries the portfolio_id value (REQ-018:
@@ -21,11 +23,15 @@ import httpx
from trex_core.application.ledger_port import (
LedgerConflictError,
LedgerPermanentError,
LedgerRetryableError,
LedgerTrade,
LedgerWriteResult,
)
# dcy-015 / REQ-017: client errors that retrying cannot resolve.
_PERMANENT_STATUS = frozenset({400, 404, 422})
class PortfolioHubLedger:
"""portfolio-hub ledger adapter (single endpoint, single responsibility)."""
@@ -91,6 +97,14 @@ class PortfolioHubLedger:
f"portfolio-hub 409 conflict for execution_id={trade.execution_id}: "
f"{resp.text[:200]}"
)
if resp.status_code in _PERMANENT_STATUS:
# dcy-015 / REQ-017 (issue #13): 404/400/422 won't be fixed by
# retrying — surface as permanent so the outbox terminates the row.
raise LedgerPermanentError(
f"portfolio-hub {resp.status_code} permanent error for "
f"asset_id={trade.asset_id} execution_id={trade.execution_id}: "
f"{resp.text[:200]}"
)
raise LedgerRetryableError(
f"portfolio-hub {resp.status_code} response: {resp.text[:200]}"
)
trex_core/adapters/persistence/postgresql.py
+14 -4
View File
@@ -325,7 +325,7 @@ class _PgIntentRecord:
_OUTBOX_COLUMNS = (
"correlation_id, asset_id, order_id, execution_id, side, qty, price, "
"fee, tax, executed_at, status, attempts, last_outcome, last_error, "
"created_at, updated_at"
"created_at, updated_at, next_retry_at"
)
@@ -362,7 +362,9 @@ class PostgreSQLLedgerOutbox:
with conn.cursor() as cur:
cur.execute(
f"SELECT {_OUTBOX_COLUMNS} FROM ledger_outbox "
"WHERE status = 'pending' ORDER BY created_at LIMIT %s",
"WHERE status = 'pending' "
"AND (next_retry_at IS NULL OR next_retry_at <= now()) "
"ORDER BY created_at LIMIT %s",
(limit,),
)
return [self._row_to_item(r) for r in cur.fetchall()]
@@ -383,6 +385,7 @@ class PostgreSQLLedgerOutbox:
outcome: OutboxOutcome,
*,
error: Optional[str] = None,
next_retry_at: Optional[datetime] = None,
) -> LedgerWorkItem:
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
with conn.cursor() as cur:
@@ -392,18 +395,24 @@ class PostgreSQLLedgerOutbox:
elif outcome is OutboxOutcome.CONFLICT_BLOCKED:
status_value = WorkItemStatus.CONFLICT_BLOCKED.value
attempts_delta = 1
elif outcome in (OutboxOutcome.PERMANENT_FAILURE, OutboxOutcome.DEAD_LETTERED):
# dcy-015 / REQ-019: terminal delivery_failed (reason in last_outcome).
status_value = WorkItemStatus.DELIVERY_FAILED.value
attempts_delta = 1
elif outcome is OutboxOutcome.DUPLICATE_COMPLETED:
status_value = WorkItemStatus.COMPLETED.value
attempts_delta = 0
else: # retryable
status_value = WorkItemStatus.PENDING.value
attempts_delta = 1
# next_retry_at only persists for a retryable backoff; cleared otherwise.
nra = next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None
cur.execute(
"UPDATE ledger_outbox SET status=%s, attempts=attempts+%s, "
"last_outcome=%s, last_error=%s, updated_at=%s "
"last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s "
"WHERE correlation_id=%s",
(
status_value, attempts_delta, outcome.value, error,
status_value, attempts_delta, outcome.value, error, nra,
datetime.now(tz=timezone.utc), correlation_id,
),
)
@@ -429,4 +438,5 @@ class PostgreSQLLedgerOutbox:
last_error=row[13],
created_at=row[14],
updated_at=row[15],
next_retry_at=row[16],
)
trex_core/adapters/persistence/sql/007_ledger_outbox_delivery_failed_and_backoff.sql
+25
View File
@@ -0,0 +1,25 @@
-- BAT-019 / dcy-015 / REQ-019 (issue #13): ledger_outbox terminal failure +
-- backoff scheduling. Adds:
-- - next_retry_at: earliest re-attempt time for a retryable row (backoff).
-- - status 'delivery_failed': single terminal for permanent (404/400/422)
-- and dead-lettered (max_attempts exhausted) rows; reason in last_outcome.
-- - last_outcome 'permanent_failure'/'dead_lettered'.
-- Idempotent: ADD COLUMN IF NOT EXISTS + DROP/ADD CONSTRAINT (re-runnable since
-- migrations are re-applied by glob).
ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
ALTER TABLE ledger_outbox DROP CONSTRAINT IF EXISTS ledger_outbox_status_check;
ALTER TABLE ledger_outbox ADD CONSTRAINT ledger_outbox_status_check
CHECK (status IN ('pending','completed','conflict_blocked','delivery_failed'));
ALTER TABLE ledger_outbox DROP CONSTRAINT IF EXISTS ledger_outbox_last_outcome_check;
ALTER TABLE ledger_outbox ADD CONSTRAINT ledger_outbox_last_outcome_check
CHECK (last_outcome IS NULL OR last_outcome IN (
'completed','duplicate_completed','retryable_failure','conflict_blocked',
'permanent_failure','dead_lettered'
));
-- Partial index so the writer's pending scan skips rows still in backoff cheaply.
CREATE INDEX IF NOT EXISTS idx_ledger_outbox_pending_retry
ON ledger_outbox (next_retry_at)
WHERE status = 'pending';