feat(ops): track broker balance worker failure lifecycle and backoff
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
59bb492883
commit
084e05a97d
@@ -5,8 +5,11 @@ In-memory adapters; deterministic paper double drives the full async path.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from trex_core.adapters.broker.paper import PaperBroker
|
||||
from trex_core.adapters.persistence.in_memory_intent_store import InMemoryIntentStore
|
||||
from trex_core.application.account_snapshot import AccountSnapshotUnavailable
|
||||
from trex_core.application.broker_port import (
|
||||
BalanceItem,
|
||||
CancelOutcome,
|
||||
@@ -214,6 +217,61 @@ def test_pending_sell_uses_injected_bounded_balance_reader():
|
||||
assert repo.fetch_slot(sid).status is SlotState.PENDING_SELL
|
||||
|
||||
|
||||
def test_pending_sell_balance_failure_is_fail_closed_with_backoff_and_recovery():
|
||||
now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
|
||||
broker = PaperBroker(fill_after_polls=0)
|
||||
repo, svc, intents, _ = _wire(broker)
|
||||
sid = _buy(intents, qty=3).slot_id
|
||||
|
||||
execution = ExecutionService(
|
||||
svc,
|
||||
repo,
|
||||
broker,
|
||||
instance_key=INST,
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
execution.run_once()
|
||||
svc.sell(sid, actor="t", reason="full")
|
||||
broker._fill_after = 5
|
||||
|
||||
calls = {"n": 0}
|
||||
reader_state = {"ok": False}
|
||||
|
||||
def failing_reader():
|
||||
calls["n"] += 1
|
||||
if not reader_state["ok"]:
|
||||
raise AccountSnapshotUnavailable("expired")
|
||||
return [BalanceItem("005931", 3)]
|
||||
|
||||
execution = ExecutionService(
|
||||
svc,
|
||||
repo,
|
||||
broker,
|
||||
instance_key=INST,
|
||||
balance_reader=failing_reader,
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
first = execution.run_once()
|
||||
assert first.sell_skipped_no_balance == 1
|
||||
assert calls["n"] == 1
|
||||
health = execution.balance_worker_health(stale_after_sec=600)
|
||||
assert health["state"] == "unavailable"
|
||||
assert health["failure_code"] == "stale_snapshot"
|
||||
assert health["consecutive_failures"] == 1
|
||||
|
||||
second = execution.run_once()
|
||||
assert second.sell_skipped_no_balance == 1
|
||||
assert calls["n"] == 1 # bounded backoff prevented another live read
|
||||
|
||||
now["t"] += timedelta(seconds=16)
|
||||
reader_state["ok"] = True
|
||||
third = execution.run_once()
|
||||
assert third.dispatched == 1
|
||||
health = execution.balance_worker_health(stale_after_sec=600)
|
||||
assert health["state"] == "ok"
|
||||
assert health["consecutive_failures"] == 0
|
||||
|
||||
|
||||
def test_partial_sell_qty_is_min_intent_broker():
|
||||
"""dcy-021/REQ-062 (D4): partial sell dispatches min(intent qty, broker);
|
||||
remainder stays holding (broker-authoritative qty), cost basis preserved."""
|
||||
|
||||
Reference in New Issue
Block a user