feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-12 13:33:12 +00:00
committed by GitHub
co-authored by genichin
parent 59bb492883
commit 084e05a97d
9 changed files with 453 additions and 27 deletions
+58
View File
@@ -5,8 +5,11 @@ In-memory adapters; deterministic paper double drives the full async path.
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from trex_core.adapters.broker.paper import PaperBroker
from trex_core.adapters.persistence.in_memory_intent_store import InMemoryIntentStore
from trex_core.application.account_snapshot import AccountSnapshotUnavailable
from trex_core.application.broker_port import (
BalanceItem,
CancelOutcome,
@@ -214,6 +217,61 @@ def test_pending_sell_uses_injected_bounded_balance_reader():
assert repo.fetch_slot(sid).status is SlotState.PENDING_SELL
def test_pending_sell_balance_failure_is_fail_closed_with_backoff_and_recovery():
now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
broker = PaperBroker(fill_after_polls=0)
repo, svc, intents, _ = _wire(broker)
sid = _buy(intents, qty=3).slot_id
execution = ExecutionService(
svc,
repo,
broker,
instance_key=INST,
clock=lambda: now["t"],
)
execution.run_once()
svc.sell(sid, actor="t", reason="full")
broker._fill_after = 5
calls = {"n": 0}
reader_state = {"ok": False}
def failing_reader():
calls["n"] += 1
if not reader_state["ok"]:
raise AccountSnapshotUnavailable("expired")
return [BalanceItem("005931", 3)]
execution = ExecutionService(
svc,
repo,
broker,
instance_key=INST,
balance_reader=failing_reader,
clock=lambda: now["t"],
)
first = execution.run_once()
assert first.sell_skipped_no_balance == 1
assert calls["n"] == 1
health = execution.balance_worker_health(stale_after_sec=600)
assert health["state"] == "unavailable"
assert health["failure_code"] == "stale_snapshot"
assert health["consecutive_failures"] == 1
second = execution.run_once()
assert second.sell_skipped_no_balance == 1
assert calls["n"] == 1 # bounded backoff prevented another live read
now["t"] += timedelta(seconds=16)
reader_state["ok"] = True
third = execution.run_once()
assert third.dispatched == 1
health = execution.balance_worker_health(stale_after_sec=600)
assert health["state"] == "ok"
assert health["consecutive_failures"] == 0
def test_partial_sell_qty_is_min_intent_broker():
"""dcy-021/REQ-062 (D4): partial sell dispatches min(intent qty, broker);
remainder stays holding (broker-authoritative qty), cost basis preserved."""