feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-12 13:33:12 +00:00
committed by GitHub
co-authored by genichin
parent 59bb492883
commit 084e05a97d
9 changed files with 453 additions and 27 deletions
@@ -665,3 +665,60 @@ def test_build_kis_reinit_callback_returns_callable_for_kis():
result(new_cred)
assert broker._account_ref == "new-acct"
assert broker._app_key == "new-k"
def test_balance_worker_health_isolated_backoff_and_recovery():
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
from trex_core.domain.slot import SlotService
from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
class SequencedBroker(PaperBroker):
def __init__(self):
super().__init__()
self.steps = []
def get_balance(self):
if not self.steps:
return []
step = self.steps.pop(0)
if isinstance(step, Exception):
raise step
return step
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=timezone.utc)}
read = _make_stub()
store = SnapshotStore()
slot_repo = InMemorySlotRepository()
slot_service = SlotService(slot_repo, instance_key="default")
slot_repo.reconcile_universe("default", {"pf-X": "X"})
broker = SequencedBroker()
sched = SnapshotScheduler(
read_port=read,
store=store,
broker=broker,
slot_repo=slot_repo,
slot_service=slot_service,
clock=lambda: now["t"],
)
broker.steps = [BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
assert sched.run_once() is True
state = sched.balance_worker_health(stale_after_sec=86_400)
assert state["position_reconciliation"]["state"] == "unavailable"
assert state["position_reconciliation"]["failure_code"] == "http_5xx"
assert state["position_reconciliation"]["consecutive_failures"] == 1
assert state["core_holding_reconciliation"]["state"] == "ok"
broker.steps = [[BalanceItem("X", 0)]]
assert sched.run_once() is True
assert broker.steps == [] # position worker stayed in backoff, core worker still ran
state = sched.balance_worker_health(stale_after_sec=86_400)
assert state["position_reconciliation"]["consecutive_failures"] == 1
assert state["core_holding_reconciliation"]["state"] == "ok"
now["t"] += timedelta(seconds=16)
broker.steps = [[BalanceItem("X", 0)], [BalanceItem("X", 0)]]
assert sched.run_once() is True
state = sched.balance_worker_health(stale_after_sec=86_400)
assert state["position_reconciliation"]["state"] == "ok"
assert state["position_reconciliation"]["consecutive_failures"] == 0