feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

tests/application/test_execution.py

@@ -5,8 +5,11 @@ In-memory adapters; deterministic paper double drives the full async path.
 
 from __future__ import annotations
 
+from datetime import UTC, datetime, timedelta
+
 from trex_core.adapters.broker.paper import PaperBroker
 from trex_core.adapters.persistence.in_memory_intent_store import InMemoryIntentStore
+from trex_core.application.account_snapshot import AccountSnapshotUnavailable
 from trex_core.application.broker_port import (
     BalanceItem,
     CancelOutcome,
@@ -214,6 +217,61 @@ def test_pending_sell_uses_injected_bounded_balance_reader():
     assert repo.fetch_slot(sid).status is SlotState.PENDING_SELL
 
 
+def test_pending_sell_balance_failure_is_fail_closed_with_backoff_and_recovery():
+    now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
+    broker = PaperBroker(fill_after_polls=0)
+    repo, svc, intents, _ = _wire(broker)
+    sid = _buy(intents, qty=3).slot_id
+
+    execution = ExecutionService(
+        svc,
+        repo,
+        broker,
+        instance_key=INST,
+        clock=lambda: now["t"],
+    )
+    execution.run_once()
+    svc.sell(sid, actor="t", reason="full")
+    broker._fill_after = 5
+
+    calls = {"n": 0}
+    reader_state = {"ok": False}
+
+    def failing_reader():
+        calls["n"] += 1
+        if not reader_state["ok"]:
+            raise AccountSnapshotUnavailable("expired")
+        return [BalanceItem("005931", 3)]
+
+    execution = ExecutionService(
+        svc,
+        repo,
+        broker,
+        instance_key=INST,
+        balance_reader=failing_reader,
+        clock=lambda: now["t"],
+    )
+    first = execution.run_once()
+    assert first.sell_skipped_no_balance == 1
+    assert calls["n"] == 1
+    health = execution.balance_worker_health(stale_after_sec=600)
+    assert health["state"] == "unavailable"
+    assert health["failure_code"] == "stale_snapshot"
+    assert health["consecutive_failures"] == 1
+
+    second = execution.run_once()
+    assert second.sell_skipped_no_balance == 1
+    assert calls["n"] == 1  # bounded backoff prevented another live read
+
+    now["t"] += timedelta(seconds=16)
+    reader_state["ok"] = True
+    third = execution.run_once()
+    assert third.dispatched == 1
+    health = execution.balance_worker_health(stale_after_sec=600)
+    assert health["state"] == "ok"
+    assert health["consecutive_failures"] == 0
+
+
 def test_partial_sell_qty_is_min_intent_broker():
     """dcy-021/REQ-062 (D4): partial sell dispatches min(intent qty, broker);
     remainder stays holding (broker-authoritative qty), cost basis preserved."""

tests/application/test_snapshot_scheduler.py

@@ -665,3 +665,60 @@ def test_build_kis_reinit_callback_returns_callable_for_kis():
     result(new_cred)
     assert broker._account_ref == "new-acct"
     assert broker._app_key == "new-k"
+
+
+def test_balance_worker_health_isolated_backoff_and_recovery():
+    from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
+    from trex_core.domain.slot import SlotService
+    from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
+
+    class SequencedBroker(PaperBroker):
+        def __init__(self):
+            super().__init__()
+            self.steps = []
+
+        def get_balance(self):
+            if not self.steps:
+                return []
+            step = self.steps.pop(0)
+            if isinstance(step, Exception):
+                raise step
+            return step
+
+    now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=timezone.utc)}
+    read = _make_stub()
+    store = SnapshotStore()
+    slot_repo = InMemorySlotRepository()
+    slot_service = SlotService(slot_repo, instance_key="default")
+    slot_repo.reconcile_universe("default", {"pf-X": "X"})
+    broker = SequencedBroker()
+    sched = SnapshotScheduler(
+        read_port=read,
+        store=store,
+        broker=broker,
+        slot_repo=slot_repo,
+        slot_service=slot_service,
+        clock=lambda: now["t"],
+    )
+
+    broker.steps = [BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["state"] == "unavailable"
+    assert state["position_reconciliation"]["failure_code"] == "http_5xx"
+    assert state["position_reconciliation"]["consecutive_failures"] == 1
+    assert state["core_holding_reconciliation"]["state"] == "ok"
+
+    broker.steps = [[BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    assert broker.steps == []  # position worker stayed in backoff, core worker still ran
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["consecutive_failures"] == 1
+    assert state["core_holding_reconciliation"]["state"] == "ok"
+
+    now["t"] += timedelta(seconds=16)
+    broker.steps = [[BalanceItem("X", 0)], [BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["state"] == "ok"
+    assert state["position_reconciliation"]["consecutive_failures"] == 0