feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

tests/adapters/broker/test_kis.py

@@ -59,7 +59,8 @@ def test_account_scope_hash_separates_endpoint_and_application_identity():
         (403, "authorization_failed"),
         (422, "broker_rejected"),
         (429, "transport_failed"),
-        (500, "transport_failed"),
+        (500, "http_5xx"),
+        (504, "timeout"),
         (None, "broker_rejected"),
     ],
 )
@@ -350,8 +351,8 @@ def test_get_balance_fails_closed_on_later_transport_and_http_errors(monkeypatch
     request = httpx.Request("GET", "https://example.test/balance")
     response = httpx.Response(503, request=request)
     for failure, expected_code in [
-        (httpx.ConnectTimeout("unavailable"), "transport_failed"),
-        (httpx.HTTPStatusError("bad gateway", request=request, response=response), "transport_failed"),
+        (httpx.ConnectTimeout("unavailable"), "timeout"),
+        (httpx.HTTPStatusError("bad gateway", request=request, response=response), "http_5xx"),
     ]:
         broker, calls = _balance_broker(monkeypatch, [
             _BalanceResponse(_balance_payload([{"pdno": "005930", "hldg_qty": "2"}], fk="A", nk="1"), "F"),
@@ -407,6 +408,29 @@ def test_get_available_cash_uses_cash_only_amount(monkeypatch):
     assert b.get_available_cash() == 695_591
 
 
+@pytest.mark.parametrize(
+    ("failure", "expected_code"),
+    [
+        (httpx.ReadTimeout("timed out"), "timeout"),
+        (
+            httpx.HTTPStatusError(
+                "bad gateway",
+                request=httpx.Request("GET", "https://example.test"),
+                response=httpx.Response(502, request=httpx.Request("GET", "https://example.test")),
+            ),
+            "http_5xx",
+        ),
+    ],
+)
+def test_get_available_cash_classifies_timeout_and_http_5xx(monkeypatch, failure, expected_code):
+    b = KISBroker("12345678-01", app_key="k", app_secret="s", base_url=_MOCK_URL)
+    monkeypatch.setattr(b, "_headers", lambda _tr: {})
+    monkeypatch.setattr("trex_core.adapters.broker.kis.httpx.get", lambda *a, **kw: (_ for _ in ()).throw(failure))
+    with pytest.raises(BrokerBalanceError) as raised:
+        b.get_available_cash()
+    assert raised.value.code == expected_code
+
+
 def test_submit_transport_failure_blocks_automatic_duplicate(monkeypatch):
     b = KISBroker("12345678-01", app_key="k", app_secret="s", base_url=_MOCK_URL)
     monkeypatch.setattr(b, "_headers", lambda _tr: {})

tests/application/test_execution.py

@@ -5,8 +5,11 @@ In-memory adapters; deterministic paper double drives the full async path.
 
 from __future__ import annotations
 
+from datetime import UTC, datetime, timedelta
+
 from trex_core.adapters.broker.paper import PaperBroker
 from trex_core.adapters.persistence.in_memory_intent_store import InMemoryIntentStore
+from trex_core.application.account_snapshot import AccountSnapshotUnavailable
 from trex_core.application.broker_port import (
     BalanceItem,
     CancelOutcome,
@@ -214,6 +217,61 @@ def test_pending_sell_uses_injected_bounded_balance_reader():
     assert repo.fetch_slot(sid).status is SlotState.PENDING_SELL
 
 
+def test_pending_sell_balance_failure_is_fail_closed_with_backoff_and_recovery():
+    now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
+    broker = PaperBroker(fill_after_polls=0)
+    repo, svc, intents, _ = _wire(broker)
+    sid = _buy(intents, qty=3).slot_id
+
+    execution = ExecutionService(
+        svc,
+        repo,
+        broker,
+        instance_key=INST,
+        clock=lambda: now["t"],
+    )
+    execution.run_once()
+    svc.sell(sid, actor="t", reason="full")
+    broker._fill_after = 5
+
+    calls = {"n": 0}
+    reader_state = {"ok": False}
+
+    def failing_reader():
+        calls["n"] += 1
+        if not reader_state["ok"]:
+            raise AccountSnapshotUnavailable("expired")
+        return [BalanceItem("005931", 3)]
+
+    execution = ExecutionService(
+        svc,
+        repo,
+        broker,
+        instance_key=INST,
+        balance_reader=failing_reader,
+        clock=lambda: now["t"],
+    )
+    first = execution.run_once()
+    assert first.sell_skipped_no_balance == 1
+    assert calls["n"] == 1
+    health = execution.balance_worker_health(stale_after_sec=600)
+    assert health["state"] == "unavailable"
+    assert health["failure_code"] == "stale_snapshot"
+    assert health["consecutive_failures"] == 1
+
+    second = execution.run_once()
+    assert second.sell_skipped_no_balance == 1
+    assert calls["n"] == 1  # bounded backoff prevented another live read
+
+    now["t"] += timedelta(seconds=16)
+    reader_state["ok"] = True
+    third = execution.run_once()
+    assert third.dispatched == 1
+    health = execution.balance_worker_health(stale_after_sec=600)
+    assert health["state"] == "ok"
+    assert health["consecutive_failures"] == 0
+
+
 def test_partial_sell_qty_is_min_intent_broker():
     """dcy-021/REQ-062 (D4): partial sell dispatches min(intent qty, broker);
     remainder stays holding (broker-authoritative qty), cost basis preserved."""

tests/application/test_snapshot_scheduler.py

@@ -665,3 +665,60 @@ def test_build_kis_reinit_callback_returns_callable_for_kis():
     result(new_cred)
     assert broker._account_ref == "new-acct"
     assert broker._app_key == "new-k"
+
+
+def test_balance_worker_health_isolated_backoff_and_recovery():
+    from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
+    from trex_core.domain.slot import SlotService
+    from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
+
+    class SequencedBroker(PaperBroker):
+        def __init__(self):
+            super().__init__()
+            self.steps = []
+
+        def get_balance(self):
+            if not self.steps:
+                return []
+            step = self.steps.pop(0)
+            if isinstance(step, Exception):
+                raise step
+            return step
+
+    now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=timezone.utc)}
+    read = _make_stub()
+    store = SnapshotStore()
+    slot_repo = InMemorySlotRepository()
+    slot_service = SlotService(slot_repo, instance_key="default")
+    slot_repo.reconcile_universe("default", {"pf-X": "X"})
+    broker = SequencedBroker()
+    sched = SnapshotScheduler(
+        read_port=read,
+        store=store,
+        broker=broker,
+        slot_repo=slot_repo,
+        slot_service=slot_service,
+        clock=lambda: now["t"],
+    )
+
+    broker.steps = [BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["state"] == "unavailable"
+    assert state["position_reconciliation"]["failure_code"] == "http_5xx"
+    assert state["position_reconciliation"]["consecutive_failures"] == 1
+    assert state["core_holding_reconciliation"]["state"] == "ok"
+
+    broker.steps = [[BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    assert broker.steps == []  # position worker stayed in backoff, core worker still ran
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["consecutive_failures"] == 1
+    assert state["core_holding_reconciliation"]["state"] == "ok"
+
+    now["t"] += timedelta(seconds=16)
+    broker.steps = [[BalanceItem("X", 0)], [BalanceItem("X", 0)]]
+    assert sched.run_once() is True
+    state = sched.balance_worker_health(stale_after_sec=86_400)
+    assert state["position_reconciliation"]["state"] == "ok"
+    assert state["position_reconciliation"]["consecutive_failures"] == 0