feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

trex_core/application/execution.py

@@ -30,13 +30,16 @@ from dataclasses import dataclass, replace
 from datetime import UTC, datetime, timedelta
 
 from trex_core.application.alarm_emitter import AlarmEmitter
+from trex_core.application.balance_health import BalanceHealthTracker
 from trex_core.application.broker_port import (
     BalanceItem,
+    BrokerBalanceError,
     BrokerPort,
     CancelOutcome,
     CancelState,
     OrderRequest,
 )
+from trex_core.application.account_snapshot import AccountSnapshotUnavailable
 from trex_core.application.ledger_outbox import LedgerOutbox
 from trex_core.application.ledger_port import LedgerTrade
 from trex_core.application.ledger_skip import publish_ledger_skip_alarm
@@ -139,6 +142,8 @@ class ExecutionService:
         # account snapshot only through this injected reader. Reconciliation
         # remains wired directly to the live broker.
         self._sell_dispatch_balance_reader = balance_reader or broker.get_balance
+        self._balance_health = BalanceHealthTracker(clock=self._clock)
+        self._balance_worker = "execution"
 
     # ------------------------------------------------------------
     def run_once(self) -> ExecutionResult:
@@ -171,7 +176,22 @@ class ExecutionService:
                 # surfaces via reconcile_core_holdings (REQ-062 A). No mirror
                 # fallback by design.
                 if sell_balance is None:
-                    sell_balance = self._sell_dispatch_balance_map()
+                    ready, backoff_left = self._balance_health.ready(self._balance_worker)
+                    if not ready:
+                        r.sell_skipped_no_balance += 1
+                        _log.info(
+                            "sell dispatch deferred: broker balance backoff active "
+                            "instance_key=%s worker=%s backoff_left_sec=%.1f",
+                            self._instance_key,
+                            self._balance_worker,
+                            backoff_left,
+                        )
+                        return
+                    try:
+                        sell_balance = self._sell_dispatch_balance_map()
+                    except (AccountSnapshotUnavailable, BrokerBalanceError):
+                        r.sell_skipped_no_balance += 1
+                        return
                 bal = sell_balance.get(slot.symbol or "", 0)
                 # dcy-021 / REQ-062 (D4): full sell (slot.qty is None) => broker
                 # balance; partial sell => min(order qty, broker balance). Broker
@@ -420,10 +440,60 @@ class ExecutionService:
 
     def _sell_dispatch_balance_map(self) -> dict:
         """Balance used only to size a not-yet-submitted pending sell."""
-        return {
-            b.symbol: int(b.hldg_qty)
-            for b in self._sell_dispatch_balance_reader()
-        }
+        try:
+            holdings = self._sell_dispatch_balance_reader()
+        except AccountSnapshotUnavailable as exc:
+            code = "stale_snapshot" if exc.code == "expired" else (
+                getattr(exc, "detail_code", None) or exc.code
+            )
+            lifecycle, backoff = self._balance_health.record_failure(
+                self._balance_worker, code=code
+            )
+            if lifecycle is not None:
+                _log.warning(
+                    "broker balance unavailable: lifecycle=%s worker=%s instance_key=%s "
+                    "failure_code=%s consecutive_failures=%d backoff_sec=%.1f",
+                    lifecycle,
+                    self._balance_worker,
+                    self._instance_key,
+                    code,
+                    self._balance_health.status(self._balance_worker, stale_after_sec=600)[
+                        "consecutive_failures"
+                    ],
+                    backoff,
+                )
+            raise
+        except BrokerBalanceError as exc:
+            lifecycle, backoff = self._balance_health.record_failure(
+                self._balance_worker, code=exc.code
+            )
+            if lifecycle is not None:
+                _log.warning(
+                    "broker balance unavailable: lifecycle=%s worker=%s instance_key=%s "
+                    "failure_code=%s consecutive_failures=%d backoff_sec=%.1f",
+                    lifecycle,
+                    self._balance_worker,
+                    self._instance_key,
+                    exc.code,
+                    self._balance_health.status(self._balance_worker, stale_after_sec=600)[
+                        "consecutive_failures"
+                    ],
+                    backoff,
+                )
+            raise
+        lifecycle = self._balance_health.record_success(self._balance_worker)
+        if lifecycle == "recovered":
+            state = self._balance_health.status(self._balance_worker, stale_after_sec=600)
+            _log.info(
+                "broker balance recovered: worker=%s instance_key=%s last_success_at=%s",
+                self._balance_worker,
+                self._instance_key,
+                state["last_success_at"],
+            )
+        return {b.symbol: int(b.hldg_qty) for b in holdings}
+
+    def balance_worker_health(self, *, stale_after_sec: float = 600.0) -> dict[str, object]:
+        return self._balance_health.status(self._balance_worker, stale_after_sec=stale_after_sec)
 
     def _resolve_fill_price(self, fs, slot) -> tuple[float | None, bool]:
         """Resolve the fill price from FillStatus, with limit_price backfill.