fix(broker): restore main wiring and add production retry cadence + status surface

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:25:28 +00:00
committed by GitHub
co-authored by genichin
parent 2b5d9043b9
commit 1f728f4d7d
9 changed files with 464 additions and 36 deletions
@@ -192,6 +192,63 @@ class SlotsBrokerQuantityReader(Protocol):
def holdings_for_slots_rpc(self) -> tuple[list[BalanceItem], datetime]: ...
class SlotsBrokerBalanceHealthReader(Protocol):
def balance_health_for_slots_rpc(self) -> dict[str, dict[str, object]]: ...
_BALANCE_HEALTH_STATES = frozenset({"ok", "unavailable", "stale"})
_BALANCE_HEALTH_FAILURE_CODES = frozenset({
"token_issuance_failed",
"authentication_failed",
"authorization_failed",
"transport_failed",
"timeout",
"http_5xx",
"malformed_response",
"broker_rejected",
"ownership_mismatch",
"stale_snapshot",
"account_mismatch",
"not_ready",
"expired",
"live_read_failed",
"read_failed",
})
def _sanitize_balance_worker_status(
*,
instance_key: str,
worker: str,
raw: dict[str, object],
) -> dict[str, object]:
state = str(raw.get("state") or "unavailable")
if state not in _BALANCE_HEALTH_STATES:
state = "unavailable"
try:
consecutive_failures = max(0, int(raw.get("consecutive_failures", 0)))
except (TypeError, ValueError):
consecutive_failures = 0
failure_code_raw = raw.get("failure_code")
failure_code = (
str(failure_code_raw)
if isinstance(failure_code_raw, str)
and failure_code_raw in _BALANCE_HEALTH_FAILURE_CODES
else None
)
last_success_at = raw.get("last_success_at")
next_retry_at = raw.get("next_retry_at")
return {
"instance_key": instance_key,
"worker": worker,
"state": state,
"last_success_at": last_success_at if isinstance(last_success_at, str) else None,
"consecutive_failures": consecutive_failures,
"failure_code": failure_code,
"next_retry_at": next_retry_at if isinstance(next_retry_at, str) else None,
}
def make_slots_rpc_handler(
slot_repo: SlotRepository,
*,
@@ -205,6 +262,7 @@ def make_slots_rpc_handler(
max_active_slots: Optional[int] = None,
strategy_state_store=None,
broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
broker_balance_health_reader: Optional[SlotsBrokerBalanceHealthReader] = None,
):
"""Build the RPC handler for `trex.slots.req` (REQ-073/074/116).
@@ -437,6 +495,23 @@ def make_slots_rpc_handler(
"reason": reason,
"message": message,
}
if broker_balance_health_reader is not None:
try:
raw_health = broker_balance_health_reader.balance_health_for_slots_rpc()
except Exception: # noqa: BLE001 - status surface must stay available
raw_health = {}
health_payload: dict[str, dict[str, object]] = {}
if isinstance(raw_health, dict):
for worker, raw in raw_health.items():
if not isinstance(worker, str) or not worker or not isinstance(raw, dict):
continue
health_payload[worker] = _sanitize_balance_worker_status(
instance_key=instance_key,
worker=worker,
raw=raw,
)
if health_payload:
base_payload["broker_balance_health"] = health_payload
if warn_store is not None:
base_payload["excluded_slots"] = {
"reconcile_quantity_mismatch": excluded_count,