fix(broker): restore main wiring and add production retry cadence + status surface
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
2b5d9043b9
commit
1f728f4d7d
@@ -192,6 +192,63 @@ class SlotsBrokerQuantityReader(Protocol):
|
||||
def holdings_for_slots_rpc(self) -> tuple[list[BalanceItem], datetime]: ...
|
||||
|
||||
|
||||
class SlotsBrokerBalanceHealthReader(Protocol):
|
||||
def balance_health_for_slots_rpc(self) -> dict[str, dict[str, object]]: ...
|
||||
|
||||
|
||||
_BALANCE_HEALTH_STATES = frozenset({"ok", "unavailable", "stale"})
|
||||
_BALANCE_HEALTH_FAILURE_CODES = frozenset({
|
||||
"token_issuance_failed",
|
||||
"authentication_failed",
|
||||
"authorization_failed",
|
||||
"transport_failed",
|
||||
"timeout",
|
||||
"http_5xx",
|
||||
"malformed_response",
|
||||
"broker_rejected",
|
||||
"ownership_mismatch",
|
||||
"stale_snapshot",
|
||||
"account_mismatch",
|
||||
"not_ready",
|
||||
"expired",
|
||||
"live_read_failed",
|
||||
"read_failed",
|
||||
})
|
||||
|
||||
|
||||
def _sanitize_balance_worker_status(
|
||||
*,
|
||||
instance_key: str,
|
||||
worker: str,
|
||||
raw: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
state = str(raw.get("state") or "unavailable")
|
||||
if state not in _BALANCE_HEALTH_STATES:
|
||||
state = "unavailable"
|
||||
try:
|
||||
consecutive_failures = max(0, int(raw.get("consecutive_failures", 0)))
|
||||
except (TypeError, ValueError):
|
||||
consecutive_failures = 0
|
||||
failure_code_raw = raw.get("failure_code")
|
||||
failure_code = (
|
||||
str(failure_code_raw)
|
||||
if isinstance(failure_code_raw, str)
|
||||
and failure_code_raw in _BALANCE_HEALTH_FAILURE_CODES
|
||||
else None
|
||||
)
|
||||
last_success_at = raw.get("last_success_at")
|
||||
next_retry_at = raw.get("next_retry_at")
|
||||
return {
|
||||
"instance_key": instance_key,
|
||||
"worker": worker,
|
||||
"state": state,
|
||||
"last_success_at": last_success_at if isinstance(last_success_at, str) else None,
|
||||
"consecutive_failures": consecutive_failures,
|
||||
"failure_code": failure_code,
|
||||
"next_retry_at": next_retry_at if isinstance(next_retry_at, str) else None,
|
||||
}
|
||||
|
||||
|
||||
def make_slots_rpc_handler(
|
||||
slot_repo: SlotRepository,
|
||||
*,
|
||||
@@ -205,6 +262,7 @@ def make_slots_rpc_handler(
|
||||
max_active_slots: Optional[int] = None,
|
||||
strategy_state_store=None,
|
||||
broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
|
||||
broker_balance_health_reader: Optional[SlotsBrokerBalanceHealthReader] = None,
|
||||
):
|
||||
"""Build the RPC handler for `trex.slots.req` (REQ-073/074/116).
|
||||
|
||||
@@ -437,6 +495,23 @@ def make_slots_rpc_handler(
|
||||
"reason": reason,
|
||||
"message": message,
|
||||
}
|
||||
if broker_balance_health_reader is not None:
|
||||
try:
|
||||
raw_health = broker_balance_health_reader.balance_health_for_slots_rpc()
|
||||
except Exception: # noqa: BLE001 - status surface must stay available
|
||||
raw_health = {}
|
||||
health_payload: dict[str, dict[str, object]] = {}
|
||||
if isinstance(raw_health, dict):
|
||||
for worker, raw in raw_health.items():
|
||||
if not isinstance(worker, str) or not worker or not isinstance(raw, dict):
|
||||
continue
|
||||
health_payload[worker] = _sanitize_balance_worker_status(
|
||||
instance_key=instance_key,
|
||||
worker=worker,
|
||||
raw=raw,
|
||||
)
|
||||
if health_payload:
|
||||
base_payload["broker_balance_health"] = health_payload
|
||||
if warn_store is not None:
|
||||
base_payload["excluded_slots"] = {
|
||||
"reconcile_quantity_mismatch": excluded_count,
|
||||
|
||||
Reference in New Issue
Block a user