feat(ops): track broker balance worker failure lifecycle and backoff
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
co-authored by
genichin
genichin
parent
59bb492883
commit
084e05a97d
@@ -16,7 +16,8 @@ from datetime import date, datetime, time, timedelta, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort
|
||||
from trex_core.application.balance_health import BalanceHealthTracker
|
||||
from trex_core.application.broker_port import BrokerBalanceError, BrokerPort
|
||||
from trex_core.application.portfolio import (
|
||||
BrokerCredentialPayload,
|
||||
DegradedReason,
|
||||
@@ -95,6 +96,7 @@ class SnapshotScheduler:
|
||||
# REQ-114: entry gate. True = new entries allowed (default).
|
||||
# credential_fetch_enabled=False keeps this always True (no gate).
|
||||
self._entry_allowed: bool = True
|
||||
self._balance_health = BalanceHealthTracker(clock=self._clock)
|
||||
|
||||
# ---- callable from a daemon thread loop --------------------------------
|
||||
def run_forever(self, stop: threading.Event, *, tick_sec: float = 60.0) -> None:
|
||||
@@ -310,14 +312,18 @@ class SnapshotScheduler:
|
||||
self._store.swap(items)
|
||||
try:
|
||||
# BAT-050 / REQ-103: pass warn_store + slot_repo for C-case durable warning.
|
||||
reconcile_positions(
|
||||
store=self._store,
|
||||
broker=self._broker,
|
||||
slot_repo=self._slot_repo,
|
||||
instance_key=self._instance_key,
|
||||
warn_store=self._warn_store,
|
||||
alarm_emitter=self._alarm_emitter,
|
||||
notification_store=self._notification_store,
|
||||
self._run_balance_worker(
|
||||
worker="position_reconciliation",
|
||||
stale_after_sec=24 * 60 * 60,
|
||||
run=lambda: reconcile_positions(
|
||||
store=self._store,
|
||||
broker=self._broker,
|
||||
slot_repo=self._slot_repo,
|
||||
instance_key=self._instance_key,
|
||||
warn_store=self._warn_store,
|
||||
alarm_emitter=self._alarm_emitter,
|
||||
notification_store=self._notification_store,
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("reconciliation pass failed")
|
||||
@@ -330,18 +336,79 @@ class SnapshotScheduler:
|
||||
try:
|
||||
from trex_core.application.reconciliation import reconcile_core_holdings
|
||||
# BAT-050 / REQ-103: pass warn_store for B-case durable warning.
|
||||
reconcile_core_holdings(
|
||||
slot_repo=self._slot_repo, slot_service=self._slot_service,
|
||||
broker=self._broker, instance_key=self._instance_key,
|
||||
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
|
||||
warn_store=self._warn_store, # BAT-050 / REQ-103
|
||||
notification_store=self._notification_store, # req-0007 / ac-0024
|
||||
self._run_balance_worker(
|
||||
worker="core_holding_reconciliation",
|
||||
stale_after_sec=24 * 60 * 60,
|
||||
run=lambda: reconcile_core_holdings(
|
||||
slot_repo=self._slot_repo, slot_service=self._slot_service,
|
||||
broker=self._broker, instance_key=self._instance_key,
|
||||
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
|
||||
warn_store=self._warn_store, # BAT-050 / REQ-103
|
||||
notification_store=self._notification_store, # req-0007 / ac-0024
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("core-holding divergence reconcile failed")
|
||||
self._last_fired_date = self._clock()
|
||||
return True
|
||||
|
||||
def _run_balance_worker(
|
||||
self,
|
||||
*,
|
||||
worker: str,
|
||||
stale_after_sec: float,
|
||||
run: Callable[[], object],
|
||||
) -> None:
|
||||
ready, backoff_left = self._balance_health.ready(worker)
|
||||
if not ready:
|
||||
_log.info(
|
||||
"broker balance worker deferred: worker=%s instance_key=%s backoff_left_sec=%.1f",
|
||||
worker,
|
||||
self._instance_key,
|
||||
backoff_left,
|
||||
)
|
||||
return
|
||||
try:
|
||||
run()
|
||||
except BrokerBalanceError as exc:
|
||||
lifecycle, backoff = self._balance_health.record_failure(worker, code=exc.code)
|
||||
if lifecycle is not None:
|
||||
state = self._balance_health.status(worker, stale_after_sec=stale_after_sec)
|
||||
_log.warning(
|
||||
"broker balance unavailable: lifecycle=%s worker=%s instance_key=%s "
|
||||
"failure_code=%s consecutive_failures=%s backoff_sec=%.1f",
|
||||
lifecycle,
|
||||
worker,
|
||||
self._instance_key,
|
||||
exc.code,
|
||||
state["consecutive_failures"],
|
||||
backoff,
|
||||
)
|
||||
return
|
||||
lifecycle = self._balance_health.record_success(worker)
|
||||
if lifecycle == "recovered":
|
||||
state = self._balance_health.status(worker, stale_after_sec=stale_after_sec)
|
||||
_log.info(
|
||||
"broker balance recovered: worker=%s instance_key=%s last_success_at=%s",
|
||||
worker,
|
||||
self._instance_key,
|
||||
state["last_success_at"],
|
||||
)
|
||||
|
||||
def balance_worker_health(
|
||||
self,
|
||||
*,
|
||||
stale_after_sec: float = 24 * 60 * 60,
|
||||
) -> dict[str, dict[str, object]]:
|
||||
return {
|
||||
"position_reconciliation": self._balance_health.status(
|
||||
"position_reconciliation", stale_after_sec=stale_after_sec
|
||||
),
|
||||
"core_holding_reconciliation": self._balance_health.status(
|
||||
"core_holding_reconciliation", stale_after_sec=stale_after_sec
|
||||
),
|
||||
}
|
||||
|
||||
# ---- asset-centric slot reconcile (dcy-020/REQ-058, BAT-028) ----------
|
||||
def _reconcile_slots(self, items: list) -> None:
|
||||
universe = {
|
||||
|
||||
Reference in New Issue
Block a user