trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(broker): restore main wiring and add production retry cadence + status surface

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:25:28 +00:00
committed by GitHub
co-authored by genichin
parent 2b5d9043b9
commit 1f728f4d7d
9 changed files with 464 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/application/messaging_handlers.py
+75
View File
@@ -192,6 +192,63 @@ class SlotsBrokerQuantityReader(Protocol):
def holdings_for_slots_rpc(self) -> tuple[list[BalanceItem], datetime]: ...
class SlotsBrokerBalanceHealthReader(Protocol):
def balance_health_for_slots_rpc(self) -> dict[str, dict[str, object]]: ...
_BALANCE_HEALTH_STATES = frozenset({"ok", "unavailable", "stale"})
_BALANCE_HEALTH_FAILURE_CODES = frozenset({
"token_issuance_failed",
"authentication_failed",
"authorization_failed",
"transport_failed",
"timeout",
"http_5xx",
"malformed_response",
"broker_rejected",
"ownership_mismatch",
"stale_snapshot",
"account_mismatch",
"not_ready",
"expired",
"live_read_failed",
"read_failed",
})
def _sanitize_balance_worker_status(
*,
instance_key: str,
worker: str,
raw: dict[str, object],
) -> dict[str, object]:
state = str(raw.get("state") or "unavailable")
if state not in _BALANCE_HEALTH_STATES:
state = "unavailable"
try:
consecutive_failures = max(0, int(raw.get("consecutive_failures", 0)))
except (TypeError, ValueError):
consecutive_failures = 0
failure_code_raw = raw.get("failure_code")
failure_code = (
str(failure_code_raw)
if isinstance(failure_code_raw, str)
and failure_code_raw in _BALANCE_HEALTH_FAILURE_CODES
else None
)
last_success_at = raw.get("last_success_at")
next_retry_at = raw.get("next_retry_at")
return {
"instance_key": instance_key,
"worker": worker,
"state": state,
"last_success_at": last_success_at if isinstance(last_success_at, str) else None,
"consecutive_failures": consecutive_failures,
"failure_code": failure_code,
"next_retry_at": next_retry_at if isinstance(next_retry_at, str) else None,
}
def make_slots_rpc_handler(
slot_repo: SlotRepository,
*,
@@ -205,6 +262,7 @@ def make_slots_rpc_handler(
max_active_slots: Optional[int] = None,
strategy_state_store=None,
broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
broker_balance_health_reader: Optional[SlotsBrokerBalanceHealthReader] = None,
):
"""Build the RPC handler for `trex.slots.req` (REQ-073/074/116).
@@ -437,6 +495,23 @@ def make_slots_rpc_handler(
"reason": reason,
"message": message,
}
if broker_balance_health_reader is not None:
try:
raw_health = broker_balance_health_reader.balance_health_for_slots_rpc()
except Exception: # noqa: BLE001 - status surface must stay available
raw_health = {}
health_payload: dict[str, dict[str, object]] = {}
if isinstance(raw_health, dict):
for worker, raw in raw_health.items():
if not isinstance(worker, str) or not worker or not isinstance(raw, dict):
continue
health_payload[worker] = _sanitize_balance_worker_status(
instance_key=instance_key,
worker=worker,
raw=raw,
)
if health_payload:
base_payload["broker_balance_health"] = health_payload
if warn_store is not None:
base_payload["excluded_slots"] = {
"reconcile_quantity_mismatch": excluded_count,
trex_core/application/snapshot_scheduler.py
+56 -35
View File
@@ -117,6 +117,7 @@ class SnapshotScheduler:
_log.exception("snapshot scheduler tick failed")
def _maybe_tick(self) -> None:
self._maybe_retry_balance_workers()
now_kst = self._clock().astimezone(_KST)
target = datetime.combine(now_kst.date(), self._schedule_kst, tzinfo=_KST)
if now_kst < target:
@@ -319,19 +320,7 @@ class SnapshotScheduler:
self._store.swap(items)
try:
# BAT-050 / REQ-103: pass warn_store + slot_repo for C-case durable warning.
self._run_balance_worker(
worker="position_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_positions(
store=self._store,
broker=self._broker,
slot_repo=self._slot_repo,
instance_key=self._instance_key,
warn_store=self._warn_store,
alarm_emitter=self._alarm_emitter,
notification_store=self._notification_store,
),
)
self._run_position_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("reconciliation pass failed")
if self._slot_repo is not None:
@@ -341,34 +330,66 @@ class SnapshotScheduler:
_log.exception("slot reconcile pass failed")
if self._slot_service is not None:
try:
from trex_core.application.reconciliation import reconcile_core_holdings
# BAT-050 / REQ-103: pass warn_store for B-case durable warning.
<<<<<<< HEAD
self._run_balance_worker(
worker="core_holding_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
),
=======
reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._core_holdings_broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
>>>>>>> origin/main
)
self._run_core_holding_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")
self._last_fired_date = self._clock()
return True
def _run_position_reconciliation_worker(self) -> None:
self._run_balance_worker(
worker="position_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_positions(
store=self._store,
broker=self._broker,
slot_repo=self._slot_repo,
instance_key=self._instance_key,
warn_store=self._warn_store,
alarm_emitter=self._alarm_emitter,
notification_store=self._notification_store,
),
)
def _run_core_holding_reconciliation_worker(self) -> None:
from trex_core.application.reconciliation import reconcile_core_holdings
self._run_balance_worker(
worker="core_holding_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_core_holdings(
slot_repo=self._slot_repo,
slot_service=self._slot_service,
broker=self._core_holdings_broker,
instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
),
)
def _has_pending_balance_failure(self, worker: str) -> bool:
status = self._balance_health.status(worker, stale_after_sec=24 * 60 * 60)
return int(status.get("consecutive_failures", 0)) > 0
def _maybe_retry_balance_workers(self) -> None:
if self._has_pending_balance_failure("position_reconciliation"):
try:
self._run_position_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("reconciliation retry pass failed")
if (
self._slot_repo is not None
and self._slot_service is not None
and self._has_pending_balance_failure("core_holding_reconciliation")
):
try:
self._run_core_holding_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("core-holding divergence retry failed")
def _run_balance_worker(
self,
*,
trex_core/runtime/daemon.py
+11
View File
@@ -466,10 +466,21 @@ class Daemon:
max_active_slots=self.config.max_active_slots,
strategy_state_store=self.strategy_state_store, # dcy-028 C-1
broker_quantity_reader=self.account_snapshot_service,
broker_balance_health_reader=self,
)
transport.consume_rpc_forever(
topology.slots_queue(self.config.instance_key), handler, self.stop)
def balance_health_for_slots_rpc(self) -> dict[str, dict[str, object]]:
reconcile = self.snapshot_scheduler.balance_worker_health(
stale_after_sec=24 * 60 * 60,
)
return {
"execution": self.execution.balance_worker_health(stale_after_sec=600),
"position_reconciliation": reconcile["position_reconciliation"],
"core_holding_reconciliation": reconcile["core_holding_reconciliation"],
}
def _strategy_state_set_loop(self) -> None:
# dcy-028 / REQ-080: opaque strategy-state write RPC (entry-version CAS).
transport = _build_transport(self.config)