trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(broker): restore main wiring and add production retry cadence + status surface

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:25:28 +00:00
committed by GitHub
co-authored by genichin
parent 2b5d9043b9
commit 1f728f4d7d
9 changed files with 464 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/application/snapshot_scheduler.py
+56 -35
View File
@@ -117,6 +117,7 @@ class SnapshotScheduler:
_log.exception("snapshot scheduler tick failed")
def _maybe_tick(self) -> None:
self._maybe_retry_balance_workers()
now_kst = self._clock().astimezone(_KST)
target = datetime.combine(now_kst.date(), self._schedule_kst, tzinfo=_KST)
if now_kst < target:
@@ -319,19 +320,7 @@ class SnapshotScheduler:
self._store.swap(items)
try:
# BAT-050 / REQ-103: pass warn_store + slot_repo for C-case durable warning.
self._run_balance_worker(
worker="position_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_positions(
store=self._store,
broker=self._broker,
slot_repo=self._slot_repo,
instance_key=self._instance_key,
warn_store=self._warn_store,
alarm_emitter=self._alarm_emitter,
notification_store=self._notification_store,
),
)
self._run_position_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("reconciliation pass failed")
if self._slot_repo is not None:
@@ -341,34 +330,66 @@ class SnapshotScheduler:
_log.exception("slot reconcile pass failed")
if self._slot_service is not None:
try:
from trex_core.application.reconciliation import reconcile_core_holdings
# BAT-050 / REQ-103: pass warn_store for B-case durable warning.
<<<<<<< HEAD
self._run_balance_worker(
worker="core_holding_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
),
=======
reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._core_holdings_broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
>>>>>>> origin/main
)
self._run_core_holding_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")
self._last_fired_date = self._clock()
return True
def _run_position_reconciliation_worker(self) -> None:
self._run_balance_worker(
worker="position_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_positions(
store=self._store,
broker=self._broker,
slot_repo=self._slot_repo,
instance_key=self._instance_key,
warn_store=self._warn_store,
alarm_emitter=self._alarm_emitter,
notification_store=self._notification_store,
),
)
def _run_core_holding_reconciliation_worker(self) -> None:
from trex_core.application.reconciliation import reconcile_core_holdings
self._run_balance_worker(
worker="core_holding_reconciliation",
stale_after_sec=24 * 60 * 60,
run=lambda: reconcile_core_holdings(
slot_repo=self._slot_repo,
slot_service=self._slot_service,
broker=self._core_holdings_broker,
instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
),
)
def _has_pending_balance_failure(self, worker: str) -> bool:
status = self._balance_health.status(worker, stale_after_sec=24 * 60 * 60)
return int(status.get("consecutive_failures", 0)) > 0
def _maybe_retry_balance_workers(self) -> None:
if self._has_pending_balance_failure("position_reconciliation"):
try:
self._run_position_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("reconciliation retry pass failed")
if (
self._slot_repo is not None
and self._slot_service is not None
and self._has_pending_balance_failure("core_holding_reconciliation")
):
try:
self._run_core_holding_reconciliation_worker()
except Exception: # noqa: BLE001
_log.exception("core-holding divergence retry failed")
def _run_balance_worker(
self,
*,