feat(ops): track broker balance worker failure lifecycle and backoff
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
59bb492883
commit
084e05a97d
@@ -665,3 +665,60 @@ def test_build_kis_reinit_callback_returns_callable_for_kis():
|
||||
result(new_cred)
|
||||
assert broker._account_ref == "new-acct"
|
||||
assert broker._app_key == "new-k"
|
||||
|
||||
|
||||
def test_balance_worker_health_isolated_backoff_and_recovery():
|
||||
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
|
||||
from trex_core.domain.slot import SlotService
|
||||
from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
|
||||
|
||||
class SequencedBroker(PaperBroker):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.steps = []
|
||||
|
||||
def get_balance(self):
|
||||
if not self.steps:
|
||||
return []
|
||||
step = self.steps.pop(0)
|
||||
if isinstance(step, Exception):
|
||||
raise step
|
||||
return step
|
||||
|
||||
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=timezone.utc)}
|
||||
read = _make_stub()
|
||||
store = SnapshotStore()
|
||||
slot_repo = InMemorySlotRepository()
|
||||
slot_service = SlotService(slot_repo, instance_key="default")
|
||||
slot_repo.reconcile_universe("default", {"pf-X": "X"})
|
||||
broker = SequencedBroker()
|
||||
sched = SnapshotScheduler(
|
||||
read_port=read,
|
||||
store=store,
|
||||
broker=broker,
|
||||
slot_repo=slot_repo,
|
||||
slot_service=slot_service,
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
|
||||
broker.steps = [BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
|
||||
assert sched.run_once() is True
|
||||
state = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert state["position_reconciliation"]["state"] == "unavailable"
|
||||
assert state["position_reconciliation"]["failure_code"] == "http_5xx"
|
||||
assert state["position_reconciliation"]["consecutive_failures"] == 1
|
||||
assert state["core_holding_reconciliation"]["state"] == "ok"
|
||||
|
||||
broker.steps = [[BalanceItem("X", 0)]]
|
||||
assert sched.run_once() is True
|
||||
assert broker.steps == [] # position worker stayed in backoff, core worker still ran
|
||||
state = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert state["position_reconciliation"]["consecutive_failures"] == 1
|
||||
assert state["core_holding_reconciliation"]["state"] == "ok"
|
||||
|
||||
now["t"] += timedelta(seconds=16)
|
||||
broker.steps = [[BalanceItem("X", 0)], [BalanceItem("X", 0)]]
|
||||
assert sched.run_once() is True
|
||||
state = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert state["position_reconciliation"]["state"] == "ok"
|
||||
assert state["position_reconciliation"]["consecutive_failures"] == 0
|
||||
|
||||
Reference in New Issue
Block a user