fix(broker): restore main wiring and add production retry cadence + status surface
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
2b5d9043b9
commit
1f728f4d7d
@@ -57,10 +57,12 @@ def test_run_once_uses_authoritative_core_holdings_broker(monkeypatch):
|
||||
store = SnapshotStore()
|
||||
projected_broker = object()
|
||||
authoritative_broker = object()
|
||||
unmanaged_store = object()
|
||||
captured = {}
|
||||
|
||||
def _capture(**kwargs):
|
||||
captured["broker"] = kwargs["broker"]
|
||||
captured["unmanaged_position_store"] = kwargs.get("unmanaged_position_store")
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -74,11 +76,13 @@ def test_run_once_uses_authoritative_core_holdings_broker(monkeypatch):
|
||||
core_holdings_broker=authoritative_broker,
|
||||
slot_repo=object(),
|
||||
slot_service=SimpleNamespace(instance_key="inst-1"),
|
||||
unmanaged_position_store=unmanaged_store,
|
||||
)
|
||||
sched._reconcile_slots = lambda _items: None
|
||||
|
||||
assert sched.run_once() is True
|
||||
assert captured["broker"] is authoritative_broker
|
||||
assert captured["unmanaged_position_store"] is unmanaged_store
|
||||
|
||||
|
||||
def test_swap_preserves_warning_for_carried_over_portfolio():
|
||||
@@ -755,3 +759,140 @@ def test_balance_worker_health_isolated_backoff_and_recovery():
|
||||
state = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert state["position_reconciliation"]["state"] == "ok"
|
||||
assert state["position_reconciliation"]["consecutive_failures"] == 0
|
||||
|
||||
|
||||
def test_maybe_tick_retries_due_worker_without_daily_refresh():
|
||||
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
|
||||
from trex_core.domain.slot import SlotService
|
||||
from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
|
||||
|
||||
class CountingRead:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def whoami(self):
|
||||
return "d-1"
|
||||
|
||||
def get_portfolios(self, _daemon_id):
|
||||
self.calls += 1
|
||||
return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
|
||||
|
||||
class SequenceBroker:
|
||||
def __init__(self, steps):
|
||||
self.steps = list(steps)
|
||||
self.calls = 0
|
||||
|
||||
def get_balance(self):
|
||||
self.calls += 1
|
||||
step = self.steps.pop(0)
|
||||
if isinstance(step, Exception):
|
||||
raise step
|
||||
return step
|
||||
|
||||
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=UTC)}
|
||||
read = CountingRead()
|
||||
position_broker = SequenceBroker(
|
||||
[BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
|
||||
)
|
||||
core_broker = SequenceBroker([[BalanceItem("X", 0)]])
|
||||
store = SnapshotStore()
|
||||
slot_repo = InMemorySlotRepository()
|
||||
slot_service = SlotService(slot_repo, instance_key="default")
|
||||
slot_repo.reconcile_universe("default", {"pf-X": "X"})
|
||||
|
||||
sched = SnapshotScheduler(
|
||||
read_port=read,
|
||||
store=store,
|
||||
broker=position_broker,
|
||||
core_holdings_broker=core_broker,
|
||||
slot_repo=slot_repo,
|
||||
slot_service=slot_service,
|
||||
schedule_kst=datetime(2026, 9, 12, 23, 59).time(),
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
|
||||
assert sched.run_once() is True
|
||||
assert read.calls == 1
|
||||
assert position_broker.calls == 1
|
||||
assert core_broker.calls == 1
|
||||
|
||||
now["t"] += timedelta(seconds=10)
|
||||
sched._maybe_tick()
|
||||
assert read.calls == 1
|
||||
assert position_broker.calls == 1 # backoff window still active
|
||||
assert core_broker.calls == 1 # no pending core failure, so no retry call
|
||||
|
||||
now["t"] += timedelta(seconds=6)
|
||||
sched._maybe_tick()
|
||||
assert read.calls == 1
|
||||
assert position_broker.calls == 2 # due retry fired without daily snapshot refresh
|
||||
health = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert health["position_reconciliation"]["state"] == "ok"
|
||||
assert health["position_reconciliation"]["consecutive_failures"] == 0
|
||||
assert health["core_holding_reconciliation"]["state"] == "ok"
|
||||
|
||||
|
||||
def test_run_forever_retries_due_worker_on_production_loop_path():
|
||||
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
|
||||
|
||||
class LoopRead:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def whoami(self):
|
||||
return "d-1"
|
||||
|
||||
def get_portfolios(self, _daemon_id):
|
||||
self.calls += 1
|
||||
return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
|
||||
|
||||
class SequenceBroker:
|
||||
def __init__(self, steps):
|
||||
self.steps = list(steps)
|
||||
self.calls = 0
|
||||
|
||||
def get_balance(self):
|
||||
self.calls += 1
|
||||
step = self.steps.pop(0)
|
||||
if isinstance(step, Exception):
|
||||
raise step
|
||||
return step
|
||||
|
||||
class StopAfter:
|
||||
def __init__(self, waits: int, tick):
|
||||
self._waits = waits
|
||||
self._tick = tick
|
||||
self._count = 0
|
||||
|
||||
def wait(self, _sec: float):
|
||||
if self._count >= self._waits:
|
||||
return True
|
||||
self._tick(self._count)
|
||||
self._count += 1
|
||||
return False
|
||||
|
||||
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=UTC)}
|
||||
read = LoopRead()
|
||||
position_broker = SequenceBroker(
|
||||
[BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
|
||||
)
|
||||
sched = SnapshotScheduler(
|
||||
read_port=read,
|
||||
store=SnapshotStore(),
|
||||
broker=position_broker,
|
||||
schedule_kst=datetime(2026, 9, 12, 23, 59).time(),
|
||||
clock=lambda: now["t"],
|
||||
)
|
||||
stop = StopAfter(
|
||||
waits=3,
|
||||
tick=lambda i: now.__setitem__("t", now["t"] + timedelta(seconds=(10 if i == 0 else 6))),
|
||||
)
|
||||
|
||||
sched.run_forever(stop, tick_sec=1.0)
|
||||
|
||||
# Boot refresh once + no extra daily refresh, but one due retry after backoff deadline.
|
||||
assert read.calls == 1
|
||||
assert position_broker.calls == 2
|
||||
state = sched.balance_worker_health(stale_after_sec=86_400)
|
||||
assert state["position_reconciliation"]["state"] == "ok"
|
||||
assert state["position_reconciliation"]["consecutive_failures"] == 0
|
||||
|
||||
Reference in New Issue
Block a user