fix(broker): restore main wiring and add production retry cadence + status surface

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:25:28 +00:00
committed by GitHub
co-authored by genichin
parent 2b5d9043b9
commit 1f728f4d7d
9 changed files with 464 additions and 36 deletions
@@ -57,10 +57,12 @@ def test_run_once_uses_authoritative_core_holdings_broker(monkeypatch):
store = SnapshotStore()
projected_broker = object()
authoritative_broker = object()
unmanaged_store = object()
captured = {}
def _capture(**kwargs):
captured["broker"] = kwargs["broker"]
captured["unmanaged_position_store"] = kwargs.get("unmanaged_position_store")
return {}
monkeypatch.setattr(
@@ -74,11 +76,13 @@ def test_run_once_uses_authoritative_core_holdings_broker(monkeypatch):
core_holdings_broker=authoritative_broker,
slot_repo=object(),
slot_service=SimpleNamespace(instance_key="inst-1"),
unmanaged_position_store=unmanaged_store,
)
sched._reconcile_slots = lambda _items: None
assert sched.run_once() is True
assert captured["broker"] is authoritative_broker
assert captured["unmanaged_position_store"] is unmanaged_store
def test_swap_preserves_warning_for_carried_over_portfolio():
@@ -755,3 +759,140 @@ def test_balance_worker_health_isolated_backoff_and_recovery():
state = sched.balance_worker_health(stale_after_sec=86_400)
assert state["position_reconciliation"]["state"] == "ok"
assert state["position_reconciliation"]["consecutive_failures"] == 0
def test_maybe_tick_retries_due_worker_without_daily_refresh():
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
from trex_core.domain.slot import SlotService
from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
class CountingRead:
def __init__(self):
self.calls = 0
def whoami(self):
return "d-1"
def get_portfolios(self, _daemon_id):
self.calls += 1
return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
class SequenceBroker:
def __init__(self, steps):
self.steps = list(steps)
self.calls = 0
def get_balance(self):
self.calls += 1
step = self.steps.pop(0)
if isinstance(step, Exception):
raise step
return step
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=UTC)}
read = CountingRead()
position_broker = SequenceBroker(
[BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
)
core_broker = SequenceBroker([[BalanceItem("X", 0)]])
store = SnapshotStore()
slot_repo = InMemorySlotRepository()
slot_service = SlotService(slot_repo, instance_key="default")
slot_repo.reconcile_universe("default", {"pf-X": "X"})
sched = SnapshotScheduler(
read_port=read,
store=store,
broker=position_broker,
core_holdings_broker=core_broker,
slot_repo=slot_repo,
slot_service=slot_service,
schedule_kst=datetime(2026, 9, 12, 23, 59).time(),
clock=lambda: now["t"],
)
assert sched.run_once() is True
assert read.calls == 1
assert position_broker.calls == 1
assert core_broker.calls == 1
now["t"] += timedelta(seconds=10)
sched._maybe_tick()
assert read.calls == 1
assert position_broker.calls == 1 # backoff window still active
assert core_broker.calls == 1 # no pending core failure, so no retry call
now["t"] += timedelta(seconds=6)
sched._maybe_tick()
assert read.calls == 1
assert position_broker.calls == 2 # due retry fired without daily snapshot refresh
health = sched.balance_worker_health(stale_after_sec=86_400)
assert health["position_reconciliation"]["state"] == "ok"
assert health["position_reconciliation"]["consecutive_failures"] == 0
assert health["core_holding_reconciliation"]["state"] == "ok"
def test_run_forever_retries_due_worker_on_production_loop_path():
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
class LoopRead:
def __init__(self):
self.calls = 0
def whoami(self):
return "d-1"
def get_portfolios(self, _daemon_id):
self.calls += 1
return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
class SequenceBroker:
def __init__(self, steps):
self.steps = list(steps)
self.calls = 0
def get_balance(self):
self.calls += 1
step = self.steps.pop(0)
if isinstance(step, Exception):
raise step
return step
class StopAfter:
def __init__(self, waits: int, tick):
self._waits = waits
self._tick = tick
self._count = 0
def wait(self, _sec: float):
if self._count >= self._waits:
return True
self._tick(self._count)
self._count += 1
return False
now = {"t": datetime(2026, 9, 12, 0, 0, tzinfo=UTC)}
read = LoopRead()
position_broker = SequenceBroker(
[BrokerBalanceError("http_5xx"), [BalanceItem("X", 0)]]
)
sched = SnapshotScheduler(
read_port=read,
store=SnapshotStore(),
broker=position_broker,
schedule_kst=datetime(2026, 9, 12, 23, 59).time(),
clock=lambda: now["t"],
)
stop = StopAfter(
waits=3,
tick=lambda i: now.__setitem__("t", now["t"] + timedelta(seconds=(10 if i == 0 else 6))),
)
sched.run_forever(stop, tick_sec=1.0)
# Boot refresh once + no extra daily refresh, but one due retry after backoff deadline.
assert read.calls == 1
assert position_broker.calls == 2
state = sched.balance_worker_health(stale_after_sec=86_400)
assert state["position_reconciliation"]["state"] == "ok"
assert state["position_reconciliation"]["consecutive_failures"] == 0