fix(broker): cap outage backoff overflow and expand worker matrix coverage

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

tests/application/test_snapshot_scheduler.py

@@ -5,6 +5,8 @@ from datetime import UTC, datetime, timedelta, timezone
 from types import SimpleNamespace
 from unittest.mock import MagicMock
 
+import pytest
+
 from trex_core.adapters.broker.paper import PaperBroker
 from trex_core.adapters.portfolio.stub_read import StubPortfolioRead
 from trex_core.application.alarm_emitter import AlarmEmitter
@@ -896,3 +898,172 @@ def test_run_forever_retries_due_worker_on_production_loop_path():
     state = sched.balance_worker_health(stale_after_sec=86_400)
     assert state["position_reconciliation"]["state"] == "ok"
     assert state["position_reconciliation"]["consecutive_failures"] == 0
+
+
+@pytest.mark.parametrize(
+    "error_code",
+    [
+        "timeout",
+        "http_5xx",
+        "authentication_failed",
+        "token_issuance_failed",
+        "malformed_response",
+    ],
+    ids=["timeout", "http_5xx", "auth", "token", "pagination_middle_malformed"],
+)
+def test_position_worker_classification_matrix_is_fail_closed(error_code):
+    from trex_core.application.broker_port import BrokerBalanceError
+    from trex_core.domain.slot import SlotService
+    from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
+
+    class ReadOnce:
+        def whoami(self):
+            return "d-1"
+
+        def get_portfolios(self, _daemon_id):
+            return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=1)]
+
+    class RaiseOrListBroker:
+        def __init__(self, failure: str | None):
+            self.failure = failure
+
+        def get_balance(self):
+            if self.failure is not None:
+                raise BrokerBalanceError(self.failure)
+            return []
+
+    store = SnapshotStore()
+    slot_repo = InMemorySlotRepository()
+    slot_service = SlotService(slot_repo, instance_key="default")
+    slot_repo.reconcile_universe("default", {"pf-X": "X"})
+    store.swap([PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=1)])
+    store.set_warning("pf-X", True)
+    sched = SnapshotScheduler(
+        read_port=ReadOnce(),
+        store=store,
+        broker=RaiseOrListBroker(error_code),
+        core_holdings_broker=RaiseOrListBroker(None),
+        slot_repo=slot_repo,
+        slot_service=slot_service,
+    )
+
+    assert sched.run_once() is True
+    health = sched.balance_worker_health(stale_after_sec=86_400)
+    assert health["position_reconciliation"]["state"] == "unavailable"
+    assert health["position_reconciliation"]["failure_code"] == error_code
+    assert health["position_reconciliation"]["consecutive_failures"] == 1
+    assert health["core_holding_reconciliation"]["state"] == "ok"
+    # fail-closed on failure: warning clear path did not run.
+    assert store.get("pf-X").warning is True
+
+
+def test_position_worker_validated_empty_success_does_not_create_failure_state():
+    from trex_core.domain.slot import SlotService
+    from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
+
+    class ReadOnce:
+        def whoami(self):
+            return "d-1"
+
+        def get_portfolios(self, _daemon_id):
+            return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
+
+    class EmptyBalanceBroker:
+        def get_balance(self):
+            return []
+
+    store = SnapshotStore()
+    slot_repo = InMemorySlotRepository()
+    slot_service = SlotService(slot_repo, instance_key="default")
+    slot_repo.reconcile_universe("default", {"pf-X": "X"})
+    sched = SnapshotScheduler(
+        read_port=ReadOnce(),
+        store=store,
+        broker=EmptyBalanceBroker(),
+        core_holdings_broker=EmptyBalanceBroker(),
+        slot_repo=slot_repo,
+        slot_service=slot_service,
+    )
+
+    assert sched.run_once() is True
+    health = sched.balance_worker_health(stale_after_sec=86_400)
+    assert health["position_reconciliation"]["state"] == "ok"
+    assert health["position_reconciliation"]["consecutive_failures"] == 0
+    assert health["position_reconciliation"]["failure_code"] is None
+
+
+@pytest.mark.parametrize(
+    "error_code",
+    [
+        "timeout",
+        "http_5xx",
+        "authentication_failed",
+        "token_issuance_failed",
+        "malformed_response",
+    ],
+    ids=["timeout", "http_5xx", "auth", "token", "pagination_middle_malformed"],
+)
+def test_core_worker_classification_matrix_is_fail_closed(error_code):
+    from trex_core.application.broker_port import BrokerBalanceError
+    from trex_core.domain.slot import SlotService, SlotState
+    from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
+
+    class ReadOnce:
+        def whoami(self):
+            return "d-1"
+
+        def get_portfolios(self, _daemon_id):
+            return [PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)]
+
+    class EmptyBalanceBroker:
+        def get_balance(self):
+            return []
+
+    class ErrorBroker:
+        def __init__(self, failure: str):
+            self.failure = failure
+
+        def get_balance(self):
+            raise BrokerBalanceError(self.failure)
+
+    slot_repo = InMemorySlotRepository()
+    slot_service = SlotService(slot_repo, instance_key="default")
+    slot_repo.reconcile_universe("default", {"pf-X": "X"})
+    slot = slot_repo.find_slot_by_portfolio("default", "pf-X")
+    pending = slot_service.transit_slot(
+        slot.slot_id,
+        SlotState.INACTIVE,
+        SlotState.PENDING_BUY,
+        reason="buy",
+        actor="test",
+        symbol="X",
+    )
+    slot_service.transit_slot(
+        pending.slot_id,
+        SlotState.PENDING_BUY,
+        SlotState.HOLDING,
+        reason="filled",
+        actor="test",
+        avg_price=100.0,
+        qty=1.0,
+    )
+    before = slot_repo.fetch_slot(slot.slot_id)
+    sched = SnapshotScheduler(
+        read_port=ReadOnce(),
+        store=SnapshotStore(),
+        broker=EmptyBalanceBroker(),
+        core_holdings_broker=ErrorBroker(error_code),
+        slot_repo=slot_repo,
+        slot_service=slot_service,
+    )
+
+    assert sched.run_once() is True
+    health = sched.balance_worker_health(stale_after_sec=86_400)
+    assert health["position_reconciliation"]["state"] == "ok"
+    assert health["core_holding_reconciliation"]["state"] == "unavailable"
+    assert health["core_holding_reconciliation"]["failure_code"] == error_code
+    assert health["core_holding_reconciliation"]["consecutive_failures"] == 1
+    # fail-closed on failure: no slot mutation on this failed core reconcile.
+    after = slot_repo.fetch_slot(slot.slot_id)
+    assert after.status is before.status
+    assert after.qty == before.qty