feat: add kill-switch control flow
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""Kill-switch execution service (REQ-045 / BAT-012).
|
||||
|
||||
Operator emergency stop: liquidate all `holding` slots in a scope
|
||||
(portfolio or global) by reusing SlotService.sell() (REQ-036), then mark the
|
||||
scope day-excluded (REQ-044) so strategy execution is masked for the day
|
||||
(REQ-046). Independent of the dcy-007 safety-sell worker — concurrent sells
|
||||
on the same slot are idempotent (REQ-036 PENDING_SELL guard).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date as date_cls
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
|
||||
from trex_core.domain.slot import SlotRepository, SlotService, SlotState
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
_ACTOR = "kill-switch"
|
||||
|
||||
|
||||
class KillSwitchService:
|
||||
"""Range-scoped emergency liquidation + day-scoped exclusion (REQ-045)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
slot_repo: SlotRepository,
|
||||
slot_service: SlotService,
|
||||
killswitch_store: KillSwitchStore,
|
||||
) -> None:
|
||||
self._repo = slot_repo
|
||||
self._svc = slot_service
|
||||
self._store = killswitch_store
|
||||
|
||||
@staticmethod
|
||||
def _trading_date(now: datetime) -> date_cls:
|
||||
return now.astimezone(timezone.utc).date()
|
||||
|
||||
def fire(
|
||||
self,
|
||||
*,
|
||||
scope: str,
|
||||
portfolio_id: str | None = None,
|
||||
actor: str = "operator",
|
||||
now: datetime | None = None,
|
||||
) -> int:
|
||||
"""Liquidate holding slots in scope + mark day-excluded. Returns count.
|
||||
|
||||
scope: "portfolio" (requires portfolio_id) | "global".
|
||||
"""
|
||||
now = now or datetime.now(tz=timezone.utc)
|
||||
day = self._trading_date(now)
|
||||
holding = [s for s in self._repo.list_slots(day) if s.status is SlotState.HOLDING]
|
||||
if scope == "portfolio":
|
||||
holding = [s for s in holding if s.portfolio_id == portfolio_id]
|
||||
|
||||
liquidated = 0
|
||||
for s in holding:
|
||||
try:
|
||||
self._svc.sell(s.slot_id, actor=_ACTOR, reason="emergency_stop")
|
||||
liquidated += 1
|
||||
except Exception: # noqa: BLE001 - one slot must not stop the sweep
|
||||
_log.warning("kill-switch sell failed (slot=%s)", s.slot_id, exc_info=True)
|
||||
|
||||
# Mark the scope day-excluded so strategy execution is masked (REQ-046).
|
||||
key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
|
||||
self._store.exclude(key, day, actor=actor)
|
||||
|
||||
_log.warning(
|
||||
"kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
|
||||
actor, scope, portfolio_id, liquidated,
|
||||
)
|
||||
return liquidated
|
||||
|
||||
def resume(
|
||||
self,
|
||||
*,
|
||||
scope: str,
|
||||
portfolio_id: str | None = None,
|
||||
now: datetime | None = None,
|
||||
) -> None:
|
||||
"""Manually clear the day-scoped exclusion (REQ-046). Sold slots stay sold."""
|
||||
now = now or datetime.now(tz=timezone.utc)
|
||||
day = self._trading_date(now)
|
||||
key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
|
||||
self._store.clear(key, day)
|
||||
_log.info("kill-switch resume: scope=%s portfolio_id=%s", scope, portfolio_id)
|
||||
@@ -164,6 +164,7 @@ def make_snapshot_rpc_handler(
|
||||
pause_store: Optional[PauseStore] = None,
|
||||
default_total_slots: int = 10,
|
||||
watchdog_store: Optional[WatchdogStore] = None,
|
||||
killswitch_store=None,
|
||||
):
|
||||
"""Build the RPC handler for `trex.snapshot.req` (REQ-028).
|
||||
|
||||
@@ -211,6 +212,7 @@ def make_snapshot_rpc_handler(
|
||||
view=view,
|
||||
strategy_key=strategy_key,
|
||||
trading_date=day,
|
||||
killswitch_store=killswitch_store,
|
||||
)
|
||||
return RpcResponse(payload=envelope.to_dict())
|
||||
|
||||
@@ -282,6 +284,78 @@ def _err(reason: str) -> RpcResponse:
|
||||
return RpcResponse(payload={"status": "error", "reason": reason})
|
||||
|
||||
|
||||
# ---- control RPC handler (REQ-043, BAT-012) ---------------------------------
|
||||
|
||||
def make_control_rpc_handler(pause_store: PauseStore, *, kill_switch_service=None):
|
||||
"""RPC handler for `trex.control.req` — action discriminator dispatch (REQ-043).
|
||||
|
||||
Actions: `pause`/`resume`/`list` (REQ-032, delegated), `kill_switch` (REQ-045),
|
||||
`resume_kill_switch` (REQ-046). Unknown actions return an error response.
|
||||
"""
|
||||
pause_handler = make_pause_rpc_handler(pause_store)
|
||||
|
||||
def handler(req: RpcRequest) -> RpcResponse:
|
||||
body = req.payload or {}
|
||||
action = str(body.get("action") or "").lower()
|
||||
if action == "kill_switch":
|
||||
return _handle_kill_switch(body, kill_switch_service)
|
||||
if action == "resume_kill_switch":
|
||||
return _handle_resume_kill_switch(body, kill_switch_service)
|
||||
if action in ("pause", "resume", "list"):
|
||||
return pause_handler(req)
|
||||
return _err(f"unknown_action:{action}")
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
def _killswitch_scope(body: dict):
|
||||
"""Validate scope/portfolio_id. Returns (scope, pid) or (None, RpcResponse error)."""
|
||||
scope = str(body.get("scope") or "").lower()
|
||||
pid = _opt_str(body.get("portfolio_id"))
|
||||
if scope not in ("portfolio", "global"):
|
||||
return None, _err(f"bad_scope:{scope}")
|
||||
if scope == "portfolio" and not pid:
|
||||
return None, _err("portfolio_id_required")
|
||||
return (scope, pid), None
|
||||
|
||||
|
||||
def _handle_kill_switch(body: dict, svc) -> RpcResponse:
|
||||
"""REQ-045: liquidate holding slots in scope + day-exclude."""
|
||||
if svc is None:
|
||||
return _err("kill_switch_unsupported")
|
||||
parsed, err = _killswitch_scope(body)
|
||||
if err is not None:
|
||||
return err
|
||||
scope, pid = parsed
|
||||
actor = str(body.get("actor") or "operator")
|
||||
try:
|
||||
liquidated = svc.fire(scope=scope, portfolio_id=pid, actor=actor)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return _err(f"kill_switch_failed:{exc.__class__.__name__}")
|
||||
payload = {"status": "ok", "scope": scope, "liquidated": liquidated}
|
||||
if pid:
|
||||
payload["portfolio_id"] = pid
|
||||
return RpcResponse(payload=payload)
|
||||
|
||||
|
||||
def _handle_resume_kill_switch(body: dict, svc) -> RpcResponse:
|
||||
"""REQ-046: clear the day-scoped exclusion (sold slots stay sold)."""
|
||||
if svc is None:
|
||||
return _err("resume_kill_switch_unsupported")
|
||||
parsed, err = _killswitch_scope(body)
|
||||
if err is not None:
|
||||
return err
|
||||
scope, pid = parsed
|
||||
try:
|
||||
svc.resume(scope=scope, portfolio_id=pid)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return _err(f"resume_kill_switch_failed:{exc.__class__.__name__}")
|
||||
payload = {"status": "ok", "scope": scope}
|
||||
if pid:
|
||||
payload["portfolio_id"] = pid
|
||||
return RpcResponse(payload=payload)
|
||||
|
||||
|
||||
# ---- event payload builder --------------------------------------------------
|
||||
|
||||
def slot_transition_event_payload(slot, *, event_type: str) -> dict:
|
||||
|
||||
@@ -216,6 +216,45 @@ class InMemoryPauseStore:
|
||||
self._paused.discard(portfolio_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- kill-switch
|
||||
# Day-scoped exclusion overlay (REQ-044/046 / BAT-012). Distinct from pause:
|
||||
# keyed by (portfolio_id, trading_date) so it auto-clears next trading day.
|
||||
GLOBAL_KILLSWITCH = "*" # sentinel: whole-universe exclusion for the day
|
||||
|
||||
|
||||
class KillSwitchStore(Protocol):
|
||||
"""Day-scoped kill-switch exclusion overlay (REQ-044)."""
|
||||
|
||||
def list_excluded(self, trading_date) -> set[str]:
|
||||
...
|
||||
|
||||
def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
|
||||
...
|
||||
|
||||
def clear(self, portfolio_id: str, trading_date) -> None:
|
||||
...
|
||||
|
||||
|
||||
class InMemoryKillSwitchStore:
|
||||
"""Process-local day-scoped kill-switch store for tests + CI."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._excluded: set[tuple[str, object]] = set() # (portfolio_id, trading_date)
|
||||
|
||||
def list_excluded(self, trading_date) -> set[str]:
|
||||
with self._lock:
|
||||
return {pid for pid, d in self._excluded if d == trading_date}
|
||||
|
||||
def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
|
||||
with self._lock:
|
||||
self._excluded.add((portfolio_id, trading_date))
|
||||
|
||||
def clear(self, portfolio_id: str, trading_date) -> None:
|
||||
with self._lock:
|
||||
self._excluded.discard((portfolio_id, trading_date))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- selection
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -274,11 +313,14 @@ def build_snapshot_envelope(
|
||||
view: View = View.ALL,
|
||||
strategy_key: Optional[str] = None,
|
||||
trading_date=None,
|
||||
killswitch_store: "Optional[KillSwitchStore]" = None,
|
||||
) -> SnapshotEnvelope:
|
||||
"""Compose base(view) − mask(pending·warning·pause) − strategy_key (REQ-029).
|
||||
"""Compose base(view) − mask(pending·warning·pause·killswitch) − strategy_key.
|
||||
|
||||
`slot_repo` is the SlotRepository used to read holding/pending state.
|
||||
`trading_date` defaults to UTC today (matches the daemon's preallocate).
|
||||
`killswitch_store` (REQ-046) adds the day-scoped kill-switch exclusion to
|
||||
the mask: a `GLOBAL_KILLSWITCH` ("*") entry masks the whole universe.
|
||||
"""
|
||||
# Read core authoritative slot state once.
|
||||
day = trading_date or datetime.now(tz=timezone.utc).date()
|
||||
@@ -298,6 +340,9 @@ def build_snapshot_envelope(
|
||||
assets = store.list_assets()
|
||||
paused = pause_store.list_paused()
|
||||
warned = {a.item.portfolio_id for a in assets if a.warning}
|
||||
# REQ-046: day-scoped kill-switch exclusion. '*' sentinel = whole universe.
|
||||
excluded = killswitch_store.list_excluded(day) if killswitch_store is not None else set()
|
||||
killswitch_global = GLOBAL_KILLSWITCH in excluded
|
||||
|
||||
# Build the base set (view).
|
||||
by_pid = {a.item.portfolio_id: a for a in assets}
|
||||
@@ -316,13 +361,13 @@ def build_snapshot_envelope(
|
||||
base_pids = list({a.item.portfolio_id for a in assets} | holding_pids)
|
||||
base_pids.sort()
|
||||
|
||||
# Apply masks: pending OR warning OR pause.
|
||||
masked = pending_pids | warned | paused
|
||||
# Apply masks: pending OR warning OR pause OR kill-switch day-exclusion.
|
||||
masked = pending_pids | warned | paused | excluded
|
||||
|
||||
# Filter and shape.
|
||||
out: list[dict] = []
|
||||
for pid in base_pids:
|
||||
if pid in masked:
|
||||
if killswitch_global or pid in masked:
|
||||
continue
|
||||
if pid in by_pid:
|
||||
item = by_pid[pid].item
|
||||
|
||||
Reference in New Issue
Block a user