feat: add kill-switch control flow

This commit is contained in:
2026-06-07 14:00:20 +09:00
parent c912c08071
commit 7e23648489
30 changed files with 1424 additions and 51 deletions
+88
View File
@@ -0,0 +1,88 @@
"""Kill-switch execution service (REQ-045 / BAT-012).
Operator emergency stop: liquidate all `holding` slots in a scope
(portfolio or global) by reusing SlotService.sell() (REQ-036), then mark the
scope day-excluded (REQ-044) so strategy execution is masked for the day
(REQ-046). Independent of the dcy-007 safety-sell worker — concurrent sells
on the same slot are idempotent (REQ-036 PENDING_SELL guard).
"""
from __future__ import annotations
import logging
from datetime import date as date_cls
from datetime import datetime, timezone
from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
from trex_core.domain.slot import SlotRepository, SlotService, SlotState
_log = logging.getLogger(__name__)
_ACTOR = "kill-switch"
class KillSwitchService:
"""Range-scoped emergency liquidation + day-scoped exclusion (REQ-045)."""
def __init__(
self,
*,
slot_repo: SlotRepository,
slot_service: SlotService,
killswitch_store: KillSwitchStore,
) -> None:
self._repo = slot_repo
self._svc = slot_service
self._store = killswitch_store
@staticmethod
def _trading_date(now: datetime) -> date_cls:
return now.astimezone(timezone.utc).date()
def fire(
self,
*,
scope: str,
portfolio_id: str | None = None,
actor: str = "operator",
now: datetime | None = None,
) -> int:
"""Liquidate holding slots in scope + mark day-excluded. Returns count.
scope: "portfolio" (requires portfolio_id) | "global".
"""
now = now or datetime.now(tz=timezone.utc)
day = self._trading_date(now)
holding = [s for s in self._repo.list_slots(day) if s.status is SlotState.HOLDING]
if scope == "portfolio":
holding = [s for s in holding if s.portfolio_id == portfolio_id]
liquidated = 0
for s in holding:
try:
self._svc.sell(s.slot_id, actor=_ACTOR, reason="emergency_stop")
liquidated += 1
except Exception: # noqa: BLE001 - one slot must not stop the sweep
_log.warning("kill-switch sell failed (slot=%s)", s.slot_id, exc_info=True)
# Mark the scope day-excluded so strategy execution is masked (REQ-046).
key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
self._store.exclude(key, day, actor=actor)
_log.warning(
"kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
actor, scope, portfolio_id, liquidated,
)
return liquidated
def resume(
self,
*,
scope: str,
portfolio_id: str | None = None,
now: datetime | None = None,
) -> None:
"""Manually clear the day-scoped exclusion (REQ-046). Sold slots stay sold."""
now = now or datetime.now(tz=timezone.utc)
day = self._trading_date(now)
key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
self._store.clear(key, day)
_log.info("kill-switch resume: scope=%s portfolio_id=%s", scope, portfolio_id)
@@ -164,6 +164,7 @@ def make_snapshot_rpc_handler(
pause_store: Optional[PauseStore] = None,
default_total_slots: int = 10,
watchdog_store: Optional[WatchdogStore] = None,
killswitch_store=None,
):
"""Build the RPC handler for `trex.snapshot.req` (REQ-028).
@@ -211,6 +212,7 @@ def make_snapshot_rpc_handler(
view=view,
strategy_key=strategy_key,
trading_date=day,
killswitch_store=killswitch_store,
)
return RpcResponse(payload=envelope.to_dict())
@@ -282,6 +284,78 @@ def _err(reason: str) -> RpcResponse:
return RpcResponse(payload={"status": "error", "reason": reason})
# ---- control RPC handler (REQ-043, BAT-012) ---------------------------------
def make_control_rpc_handler(pause_store: PauseStore, *, kill_switch_service=None):
"""RPC handler for `trex.control.req` — action discriminator dispatch (REQ-043).
Actions: `pause`/`resume`/`list` (REQ-032, delegated), `kill_switch` (REQ-045),
`resume_kill_switch` (REQ-046). Unknown actions return an error response.
"""
pause_handler = make_pause_rpc_handler(pause_store)
def handler(req: RpcRequest) -> RpcResponse:
body = req.payload or {}
action = str(body.get("action") or "").lower()
if action == "kill_switch":
return _handle_kill_switch(body, kill_switch_service)
if action == "resume_kill_switch":
return _handle_resume_kill_switch(body, kill_switch_service)
if action in ("pause", "resume", "list"):
return pause_handler(req)
return _err(f"unknown_action:{action}")
return handler
def _killswitch_scope(body: dict):
"""Validate scope/portfolio_id. Returns (scope, pid) or (None, RpcResponse error)."""
scope = str(body.get("scope") or "").lower()
pid = _opt_str(body.get("portfolio_id"))
if scope not in ("portfolio", "global"):
return None, _err(f"bad_scope:{scope}")
if scope == "portfolio" and not pid:
return None, _err("portfolio_id_required")
return (scope, pid), None
def _handle_kill_switch(body: dict, svc) -> RpcResponse:
"""REQ-045: liquidate holding slots in scope + day-exclude."""
if svc is None:
return _err("kill_switch_unsupported")
parsed, err = _killswitch_scope(body)
if err is not None:
return err
scope, pid = parsed
actor = str(body.get("actor") or "operator")
try:
liquidated = svc.fire(scope=scope, portfolio_id=pid, actor=actor)
except Exception as exc: # noqa: BLE001
return _err(f"kill_switch_failed:{exc.__class__.__name__}")
payload = {"status": "ok", "scope": scope, "liquidated": liquidated}
if pid:
payload["portfolio_id"] = pid
return RpcResponse(payload=payload)
def _handle_resume_kill_switch(body: dict, svc) -> RpcResponse:
"""REQ-046: clear the day-scoped exclusion (sold slots stay sold)."""
if svc is None:
return _err("resume_kill_switch_unsupported")
parsed, err = _killswitch_scope(body)
if err is not None:
return err
scope, pid = parsed
try:
svc.resume(scope=scope, portfolio_id=pid)
except Exception as exc: # noqa: BLE001
return _err(f"resume_kill_switch_failed:{exc.__class__.__name__}")
payload = {"status": "ok", "scope": scope}
if pid:
payload["portfolio_id"] = pid
return RpcResponse(payload=payload)
# ---- event payload builder --------------------------------------------------
def slot_transition_event_payload(slot, *, event_type: str) -> dict:
+49 -4
View File
@@ -216,6 +216,45 @@ class InMemoryPauseStore:
self._paused.discard(portfolio_id)
# ---------------------------------------------------------------- kill-switch
# Day-scoped exclusion overlay (REQ-044/046 / BAT-012). Distinct from pause:
# keyed by (portfolio_id, trading_date) so it auto-clears next trading day.
GLOBAL_KILLSWITCH = "*" # sentinel: whole-universe exclusion for the day
class KillSwitchStore(Protocol):
"""Day-scoped kill-switch exclusion overlay (REQ-044)."""
def list_excluded(self, trading_date) -> set[str]:
...
def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
...
def clear(self, portfolio_id: str, trading_date) -> None:
...
class InMemoryKillSwitchStore:
"""Process-local day-scoped kill-switch store for tests + CI."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._excluded: set[tuple[str, object]] = set() # (portfolio_id, trading_date)
def list_excluded(self, trading_date) -> set[str]:
with self._lock:
return {pid for pid, d in self._excluded if d == trading_date}
def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
with self._lock:
self._excluded.add((portfolio_id, trading_date))
def clear(self, portfolio_id: str, trading_date) -> None:
with self._lock:
self._excluded.discard((portfolio_id, trading_date))
# ---------------------------------------------------------------- selection
@dataclass(frozen=True)
@@ -274,11 +313,14 @@ def build_snapshot_envelope(
view: View = View.ALL,
strategy_key: Optional[str] = None,
trading_date=None,
killswitch_store: "Optional[KillSwitchStore]" = None,
) -> SnapshotEnvelope:
"""Compose base(view) − mask(pending·warning·pause) − strategy_key (REQ-029).
"""Compose base(view) − mask(pending·warning·pause·killswitch) − strategy_key.
`slot_repo` is the SlotRepository used to read holding/pending state.
`trading_date` defaults to UTC today (matches the daemon's preallocate).
`killswitch_store` (REQ-046) adds the day-scoped kill-switch exclusion to
the mask: a `GLOBAL_KILLSWITCH` ("*") entry masks the whole universe.
"""
# Read core authoritative slot state once.
day = trading_date or datetime.now(tz=timezone.utc).date()
@@ -298,6 +340,9 @@ def build_snapshot_envelope(
assets = store.list_assets()
paused = pause_store.list_paused()
warned = {a.item.portfolio_id for a in assets if a.warning}
# REQ-046: day-scoped kill-switch exclusion. '*' sentinel = whole universe.
excluded = killswitch_store.list_excluded(day) if killswitch_store is not None else set()
killswitch_global = GLOBAL_KILLSWITCH in excluded
# Build the base set (view).
by_pid = {a.item.portfolio_id: a for a in assets}
@@ -316,13 +361,13 @@ def build_snapshot_envelope(
base_pids = list({a.item.portfolio_id for a in assets} | holding_pids)
base_pids.sort()
# Apply masks: pending OR warning OR pause.
masked = pending_pids | warned | paused
# Apply masks: pending OR warning OR pause OR kill-switch day-exclusion.
masked = pending_pids | warned | paused | excluded
# Filter and shape.
out: list[dict] = []
for pid in base_pids:
if pid in masked:
if killswitch_global or pid in masked:
continue
if pid in by_pid:
item = by_pid[pid].item