feat: add kill-switch control flow

trex_core/adapters/messaging/rabbitmq.py

@@ -28,6 +28,7 @@ import pika
 import pika.exceptions
 
 from trex_core.adapters.messaging.topology import (
+    CONTROL_REQ_QUEUE,
     EVENTS_DLQ_EXCHANGE,
     EVENTS_DLQ_QUEUE,
     EVENTS_EXCHANGE,
@@ -142,6 +143,12 @@ class RabbitMQTransport:
             durable=True,
             arguments={"x-dead-letter-exchange": EVENTS_DLQ_EXCHANGE},
         )
+        # BAT-012 / REQ-043: operator control channel (action dispatch).
+        ch.queue_declare(
+            CONTROL_REQ_QUEUE,
+            durable=True,
+            arguments={"x-dead-letter-exchange": EVENTS_DLQ_EXCHANGE},
+        )
 
     def declare_strategy_queue(self, queue: str, binding_pattern: str) -> None:
         """Per-strategy durable queue (REQ-023 AC-4) bound to events exchange."""

trex_core/adapters/messaging/topology.py

@@ -15,7 +15,9 @@ from __future__ import annotations
 #     durable for simplicity since the queue itself is shared) ---
 SNAPSHOT_REQ_QUEUE = "trex.snapshot.req"
 INTENT_REQ_QUEUE = "trex.intent.req"
-PAUSE_REQ_QUEUE = "trex.pause.req"      # BAT-008 / REQ-032 control RPC
+PAUSE_REQ_QUEUE = "trex.pause.req"      # BAT-008 / REQ-032 (legacy alias of control)
+CONTROL_REQ_QUEUE = "trex.control.req"  # BAT-012 / REQ-043 operator control channel
+                                        # (action: pause|resume|list|kill_switch|resume_kill_switch)
 
 # --- Events ---
 EVENTS_EXCHANGE = "trex.events"            # topic, durable

trex_core/adapters/persistence/sql/005_create_killswitch_exclusion.sql

@@ -0,0 +1,15 @@
+-- BAT-012 / REQ-044: day-scoped kill-switch exclusion overlay.
+-- Distinct from snapshot_pause (permanent, resume-until). A row excludes a
+-- portfolio (or '*' for global) from strategy execution for ONE trading_date;
+-- the next trading day is automatically clear because reads filter by
+-- trading_date. Manual resume (REQ-046) deletes the row.
+CREATE TABLE IF NOT EXISTS killswitch_exclusion (
+    portfolio_id  TEXT NOT NULL,   -- '*' = global exclusion sentinel
+    trading_date  DATE NOT NULL,
+    actor         TEXT,
+    created_at    TIMESTAMPTZ NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'),
+    PRIMARY KEY (portfolio_id, trading_date)
+);
+
+CREATE INDEX IF NOT EXISTS idx_killswitch_exclusion_trading_date
+    ON killswitch_exclusion (trading_date);

trex_core/adapters/portfolio/killswitch_store.py

@@ -0,0 +1,52 @@
+"""PostgreSQL-backed day-scoped kill-switch exclusion store (REQ-044 / BAT-012).
+
+Distinct from `PostgreSQLPauseStore`: pause is permanent (resume-until),
+kill-switch exclusion is day-scoped — keyed by `(portfolio_id, trading_date)`
+so the next trading day is automatically clear (reads filter by trading_date).
+`portfolio_id == "*"` is the global-exclusion sentinel.
+"""
+from __future__ import annotations
+
+from datetime import date
+from typing import Any
+
+from trex_core.adapters.persistence.postgresql import _require_psycopg
+from trex_core.application.portfolio import GLOBAL_KILLSWITCH  # noqa: F401  (re-export)
+
+
+class PostgreSQLKillSwitchStore:
+    """`killswitch_exclusion` table CRUD. (portfolio_id, trading_date) PK."""
+
+    def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None:
+        self._psycopg = _require_psycopg(psycopg_module)
+        self._dsn = dsn
+
+    def exclude(self, portfolio_id: str, trading_date: date, *, actor: str = "kill-switch") -> None:
+        with self._psycopg.connect(self._dsn, autocommit=False) as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "INSERT INTO killswitch_exclusion (portfolio_id, trading_date, actor) "
+                    "VALUES (%s, %s, %s) "
+                    "ON CONFLICT (portfolio_id, trading_date) DO NOTHING",
+                    (portfolio_id, trading_date, actor),
+                )
+                conn.commit()
+
+    def clear(self, portfolio_id: str, trading_date: date) -> None:
+        with self._psycopg.connect(self._dsn, autocommit=False) as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "DELETE FROM killswitch_exclusion "
+                    "WHERE portfolio_id = %s AND trading_date = %s",
+                    (portfolio_id, trading_date),
+                )
+                conn.commit()
+
+    def list_excluded(self, trading_date: date) -> set[str]:
+        with self._psycopg.connect(self._dsn) as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "SELECT portfolio_id FROM killswitch_exclusion WHERE trading_date = %s",
+                    (trading_date,),
+                )
+                return {row[0] for row in cur.fetchall()}

trex_core/application/killswitch.py

@@ -0,0 +1,88 @@
+"""Kill-switch execution service (REQ-045 / BAT-012).
+
+Operator emergency stop: liquidate all `holding` slots in a scope
+(portfolio or global) by reusing SlotService.sell() (REQ-036), then mark the
+scope day-excluded (REQ-044) so strategy execution is masked for the day
+(REQ-046). Independent of the dcy-007 safety-sell worker — concurrent sells
+on the same slot are idempotent (REQ-036 PENDING_SELL guard).
+"""
+from __future__ import annotations
+
+import logging
+from datetime import date as date_cls
+from datetime import datetime, timezone
+
+from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
+from trex_core.domain.slot import SlotRepository, SlotService, SlotState
+
+_log = logging.getLogger(__name__)
+_ACTOR = "kill-switch"
+
+
+class KillSwitchService:
+    """Range-scoped emergency liquidation + day-scoped exclusion (REQ-045)."""
+
+    def __init__(
+        self,
+        *,
+        slot_repo: SlotRepository,
+        slot_service: SlotService,
+        killswitch_store: KillSwitchStore,
+    ) -> None:
+        self._repo = slot_repo
+        self._svc = slot_service
+        self._store = killswitch_store
+
+    @staticmethod
+    def _trading_date(now: datetime) -> date_cls:
+        return now.astimezone(timezone.utc).date()
+
+    def fire(
+        self,
+        *,
+        scope: str,
+        portfolio_id: str | None = None,
+        actor: str = "operator",
+        now: datetime | None = None,
+    ) -> int:
+        """Liquidate holding slots in scope + mark day-excluded. Returns count.
+
+        scope: "portfolio" (requires portfolio_id) | "global".
+        """
+        now = now or datetime.now(tz=timezone.utc)
+        day = self._trading_date(now)
+        holding = [s for s in self._repo.list_slots(day) if s.status is SlotState.HOLDING]
+        if scope == "portfolio":
+            holding = [s for s in holding if s.portfolio_id == portfolio_id]
+
+        liquidated = 0
+        for s in holding:
+            try:
+                self._svc.sell(s.slot_id, actor=_ACTOR, reason="emergency_stop")
+                liquidated += 1
+            except Exception:  # noqa: BLE001 - one slot must not stop the sweep
+                _log.warning("kill-switch sell failed (slot=%s)", s.slot_id, exc_info=True)
+
+        # Mark the scope day-excluded so strategy execution is masked (REQ-046).
+        key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
+        self._store.exclude(key, day, actor=actor)
+
+        _log.warning(
+            "kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
+            actor, scope, portfolio_id, liquidated,
+        )
+        return liquidated
+
+    def resume(
+        self,
+        *,
+        scope: str,
+        portfolio_id: str | None = None,
+        now: datetime | None = None,
+    ) -> None:
+        """Manually clear the day-scoped exclusion (REQ-046). Sold slots stay sold."""
+        now = now or datetime.now(tz=timezone.utc)
+        day = self._trading_date(now)
+        key = GLOBAL_KILLSWITCH if scope == "global" else (portfolio_id or "")
+        self._store.clear(key, day)
+        _log.info("kill-switch resume: scope=%s portfolio_id=%s", scope, portfolio_id)

trex_core/application/messaging_handlers.py

@@ -164,6 +164,7 @@ def make_snapshot_rpc_handler(
     pause_store: Optional[PauseStore] = None,
     default_total_slots: int = 10,
     watchdog_store: Optional[WatchdogStore] = None,
+    killswitch_store=None,
 ):
     """Build the RPC handler for `trex.snapshot.req` (REQ-028).
 
@@ -211,6 +212,7 @@ def make_snapshot_rpc_handler(
                 view=view,
                 strategy_key=strategy_key,
                 trading_date=day,
+                killswitch_store=killswitch_store,
             )
             return RpcResponse(payload=envelope.to_dict())
 
@@ -282,6 +284,78 @@ def _err(reason: str) -> RpcResponse:
     return RpcResponse(payload={"status": "error", "reason": reason})
 
 
+# ---- control RPC handler (REQ-043, BAT-012) ---------------------------------
+
+def make_control_rpc_handler(pause_store: PauseStore, *, kill_switch_service=None):
+    """RPC handler for `trex.control.req` — action discriminator dispatch (REQ-043).
+
+    Actions: `pause`/`resume`/`list` (REQ-032, delegated), `kill_switch` (REQ-045),
+    `resume_kill_switch` (REQ-046). Unknown actions return an error response.
+    """
+    pause_handler = make_pause_rpc_handler(pause_store)
+
+    def handler(req: RpcRequest) -> RpcResponse:
+        body = req.payload or {}
+        action = str(body.get("action") or "").lower()
+        if action == "kill_switch":
+            return _handle_kill_switch(body, kill_switch_service)
+        if action == "resume_kill_switch":
+            return _handle_resume_kill_switch(body, kill_switch_service)
+        if action in ("pause", "resume", "list"):
+            return pause_handler(req)
+        return _err(f"unknown_action:{action}")
+
+    return handler
+
+
+def _killswitch_scope(body: dict):
+    """Validate scope/portfolio_id. Returns (scope, pid) or (None, RpcResponse error)."""
+    scope = str(body.get("scope") or "").lower()
+    pid = _opt_str(body.get("portfolio_id"))
+    if scope not in ("portfolio", "global"):
+        return None, _err(f"bad_scope:{scope}")
+    if scope == "portfolio" and not pid:
+        return None, _err("portfolio_id_required")
+    return (scope, pid), None
+
+
+def _handle_kill_switch(body: dict, svc) -> RpcResponse:
+    """REQ-045: liquidate holding slots in scope + day-exclude."""
+    if svc is None:
+        return _err("kill_switch_unsupported")
+    parsed, err = _killswitch_scope(body)
+    if err is not None:
+        return err
+    scope, pid = parsed
+    actor = str(body.get("actor") or "operator")
+    try:
+        liquidated = svc.fire(scope=scope, portfolio_id=pid, actor=actor)
+    except Exception as exc:  # noqa: BLE001
+        return _err(f"kill_switch_failed:{exc.__class__.__name__}")
+    payload = {"status": "ok", "scope": scope, "liquidated": liquidated}
+    if pid:
+        payload["portfolio_id"] = pid
+    return RpcResponse(payload=payload)
+
+
+def _handle_resume_kill_switch(body: dict, svc) -> RpcResponse:
+    """REQ-046: clear the day-scoped exclusion (sold slots stay sold)."""
+    if svc is None:
+        return _err("resume_kill_switch_unsupported")
+    parsed, err = _killswitch_scope(body)
+    if err is not None:
+        return err
+    scope, pid = parsed
+    try:
+        svc.resume(scope=scope, portfolio_id=pid)
+    except Exception as exc:  # noqa: BLE001
+        return _err(f"resume_kill_switch_failed:{exc.__class__.__name__}")
+    payload = {"status": "ok", "scope": scope}
+    if pid:
+        payload["portfolio_id"] = pid
+    return RpcResponse(payload=payload)
+
+
 # ---- event payload builder --------------------------------------------------
 
 def slot_transition_event_payload(slot, *, event_type: str) -> dict:

trex_core/application/portfolio.py

@@ -216,6 +216,45 @@ class InMemoryPauseStore:
             self._paused.discard(portfolio_id)
 
 
+# ---------------------------------------------------------------- kill-switch
+# Day-scoped exclusion overlay (REQ-044/046 / BAT-012). Distinct from pause:
+# keyed by (portfolio_id, trading_date) so it auto-clears next trading day.
+GLOBAL_KILLSWITCH = "*"  # sentinel: whole-universe exclusion for the day
+
+
+class KillSwitchStore(Protocol):
+    """Day-scoped kill-switch exclusion overlay (REQ-044)."""
+
+    def list_excluded(self, trading_date) -> set[str]:
+        ...
+
+    def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
+        ...
+
+    def clear(self, portfolio_id: str, trading_date) -> None:
+        ...
+
+
+class InMemoryKillSwitchStore:
+    """Process-local day-scoped kill-switch store for tests + CI."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._excluded: set[tuple[str, object]] = set()  # (portfolio_id, trading_date)
+
+    def list_excluded(self, trading_date) -> set[str]:
+        with self._lock:
+            return {pid for pid, d in self._excluded if d == trading_date}
+
+    def exclude(self, portfolio_id: str, trading_date, *, actor: str = "kill-switch") -> None:
+        with self._lock:
+            self._excluded.add((portfolio_id, trading_date))
+
+    def clear(self, portfolio_id: str, trading_date) -> None:
+        with self._lock:
+            self._excluded.discard((portfolio_id, trading_date))
+
+
 # ---------------------------------------------------------------- selection
 
 @dataclass(frozen=True)
@@ -274,11 +313,14 @@ def build_snapshot_envelope(
     view: View = View.ALL,
     strategy_key: Optional[str] = None,
     trading_date=None,
+    killswitch_store: "Optional[KillSwitchStore]" = None,
 ) -> SnapshotEnvelope:
-    """Compose base(view) − mask(pending·warning·pause) − strategy_key (REQ-029).
+    """Compose base(view) − mask(pending·warning·pause·killswitch) − strategy_key.
 
     `slot_repo` is the SlotRepository used to read holding/pending state.
     `trading_date` defaults to UTC today (matches the daemon's preallocate).
+    `killswitch_store` (REQ-046) adds the day-scoped kill-switch exclusion to
+    the mask: a `GLOBAL_KILLSWITCH` ("*") entry masks the whole universe.
     """
     # Read core authoritative slot state once.
     day = trading_date or datetime.now(tz=timezone.utc).date()
@@ -298,6 +340,9 @@ def build_snapshot_envelope(
     assets = store.list_assets()
     paused = pause_store.list_paused()
     warned = {a.item.portfolio_id for a in assets if a.warning}
+    # REQ-046: day-scoped kill-switch exclusion. '*' sentinel = whole universe.
+    excluded = killswitch_store.list_excluded(day) if killswitch_store is not None else set()
+    killswitch_global = GLOBAL_KILLSWITCH in excluded
 
     # Build the base set (view).
     by_pid = {a.item.portfolio_id: a for a in assets}
@@ -316,13 +361,13 @@ def build_snapshot_envelope(
         base_pids = list({a.item.portfolio_id for a in assets} | holding_pids)
         base_pids.sort()
 
-    # Apply masks: pending OR warning OR pause.
-    masked = pending_pids | warned | paused
+    # Apply masks: pending OR warning OR pause OR kill-switch day-exclusion.
+    masked = pending_pids | warned | paused | excluded
 
     # Filter and shape.
     out: list[dict] = []
     for pid in base_pids:
-        if pid in masked:
+        if killswitch_global or pid in masked:
             continue
         if pid in by_pid:
             item = by_pid[pid].item

trex_core/runtime/daemon.py

@@ -6,7 +6,7 @@ strategy-facing communication runs over RabbitMQ (REQ-021/022/023).
 Threads (started in `run()`):
   - intent RPC consumer    — `trex.intent.req`,  prefetch=1, FIFO, dedup via intent_log
   - snapshot RPC consumer  — `trex.snapshot.req` (also refreshes the watchdog)
-  - pause RPC consumer      — `trex.pause.req` (REQ-032)
+  - control RPC consumer    — `trex.control.req` (pause/resume/list + kill_switch/resume_kill_switch, REQ-032/043)
   - execution loop         — broker dispatch + poll + slot transitions + event emit
   - ledger writer          — ledger outbox drain (REQ-019)
   - snapshot scheduler      — daily portfolio-hub snapshot refresh (REQ-030)
@@ -43,15 +43,17 @@ from trex_core.adapters.portfolio import (
     PostgreSQLPauseStore,
     StubPortfolioRead,
 )
+from trex_core.adapters.portfolio.killswitch_store import PostgreSQLKillSwitchStore
 from trex_core.application.broker_port import BrokerPort
 from trex_core.application.execution import ExecutionService
 from trex_core.application.intents import IntentService
 from trex_core.application.ledger_outbox import LedgerWriter
 from trex_core.application.ledger_port import LedgerPort
 from trex_core.application.messaging import MessagingTransport
+from trex_core.application.killswitch import KillSwitchService
 from trex_core.application.messaging_handlers import (
+    make_control_rpc_handler,
     make_intent_rpc_handler,
-    make_pause_rpc_handler,
     make_snapshot_rpc_handler,
     slot_transition_event_payload,
 )
@@ -148,6 +150,13 @@ class Daemon:
         # BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader).
         self.watchdog = WatchdogStore()
         self.pause_store = PostgreSQLPauseStore(config.dsn)
+        # BAT-012: day-scoped kill-switch exclusion store + service.
+        self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn)
+        self.kill_switch = KillSwitchService(
+            slot_repo=self.slot_repo,
+            slot_service=self.slot_service,
+            killswitch_store=self.killswitch_store,
+        )
         self.portfolio_read = _build_portfolio_read(config)
         self.snapshot_scheduler = SnapshotScheduler(
             read_port=self.portfolio_read,
@@ -231,18 +240,22 @@ class Daemon:
             pause_store=self.pause_store,
             default_total_slots=self.config.total_slots,
             watchdog_store=self.watchdog,
+            killswitch_store=self.killswitch_store,
         )
         transport.consume_rpc_forever(topology.SNAPSHOT_REQ_QUEUE, handler, self.stop)
 
-    def _pause_rpc_loop(self) -> None:
+    def _control_rpc_loop(self) -> None:
+        # REQ-043: operator control channel (pause/resume/list + kill_switch/resume_kill_switch).
         transport = _build_transport(self.config)
         self._transports.append(transport)
         try:
             transport.declare_topology()
         except Exception:  # noqa: BLE001
             pass
-        handler = make_pause_rpc_handler(self.pause_store)
-        transport.consume_rpc_forever(topology.PAUSE_REQ_QUEUE, handler, self.stop)
+        handler = make_control_rpc_handler(
+            self.pause_store, kill_switch_service=self.kill_switch,
+        )
+        transport.consume_rpc_forever(topology.CONTROL_REQ_QUEUE, handler, self.stop)
 
     def _safety_sell_loop(self) -> None:
         # REQ-037/038/039: forced_close / stop-loss·take-profit / market-close.
@@ -278,7 +291,7 @@ class Daemon:
             ("trex-ledger", self._ledger_loop),
             ("trex-intent-rpc", self._intent_rpc_loop),
             ("trex-snapshot-rpc", self._snapshot_rpc_loop),
-            ("trex-pause-rpc", self._pause_rpc_loop),
+            ("trex-control-rpc", self._control_rpc_loop),
             ("trex-snapshot-scheduler", self._snapshot_scheduler_loop),
             ("trex-safety-sell", self._safety_sell_loop),
         ]