feat(alarm): trex-core 알람 emitter + trex.alerts exchange (dcy-033 BAT-049, Implemented)

AlarmEmitter(application, best-effort 비차단)+trex.alerts topic exchange 가산·격리+8 CRITICAL code 배선. 608 passed, opus QC PASS, 팀장 e2e(텔레그램 HTTP200) 통과. REQ-098~101 Implemented.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5
This commit is contained in:
2026-06-21 19:25:55 +09:00
co-authored by Claude Opus 4.8
parent b2165eac25
commit d4c5fcbba1
29 changed files with 1195 additions and 47 deletions
+16
View File
@@ -29,6 +29,7 @@ class InMemoryTransport:
self._rpc_handlers: dict[str, RpcHandler] = {}
self._event_subscribers: list[tuple[str, EventHandler]] = [] # (pattern, handler)
self._dlq: list[tuple[str, dict]] = [] # (routing_key, payload)
self._alarm_events: list[tuple[str, dict]] = [] # (routing_key, payload) dcy-033
self._available = True
self._health = TransportHealth.CONNECTED
@@ -53,6 +54,21 @@ class InMemoryTransport:
with self._lock:
return list(self._dlq)
def alarm_events(self) -> list[tuple[str, dict]]:
"""Return recorded alarm events (test affordance, dcy-033)."""
with self._lock:
return list(self._alarm_events)
# --- alarm events (dcy-033 / REQ-099) ---
def publish_alarm_event(self, routing_key: str, payload: dict) -> None:
"""Stub for trex.alerts publish. Records event; raises when unavailable."""
with self._lock:
if not self._available:
raise TransportUnavailableError(
"transport disconnected; alarm event dropped"
)
self._alarm_events.append((routing_key, dict(payload)))
# --- events ---
def publish_event(
self,
+78 -2
View File
@@ -22,12 +22,13 @@ import json
import logging
import threading
import time
from typing import Optional
from typing import Callable, Optional
import pika
import pika.exceptions
from trex_core.adapters.messaging.topology import (
ALERTS_EXCHANGE,
BROADCAST_EXCHANGE,
DEFAULT_INSTANCE_KEY,
DIRECT_EXCHANGE,
@@ -65,6 +66,7 @@ class RabbitMQTransport:
timeout: float = 5.0,
intent_prefetch: int = 1,
snapshot_prefetch: int = 8,
alarm_callback: Optional[Callable[[str, str, dict], None]] = None,
) -> None:
self._url = url
self._timeout = timeout
@@ -74,6 +76,20 @@ class RabbitMQTransport:
self._conn: Optional[pika.BlockingConnection] = None
self._channel: Optional[pika.adapters.blocking_connection.BlockingChannel] = None
self._health = TransportHealth.DISCONNECTED
# dcy-033 / REQ-100 (KD-1): alarm callback for adapter-layer alarm codes.
# Injected by daemon.py after AlarmEmitter construction (avoid circular init).
# Signature: (code: str, title: str, fields: dict) -> None
self._alarm_callback: Optional[Callable[[str, str, dict], None]] = alarm_callback
def set_alarm_callback(
self, callback: Optional[Callable[[str, str, dict], None]]
) -> None:
"""Late-bind alarm callback (dcy-033 / REQ-100 KD-1).
Allows daemon.py to inject the callback after AlarmEmitter is created,
avoiding circular initialization between transport and emitter.
"""
self._alarm_callback = callback
# ------------------------------------------------------------ lifecycle
def _connect(self, connect_timeout: Optional[float] = None) -> None:
@@ -163,6 +179,9 @@ class RabbitMQTransport:
# Events topic + DLQ
ch.exchange_declare(EVENTS_EXCHANGE, exchange_type="topic", durable=True)
ch.exchange_declare(EVENTS_DLQ_EXCHANGE, exchange_type="fanout", durable=True)
# dcy-033 / REQ-099: alarm channel — independent of trex.events, no DLQ,
# mandatory=False (notifier down = silent drop, no core-path blocker).
ch.exchange_declare(ALERTS_EXCHANGE, exchange_type="topic", durable=True)
ch.queue_declare(EVENTS_DLQ_QUEUE, durable=True)
ch.queue_bind(EVENTS_DLQ_QUEUE, EVENTS_DLQ_EXCHANGE)
# BAT-013 / REQ-050: multi-instance control exchanges.
@@ -259,6 +278,12 @@ class RabbitMQTransport:
self._publish_dlq(routing_key, payload, reason="unroutable")
except (pika.exceptions.AMQPError, OSError) as retry_exc:
self._mark_disconnected()
# dcy-033 / REQ-100: fire alarm for publish unavailable after reconnect.
self._fire_alarm_callback(
"publish_unavailable_after_reconnect",
f"재연결 후에도 이벤트 발행 불가: rk={routing_key}",
{"routing_key": routing_key, "error": str(retry_exc)},
)
raise TransportUnavailableError(
f"publish_event failed after reconnect+retry: "
f"{retry_exc.__class__.__name__}: {retry_exc}"
@@ -289,8 +314,59 @@ class RabbitMQTransport:
content_type="application/json", delivery_mode=2
),
)
except Exception: # noqa: BLE001 - DLQ best-effort
except Exception as dlq_exc: # noqa: BLE001 - DLQ best-effort
_log.error("DLQ publish failed (rk=%s)", routing_key, exc_info=True)
# dcy-033 / REQ-100: fire alarm for DLQ publish failure.
self._fire_alarm_callback(
"dlq_publish_failed",
f"DLQ 발행 실패: rk={routing_key}",
{"routing_key": routing_key, "error": str(dlq_exc)},
)
def publish_alarm_event(self, routing_key: str, payload: dict) -> None:
"""Publish to trex.alerts exchange (dcy-033 / REQ-099).
Best-effort: mandatory=False, delivery_mode=1 (non-persistent).
No retry on failure — alarm must never block the execution path.
Raises TransportUnavailableError on connect failure (caller swallows).
"""
body = json.dumps(payload).encode("utf-8")
try:
self._ensure_connected()
ch = self._channel
assert ch is not None
ch.basic_publish(
exchange=ALERTS_EXCHANGE,
routing_key=routing_key,
body=body,
properties=pika.BasicProperties(
content_type="application/json",
delivery_mode=1, # non-persistent — best-effort alarm
),
mandatory=False, # silent drop when notifier not bound
)
except (pika.exceptions.AMQPError, OSError) as exc:
self._mark_disconnected()
raise TransportUnavailableError(
f"publish_alarm_event failed: {exc.__class__.__name__}: {exc}"
) from exc
def _fire_alarm_callback(
self, code: str, title: str, fields: dict | None = None
) -> None:
"""Invoke the injected alarm callback (dcy-033 / REQ-100 KD-1).
Used by adapter-layer alarm sources (publish_unavailable_after_reconnect,
dlq_publish_failed) that cannot call AlarmEmitter directly.
No-op when callback is not set. Swallows callback exceptions.
"""
cb = self._alarm_callback
if cb is None:
return
try:
cb(code, title, fields or {})
except Exception: # noqa: BLE001 — callback must never crash the adapter
_log.debug("alarm callback raised (code=%s)", code, exc_info=True)
# ------------------------------------------------------------ RPC client
def call_rpc(
+14
View File
@@ -62,6 +62,20 @@ EVENTS_EXCHANGE = "trex.events" # topic, durable
EVENTS_DLQ_EXCHANGE = "trex.events.dlq" # fanout, durable; receives unroutable/NACKed
EVENTS_DLQ_QUEUE = "trex.events.dlq" # one collector queue bound to DLQ exchange
# --- Alarm channel (dcy-033 / REQ-099) ---
# Independent of trex.events. mandatory=False (best-effort, notifier down = silent drop).
ALERTS_EXCHANGE = "trex.alerts" # topic, durable; alarm envelope channel
def alarm_routing_key(severity: str, source: str) -> str:
"""Build the routing key for an alarm event (dcy-033 / REQ-098).
Pattern: alarm.{severity}.{source} (e.g. alarm.critical.trex-core)
Severity is always lower-case. Source tokens containing '.' are safe
because notifier binds with alarm.# pattern (zero-or-more tail words).
"""
return f"alarm.{severity}.{source}"
def event_routing_key(event_type: str, strategy_id: str, symbol: str) -> str:
"""Build the routing key for an event (dcy-006 OQ-2 mapping)."""
+93
View File
@@ -0,0 +1,93 @@
"""Alarm emitter — best-effort CRITICAL alarm publisher (dcy-033 / REQ-100).
Publishes REQ-098 alarm envelopes to the `trex.alerts` exchange via the
MessagingTransport port. No pika import here — dependency inversion
preserved (NFR-1). Domain layer is never touched.
Caller contract:
- Inject into execution / reconciliation / killswitch / ledger_outbox / etc.
- Call publish_alarm(code, title, fields=...) at each alarm point.
- publish_alarm() NEVER re-raises. Alarm failure is ERROR-logged and swallowed
so the core business path (slot transition / order execution / ledger) is
never blocked by alarm delivery.
transport=None is explicitly supported for CLI entry points (e.g. reset.py)
where no broker connection exists. In that case publish_alarm() logs and skips.
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone, timedelta
from typing import Optional
from trex_core.application.messaging import MessagingTransport, TransportUnavailableError
from trex_core.adapters.messaging.topology import alarm_routing_key
_log = logging.getLogger(__name__)
_KST = timezone(timedelta(hours=9))
# Wire constants (REQ-098)
_SOURCE = "trex-core"
_SCHEMA_VERSION = "1"
class AlarmEmitter:
"""Application-layer alarm publisher (dcy-033 / REQ-100).
Wraps MessagingTransport.publish_alarm_event() with:
- envelope construction (source/severity/occurred_at/schema_version fixed)
- exception swallowing + ERROR log on failure
- None-transport guard for non-daemon entry points
"""
def __init__(self, transport: Optional[MessagingTransport]) -> None:
self._transport = transport
def publish_alarm(
self,
code: str,
title: str,
*,
fields: Optional[dict] = None,
severity: str = "critical",
) -> None:
"""Publish a single alarm envelope. Best-effort: never re-raises.
Args:
code: Alarm identifier code (see ALARM-INVENTORY.md §A).
title: Human-readable 1-line summary. No credentials or raw
exception bodies — callers must sanitize.
fields: Optional structured context dict (opaque, default {}).
severity: 'critical' | 'warn' | 'info'. Default 'critical'.
"""
if self._transport is None:
_log.info(
"alarm skipped (no transport): code=%s title=%s",
code, title,
)
return
routing_key = alarm_routing_key(severity, _SOURCE)
occurred_at = datetime.now(tz=_KST).isoformat()
payload = {
"source": _SOURCE,
"severity": severity,
"code": code,
"title": title,
"fields": fields or {},
"occurred_at": occurred_at,
"schema_version": _SCHEMA_VERSION,
}
try:
self._transport.publish_alarm_event(routing_key, payload)
except TransportUnavailableError as exc:
_log.error(
"alarm publish skipped (transport unavailable): code=%s: %s",
code, exc,
)
except Exception: # noqa: BLE001 — alarm must never crash the caller
_log.error(
"alarm publish failed: code=%s title=%s",
code, title, exc_info=True,
)
+19
View File
@@ -27,6 +27,7 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Callable, Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort, OrderRequest
from trex_core.application.ledger_outbox import LedgerOutbox
from trex_core.application.ledger_port import LedgerTrade
@@ -67,6 +68,7 @@ class ExecutionService:
event_emitter: Optional[SlotEventEmitter] = None,
stale_pending_threshold_sec: Optional[int] = None,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._svc = slot_service
self._repo = repo
@@ -85,6 +87,8 @@ class ExecutionService:
if stale_pending_threshold_sec and stale_pending_threshold_sec > 0
else None
)
# dcy-033 / REQ-100: best-effort alarm emitter (None = no alarms sent).
self._alarm_emitter = alarm_emitter
# ------------------------------------------------------------
def run_once(self) -> ExecutionResult:
@@ -130,6 +134,13 @@ class ExecutionService:
"(divergence — no order submitted; operator/reconcile attention)",
slot.slot_id, slot.symbol,
)
# dcy-033 / REQ-100: CRITICAL alarm — PENDING_SELL may stick.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"sell_skipped_no_balance",
f"매도 dispatch skip: 브로커 잔고 0 (slot={slot.slot_id} symbol={slot.symbol})",
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol},
)
continue
else:
qty = int(slot.qty or 0)
@@ -265,6 +276,14 @@ class ExecutionService:
slot.slot_id, slot.symbol, slot.broker_order_id,
avg, limit,
)
# dcy-033 / REQ-100: CRITICAL alarm — slot stuck in PENDING.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"fill_price_unresolvable",
f"체결가 미해결: 슬롯 PENDING 고착 위험 (slot={slot.slot_id} symbol={slot.symbol})",
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol,
"broker_order_id": slot.broker_order_id},
)
return None, False
def _on_filled(self, slot, fs, side: str, r: ExecutionResult) -> None:
+13
View File
@@ -11,7 +11,9 @@ from __future__ import annotations
import logging
from datetime import date as date_cls
from datetime import datetime, timezone
from typing import Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
from trex_core.domain.slot import SlotRepository, SlotService
@@ -29,11 +31,14 @@ class KillSwitchService:
slot_service: SlotService,
killswitch_store: KillSwitchStore,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._repo = slot_repo
self._svc = slot_service
self._store = killswitch_store
self._instance_key = instance_key # dcy-020/REQ-056: instance-scoped sweep
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
@staticmethod
def _trading_date(now: datetime) -> date_cls:
@@ -73,6 +78,14 @@ class KillSwitchService:
"kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
actor, scope, portfolio_id, liquidated,
)
# dcy-033 / REQ-100: CRITICAL alarm — emergency liquidation fired.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"kill_switch_fired",
f"긴급 청산 발동: scope={scope} liquidated={liquidated}",
fields={"scope": scope, "portfolio_id": portfolio_id, "liquidated": liquidated,
"actor": actor},
)
return liquidated
def resume(
+32
View File
@@ -30,6 +30,7 @@ from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Callable, Optional, Protocol
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.ledger_port import (
LedgerConflictError,
LedgerPermanentError,
@@ -169,6 +170,7 @@ class LedgerWriter:
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
clock: Optional[Callable[[], datetime]] = None,
rng: Optional[Callable[[], float]] = None,
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._outbox = outbox
self._ledger = ledger
@@ -181,6 +183,8 @@ class LedgerWriter:
self._max_attempts = max_attempts
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
self._rng = rng or random.random # jitter source; inject for determinism
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
@@ -219,6 +223,15 @@ class LedgerWriter:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)
)
# dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 409 충돌 (conflict_blocked): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"reason": "conflict_blocked"},
)
return OutboxOutcome.CONFLICT_BLOCKED
except LedgerPermanentError as exc:
# dcy-015 / REQ-017/018 (issue #13): non-retryable client error →
@@ -230,6 +243,15 @@ class LedgerWriter:
"ledger delivery_failed (permanent): correlation_id=%s reason=%s",
item.correlation_id, exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 영구 실패 (permanent): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"reason": "permanent_failure"},
)
return OutboxOutcome.PERMANENT_FAILURE
except LedgerRetryableError as exc:
# dcy-015 / REQ-019: backoff, or dead-letter once max_attempts hit.
@@ -243,6 +265,16 @@ class LedgerWriter:
"correlation_id=%s reason=%s",
new_attempts, item.correlation_id, exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 데드레터 (attempts={new_attempts}): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"attempts": new_attempts,
"reason": "dead_lettered"},
)
return OutboxOutcome.DEAD_LETTERED
next_retry_at = self._clock() + timedelta(
seconds=self._backoff_delay(new_attempts)
+13
View File
@@ -79,6 +79,19 @@ class MessagingTransport(Protocol):
Raises TransportUnavailableError if not connected."""
...
def publish_alarm_event(
self,
routing_key: str,
payload: dict,
) -> None:
"""Publish to the trex.alerts exchange (dcy-033 / REQ-099).
Best-effort: mandatory=False so an unroutable message (notifier not
connected) is silently dropped — no DLQ, no retry.
Raises TransportUnavailableError if not connected (caller must swallow).
"""
...
def call_rpc(
self,
queue: str,
+17 -2
View File
@@ -9,7 +9,9 @@ manual holds).
from __future__ import annotations
import logging
from typing import Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import SnapshotStore
from trex_core.domain.slot.state import SlotState
@@ -17,8 +19,14 @@ from trex_core.domain.slot.state import SlotState
_log = logging.getLogger(__name__)
def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
instance_key: str) -> dict[str, str]:
def reconcile_core_holdings(
*,
slot_repo,
slot_service,
broker: BrokerPort,
instance_key: str,
alarm_emitter: Optional[AlarmEmitter] = None,
) -> dict[str, str]:
"""dcy-020 / REQ-062 (BAT-029): surface core-slot holding ↔ broker divergence.
Broker balance is the reconcile ground truth. Decisions (no silent auto-fix,
@@ -48,6 +56,13 @@ def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
"core holding but broker flat: slot=%s symbol=%s -> error",
slot.slot_id, sym,
)
# dcy-033 / REQ-100: CRITICAL alarm — position mismatch.
if alarm_emitter is not None:
alarm_emitter.publish_alarm(
"core_holding_broker_flat",
f"슬롯 보유인데 브로커 잔고 0 (slot={slot.slot_id} symbol={sym})",
fields={"slot_id": str(slot.slot_id), "symbol": sym},
)
except Exception: # noqa: BLE001 - never let reconcile crash the loop
_log.exception("failed to error-quarantine slot %s", slot.slot_id)
elif slot.qty is not None and int(slot.qty) != bq: # C — qty mismatch
@@ -12,6 +12,7 @@ import threading
from datetime import datetime, time, timedelta, timezone
from typing import Callable, Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import (
DegradedReason,
@@ -40,6 +41,7 @@ class SnapshotScheduler:
slot_repo=None,
slot_service=None,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._read = read_port
self._store = store
@@ -55,6 +57,8 @@ class SnapshotScheduler:
self._slot_service = slot_service
self._instance_key = instance_key
self._last_universe_size: Optional[int] = None
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
# ---- callable from a daemon thread loop --------------------------------
def run_forever(self, stop: threading.Event, *, tick_sec: float = 60.0) -> None:
@@ -88,6 +92,13 @@ class SnapshotScheduler:
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
)
_log.warning("whoami failed: %s", exc)
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"portfolio_hub_unavailable",
"portfolio-hub whoami 실패: 포트폴리오 정보 부실화",
fields={"reason": "whoami_failed"},
)
return False
try:
items = self._read.get_portfolios(self._daemon_id)
@@ -96,6 +107,13 @@ class SnapshotScheduler:
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
)
_log.warning("get_portfolios failed: %s", exc)
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"portfolio_hub_unavailable",
"portfolio-hub get_portfolios 실패: 포트폴리오 정보 부실화",
fields={"daemon_id": self._daemon_id, "reason": "get_portfolios_failed"},
)
return False
except Exception as exc: # noqa: BLE001 - shape/parsing failure
self._store.set_degraded(
@@ -120,6 +138,7 @@ class SnapshotScheduler:
reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
)
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")
+26 -12
View File
@@ -44,6 +44,7 @@ from trex_core.adapters.portfolio import (
StubPortfolioRead,
)
from trex_core.adapters.portfolio.killswitch_store import PostgreSQLKillSwitchStore
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.execution import ExecutionService
from trex_core.application.intents import IntentService
@@ -169,25 +170,42 @@ class Daemon:
# BAT-008: portfolio-hub read + snapshot store + durable pause + scheduler.
# (Created before the ledger writer so option-D write-back can target it.)
self.snapshot_store = SnapshotStore()
# BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader).
self.watchdog = WatchdogStore()
self.pause_store = PostgreSQLPauseStore(config.dsn)
# BAT-012: day-scoped kill-switch exclusion store + service.
self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn)
self.portfolio_read = _build_portfolio_read(config)
# Event publisher transport — one dedicated instance for the
# execution-loop thread (pika is not thread-safe).
self.event_publisher: MessagingTransport = _build_transport(config)
self._transports.append(self.event_publisher)
# dcy-033 / REQ-100: alarm emitter — shares the event_publisher transport
# (both run on the execution-loop thread; pika thread-safety rule satisfied).
# Created before services so all services receive the same emitter instance.
self.alarm_emitter = AlarmEmitter(self.event_publisher)
# Late-bind alarm callback into event_publisher for adapter-layer alarm codes
# (KD-1: publish_unavailable_after_reconnect / dlq_publish_failed).
if hasattr(self.event_publisher, "set_alarm_callback"):
self.event_publisher.set_alarm_callback(self.alarm_emitter.publish_alarm)
self.ledger_writer = LedgerWriter(
self.outbox, _build_ledger(config),
snapshot_store=self.snapshot_store, # dcy-018 / REQ-030: option D write-back
retry_base_sec=config.ledger_retry_base_sec,
retry_cap_sec=config.ledger_retry_cap_sec,
max_attempts=config.ledger_max_attempts,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
)
# BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader).
self.watchdog = WatchdogStore()
self.pause_store = PostgreSQLPauseStore(config.dsn)
# BAT-012: day-scoped kill-switch exclusion store + service.
self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn)
self.kill_switch = KillSwitchService(
slot_repo=self.slot_repo,
slot_service=self.slot_service,
killswitch_store=self.killswitch_store,
instance_key=config.instance_key,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
)
self.portfolio_read = _build_portfolio_read(config)
self.snapshot_scheduler = SnapshotScheduler(
read_port=self.portfolio_read,
store=self.snapshot_store,
@@ -199,19 +217,15 @@ class Daemon:
slot_repo=self.slot_repo,
slot_service=self.slot_service,
instance_key=config.instance_key,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
)
# Event publisher transport — one dedicated instance for the
# execution-loop thread (pika is not thread-safe).
self.event_publisher: MessagingTransport = _build_transport(config)
self._transports.append(self.event_publisher)
self.execution = ExecutionService(
self.slot_service, self.slot_repo, self.broker,
ledger_outbox=self.outbox,
event_emitter=self._make_event_emitter(),
stale_pending_threshold_sec=config.pending_stale_threshold_sec,
instance_key=config.instance_key,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
)
# BAT-011: safety-sell trigger service (forced_close / SL-TP / market-close).
+35 -1
View File
@@ -29,6 +29,21 @@ from typing import Any, Optional
_log = logging.getLogger(__name__)
def _emit_reset_alarm(
alarm_emitter: Optional[object],
code: str,
title: str,
fields: dict | None = None,
) -> None:
"""Best-effort alarm from reset.py (transport may be None — dcy-033 / REQ-100 KD-2)."""
if alarm_emitter is None:
return
try:
alarm_emitter.publish_alarm(code, title, fields=fields) # type: ignore[union-attr]
except Exception: # noqa: BLE001
_log.debug("reset alarm emit failed (code=%s)", code, exc_info=True)
# slot-core tables owned by trex-core migrations (drop order irrelevant — no FKs).
SLOT_CORE_TABLES: tuple[str, ...] = (
"daily_slot",
@@ -74,6 +89,7 @@ def run_reset(
*,
dry_run: bool = True,
psycopg_module: Any = None,
alarm_emitter: Optional[object] = None,
) -> ResetReport:
"""Wipe the slot-core tables and reapply migrations to the asset-centric schema.
@@ -106,7 +122,17 @@ def run_reset(
# 010 sets the inactive CHECK, 011 drops the pool-model columns.
from trex_core.adapters.persistence.migrate import apply_migrations
report.migrations_applied = apply_migrations(dsn, psycopg_module=psycopg_module)
try:
report.migrations_applied = apply_migrations(dsn, psycopg_module=psycopg_module)
except Exception as mig_exc: # noqa: BLE001
# dcy-033 / REQ-100: CRITICAL alarm — migration failed.
_emit_reset_alarm(
alarm_emitter,
"migration_failed",
f"마이그레이션 실패: {mig_exc.__class__.__name__}",
fields={"error": str(mig_exc)},
)
raise
from trex_core.domain.slot import EXPECTED_SLOT_FIELDS
@@ -118,6 +144,14 @@ def run_reset(
)
cols = {r[0] for r in cur.fetchall()}
report.schema_ok = cols == set(EXPECTED_SLOT_FIELDS)
if not report.schema_ok:
# dcy-033 / REQ-100: CRITICAL alarm — schema verification failed post-reset.
_emit_reset_alarm(
alarm_emitter,
"schema_verification_failed",
"리셋 후 schema 검증 실패: daily_slot 컬럼 불일치",
fields={"expected": sorted(EXPECTED_SLOT_FIELDS), "actual": sorted(cols)},
)
return report