feat(alarm): trex-core 알람 emitter + trex.alerts exchange (dcy-033 BAT-049, Implemented)
AlarmEmitter(application, best-effort 비차단)+trex.alerts topic exchange 가산·격리+8 CRITICAL code 배선. 608 passed, opus QC PASS, 팀장 e2e(텔레그램 HTTP200) 통과. REQ-098~101 Implemented. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5
This commit is contained in:
@@ -29,6 +29,7 @@ class InMemoryTransport:
|
||||
self._rpc_handlers: dict[str, RpcHandler] = {}
|
||||
self._event_subscribers: list[tuple[str, EventHandler]] = [] # (pattern, handler)
|
||||
self._dlq: list[tuple[str, dict]] = [] # (routing_key, payload)
|
||||
self._alarm_events: list[tuple[str, dict]] = [] # (routing_key, payload) dcy-033
|
||||
self._available = True
|
||||
self._health = TransportHealth.CONNECTED
|
||||
|
||||
@@ -53,6 +54,21 @@ class InMemoryTransport:
|
||||
with self._lock:
|
||||
return list(self._dlq)
|
||||
|
||||
def alarm_events(self) -> list[tuple[str, dict]]:
|
||||
"""Return recorded alarm events (test affordance, dcy-033)."""
|
||||
with self._lock:
|
||||
return list(self._alarm_events)
|
||||
|
||||
# --- alarm events (dcy-033 / REQ-099) ---
|
||||
def publish_alarm_event(self, routing_key: str, payload: dict) -> None:
|
||||
"""Stub for trex.alerts publish. Records event; raises when unavailable."""
|
||||
with self._lock:
|
||||
if not self._available:
|
||||
raise TransportUnavailableError(
|
||||
"transport disconnected; alarm event dropped"
|
||||
)
|
||||
self._alarm_events.append((routing_key, dict(payload)))
|
||||
|
||||
# --- events ---
|
||||
def publish_event(
|
||||
self,
|
||||
|
||||
@@ -22,12 +22,13 @@ import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
from typing import Callable, Optional
|
||||
|
||||
import pika
|
||||
import pika.exceptions
|
||||
|
||||
from trex_core.adapters.messaging.topology import (
|
||||
ALERTS_EXCHANGE,
|
||||
BROADCAST_EXCHANGE,
|
||||
DEFAULT_INSTANCE_KEY,
|
||||
DIRECT_EXCHANGE,
|
||||
@@ -65,6 +66,7 @@ class RabbitMQTransport:
|
||||
timeout: float = 5.0,
|
||||
intent_prefetch: int = 1,
|
||||
snapshot_prefetch: int = 8,
|
||||
alarm_callback: Optional[Callable[[str, str, dict], None]] = None,
|
||||
) -> None:
|
||||
self._url = url
|
||||
self._timeout = timeout
|
||||
@@ -74,6 +76,20 @@ class RabbitMQTransport:
|
||||
self._conn: Optional[pika.BlockingConnection] = None
|
||||
self._channel: Optional[pika.adapters.blocking_connection.BlockingChannel] = None
|
||||
self._health = TransportHealth.DISCONNECTED
|
||||
# dcy-033 / REQ-100 (KD-1): alarm callback for adapter-layer alarm codes.
|
||||
# Injected by daemon.py after AlarmEmitter construction (avoid circular init).
|
||||
# Signature: (code: str, title: str, fields: dict) -> None
|
||||
self._alarm_callback: Optional[Callable[[str, str, dict], None]] = alarm_callback
|
||||
|
||||
def set_alarm_callback(
|
||||
self, callback: Optional[Callable[[str, str, dict], None]]
|
||||
) -> None:
|
||||
"""Late-bind alarm callback (dcy-033 / REQ-100 KD-1).
|
||||
|
||||
Allows daemon.py to inject the callback after AlarmEmitter is created,
|
||||
avoiding circular initialization between transport and emitter.
|
||||
"""
|
||||
self._alarm_callback = callback
|
||||
|
||||
# ------------------------------------------------------------ lifecycle
|
||||
def _connect(self, connect_timeout: Optional[float] = None) -> None:
|
||||
@@ -163,6 +179,9 @@ class RabbitMQTransport:
|
||||
# Events topic + DLQ
|
||||
ch.exchange_declare(EVENTS_EXCHANGE, exchange_type="topic", durable=True)
|
||||
ch.exchange_declare(EVENTS_DLQ_EXCHANGE, exchange_type="fanout", durable=True)
|
||||
# dcy-033 / REQ-099: alarm channel — independent of trex.events, no DLQ,
|
||||
# mandatory=False (notifier down = silent drop, no core-path blocker).
|
||||
ch.exchange_declare(ALERTS_EXCHANGE, exchange_type="topic", durable=True)
|
||||
ch.queue_declare(EVENTS_DLQ_QUEUE, durable=True)
|
||||
ch.queue_bind(EVENTS_DLQ_QUEUE, EVENTS_DLQ_EXCHANGE)
|
||||
# BAT-013 / REQ-050: multi-instance control exchanges.
|
||||
@@ -259,6 +278,12 @@ class RabbitMQTransport:
|
||||
self._publish_dlq(routing_key, payload, reason="unroutable")
|
||||
except (pika.exceptions.AMQPError, OSError) as retry_exc:
|
||||
self._mark_disconnected()
|
||||
# dcy-033 / REQ-100: fire alarm for publish unavailable after reconnect.
|
||||
self._fire_alarm_callback(
|
||||
"publish_unavailable_after_reconnect",
|
||||
f"재연결 후에도 이벤트 발행 불가: rk={routing_key}",
|
||||
{"routing_key": routing_key, "error": str(retry_exc)},
|
||||
)
|
||||
raise TransportUnavailableError(
|
||||
f"publish_event failed after reconnect+retry: "
|
||||
f"{retry_exc.__class__.__name__}: {retry_exc}"
|
||||
@@ -289,8 +314,59 @@ class RabbitMQTransport:
|
||||
content_type="application/json", delivery_mode=2
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 - DLQ best-effort
|
||||
except Exception as dlq_exc: # noqa: BLE001 - DLQ best-effort
|
||||
_log.error("DLQ publish failed (rk=%s)", routing_key, exc_info=True)
|
||||
# dcy-033 / REQ-100: fire alarm for DLQ publish failure.
|
||||
self._fire_alarm_callback(
|
||||
"dlq_publish_failed",
|
||||
f"DLQ 발행 실패: rk={routing_key}",
|
||||
{"routing_key": routing_key, "error": str(dlq_exc)},
|
||||
)
|
||||
|
||||
def publish_alarm_event(self, routing_key: str, payload: dict) -> None:
|
||||
"""Publish to trex.alerts exchange (dcy-033 / REQ-099).
|
||||
|
||||
Best-effort: mandatory=False, delivery_mode=1 (non-persistent).
|
||||
No retry on failure — alarm must never block the execution path.
|
||||
Raises TransportUnavailableError on connect failure (caller swallows).
|
||||
"""
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
try:
|
||||
self._ensure_connected()
|
||||
ch = self._channel
|
||||
assert ch is not None
|
||||
ch.basic_publish(
|
||||
exchange=ALERTS_EXCHANGE,
|
||||
routing_key=routing_key,
|
||||
body=body,
|
||||
properties=pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=1, # non-persistent — best-effort alarm
|
||||
),
|
||||
mandatory=False, # silent drop when notifier not bound
|
||||
)
|
||||
except (pika.exceptions.AMQPError, OSError) as exc:
|
||||
self._mark_disconnected()
|
||||
raise TransportUnavailableError(
|
||||
f"publish_alarm_event failed: {exc.__class__.__name__}: {exc}"
|
||||
) from exc
|
||||
|
||||
def _fire_alarm_callback(
|
||||
self, code: str, title: str, fields: dict | None = None
|
||||
) -> None:
|
||||
"""Invoke the injected alarm callback (dcy-033 / REQ-100 KD-1).
|
||||
|
||||
Used by adapter-layer alarm sources (publish_unavailable_after_reconnect,
|
||||
dlq_publish_failed) that cannot call AlarmEmitter directly.
|
||||
No-op when callback is not set. Swallows callback exceptions.
|
||||
"""
|
||||
cb = self._alarm_callback
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
cb(code, title, fields or {})
|
||||
except Exception: # noqa: BLE001 — callback must never crash the adapter
|
||||
_log.debug("alarm callback raised (code=%s)", code, exc_info=True)
|
||||
|
||||
# ------------------------------------------------------------ RPC client
|
||||
def call_rpc(
|
||||
|
||||
@@ -62,6 +62,20 @@ EVENTS_EXCHANGE = "trex.events" # topic, durable
|
||||
EVENTS_DLQ_EXCHANGE = "trex.events.dlq" # fanout, durable; receives unroutable/NACKed
|
||||
EVENTS_DLQ_QUEUE = "trex.events.dlq" # one collector queue bound to DLQ exchange
|
||||
|
||||
# --- Alarm channel (dcy-033 / REQ-099) ---
|
||||
# Independent of trex.events. mandatory=False (best-effort, notifier down = silent drop).
|
||||
ALERTS_EXCHANGE = "trex.alerts" # topic, durable; alarm envelope channel
|
||||
|
||||
|
||||
def alarm_routing_key(severity: str, source: str) -> str:
|
||||
"""Build the routing key for an alarm event (dcy-033 / REQ-098).
|
||||
|
||||
Pattern: alarm.{severity}.{source} (e.g. alarm.critical.trex-core)
|
||||
Severity is always lower-case. Source tokens containing '.' are safe
|
||||
because notifier binds with alarm.# pattern (zero-or-more tail words).
|
||||
"""
|
||||
return f"alarm.{severity}.{source}"
|
||||
|
||||
|
||||
def event_routing_key(event_type: str, strategy_id: str, symbol: str) -> str:
|
||||
"""Build the routing key for an event (dcy-006 OQ-2 mapping)."""
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Alarm emitter — best-effort CRITICAL alarm publisher (dcy-033 / REQ-100).
|
||||
|
||||
Publishes REQ-098 alarm envelopes to the `trex.alerts` exchange via the
|
||||
MessagingTransport port. No pika import here — dependency inversion
|
||||
preserved (NFR-1). Domain layer is never touched.
|
||||
|
||||
Caller contract:
|
||||
- Inject into execution / reconciliation / killswitch / ledger_outbox / etc.
|
||||
- Call publish_alarm(code, title, fields=...) at each alarm point.
|
||||
- publish_alarm() NEVER re-raises. Alarm failure is ERROR-logged and swallowed
|
||||
so the core business path (slot transition / order execution / ledger) is
|
||||
never blocked by alarm delivery.
|
||||
|
||||
transport=None is explicitly supported for CLI entry points (e.g. reset.py)
|
||||
where no broker connection exists. In that case publish_alarm() logs and skips.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from trex_core.application.messaging import MessagingTransport, TransportUnavailableError
|
||||
from trex_core.adapters.messaging.topology import alarm_routing_key
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
_KST = timezone(timedelta(hours=9))
|
||||
|
||||
# Wire constants (REQ-098)
|
||||
_SOURCE = "trex-core"
|
||||
_SCHEMA_VERSION = "1"
|
||||
|
||||
|
||||
class AlarmEmitter:
|
||||
"""Application-layer alarm publisher (dcy-033 / REQ-100).
|
||||
|
||||
Wraps MessagingTransport.publish_alarm_event() with:
|
||||
- envelope construction (source/severity/occurred_at/schema_version fixed)
|
||||
- exception swallowing + ERROR log on failure
|
||||
- None-transport guard for non-daemon entry points
|
||||
"""
|
||||
|
||||
def __init__(self, transport: Optional[MessagingTransport]) -> None:
|
||||
self._transport = transport
|
||||
|
||||
def publish_alarm(
|
||||
self,
|
||||
code: str,
|
||||
title: str,
|
||||
*,
|
||||
fields: Optional[dict] = None,
|
||||
severity: str = "critical",
|
||||
) -> None:
|
||||
"""Publish a single alarm envelope. Best-effort: never re-raises.
|
||||
|
||||
Args:
|
||||
code: Alarm identifier code (see ALARM-INVENTORY.md §A).
|
||||
title: Human-readable 1-line summary. No credentials or raw
|
||||
exception bodies — callers must sanitize.
|
||||
fields: Optional structured context dict (opaque, default {}).
|
||||
severity: 'critical' | 'warn' | 'info'. Default 'critical'.
|
||||
"""
|
||||
if self._transport is None:
|
||||
_log.info(
|
||||
"alarm skipped (no transport): code=%s title=%s",
|
||||
code, title,
|
||||
)
|
||||
return
|
||||
|
||||
routing_key = alarm_routing_key(severity, _SOURCE)
|
||||
occurred_at = datetime.now(tz=_KST).isoformat()
|
||||
payload = {
|
||||
"source": _SOURCE,
|
||||
"severity": severity,
|
||||
"code": code,
|
||||
"title": title,
|
||||
"fields": fields or {},
|
||||
"occurred_at": occurred_at,
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
}
|
||||
try:
|
||||
self._transport.publish_alarm_event(routing_key, payload)
|
||||
except TransportUnavailableError as exc:
|
||||
_log.error(
|
||||
"alarm publish skipped (transport unavailable): code=%s: %s",
|
||||
code, exc,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — alarm must never crash the caller
|
||||
_log.error(
|
||||
"alarm publish failed: code=%s title=%s",
|
||||
code, title, exc_info=True,
|
||||
)
|
||||
@@ -27,6 +27,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort, OrderRequest
|
||||
from trex_core.application.ledger_outbox import LedgerOutbox
|
||||
from trex_core.application.ledger_port import LedgerTrade
|
||||
@@ -67,6 +68,7 @@ class ExecutionService:
|
||||
event_emitter: Optional[SlotEventEmitter] = None,
|
||||
stale_pending_threshold_sec: Optional[int] = None,
|
||||
instance_key: str = "default",
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
) -> None:
|
||||
self._svc = slot_service
|
||||
self._repo = repo
|
||||
@@ -85,6 +87,8 @@ class ExecutionService:
|
||||
if stale_pending_threshold_sec and stale_pending_threshold_sec > 0
|
||||
else None
|
||||
)
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter (None = no alarms sent).
|
||||
self._alarm_emitter = alarm_emitter
|
||||
|
||||
# ------------------------------------------------------------
|
||||
def run_once(self) -> ExecutionResult:
|
||||
@@ -130,6 +134,13 @@ class ExecutionService:
|
||||
"(divergence — no order submitted; operator/reconcile attention)",
|
||||
slot.slot_id, slot.symbol,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — PENDING_SELL may stick.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"sell_skipped_no_balance",
|
||||
f"매도 dispatch skip: 브로커 잔고 0 (slot={slot.slot_id} symbol={slot.symbol})",
|
||||
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol},
|
||||
)
|
||||
continue
|
||||
else:
|
||||
qty = int(slot.qty or 0)
|
||||
@@ -265,6 +276,14 @@ class ExecutionService:
|
||||
slot.slot_id, slot.symbol, slot.broker_order_id,
|
||||
avg, limit,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — slot stuck in PENDING.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"fill_price_unresolvable",
|
||||
f"체결가 미해결: 슬롯 PENDING 고착 위험 (slot={slot.slot_id} symbol={slot.symbol})",
|
||||
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol,
|
||||
"broker_order_id": slot.broker_order_id},
|
||||
)
|
||||
return None, False
|
||||
|
||||
def _on_filled(self, slot, fs, side: str, r: ExecutionResult) -> None:
|
||||
|
||||
@@ -11,7 +11,9 @@ from __future__ import annotations
|
||||
import logging
|
||||
from datetime import date as date_cls
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
|
||||
from trex_core.domain.slot import SlotRepository, SlotService
|
||||
|
||||
@@ -29,11 +31,14 @@ class KillSwitchService:
|
||||
slot_service: SlotService,
|
||||
killswitch_store: KillSwitchStore,
|
||||
instance_key: str = "default",
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
) -> None:
|
||||
self._repo = slot_repo
|
||||
self._svc = slot_service
|
||||
self._store = killswitch_store
|
||||
self._instance_key = instance_key # dcy-020/REQ-056: instance-scoped sweep
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter.
|
||||
self._alarm_emitter = alarm_emitter
|
||||
|
||||
@staticmethod
|
||||
def _trading_date(now: datetime) -> date_cls:
|
||||
@@ -73,6 +78,14 @@ class KillSwitchService:
|
||||
"kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
|
||||
actor, scope, portfolio_id, liquidated,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — emergency liquidation fired.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"kill_switch_fired",
|
||||
f"긴급 청산 발동: scope={scope} liquidated={liquidated}",
|
||||
fields={"scope": scope, "portfolio_id": portfolio_id, "liquidated": liquidated,
|
||||
"actor": actor},
|
||||
)
|
||||
return liquidated
|
||||
|
||||
def resume(
|
||||
|
||||
@@ -30,6 +30,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from typing import Callable, Optional, Protocol
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.ledger_port import (
|
||||
LedgerConflictError,
|
||||
LedgerPermanentError,
|
||||
@@ -169,6 +170,7 @@ class LedgerWriter:
|
||||
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
|
||||
clock: Optional[Callable[[], datetime]] = None,
|
||||
rng: Optional[Callable[[], float]] = None,
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
) -> None:
|
||||
self._outbox = outbox
|
||||
self._ledger = ledger
|
||||
@@ -181,6 +183,8 @@ class LedgerWriter:
|
||||
self._max_attempts = max_attempts
|
||||
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
|
||||
self._rng = rng or random.random # jitter source; inject for determinism
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter.
|
||||
self._alarm_emitter = alarm_emitter
|
||||
|
||||
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
|
||||
return self._outbox.enqueue(trade)
|
||||
@@ -219,6 +223,15 @@ class LedgerWriter:
|
||||
self._outbox.mark_outcome(
|
||||
item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"ledger_delivery_failed",
|
||||
f"원장 409 충돌 (conflict_blocked): {item.correlation_id}",
|
||||
fields={"correlation_id": item.correlation_id,
|
||||
"asset_id": item.trade.asset_id,
|
||||
"reason": "conflict_blocked"},
|
||||
)
|
||||
return OutboxOutcome.CONFLICT_BLOCKED
|
||||
except LedgerPermanentError as exc:
|
||||
# dcy-015 / REQ-017/018 (issue #13): non-retryable client error →
|
||||
@@ -230,6 +243,15 @@ class LedgerWriter:
|
||||
"ledger delivery_failed (permanent): correlation_id=%s reason=%s",
|
||||
item.correlation_id, exc,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"ledger_delivery_failed",
|
||||
f"원장 영구 실패 (permanent): {item.correlation_id}",
|
||||
fields={"correlation_id": item.correlation_id,
|
||||
"asset_id": item.trade.asset_id,
|
||||
"reason": "permanent_failure"},
|
||||
)
|
||||
return OutboxOutcome.PERMANENT_FAILURE
|
||||
except LedgerRetryableError as exc:
|
||||
# dcy-015 / REQ-019: backoff, or dead-letter once max_attempts hit.
|
||||
@@ -243,6 +265,16 @@ class LedgerWriter:
|
||||
"correlation_id=%s reason=%s",
|
||||
new_attempts, item.correlation_id, exc,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"ledger_delivery_failed",
|
||||
f"원장 데드레터 (attempts={new_attempts}): {item.correlation_id}",
|
||||
fields={"correlation_id": item.correlation_id,
|
||||
"asset_id": item.trade.asset_id,
|
||||
"attempts": new_attempts,
|
||||
"reason": "dead_lettered"},
|
||||
)
|
||||
return OutboxOutcome.DEAD_LETTERED
|
||||
next_retry_at = self._clock() + timedelta(
|
||||
seconds=self._backoff_delay(new_attempts)
|
||||
|
||||
@@ -79,6 +79,19 @@ class MessagingTransport(Protocol):
|
||||
Raises TransportUnavailableError if not connected."""
|
||||
...
|
||||
|
||||
def publish_alarm_event(
|
||||
self,
|
||||
routing_key: str,
|
||||
payload: dict,
|
||||
) -> None:
|
||||
"""Publish to the trex.alerts exchange (dcy-033 / REQ-099).
|
||||
|
||||
Best-effort: mandatory=False so an unroutable message (notifier not
|
||||
connected) is silently dropped — no DLQ, no retry.
|
||||
Raises TransportUnavailableError if not connected (caller must swallow).
|
||||
"""
|
||||
...
|
||||
|
||||
def call_rpc(
|
||||
self,
|
||||
queue: str,
|
||||
|
||||
@@ -9,7 +9,9 @@ manual holds).
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort
|
||||
from trex_core.application.portfolio import SnapshotStore
|
||||
from trex_core.domain.slot.state import SlotState
|
||||
@@ -17,8 +19,14 @@ from trex_core.domain.slot.state import SlotState
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
|
||||
instance_key: str) -> dict[str, str]:
|
||||
def reconcile_core_holdings(
|
||||
*,
|
||||
slot_repo,
|
||||
slot_service,
|
||||
broker: BrokerPort,
|
||||
instance_key: str,
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
) -> dict[str, str]:
|
||||
"""dcy-020 / REQ-062 (BAT-029): surface core-slot holding ↔ broker divergence.
|
||||
|
||||
Broker balance is the reconcile ground truth. Decisions (no silent auto-fix,
|
||||
@@ -48,6 +56,13 @@ def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
|
||||
"core holding but broker flat: slot=%s symbol=%s -> error",
|
||||
slot.slot_id, sym,
|
||||
)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — position mismatch.
|
||||
if alarm_emitter is not None:
|
||||
alarm_emitter.publish_alarm(
|
||||
"core_holding_broker_flat",
|
||||
f"슬롯 보유인데 브로커 잔고 0 (slot={slot.slot_id} symbol={sym})",
|
||||
fields={"slot_id": str(slot.slot_id), "symbol": sym},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - never let reconcile crash the loop
|
||||
_log.exception("failed to error-quarantine slot %s", slot.slot_id)
|
||||
elif slot.qty is not None and int(slot.qty) != bq: # C — qty mismatch
|
||||
|
||||
@@ -12,6 +12,7 @@ import threading
|
||||
from datetime import datetime, time, timedelta, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort
|
||||
from trex_core.application.portfolio import (
|
||||
DegradedReason,
|
||||
@@ -40,6 +41,7 @@ class SnapshotScheduler:
|
||||
slot_repo=None,
|
||||
slot_service=None,
|
||||
instance_key: str = "default",
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
) -> None:
|
||||
self._read = read_port
|
||||
self._store = store
|
||||
@@ -55,6 +57,8 @@ class SnapshotScheduler:
|
||||
self._slot_service = slot_service
|
||||
self._instance_key = instance_key
|
||||
self._last_universe_size: Optional[int] = None
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter.
|
||||
self._alarm_emitter = alarm_emitter
|
||||
|
||||
# ---- callable from a daemon thread loop --------------------------------
|
||||
def run_forever(self, stop: threading.Event, *, tick_sec: float = 60.0) -> None:
|
||||
@@ -88,6 +92,13 @@ class SnapshotScheduler:
|
||||
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
|
||||
)
|
||||
_log.warning("whoami failed: %s", exc)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"portfolio_hub_unavailable",
|
||||
"portfolio-hub whoami 실패: 포트폴리오 정보 부실화",
|
||||
fields={"reason": "whoami_failed"},
|
||||
)
|
||||
return False
|
||||
try:
|
||||
items = self._read.get_portfolios(self._daemon_id)
|
||||
@@ -96,6 +107,13 @@ class SnapshotScheduler:
|
||||
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
|
||||
)
|
||||
_log.warning("get_portfolios failed: %s", exc)
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"portfolio_hub_unavailable",
|
||||
"portfolio-hub get_portfolios 실패: 포트폴리오 정보 부실화",
|
||||
fields={"daemon_id": self._daemon_id, "reason": "get_portfolios_failed"},
|
||||
)
|
||||
return False
|
||||
except Exception as exc: # noqa: BLE001 - shape/parsing failure
|
||||
self._store.set_degraded(
|
||||
@@ -120,6 +138,7 @@ class SnapshotScheduler:
|
||||
reconcile_core_holdings(
|
||||
slot_repo=self._slot_repo, slot_service=self._slot_service,
|
||||
broker=self._broker, instance_key=self._instance_key,
|
||||
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("core-holding divergence reconcile failed")
|
||||
|
||||
+26
-12
@@ -44,6 +44,7 @@ from trex_core.adapters.portfolio import (
|
||||
StubPortfolioRead,
|
||||
)
|
||||
from trex_core.adapters.portfolio.killswitch_store import PostgreSQLKillSwitchStore
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort
|
||||
from trex_core.application.execution import ExecutionService
|
||||
from trex_core.application.intents import IntentService
|
||||
@@ -169,25 +170,42 @@ class Daemon:
|
||||
# BAT-008: portfolio-hub read + snapshot store + durable pause + scheduler.
|
||||
# (Created before the ledger writer so option-D write-back can target it.)
|
||||
self.snapshot_store = SnapshotStore()
|
||||
# BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader).
|
||||
self.watchdog = WatchdogStore()
|
||||
self.pause_store = PostgreSQLPauseStore(config.dsn)
|
||||
# BAT-012: day-scoped kill-switch exclusion store + service.
|
||||
self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn)
|
||||
self.portfolio_read = _build_portfolio_read(config)
|
||||
|
||||
# Event publisher transport — one dedicated instance for the
|
||||
# execution-loop thread (pika is not thread-safe).
|
||||
self.event_publisher: MessagingTransport = _build_transport(config)
|
||||
self._transports.append(self.event_publisher)
|
||||
|
||||
# dcy-033 / REQ-100: alarm emitter — shares the event_publisher transport
|
||||
# (both run on the execution-loop thread; pika thread-safety rule satisfied).
|
||||
# Created before services so all services receive the same emitter instance.
|
||||
self.alarm_emitter = AlarmEmitter(self.event_publisher)
|
||||
# Late-bind alarm callback into event_publisher for adapter-layer alarm codes
|
||||
# (KD-1: publish_unavailable_after_reconnect / dlq_publish_failed).
|
||||
if hasattr(self.event_publisher, "set_alarm_callback"):
|
||||
self.event_publisher.set_alarm_callback(self.alarm_emitter.publish_alarm)
|
||||
|
||||
self.ledger_writer = LedgerWriter(
|
||||
self.outbox, _build_ledger(config),
|
||||
snapshot_store=self.snapshot_store, # dcy-018 / REQ-030: option D write-back
|
||||
retry_base_sec=config.ledger_retry_base_sec,
|
||||
retry_cap_sec=config.ledger_retry_cap_sec,
|
||||
max_attempts=config.ledger_max_attempts,
|
||||
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
|
||||
)
|
||||
# BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader).
|
||||
self.watchdog = WatchdogStore()
|
||||
self.pause_store = PostgreSQLPauseStore(config.dsn)
|
||||
# BAT-012: day-scoped kill-switch exclusion store + service.
|
||||
self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn)
|
||||
self.kill_switch = KillSwitchService(
|
||||
slot_repo=self.slot_repo,
|
||||
slot_service=self.slot_service,
|
||||
killswitch_store=self.killswitch_store,
|
||||
instance_key=config.instance_key,
|
||||
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
|
||||
)
|
||||
self.portfolio_read = _build_portfolio_read(config)
|
||||
self.snapshot_scheduler = SnapshotScheduler(
|
||||
read_port=self.portfolio_read,
|
||||
store=self.snapshot_store,
|
||||
@@ -199,19 +217,15 @@ class Daemon:
|
||||
slot_repo=self.slot_repo,
|
||||
slot_service=self.slot_service,
|
||||
instance_key=config.instance_key,
|
||||
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
|
||||
)
|
||||
|
||||
# Event publisher transport — one dedicated instance for the
|
||||
# execution-loop thread (pika is not thread-safe).
|
||||
self.event_publisher: MessagingTransport = _build_transport(config)
|
||||
self._transports.append(self.event_publisher)
|
||||
|
||||
self.execution = ExecutionService(
|
||||
self.slot_service, self.slot_repo, self.broker,
|
||||
ledger_outbox=self.outbox,
|
||||
event_emitter=self._make_event_emitter(),
|
||||
stale_pending_threshold_sec=config.pending_stale_threshold_sec,
|
||||
instance_key=config.instance_key,
|
||||
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
|
||||
)
|
||||
|
||||
# BAT-011: safety-sell trigger service (forced_close / SL-TP / market-close).
|
||||
|
||||
@@ -29,6 +29,21 @@ from typing import Any, Optional
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _emit_reset_alarm(
|
||||
alarm_emitter: Optional[object],
|
||||
code: str,
|
||||
title: str,
|
||||
fields: dict | None = None,
|
||||
) -> None:
|
||||
"""Best-effort alarm from reset.py (transport may be None — dcy-033 / REQ-100 KD-2)."""
|
||||
if alarm_emitter is None:
|
||||
return
|
||||
try:
|
||||
alarm_emitter.publish_alarm(code, title, fields=fields) # type: ignore[union-attr]
|
||||
except Exception: # noqa: BLE001
|
||||
_log.debug("reset alarm emit failed (code=%s)", code, exc_info=True)
|
||||
|
||||
# slot-core tables owned by trex-core migrations (drop order irrelevant — no FKs).
|
||||
SLOT_CORE_TABLES: tuple[str, ...] = (
|
||||
"daily_slot",
|
||||
@@ -74,6 +89,7 @@ def run_reset(
|
||||
*,
|
||||
dry_run: bool = True,
|
||||
psycopg_module: Any = None,
|
||||
alarm_emitter: Optional[object] = None,
|
||||
) -> ResetReport:
|
||||
"""Wipe the slot-core tables and reapply migrations to the asset-centric schema.
|
||||
|
||||
@@ -106,7 +122,17 @@ def run_reset(
|
||||
# 010 sets the inactive CHECK, 011 drops the pool-model columns.
|
||||
from trex_core.adapters.persistence.migrate import apply_migrations
|
||||
|
||||
report.migrations_applied = apply_migrations(dsn, psycopg_module=psycopg_module)
|
||||
try:
|
||||
report.migrations_applied = apply_migrations(dsn, psycopg_module=psycopg_module)
|
||||
except Exception as mig_exc: # noqa: BLE001
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — migration failed.
|
||||
_emit_reset_alarm(
|
||||
alarm_emitter,
|
||||
"migration_failed",
|
||||
f"마이그레이션 실패: {mig_exc.__class__.__name__}",
|
||||
fields={"error": str(mig_exc)},
|
||||
)
|
||||
raise
|
||||
|
||||
from trex_core.domain.slot import EXPECTED_SLOT_FIELDS
|
||||
|
||||
@@ -118,6 +144,14 @@ def run_reset(
|
||||
)
|
||||
cols = {r[0] for r in cur.fetchall()}
|
||||
report.schema_ok = cols == set(EXPECTED_SLOT_FIELDS)
|
||||
if not report.schema_ok:
|
||||
# dcy-033 / REQ-100: CRITICAL alarm — schema verification failed post-reset.
|
||||
_emit_reset_alarm(
|
||||
alarm_emitter,
|
||||
"schema_verification_failed",
|
||||
"리셋 후 schema 검증 실패: daily_slot 컬럼 불일치",
|
||||
fields={"expected": sorted(EXPECTED_SLOT_FIELDS), "actual": sorted(cols)},
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user