trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alarm): trex-core 알람 emitter + trex.alerts exchange (dcy-033 BAT-049, Implemented)

Browse Source 
AlarmEmitter(application, best-effort 비차단)+trex.alerts topic exchange 가산·격리+8 CRITICAL code 배선. 608 passed, opus QC PASS, 팀장 e2e(텔레그램 HTTP200) 통과. REQ-098~101 Implemented.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5
This commit is contained in:
genichin
2026-06-21 19:25:55 +09:00
co-authored by Claude Opus 4.8
parent b2165eac25
commit d4c5fcbba1
29 changed files with 1195 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/application/alarm_emitter.py
+93
View File
@@ -0,0 +1,93 @@
"""Alarm emitter — best-effort CRITICAL alarm publisher (dcy-033 / REQ-100).
Publishes REQ-098 alarm envelopes to the `trex.alerts` exchange via the
MessagingTransport port. No pika import here — dependency inversion
preserved (NFR-1). Domain layer is never touched.
Caller contract:
- Inject into execution / reconciliation / killswitch / ledger_outbox / etc.
- Call publish_alarm(code, title, fields=...) at each alarm point.
- publish_alarm() NEVER re-raises. Alarm failure is ERROR-logged and swallowed
so the core business path (slot transition / order execution / ledger) is
never blocked by alarm delivery.
transport=None is explicitly supported for CLI entry points (e.g. reset.py)
where no broker connection exists. In that case publish_alarm() logs and skips.
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone, timedelta
from typing import Optional
from trex_core.application.messaging import MessagingTransport, TransportUnavailableError
from trex_core.adapters.messaging.topology import alarm_routing_key
_log = logging.getLogger(__name__)
_KST = timezone(timedelta(hours=9))
# Wire constants (REQ-098)
_SOURCE = "trex-core"
_SCHEMA_VERSION = "1"
class AlarmEmitter:
"""Application-layer alarm publisher (dcy-033 / REQ-100).
Wraps MessagingTransport.publish_alarm_event() with:
- envelope construction (source/severity/occurred_at/schema_version fixed)
- exception swallowing + ERROR log on failure
- None-transport guard for non-daemon entry points
"""
def __init__(self, transport: Optional[MessagingTransport]) -> None:
self._transport = transport
def publish_alarm(
self,
code: str,
title: str,
*,
fields: Optional[dict] = None,
severity: str = "critical",
) -> None:
"""Publish a single alarm envelope. Best-effort: never re-raises.
Args:
code: Alarm identifier code (see ALARM-INVENTORY.md §A).
title: Human-readable 1-line summary. No credentials or raw
exception bodies — callers must sanitize.
fields: Optional structured context dict (opaque, default {}).
severity: 'critical' | 'warn' | 'info'. Default 'critical'.
"""
if self._transport is None:
_log.info(
"alarm skipped (no transport): code=%s title=%s",
code, title,
)
return
routing_key = alarm_routing_key(severity, _SOURCE)
occurred_at = datetime.now(tz=_KST).isoformat()
payload = {
"source": _SOURCE,
"severity": severity,
"code": code,
"title": title,
"fields": fields or {},
"occurred_at": occurred_at,
"schema_version": _SCHEMA_VERSION,
}
try:
self._transport.publish_alarm_event(routing_key, payload)
except TransportUnavailableError as exc:
_log.error(
"alarm publish skipped (transport unavailable): code=%s: %s",
code, exc,
)
except Exception: # noqa: BLE001 — alarm must never crash the caller
_log.error(
"alarm publish failed: code=%s title=%s",
code, title, exc_info=True,
)
trex_core/application/execution.py
+19
View File
@@ -27,6 +27,7 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Callable, Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort, OrderRequest
from trex_core.application.ledger_outbox import LedgerOutbox
from trex_core.application.ledger_port import LedgerTrade
@@ -67,6 +68,7 @@ class ExecutionService:
event_emitter: Optional[SlotEventEmitter] = None,
stale_pending_threshold_sec: Optional[int] = None,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._svc = slot_service
self._repo = repo
@@ -85,6 +87,8 @@ class ExecutionService:
if stale_pending_threshold_sec and stale_pending_threshold_sec > 0
else None
)
# dcy-033 / REQ-100: best-effort alarm emitter (None = no alarms sent).
self._alarm_emitter = alarm_emitter
# ------------------------------------------------------------
def run_once(self) -> ExecutionResult:
@@ -130,6 +134,13 @@ class ExecutionService:
"(divergence — no order submitted; operator/reconcile attention)",
slot.slot_id, slot.symbol,
)
# dcy-033 / REQ-100: CRITICAL alarm — PENDING_SELL may stick.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"sell_skipped_no_balance",
f"매도 dispatch skip: 브로커 잔고 0 (slot={slot.slot_id} symbol={slot.symbol})",
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol},
)
continue
else:
qty = int(slot.qty or 0)
@@ -265,6 +276,14 @@ class ExecutionService:
slot.slot_id, slot.symbol, slot.broker_order_id,
avg, limit,
)
# dcy-033 / REQ-100: CRITICAL alarm — slot stuck in PENDING.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"fill_price_unresolvable",
f"체결가 미해결: 슬롯 PENDING 고착 위험 (slot={slot.slot_id} symbol={slot.symbol})",
fields={"slot_id": str(slot.slot_id), "symbol": slot.symbol,
"broker_order_id": slot.broker_order_id},
)
return None, False
def _on_filled(self, slot, fs, side: str, r: ExecutionResult) -> None:
trex_core/application/killswitch.py
+13
View File
@@ -11,7 +11,9 @@ from __future__ import annotations
import logging
from datetime import date as date_cls
from datetime import datetime, timezone
from typing import Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.portfolio import GLOBAL_KILLSWITCH, KillSwitchStore
from trex_core.domain.slot import SlotRepository, SlotService
@@ -29,11 +31,14 @@ class KillSwitchService:
slot_service: SlotService,
killswitch_store: KillSwitchStore,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._repo = slot_repo
self._svc = slot_service
self._store = killswitch_store
self._instance_key = instance_key # dcy-020/REQ-056: instance-scoped sweep
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
@staticmethod
def _trading_date(now: datetime) -> date_cls:
@@ -73,6 +78,14 @@ class KillSwitchService:
"kill-switch fired: actor=%s scope=%s portfolio_id=%s liquidated=%d",
actor, scope, portfolio_id, liquidated,
)
# dcy-033 / REQ-100: CRITICAL alarm — emergency liquidation fired.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"kill_switch_fired",
f"긴급 청산 발동: scope={scope} liquidated={liquidated}",
fields={"scope": scope, "portfolio_id": portfolio_id, "liquidated": liquidated,
"actor": actor},
)
return liquidated
def resume(
trex_core/application/ledger_outbox.py
+32
View File
@@ -30,6 +30,7 @@ from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Callable, Optional, Protocol
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.ledger_port import (
LedgerConflictError,
LedgerPermanentError,
@@ -169,6 +170,7 @@ class LedgerWriter:
max_attempts: int = _DEFAULT_MAX_ATTEMPTS,
clock: Optional[Callable[[], datetime]] = None,
rng: Optional[Callable[[], float]] = None,
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._outbox = outbox
self._ledger = ledger
@@ -181,6 +183,8 @@ class LedgerWriter:
self._max_attempts = max_attempts
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
self._rng = rng or random.random # jitter source; inject for determinism
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
@@ -219,6 +223,15 @@ class LedgerWriter:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)
)
# dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 409 충돌 (conflict_blocked): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"reason": "conflict_blocked"},
)
return OutboxOutcome.CONFLICT_BLOCKED
except LedgerPermanentError as exc:
# dcy-015 / REQ-017/018 (issue #13): non-retryable client error →
@@ -230,6 +243,15 @@ class LedgerWriter:
"ledger delivery_failed (permanent): correlation_id=%s reason=%s",
item.correlation_id, exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 영구 실패 (permanent): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"reason": "permanent_failure"},
)
return OutboxOutcome.PERMANENT_FAILURE
except LedgerRetryableError as exc:
# dcy-015 / REQ-019: backoff, or dead-letter once max_attempts hit.
@@ -243,6 +265,16 @@ class LedgerWriter:
"correlation_id=%s reason=%s",
new_attempts, item.correlation_id, exc,
)
# dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"ledger_delivery_failed",
f"원장 데드레터 (attempts={new_attempts}): {item.correlation_id}",
fields={"correlation_id": item.correlation_id,
"asset_id": item.trade.asset_id,
"attempts": new_attempts,
"reason": "dead_lettered"},
)
return OutboxOutcome.DEAD_LETTERED
next_retry_at = self._clock() + timedelta(
seconds=self._backoff_delay(new_attempts)
trex_core/application/messaging.py
+13
View File
@@ -79,6 +79,19 @@ class MessagingTransport(Protocol):
Raises TransportUnavailableError if not connected."""
...
def publish_alarm_event(
self,
routing_key: str,
payload: dict,
) -> None:
"""Publish to the trex.alerts exchange (dcy-033 / REQ-099).
Best-effort: mandatory=False so an unroutable message (notifier not
connected) is silently dropped — no DLQ, no retry.
Raises TransportUnavailableError if not connected (caller must swallow).
"""
...
def call_rpc(
self,
queue: str,
trex_core/application/reconciliation.py
+17 -2
View File
@@ -9,7 +9,9 @@ manual holds).
from __future__ import annotations
import logging
from typing import Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import SnapshotStore
from trex_core.domain.slot.state import SlotState
@@ -17,8 +19,14 @@ from trex_core.domain.slot.state import SlotState
_log = logging.getLogger(__name__)
def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
instance_key: str) -> dict[str, str]:
def reconcile_core_holdings(
*,
slot_repo,
slot_service,
broker: BrokerPort,
instance_key: str,
alarm_emitter: Optional[AlarmEmitter] = None,
) -> dict[str, str]:
"""dcy-020 / REQ-062 (BAT-029): surface core-slot holding ↔ broker divergence.
Broker balance is the reconcile ground truth. Decisions (no silent auto-fix,
@@ -48,6 +56,13 @@ def reconcile_core_holdings(*, slot_repo, slot_service, broker: BrokerPort,
"core holding but broker flat: slot=%s symbol=%s -> error",
slot.slot_id, sym,
)
# dcy-033 / REQ-100: CRITICAL alarm — position mismatch.
if alarm_emitter is not None:
alarm_emitter.publish_alarm(
"core_holding_broker_flat",
f"슬롯 보유인데 브로커 잔고 0 (slot={slot.slot_id} symbol={sym})",
fields={"slot_id": str(slot.slot_id), "symbol": sym},
)
except Exception: # noqa: BLE001 - never let reconcile crash the loop
_log.exception("failed to error-quarantine slot %s", slot.slot_id)
elif slot.qty is not None and int(slot.qty) != bq: # C — qty mismatch
trex_core/application/snapshot_scheduler.py
+19
View File
@@ -12,6 +12,7 @@ import threading
from datetime import datetime, time, timedelta, timezone
from typing import Callable, Optional
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import (
DegradedReason,
@@ -40,6 +41,7 @@ class SnapshotScheduler:
slot_repo=None,
slot_service=None,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
) -> None:
self._read = read_port
self._store = store
@@ -55,6 +57,8 @@ class SnapshotScheduler:
self._slot_service = slot_service
self._instance_key = instance_key
self._last_universe_size: Optional[int] = None
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
# ---- callable from a daemon thread loop --------------------------------
def run_forever(self, stop: threading.Event, *, tick_sec: float = 60.0) -> None:
@@ -88,6 +92,13 @@ class SnapshotScheduler:
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
)
_log.warning("whoami failed: %s", exc)
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"portfolio_hub_unavailable",
"portfolio-hub whoami 실패: 포트폴리오 정보 부실화",
fields={"reason": "whoami_failed"},
)
return False
try:
items = self._read.get_portfolios(self._daemon_id)
@@ -96,6 +107,13 @@ class SnapshotScheduler:
DegradedReason.PORTFOLIO_HUB_UNAVAILABLE, str(exc)
)
_log.warning("get_portfolios failed: %s", exc)
# dcy-033 / REQ-100: CRITICAL alarm — portfolio hub unavailable.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
"portfolio_hub_unavailable",
"portfolio-hub get_portfolios 실패: 포트폴리오 정보 부실화",
fields={"daemon_id": self._daemon_id, "reason": "get_portfolios_failed"},
)
return False
except Exception as exc: # noqa: BLE001 - shape/parsing failure
self._store.set_degraded(
@@ -120,6 +138,7 @@ class SnapshotScheduler:
reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
)
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")