feat(alarm): trex-core 알람 emitter + trex.alerts exchange (dcy-033 BAT-049, Implemented)
AlarmEmitter(application, best-effort 비차단)+trex.alerts topic exchange 가산·격리+8 CRITICAL code 배선. 608 passed, opus QC PASS, 팀장 e2e(텔레그램 HTTP200) 통과. REQ-098~101 Implemented. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5
This commit is contained in:
@@ -22,12 +22,13 @@ import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
from typing import Callable, Optional
|
||||
|
||||
import pika
|
||||
import pika.exceptions
|
||||
|
||||
from trex_core.adapters.messaging.topology import (
|
||||
ALERTS_EXCHANGE,
|
||||
BROADCAST_EXCHANGE,
|
||||
DEFAULT_INSTANCE_KEY,
|
||||
DIRECT_EXCHANGE,
|
||||
@@ -65,6 +66,7 @@ class RabbitMQTransport:
|
||||
timeout: float = 5.0,
|
||||
intent_prefetch: int = 1,
|
||||
snapshot_prefetch: int = 8,
|
||||
alarm_callback: Optional[Callable[[str, str, dict], None]] = None,
|
||||
) -> None:
|
||||
self._url = url
|
||||
self._timeout = timeout
|
||||
@@ -74,6 +76,20 @@ class RabbitMQTransport:
|
||||
self._conn: Optional[pika.BlockingConnection] = None
|
||||
self._channel: Optional[pika.adapters.blocking_connection.BlockingChannel] = None
|
||||
self._health = TransportHealth.DISCONNECTED
|
||||
# dcy-033 / REQ-100 (KD-1): alarm callback for adapter-layer alarm codes.
|
||||
# Injected by daemon.py after AlarmEmitter construction (avoid circular init).
|
||||
# Signature: (code: str, title: str, fields: dict) -> None
|
||||
self._alarm_callback: Optional[Callable[[str, str, dict], None]] = alarm_callback
|
||||
|
||||
def set_alarm_callback(
|
||||
self, callback: Optional[Callable[[str, str, dict], None]]
|
||||
) -> None:
|
||||
"""Late-bind alarm callback (dcy-033 / REQ-100 KD-1).
|
||||
|
||||
Allows daemon.py to inject the callback after AlarmEmitter is created,
|
||||
avoiding circular initialization between transport and emitter.
|
||||
"""
|
||||
self._alarm_callback = callback
|
||||
|
||||
# ------------------------------------------------------------ lifecycle
|
||||
def _connect(self, connect_timeout: Optional[float] = None) -> None:
|
||||
@@ -163,6 +179,9 @@ class RabbitMQTransport:
|
||||
# Events topic + DLQ
|
||||
ch.exchange_declare(EVENTS_EXCHANGE, exchange_type="topic", durable=True)
|
||||
ch.exchange_declare(EVENTS_DLQ_EXCHANGE, exchange_type="fanout", durable=True)
|
||||
# dcy-033 / REQ-099: alarm channel — independent of trex.events, no DLQ,
|
||||
# mandatory=False (notifier down = silent drop, no core-path blocker).
|
||||
ch.exchange_declare(ALERTS_EXCHANGE, exchange_type="topic", durable=True)
|
||||
ch.queue_declare(EVENTS_DLQ_QUEUE, durable=True)
|
||||
ch.queue_bind(EVENTS_DLQ_QUEUE, EVENTS_DLQ_EXCHANGE)
|
||||
# BAT-013 / REQ-050: multi-instance control exchanges.
|
||||
@@ -259,6 +278,12 @@ class RabbitMQTransport:
|
||||
self._publish_dlq(routing_key, payload, reason="unroutable")
|
||||
except (pika.exceptions.AMQPError, OSError) as retry_exc:
|
||||
self._mark_disconnected()
|
||||
# dcy-033 / REQ-100: fire alarm for publish unavailable after reconnect.
|
||||
self._fire_alarm_callback(
|
||||
"publish_unavailable_after_reconnect",
|
||||
f"재연결 후에도 이벤트 발행 불가: rk={routing_key}",
|
||||
{"routing_key": routing_key, "error": str(retry_exc)},
|
||||
)
|
||||
raise TransportUnavailableError(
|
||||
f"publish_event failed after reconnect+retry: "
|
||||
f"{retry_exc.__class__.__name__}: {retry_exc}"
|
||||
@@ -289,8 +314,59 @@ class RabbitMQTransport:
|
||||
content_type="application/json", delivery_mode=2
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 - DLQ best-effort
|
||||
except Exception as dlq_exc: # noqa: BLE001 - DLQ best-effort
|
||||
_log.error("DLQ publish failed (rk=%s)", routing_key, exc_info=True)
|
||||
# dcy-033 / REQ-100: fire alarm for DLQ publish failure.
|
||||
self._fire_alarm_callback(
|
||||
"dlq_publish_failed",
|
||||
f"DLQ 발행 실패: rk={routing_key}",
|
||||
{"routing_key": routing_key, "error": str(dlq_exc)},
|
||||
)
|
||||
|
||||
def publish_alarm_event(self, routing_key: str, payload: dict) -> None:
|
||||
"""Publish to trex.alerts exchange (dcy-033 / REQ-099).
|
||||
|
||||
Best-effort: mandatory=False, delivery_mode=1 (non-persistent).
|
||||
No retry on failure — alarm must never block the execution path.
|
||||
Raises TransportUnavailableError on connect failure (caller swallows).
|
||||
"""
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
try:
|
||||
self._ensure_connected()
|
||||
ch = self._channel
|
||||
assert ch is not None
|
||||
ch.basic_publish(
|
||||
exchange=ALERTS_EXCHANGE,
|
||||
routing_key=routing_key,
|
||||
body=body,
|
||||
properties=pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=1, # non-persistent — best-effort alarm
|
||||
),
|
||||
mandatory=False, # silent drop when notifier not bound
|
||||
)
|
||||
except (pika.exceptions.AMQPError, OSError) as exc:
|
||||
self._mark_disconnected()
|
||||
raise TransportUnavailableError(
|
||||
f"publish_alarm_event failed: {exc.__class__.__name__}: {exc}"
|
||||
) from exc
|
||||
|
||||
def _fire_alarm_callback(
|
||||
self, code: str, title: str, fields: dict | None = None
|
||||
) -> None:
|
||||
"""Invoke the injected alarm callback (dcy-033 / REQ-100 KD-1).
|
||||
|
||||
Used by adapter-layer alarm sources (publish_unavailable_after_reconnect,
|
||||
dlq_publish_failed) that cannot call AlarmEmitter directly.
|
||||
No-op when callback is not set. Swallows callback exceptions.
|
||||
"""
|
||||
cb = self._alarm_callback
|
||||
if cb is None:
|
||||
return
|
||||
try:
|
||||
cb(code, title, fields or {})
|
||||
except Exception: # noqa: BLE001 — callback must never crash the adapter
|
||||
_log.debug("alarm callback raised (code=%s)", code, exc_info=True)
|
||||
|
||||
# ------------------------------------------------------------ RPC client
|
||||
def call_rpc(
|
||||
|
||||
Reference in New Issue
Block a user