feat(messaging): OPS-005 이벤트 발행 연결 신뢰성 (BAT-047)

발행 전용 연결의 idle heartbeat 사망 후 첫 발행 유실 결함 제거.
- REQ-091 발행 재시도 + 강제 재연결(_force_reconnect, is_open 불신)
- REQ-092 idle keepalive(execution-loop 단일스레드 틱, process_data_events time_limit=0)
- REQ-093 DLQ 가시성 + emit 실패 ERROR 경보(구조화 로그, dcy-029 재사용)
- REQ-094 baseline 문서 정합(runtime-flows/operations/project-structure)

독립 opus QC PASS-WITH-NOTES → 시정 2건(ruff F401, AC-3 프로덕션 경로) 완료.
554 passed/47 skipped/회귀 0/ruff clean. NFR-1 wire 무회귀(interface-contract diff 0).
BAT-047 release-candidate, REQ-091~094 Implemented. dcy-031. M2 트랙B.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5

trex_core/adapters/messaging/in_memory.py

@@ -144,6 +144,9 @@ class InMemoryTransport:
                 if entry in self._event_subscribers:
                     self._event_subscribers.remove(entry)
 
+    def keepalive_tick(self) -> None:
+        """No-op for in-memory transport (REQ-092 — RabbitMQ only)."""
+
     def close(self) -> None:
         with self._lock:
             self._available = False

trex_core/adapters/messaging/rabbitmq.py

@@ -76,10 +76,11 @@ class RabbitMQTransport:
         self._health = TransportHealth.DISCONNECTED
 
     # ------------------------------------------------------------ lifecycle
-    def _connect(self) -> None:
+    def _connect(self, connect_timeout: Optional[float] = None) -> None:
         params = pika.URLParameters(self._url)
-        params.socket_timeout = self._timeout
-        params.blocked_connection_timeout = self._timeout
+        t = connect_timeout if connect_timeout is not None else self._timeout
+        params.socket_timeout = t
+        params.blocked_connection_timeout = t
         params.heartbeat = 30
         with self._lock:
             self._health = TransportHealth.RECONNECTING
@@ -102,6 +103,36 @@ class RabbitMQTransport:
         if not ok:
             self._connect()
 
+    def _force_reconnect(self) -> None:
+        """Force-close current connection and reconnect (REQ-091).
+
+        Uses a shorter socket_timeout (2 s) so a single publish attempt stays
+        well within the 5 s execution-loop block budget (NFR-2).
+        """
+        self.close()
+        self._connect(connect_timeout=2.0)
+
+    def keepalive_tick(self) -> None:
+        """Process pending I/O frames on the publish connection (REQ-092).
+
+        MUST be called from the thread that owns this transport instance
+        (pika BlockingConnection is NOT thread-safe).  The execution-loop
+        calls this once per tick (interval < heartbeat/2) so the broker's
+        idle-connection reaper never fires.
+
+        Uses time_limit=0 to be non-blocking — only drains already-pending
+        frames from the socket buffer.
+        """
+        with self._lock:
+            conn = self._conn
+        if conn is None or not conn.is_open:
+            return
+        try:
+            conn.process_data_events(time_limit=0)
+        except Exception:  # noqa: BLE001 — keepalive is best-effort
+            _log.debug("keepalive_tick process_data_events failed", exc_info=True)
+            self._mark_disconnected()
+
     def close(self) -> None:
         with self._lock:
             ch, conn = self._channel, self._conn
@@ -165,6 +196,27 @@ class RabbitMQTransport:
         ch.queue_bind(queue, EVENTS_EXCHANGE, routing_key=binding_pattern)
 
     # ------------------------------------------------------------ publish
+    def _do_basic_publish(
+        self,
+        routing_key: str,
+        body: bytes,
+        *,
+        mandatory: bool,
+    ) -> None:
+        """Internal: call basic_publish on current channel (must be connected)."""
+        ch = self._channel
+        assert ch is not None
+        ch.basic_publish(
+            exchange=EVENTS_EXCHANGE,
+            routing_key=routing_key,
+            body=body,
+            properties=pika.BasicProperties(
+                content_type="application/json",
+                delivery_mode=2,  # persistent
+            ),
+            mandatory=mandatory,
+        )
+
     def publish_event(
         self,
         routing_key: str,
@@ -172,31 +224,56 @@ class RabbitMQTransport:
         *,
         mandatory: bool = True,
     ) -> None:
+        """Publish to trex.events exchange with retry-on-failure (REQ-091).
+
+        Strategy:
+          1. Attempt publish (after _ensure_connected).
+          2. On AMQPError/OSError (including half-open stale connections):
+             force-reconnect, then retry once.
+          3. If retry also fails: raise TransportUnavailableError.
+          4. UnroutableError (no subscribers) → DLQ, no retry needed.
+
+        is_open=True is NOT trusted alone — a half-open connection will pass
+        the is_open check but fail on basic_publish.  The retry path handles
+        this transparently (REQ-091 FR-1).
+        """
         body = json.dumps(payload).encode("utf-8")
         try:
             self._ensure_connected()
-            ch = self._channel
-            assert ch is not None
-            ch.basic_publish(
-                exchange=EVENTS_EXCHANGE,
-                routing_key=routing_key,
-                body=body,
-                properties=pika.BasicProperties(
-                    content_type="application/json",
-                    delivery_mode=2,  # persistent
-                ),
-                mandatory=mandatory,
-            )
+            self._do_basic_publish(routing_key, body, mandatory=mandatory)
         except pika.exceptions.UnroutableError:
             # No matching queue subscribers → push to DLQ collector (REQ-023 AC-6).
             self._publish_dlq(routing_key, payload, reason="unroutable")
-        except (pika.exceptions.AMQPError, OSError) as exc:
+        except (pika.exceptions.AMQPError, OSError) as first_exc:
+            # First attempt failed (possibly half-open). Force reconnect + retry once.
+            _log.debug(
+                "publish_event first attempt failed, force-reconnecting (rk=%s): %s",
+                routing_key, first_exc,
+            )
             self._mark_disconnected()
-            raise TransportUnavailableError(
-                f"publish_event failed: {exc.__class__.__name__}: {exc}"
-            ) from exc
+            try:
+                self._force_reconnect()
+                self._do_basic_publish(routing_key, body, mandatory=mandatory)
+            except pika.exceptions.UnroutableError:
+                # Reconnected but still unroutable after retry.
+                self._publish_dlq(routing_key, payload, reason="unroutable")
+            except (pika.exceptions.AMQPError, OSError) as retry_exc:
+                self._mark_disconnected()
+                raise TransportUnavailableError(
+                    f"publish_event failed after reconnect+retry: "
+                    f"{retry_exc.__class__.__name__}: {retry_exc}"
+                ) from retry_exc
 
     def _publish_dlq(self, routing_key: str, payload: dict, *, reason: str) -> None:
+        """Best-effort publish to DLQ (REQ-093).
+
+        Logs ERROR on both DLQ entry (visibility) and DLQ publish failure
+        so the event is never silently lost from the operations perspective.
+        """
+        _log.error(
+            "DLQ entry: reason=%s rk=%s payload_keys=%s",
+            reason, routing_key, list(payload.keys()),
+        )
         try:
             self._ensure_connected()
             ch = self._channel
@@ -213,7 +290,7 @@ class RabbitMQTransport:
                 ),
             )
         except Exception:  # noqa: BLE001 - DLQ best-effort
-            _log.warning("DLQ publish failed", exc_info=True)
+            _log.error("DLQ publish failed (rk=%s)", routing_key, exc_info=True)
 
     # ------------------------------------------------------------ RPC client
     def call_rpc(

trex_core/application/messaging.py

@@ -112,6 +112,15 @@ class MessagingTransport(Protocol):
         exchange, dispatch to handler. Blocks until `stop` is set."""
         ...
 
+    def keepalive_tick(self) -> None:
+        """Process pending I/O frames to keep the connection alive (REQ-092).
+
+        MUST be called from the thread that owns this transport instance.
+        Non-blocking (time_limit=0 equivalent). Implementations that do not
+        need keepalive (e.g. InMemoryTransport) should be a no-op.
+        """
+        ...
+
     def health(self) -> TransportHealth:
         """Current connection state (REQ-024)."""
         ...

trex_core/runtime/daemon.py

@@ -244,12 +244,24 @@ class Daemon:
             try:
                 self.event_publisher.publish_event(rk, payload)
             except Exception:  # noqa: BLE001 - publish failure must not roll back
-                _log.warning("event publish failed (rk=%s)", rk, exc_info=True)
+                # REQ-093: escalate to ERROR so operators can observe via log
+                # alert. Include routing key + slot identity for correlation.
+                _log.error(
+                    "event publish failed (rk=%s slot_id=%s strategy=%s)",
+                    rk,
+                    getattr(slot, "slot_id", None) or getattr(slot, "id", None),
+                    getattr(slot, "strategy_id", None),
+                    exc_info=True,
+                )
         return emit
 
     # ------------------------------------------------------------ workers
     def _execution_loop(self) -> None:
         while not self.stop.wait(self.config.execution_interval_sec):
+            # REQ-092: keepalive tick — process pending I/O frames on the
+            # publish connection before each execution run.  Called in the
+            # same thread that owns event_publisher (pika thread-safety rule).
+            self.event_publisher.keepalive_tick()
             try:
                 self.execution.run_once()
             except Exception:  # noqa: BLE001 - loop must survive transient errors