feat(messaging): OPS-005 이벤트 발행 연결 신뢰성 (BAT-047)

발행 전용 연결의 idle heartbeat 사망 후 첫 발행 유실 결함 제거. - REQ-091 발행 재시도 + 강제 재연결(_force_reconnect, is_open 불신) - REQ-092 idle keepalive(execution-loop 단일스레드 틱, process_data_events time_limit=0) - REQ-093 DLQ 가시성 + emit 실패 ERROR 경보(구조화 로그, dcy-029 재사용) - REQ-094 baseline 문서 정합(runtime-flows/operations/project-structure) 독립 opus QC PASS-WITH-NOTES → 시정 2건(ruff F401, AC-3 프로덕션 경로) 완료. 554 passed/47 skipped/회귀 0/ruff clean. NFR-1 wire 무회귀(interface-contract diff 0). BAT-047 release-candidate, REQ-091~094 Implemented. dcy-031. M2 트랙B. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VHtM5Ws1HKCKQ4JGi6GRb5
2026-06-19 22:39:43 +09:00
parent bc79f60274
commit d182464b7a
19 changed files with 1067 additions and 43 deletions
@@ -144,6 +144,9 @@ class InMemoryTransport:
                if entry in self._event_subscribers:
                    self._event_subscribers.remove(entry)

+    def keepalive_tick(self) -> None:
+        """No-op for in-memory transport (REQ-092 — RabbitMQ only)."""
+
    def close(self) -> None:
        with self._lock:
            self._available = False
@@ -76,10 +76,11 @@ class RabbitMQTransport:
        self._health = TransportHealth.DISCONNECTED

    # ------------------------------------------------------------ lifecycle
-    def _connect(self) -> None:
+    def _connect(self, connect_timeout: Optional[float] = None) -> None:
        params = pika.URLParameters(self._url)
-        params.socket_timeout = self._timeout
-        params.blocked_connection_timeout = self._timeout
+        t = connect_timeout if connect_timeout is not None else self._timeout
+        params.socket_timeout = t
+        params.blocked_connection_timeout = t
        params.heartbeat = 30
        with self._lock:
            self._health = TransportHealth.RECONNECTING
@@ -102,6 +103,36 @@ class RabbitMQTransport:
        if not ok:
            self._connect()

+    def _force_reconnect(self) -> None:
+        """Force-close current connection and reconnect (REQ-091).
+
+        Uses a shorter socket_timeout (2 s) so a single publish attempt stays
+        well within the 5 s execution-loop block budget (NFR-2).
+        """
+        self.close()
+        self._connect(connect_timeout=2.0)
+
+    def keepalive_tick(self) -> None:
+        """Process pending I/O frames on the publish connection (REQ-092).
+
+        MUST be called from the thread that owns this transport instance
+        (pika BlockingConnection is NOT thread-safe).  The execution-loop
+        calls this once per tick (interval < heartbeat/2) so the broker's
+        idle-connection reaper never fires.
+
+        Uses time_limit=0 to be non-blocking — only drains already-pending
+        frames from the socket buffer.
+        """
+        with self._lock:
+            conn = self._conn
+        if conn is None or not conn.is_open:
+            return
+        try:
+            conn.process_data_events(time_limit=0)
+        except Exception:  # noqa: BLE001 — keepalive is best-effort
+            _log.debug("keepalive_tick process_data_events failed", exc_info=True)
+            self._mark_disconnected()
+
    def close(self) -> None:
        with self._lock:
            ch, conn = self._channel, self._conn
@@ -165,6 +196,27 @@ class RabbitMQTransport:
        ch.queue_bind(queue, EVENTS_EXCHANGE, routing_key=binding_pattern)

    # ------------------------------------------------------------ publish
+    def _do_basic_publish(
+        self,
+        routing_key: str,
+        body: bytes,
+        *,
+        mandatory: bool,
+    ) -> None:
+        """Internal: call basic_publish on current channel (must be connected)."""
+        ch = self._channel
+        assert ch is not None
+        ch.basic_publish(
+            exchange=EVENTS_EXCHANGE,
+            routing_key=routing_key,
+            body=body,
+            properties=pika.BasicProperties(
+                content_type="application/json",
+                delivery_mode=2,  # persistent
+            ),
+            mandatory=mandatory,
+        )
+
    def publish_event(
        self,
        routing_key: str,
@@ -172,31 +224,56 @@ class RabbitMQTransport:
        *,
        mandatory: bool = True,
    ) -> None:
+        """Publish to trex.events exchange with retry-on-failure (REQ-091).
+
+        Strategy:
+          1. Attempt publish (after _ensure_connected).
+          2. On AMQPError/OSError (including half-open stale connections):
+             force-reconnect, then retry once.
+          3. If retry also fails: raise TransportUnavailableError.
+          4. UnroutableError (no subscribers) → DLQ, no retry needed.
+
+        is_open=True is NOT trusted alone — a half-open connection will pass
+        the is_open check but fail on basic_publish.  The retry path handles
+        this transparently (REQ-091 FR-1).
+        """
        body = json.dumps(payload).encode("utf-8")
        try:
            self._ensure_connected()
-            ch = self._channel
-            assert ch is not None
-            ch.basic_publish(
-                exchange=EVENTS_EXCHANGE,
-                routing_key=routing_key,
-                body=body,
-                properties=pika.BasicProperties(
-                    content_type="application/json",
-                    delivery_mode=2,  # persistent
-                ),
-                mandatory=mandatory,
-            )
+            self._do_basic_publish(routing_key, body, mandatory=mandatory)
        except pika.exceptions.UnroutableError:
            # No matching queue subscribers → push to DLQ collector (REQ-023 AC-6).
            self._publish_dlq(routing_key, payload, reason="unroutable")
-        except (pika.exceptions.AMQPError, OSError) as exc:
+        except (pika.exceptions.AMQPError, OSError) as first_exc:
+            # First attempt failed (possibly half-open). Force reconnect + retry once.
+            _log.debug(
+                "publish_event first attempt failed, force-reconnecting (rk=%s): %s",
+                routing_key, first_exc,
+            )
            self._mark_disconnected()
-            raise TransportUnavailableError(
-                f"publish_event failed: {exc.__class__.__name__}: {exc}"
-            ) from exc
+            try:
+                self._force_reconnect()
+                self._do_basic_publish(routing_key, body, mandatory=mandatory)
+            except pika.exceptions.UnroutableError:
+                # Reconnected but still unroutable after retry.
+                self._publish_dlq(routing_key, payload, reason="unroutable")
+            except (pika.exceptions.AMQPError, OSError) as retry_exc:
+                self._mark_disconnected()
+                raise TransportUnavailableError(
+                    f"publish_event failed after reconnect+retry: "
+                    f"{retry_exc.__class__.__name__}: {retry_exc}"
+                ) from retry_exc

    def _publish_dlq(self, routing_key: str, payload: dict, *, reason: str) -> None:
+        """Best-effort publish to DLQ (REQ-093).
+
+        Logs ERROR on both DLQ entry (visibility) and DLQ publish failure
+        so the event is never silently lost from the operations perspective.
+        """
+        _log.error(
+            "DLQ entry: reason=%s rk=%s payload_keys=%s",
+            reason, routing_key, list(payload.keys()),
+        )
        try:
            self._ensure_connected()
            ch = self._channel
@@ -213,7 +290,7 @@ class RabbitMQTransport:
                ),
            )
        except Exception:  # noqa: BLE001 - DLQ best-effort
-            _log.warning("DLQ publish failed", exc_info=True)
+            _log.error("DLQ publish failed (rk=%s)", routing_key, exc_info=True)

    # ------------------------------------------------------------ RPC client
    def call_rpc(