feat(ledger): quiet idle outbox polling

trex_core/application/ledger_outbox.py

@@ -23,13 +23,14 @@ critical path while making ledger writes eventually-consistent and idempotent.
 """
 from __future__ import annotations
 
+import json
 import logging
 import random
 from collections.abc import Callable
 from dataclasses import dataclass, replace
 from datetime import datetime, timedelta, timezone
 from enum import Enum
-from typing import Optional, Protocol
+from typing import Any, Optional, Protocol
 
 from trex_core.application.alarm_emitter import AlarmEmitter
 from trex_core.application.ledger_port import (
@@ -223,6 +224,41 @@ class LedgerWriter:
     def _count(self, key: str) -> None:
         self.counters[key] = self.counters.get(key, 0) + 1
 
+    def metrics_snapshot(self) -> dict[str, Any]:
+        """Return a detached process-lifetime observability snapshot."""
+        return {
+            "available": True,
+            "scope": "process_lifetime",
+            "instance_key": self._instance_key,
+            "claims": {
+                "empty_polls_total": self.counters.get("claim:empty", 0),
+                "claimed_batches_total": self.counters.get("claim:claimed", 0),
+                "unavailable_total": self.counters.get("claim:unavailable", 0),
+                "lost_total": self.counters.get("claim:lost", 0),
+                "foreign_instance_total": self.counters.get(
+                    "claim:foreign_instance", 0
+                ),
+            },
+            "outcomes": {
+                "completed_total": self.counters.get("outcome:completed", 0),
+                "duplicate_completed_total": self.counters.get(
+                    "outcome:duplicate_completed", 0
+                ),
+                "retryable_failure_total": self.counters.get(
+                    "outcome:retryable_failure", 0
+                ),
+                "conflict_blocked_total": self.counters.get(
+                    "outcome:conflict_blocked", 0
+                ),
+                "permanent_failure_total": self.counters.get(
+                    "outcome:permanent_failure", 0
+                ),
+                "dead_lettered_total": self.counters.get(
+                    "outcome:dead_lettered", 0
+                ),
+            },
+        }
+
     def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
         return self._outbox.enqueue(trade)
 
@@ -231,13 +267,20 @@ class LedgerWriter:
         items = self._outbox.claim_pending(limit)
         claim_result = "claimed" if items else "empty"
         self._count(f"claim:{claim_result}")
-        _log.info(
-            "ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d",
-            self._instance_key,
-            claim_result,
-            len(items),
-            limit,
-        )
+        if items:
+            correlation_ids = json.dumps(
+                [item.correlation_id for item in items],
+                ensure_ascii=True,
+                separators=(",", ":"),
+            )
+            _log.info(
+                "ledger outbox claim: instance_key=%s claim_result=claimed "
+                "count=%d limit=%d correlation_ids=%s",
+                self._instance_key,
+                len(items),
+                limit,
+                correlation_ids,
+            )
         for item in items:
             if not self._validate_instance(item):
                 continue

trex_core/application/messaging_handlers.py

@@ -259,6 +259,7 @@ def make_slots_rpc_handler(
     watchdog_store: Optional[WatchdogStore] = None,
     watchdog_timeout_sec: int = 120,
     ledger_outbox=None,
+    ledger_outbox_metrics_reader=None,
     max_active_slots: Optional[int] = None,
     strategy_state_store=None,
     broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
@@ -512,6 +513,23 @@ def make_slots_rpc_handler(
                     )
             if health_payload:
                 base_payload["broker_balance_health"] = health_payload
+        if spy:
+            unavailable_metrics = {
+                "available": False,
+                "scope": "process_lifetime",
+                "instance_key": instance_key,
+                "reason": "ledger_outbox_metrics_unavailable",
+            }
+            if ledger_outbox_metrics_reader is None:
+                base_payload["ledger_outbox_metrics"] = unavailable_metrics
+            else:
+                try:
+                    metrics = ledger_outbox_metrics_reader.metrics_snapshot()
+                    if not isinstance(metrics, dict):
+                        raise TypeError("metrics snapshot must be a dict")
+                    base_payload["ledger_outbox_metrics"] = metrics
+                except Exception:  # noqa: BLE001 - status response must stay available
+                    base_payload["ledger_outbox_metrics"] = unavailable_metrics
         if warn_store is not None:
             base_payload["excluded_slots"] = {
                 "reconcile_quantity_mismatch": excluded_count,

trex_core/runtime/daemon.py

@@ -452,8 +452,13 @@ class Daemon:
         while not self.stop.wait(self.config.ledger_interval_sec):
             try:
                 self.ledger_writer.run_once()
-            except Exception:  # noqa: BLE001,S112
-                continue
+            except Exception as exc:  # noqa: BLE001 - loop must survive transient errors
+                _log.error(
+                    "ledger writer loop error: reason=ledger_writer_run_once_failed "
+                    "instance_key=%s exc_type=%s",
+                    self.config.instance_key,
+                    type(exc).__name__,
+                )
 
     def _intent_rpc_loop(self) -> None:
         transport = _build_transport(self.config)
@@ -486,6 +491,7 @@ class Daemon:
             watchdog_store=self.watchdog,
             watchdog_timeout_sec=self.config.safety_sell.watchdog_timeout_sec,
             ledger_outbox=self.outbox,
+            ledger_outbox_metrics_reader=self.ledger_writer,
             max_active_slots=self.config.max_active_slots,
             strategy_state_store=self.strategy_state_store,  # dcy-028 C-1
             broker_quantity_reader=self.account_snapshot_service,