feat(ledger): quiet idle outbox polling

This commit is contained in:
Hermes Agent
2026-09-20 21:57:30 +09:00
parent 62e4f57c2b
commit a2be76b199
14 changed files with 437 additions and 15 deletions
+51 -8
View File
@@ -23,13 +23,14 @@ critical path while making ledger writes eventually-consistent and idempotent.
"""
from __future__ import annotations
import json
import logging
import random
from collections.abc import Callable
from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Optional, Protocol
from typing import Any, Optional, Protocol
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.ledger_port import (
@@ -223,6 +224,41 @@ class LedgerWriter:
def _count(self, key: str) -> None:
self.counters[key] = self.counters.get(key, 0) + 1
def metrics_snapshot(self) -> dict[str, Any]:
"""Return a detached process-lifetime observability snapshot."""
return {
"available": True,
"scope": "process_lifetime",
"instance_key": self._instance_key,
"claims": {
"empty_polls_total": self.counters.get("claim:empty", 0),
"claimed_batches_total": self.counters.get("claim:claimed", 0),
"unavailable_total": self.counters.get("claim:unavailable", 0),
"lost_total": self.counters.get("claim:lost", 0),
"foreign_instance_total": self.counters.get(
"claim:foreign_instance", 0
),
},
"outcomes": {
"completed_total": self.counters.get("outcome:completed", 0),
"duplicate_completed_total": self.counters.get(
"outcome:duplicate_completed", 0
),
"retryable_failure_total": self.counters.get(
"outcome:retryable_failure", 0
),
"conflict_blocked_total": self.counters.get(
"outcome:conflict_blocked", 0
),
"permanent_failure_total": self.counters.get(
"outcome:permanent_failure", 0
),
"dead_lettered_total": self.counters.get(
"outcome:dead_lettered", 0
),
},
}
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
@@ -231,13 +267,20 @@ class LedgerWriter:
items = self._outbox.claim_pending(limit)
claim_result = "claimed" if items else "empty"
self._count(f"claim:{claim_result}")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d",
self._instance_key,
claim_result,
len(items),
limit,
)
if items:
correlation_ids = json.dumps(
[item.correlation_id for item in items],
ensure_ascii=True,
separators=(",", ":"),
)
_log.info(
"ledger outbox claim: instance_key=%s claim_result=claimed "
"count=%d limit=%d correlation_ids=%s",
self._instance_key,
len(items),
limit,
correlation_ids,
)
for item in items:
if not self._validate_instance(item):
continue
@@ -259,6 +259,7 @@ def make_slots_rpc_handler(
watchdog_store: Optional[WatchdogStore] = None,
watchdog_timeout_sec: int = 120,
ledger_outbox=None,
ledger_outbox_metrics_reader=None,
max_active_slots: Optional[int] = None,
strategy_state_store=None,
broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
@@ -512,6 +513,23 @@ def make_slots_rpc_handler(
)
if health_payload:
base_payload["broker_balance_health"] = health_payload
if spy:
unavailable_metrics = {
"available": False,
"scope": "process_lifetime",
"instance_key": instance_key,
"reason": "ledger_outbox_metrics_unavailable",
}
if ledger_outbox_metrics_reader is None:
base_payload["ledger_outbox_metrics"] = unavailable_metrics
else:
try:
metrics = ledger_outbox_metrics_reader.metrics_snapshot()
if not isinstance(metrics, dict):
raise TypeError("metrics snapshot must be a dict")
base_payload["ledger_outbox_metrics"] = metrics
except Exception: # noqa: BLE001 - status response must stay available
base_payload["ledger_outbox_metrics"] = unavailable_metrics
if warn_store is not None:
base_payload["excluded_slots"] = {
"reconcile_quantity_mismatch": excluded_count,