feat(ledger): quiet idle outbox polling
This commit is contained in:
@@ -23,13 +23,14 @@ critical path while making ledger writes eventually-consistent and idempotent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from typing import Optional, Protocol
|
||||
from typing import Any, Optional, Protocol
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.ledger_port import (
|
||||
@@ -223,6 +224,41 @@ class LedgerWriter:
|
||||
def _count(self, key: str) -> None:
|
||||
self.counters[key] = self.counters.get(key, 0) + 1
|
||||
|
||||
def metrics_snapshot(self) -> dict[str, Any]:
|
||||
"""Return a detached process-lifetime observability snapshot."""
|
||||
return {
|
||||
"available": True,
|
||||
"scope": "process_lifetime",
|
||||
"instance_key": self._instance_key,
|
||||
"claims": {
|
||||
"empty_polls_total": self.counters.get("claim:empty", 0),
|
||||
"claimed_batches_total": self.counters.get("claim:claimed", 0),
|
||||
"unavailable_total": self.counters.get("claim:unavailable", 0),
|
||||
"lost_total": self.counters.get("claim:lost", 0),
|
||||
"foreign_instance_total": self.counters.get(
|
||||
"claim:foreign_instance", 0
|
||||
),
|
||||
},
|
||||
"outcomes": {
|
||||
"completed_total": self.counters.get("outcome:completed", 0),
|
||||
"duplicate_completed_total": self.counters.get(
|
||||
"outcome:duplicate_completed", 0
|
||||
),
|
||||
"retryable_failure_total": self.counters.get(
|
||||
"outcome:retryable_failure", 0
|
||||
),
|
||||
"conflict_blocked_total": self.counters.get(
|
||||
"outcome:conflict_blocked", 0
|
||||
),
|
||||
"permanent_failure_total": self.counters.get(
|
||||
"outcome:permanent_failure", 0
|
||||
),
|
||||
"dead_lettered_total": self.counters.get(
|
||||
"outcome:dead_lettered", 0
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
|
||||
return self._outbox.enqueue(trade)
|
||||
|
||||
@@ -231,13 +267,20 @@ class LedgerWriter:
|
||||
items = self._outbox.claim_pending(limit)
|
||||
claim_result = "claimed" if items else "empty"
|
||||
self._count(f"claim:{claim_result}")
|
||||
_log.info(
|
||||
"ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d",
|
||||
self._instance_key,
|
||||
claim_result,
|
||||
len(items),
|
||||
limit,
|
||||
)
|
||||
if items:
|
||||
correlation_ids = json.dumps(
|
||||
[item.correlation_id for item in items],
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
_log.info(
|
||||
"ledger outbox claim: instance_key=%s claim_result=claimed "
|
||||
"count=%d limit=%d correlation_ids=%s",
|
||||
self._instance_key,
|
||||
len(items),
|
||||
limit,
|
||||
correlation_ids,
|
||||
)
|
||||
for item in items:
|
||||
if not self._validate_instance(item):
|
||||
continue
|
||||
|
||||
@@ -259,6 +259,7 @@ def make_slots_rpc_handler(
|
||||
watchdog_store: Optional[WatchdogStore] = None,
|
||||
watchdog_timeout_sec: int = 120,
|
||||
ledger_outbox=None,
|
||||
ledger_outbox_metrics_reader=None,
|
||||
max_active_slots: Optional[int] = None,
|
||||
strategy_state_store=None,
|
||||
broker_quantity_reader: Optional[SlotsBrokerQuantityReader] = None,
|
||||
@@ -512,6 +513,23 @@ def make_slots_rpc_handler(
|
||||
)
|
||||
if health_payload:
|
||||
base_payload["broker_balance_health"] = health_payload
|
||||
if spy:
|
||||
unavailable_metrics = {
|
||||
"available": False,
|
||||
"scope": "process_lifetime",
|
||||
"instance_key": instance_key,
|
||||
"reason": "ledger_outbox_metrics_unavailable",
|
||||
}
|
||||
if ledger_outbox_metrics_reader is None:
|
||||
base_payload["ledger_outbox_metrics"] = unavailable_metrics
|
||||
else:
|
||||
try:
|
||||
metrics = ledger_outbox_metrics_reader.metrics_snapshot()
|
||||
if not isinstance(metrics, dict):
|
||||
raise TypeError("metrics snapshot must be a dict")
|
||||
base_payload["ledger_outbox_metrics"] = metrics
|
||||
except Exception: # noqa: BLE001 - status response must stay available
|
||||
base_payload["ledger_outbox_metrics"] = unavailable_metrics
|
||||
if warn_store is not None:
|
||||
base_payload["excluded_slots"] = {
|
||||
"reconcile_quantity_mismatch": excluded_count,
|
||||
|
||||
Reference in New Issue
Block a user