trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: persist skipped ledger fills

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-08-24 13:23:06 +00:00
committed by GitHub
co-authored by genichin
parent df0258bc36
commit 76cde2edd0
19 changed files with 1246 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/runtime/cli.py
+197 -77
View File
@@ -12,6 +12,8 @@ Subcommands:
sync — sync slot qty to broker ground truth (direct-DB; REQ-105)
verify — re-check C-case reconcile-warning (direct-DB; REQ-106)
ledger backfill — evidence-backed historical ledger recovery
ledger skips list — query durable skipped-ledger records
ledger skips resolve — explicitly resolve a skipped-ledger record
universe/slots/enter/exit go through the daemon's RabbitMQ RPC surface so the
single-writer admission + invariants are preserved (no direct slot writes,
@@ -28,24 +30,27 @@ live accounts require `--yes --live` (double ack); `--dry-run` previews
before/after without writing. sync is direct-DB (no RPC).
verify (REQ-106): read-centric (no --yes/--live gate needed).
ledger backfill: verifies KIS daily-ccld evidence, reconstructs one ledger
trade, enqueues it through ledger_outbox, and immediately drives the
LedgerWriter path with an audit JSON report.
trade, enqueues it through ledger_outbox, immediately drives the
LedgerWriter path, and resolves matching durable skipped-ledger records.
"""
from __future__ import annotations
import argparse
import json as _json
import logging
import re
import sys
import uuid
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta, timezone
from pathlib import Path
from zoneinfo import ZoneInfo
from trex_core.adapters.messaging import topology
from trex_core.adapters.portfolio.pause_store import PostgreSQLPauseStore
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.ledger_outbox import LedgerWriter, OutboxOutcome
from trex_core.application.ledger_port import LedgerTrade
from trex_core.application.ledger_skip import ledger_skip_alarm_fields, publish_ledger_skip_alarm
from trex_core.runtime.config import (
DaemonConfig,
DaemonInstanceConfig,
@@ -55,6 +60,7 @@ from trex_core.runtime.config import (
_DEFAULT_TIMEOUT = 10.0
_DEFAULT_LEDGER_BACKFILL_AUDIT_DIR = "artifacts/validation/ledger_backfill"
_log = logging.getLogger(__name__)
# --------------------------------------------------------------------------- #
@@ -91,6 +97,14 @@ def _default_rpc(cfg: DaemonConfig):
return _rpc
def _default_alarm_emitter(cfg: DaemonConfig):
from trex_core.adapters.messaging import InMemoryTransport, RabbitMQTransport
mq = getattr(cfg, "rabbitmq", None)
transport = RabbitMQTransport(mq.url, timeout=mq.timeout) if mq else InMemoryTransport()
return AlarmEmitter(transport), transport
def _default_confirm(prompt: str) -> bool:
try:
return input(f"{prompt} [y/N] ").strip().lower() in ("y", "yes")
@@ -376,7 +390,7 @@ def _sanitize_path_component(value: str, *, limit: int = 80) -> str:
def _ledger_backfill_audit_path(args, execution_id: str) -> Path:
base = Path(getattr(args, "audit_dir", _DEFAULT_LEDGER_BACKFILL_AUDIT_DIR))
stamp = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ")
stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ")
name = (
f"{stamp}--{_sanitize_path_component(_resolve_instance(args))}"
f"--{_sanitize_path_component(execution_id)}.json"
@@ -393,37 +407,30 @@ def _write_ledger_backfill_audit(args, report: dict) -> str:
def _lookup_ledger_backfill_evidence(cfg: DaemonConfig, *, trading_date: str, execution_id: str):
from trex_core.runtime.broker_provider import (
BrokerCredentialUnavailable,
build_broker_provider,
)
from trex_core.runtime.broker_provider import build_broker_provider
broker, source = build_broker_provider(cfg).get_broker_for_read()
lookup = getattr(broker, "lookup_fill_evidence", None)
if not callable(lookup):
raise RuntimeError("configured broker does not support KIS ledger backfill evidence lookup")
try:
evidence = lookup(trading_date=trading_date, execution_id=execution_id)
except BrokerCredentialUnavailable:
raise
raise TypeError("configured broker does not support KIS ledger backfill evidence lookup")
evidence = lookup(trading_date=trading_date, execution_id=execution_id)
return evidence, source
def _resolve_ledger_backfill_slot(dsn: str, *, instance: str, symbol: str) -> dict:
import psycopg
with psycopg.connect(dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT slot_id::text, portfolio_id, status "
"FROM daily_slot "
"WHERE instance_key = %s AND symbol = %s "
"AND portfolio_id IS NOT NULL AND retired_at IS NULL "
"ORDER BY updated_at DESC NULLS LAST, slot_id ASC "
"LIMIT 2",
(instance, symbol),
)
rows = cur.fetchall()
with psycopg.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT slot_id::text, portfolio_id, status "
"FROM daily_slot "
"WHERE instance_key = %s AND symbol = %s "
"AND portfolio_id IS NOT NULL AND retired_at IS NULL "
"ORDER BY updated_at DESC NULLS LAST, slot_id ASC "
"LIMIT 2",
(instance, symbol),
)
rows = cur.fetchall()
if not rows:
raise RuntimeError(f"no daemon-owned slot found for instance={instance} symbol={symbol}")
if len(rows) != 1:
@@ -439,16 +446,15 @@ def _resolve_ledger_backfill_slot(dsn: str, *, instance: str, symbol: str) -> di
def _lookup_outbox_by_execution_id(dsn: str, *, execution_id: str) -> list[dict]:
import psycopg
with psycopg.connect(dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT correlation_id, asset_id, order_id, execution_id, status, "
"attempts, last_outcome, last_error "
"FROM ledger_outbox WHERE execution_id = %s "
"ORDER BY created_at, correlation_id",
(execution_id,),
)
rows = cur.fetchall()
with psycopg.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT correlation_id, asset_id, order_id, execution_id, status, "
"attempts, last_outcome, last_error "
"FROM ledger_outbox WHERE execution_id = %s "
"ORDER BY created_at, correlation_id",
(execution_id,),
)
rows = cur.fetchall()
return [
{
"correlation_id": str(cid),
@@ -630,6 +636,33 @@ def _build_parser() -> argparse.ArgumentParser:
)
_add_common(p_backfill)
p_skips = p_ledger_sub.add_parser(
"skips",
help="Query or resolve durable skipped-ledger records",
)
p_skips_sub = p_skips.add_subparsers(dest="ledger_skips_cmd", required=True)
p_skips_list = p_skips_sub.add_parser(
"list",
help="List skipped-ledger records by instance/date/portfolio/state",
)
p_skips_list.add_argument("--date", default=None, help="UTC date filter (YYYY-MM-DD)")
p_skips_list.add_argument("--portfolio-id", dest="portfolio_id", default=None)
p_skips_list.add_argument(
"--status",
choices=("unresolved", "resolved", "all"),
default="unresolved",
)
p_skips_list.add_argument("--limit", type=int, default=100)
_add_common(p_skips_list)
p_skips_resolve = p_skips_sub.add_parser(
"resolve",
help="Mark one skipped-ledger record resolved and emit a resolved alert",
)
p_skips_resolve.add_argument("--skip-key", required=True)
p_skips_resolve.add_argument("--reason", default="operator_reconciled")
_add_common(p_skips_resolve)
return parser
@@ -696,14 +729,13 @@ def _cmd_sync(args, cfg, confirm) -> int:
# Determine target slot_ids
import psycopg
if args.portfolio_id:
with psycopg.connect(cfg.dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT slot_id FROM daily_slot "
"WHERE instance_key = %s AND portfolio_id = %s LIMIT 1",
(instance, args.portfolio_id),
)
row = cur.fetchone()
with psycopg.connect(cfg.dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT slot_id FROM daily_slot "
"WHERE instance_key = %s AND portfolio_id = %s LIMIT 1",
(instance, args.portfolio_id),
)
row = cur.fetchone()
if row is None:
_fail(f"no slot found for portfolio_id={args.portfolio_id} instance={instance}", args.json)
return 1
@@ -711,14 +743,13 @@ def _cmd_sync(args, cfg, confirm) -> int:
else:
# --all: target all warned slots (B case) plus all inactive slots with broker holding
warned = warn_store.list_warned()
with psycopg.connect(cfg.dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT slot_id FROM daily_slot "
"WHERE instance_key = %s AND status IN ('inactive', 'holding')",
(instance,),
)
all_ids = {str(r[0]) for r in cur.fetchall()}
with psycopg.connect(cfg.dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT slot_id FROM daily_slot "
"WHERE instance_key = %s AND status IN ('inactive', 'holding')",
(instance,),
)
all_ids = {str(r[0]) for r in cur.fetchall()}
slot_ids = sorted(all_ids | warned)
results = []
@@ -820,7 +851,7 @@ def _cmd_verify(args, cfg) -> int:
items = hub.get_portfolios(daemon_id)
store.swap(items)
except Exception: # noqa: BLE001 — degrade: verify with empty store
pass
_log.info("verify snapshot preload skipped", exc_info=True)
if args.portfolio_id:
target_pids = [args.portfolio_id]
@@ -903,30 +934,29 @@ def _cmd_report(args, cfg) -> int:
holdings: list[Holding] = []
pending: list[PendingOrder] = []
trades: list[TradeRow] = []
with psycopg.connect(cfg.dsn) as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT symbol, portfolio_id, qty, avg_price FROM daily_slot "
"WHERE status = 'holding' ORDER BY symbol"
)
for symbol, pid, qty, avg in cur.fetchall():
holdings.append(Holding(symbol or "", pid or "",
float(qty or 0), float(avg or 0)))
cur.execute(
"SELECT symbol, portfolio_id, status FROM daily_slot "
"WHERE status IN ('pending_buy','pending_sell') ORDER BY symbol"
)
for symbol, pid, status in cur.fetchall():
pending.append(PendingOrder(symbol or "", pid or "", status))
cur.execute(
"SELECT asset_id, side, qty, price, fee, tax FROM ledger_outbox "
"WHERE COALESCE(executed_at, created_at) >= %s "
"AND COALESCE(executed_at, created_at) < %s",
(start_utc, end_utc),
)
for asset_id, side, qty, price, fee, tax in cur.fetchall():
trades.append(TradeRow(asset_id, side, float(qty), float(price),
float(fee or 0), float(tax or 0)))
with psycopg.connect(cfg.dsn) as conn, conn.cursor() as cur:
cur.execute(
"SELECT symbol, portfolio_id, qty, avg_price FROM daily_slot "
"WHERE status = 'holding' ORDER BY symbol"
)
for symbol, pid, qty, avg in cur.fetchall():
holdings.append(Holding(symbol or "", pid or "",
float(qty or 0), float(avg or 0)))
cur.execute(
"SELECT symbol, portfolio_id, status FROM daily_slot "
"WHERE status IN ('pending_buy','pending_sell') ORDER BY symbol"
)
for symbol, pid, status in cur.fetchall():
pending.append(PendingOrder(symbol or "", pid or "", status))
cur.execute(
"SELECT asset_id, side, qty, price, fee, tax FROM ledger_outbox "
"WHERE COALESCE(executed_at, created_at) >= %s "
"AND COALESCE(executed_at, created_at) < %s",
(start_utc, end_utc),
)
for asset_id, side, qty, price, fee, tax in cur.fetchall():
trades.append(TradeRow(asset_id, side, float(qty), float(price),
float(fee or 0), float(tax or 0)))
rep = build_report(holdings, pending, trades)
@@ -989,9 +1019,69 @@ def _cmd_report(args, cfg) -> int:
return 0
def _render_ledger_skip_record(record) -> dict[str, object]:
return ledger_skip_alarm_fields(record)
def _cmd_ledger_skips_list(args, cfg) -> int:
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
instance = _resolve_instance(args)
on_date = _parse_iso_date(args.date) if args.date else None
records = PostgreSQLLedgerSkipStore(cfg.dsn).list_records(
instance_key=instance,
portfolio_id=args.portfolio_id,
on_date=on_date,
status=args.status,
limit=args.limit,
)
payload = {
"instance": instance,
"status": args.status,
"count": len(records),
"records": [_render_ledger_skip_record(record) for record in records],
}
if args.json:
_emit(payload, as_json=True)
else:
print(
f"instance={instance} status={args.status} count={len(records)} "
f"portfolio_id={args.portfolio_id or '*'} date={args.date or '*'}"
)
for record in records:
print(
f" {record.skip_key}\t{record.status}\t{record.skip_reason}\t"
f"slot={record.slot_id}\tportfolio={record.portfolio_id or '-'}\t"
f"execution={record.execution_id or '-'}"
)
return 0
def _cmd_ledger_skips_resolve(args, cfg) -> int:
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
store = PostgreSQLLedgerSkipStore(cfg.dsn)
record = store.resolve_by_key(args.skip_key, resolution_reason=args.reason)
if record is None:
_fail(f"unresolved ledger skip not found: {args.skip_key}", args.json)
return 1
emitter, transport = _default_alarm_emitter(cfg)
try:
publish_ledger_skip_alarm(emitter, record, resolved=True)
finally:
transport.close()
payload = {"result": "resolved", "record": _render_ledger_skip_record(record)}
if args.json:
_emit(payload, as_json=True)
else:
print(f"resolved {record.skip_key} reason={record.resolution_reason}")
return 0
def _cmd_ledger_backfill(args, cfg, confirm) -> int:
from trex_core.adapters.broker.kis import KISEvidenceLookupError
from trex_core.adapters.ledger.portfolio_hub import PortfolioHubLedger
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
from trex_core.adapters.persistence.postgresql import PostgreSQLLedgerOutbox
from trex_core.runtime.broker_provider import BrokerCredentialUnavailable
@@ -1020,6 +1110,7 @@ def _cmd_ledger_backfill(args, cfg, confirm) -> int:
"trade": None,
"outbox_precheck": None,
"result": None,
"resolved_skips": [],
"audit_path": None,
}
@@ -1084,7 +1175,7 @@ def _cmd_ledger_backfill(args, cfg, confirm) -> int:
"execution_id": trade.execution_id,
"existing": existing,
}
except (BrokerCredentialUnavailable, KISEvidenceLookupError, RuntimeError) as exc:
except (BrokerCredentialUnavailable, KISEvidenceLookupError, RuntimeError, TypeError) as exc:
report["result"] = {"status": "rejected", "reason": str(exc)}
report["audit_path"] = _write_ledger_backfill_audit(args, report)
if args.json:
@@ -1172,6 +1263,23 @@ def _cmd_ledger_backfill(args, cfg, confirm) -> int:
"last_outcome": item.last_outcome.value if item and item.last_outcome else None,
"last_error": item.last_error if item is not None else None,
}
if exit_code == 0:
resolved = PostgreSQLLedgerSkipStore(cfg.dsn).resolve_for_fill(
instance_key=instance,
slot_id=uuid.UUID(str(slot_binding["slot_id"])),
side=trade.side,
order_id=trade.order_id,
execution_id=trade.execution_id,
resolution_reason="manual_ledger_backfill",
)
report["resolved_skips"] = [_render_ledger_skip_record(record) for record in resolved]
if resolved:
emitter, transport = _default_alarm_emitter(cfg)
try:
for record in resolved:
publish_ledger_skip_alarm(emitter, record, resolved=True)
finally:
transport.close()
report["audit_path"] = _write_ledger_backfill_audit(args, report)
if args.json:
_emit(report, as_json=True)
@@ -1219,6 +1327,18 @@ def main(argv: list[str] | None = None, *, rpc=None, confirm=None) -> int:
if args.cmd == "ledger" and getattr(args, "ledger_cmd", None) == "backfill":
return _cmd_ledger_backfill(args, cfg, confirm)
if (
args.cmd == "ledger"
and getattr(args, "ledger_cmd", None) == "skips"
and getattr(args, "ledger_skips_cmd", None) == "list"
):
return _cmd_ledger_skips_list(args, cfg)
if (
args.cmd == "ledger"
and getattr(args, "ledger_cmd", None) == "skips"
and getattr(args, "ledger_skips_cmd", None) == "resolve"
):
return _cmd_ledger_skips_resolve(args, cfg)
# direct-DB control commands (pause/resume stay direct-DB, OQ-3).
if args.cmd in ("pause", "resume", "paused"):
trex_core/runtime/daemon.py
+3
View File
@@ -35,6 +35,7 @@ from trex_core.adapters.persistence.migrate import apply_migrations
from trex_core.adapters.persistence.account_snapshot import (
PostgreSQLAccountSnapshotStore,
)
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
from trex_core.adapters.persistence.postgresql import (
PgStrategyStateStore,
PostgreSQLIntentStore,
@@ -188,6 +189,7 @@ class Daemon:
)
self.outbox = PostgreSQLLedgerOutbox(config.dsn)
self.ledger_skip_store = PostgreSQLLedgerSkipStore(config.dsn)
# BAT-008: portfolio-hub read + snapshot store + durable pause + scheduler.
# (Created before the ledger writer so option-D write-back can target it.)
self.snapshot_store = SnapshotStore()
@@ -303,6 +305,7 @@ class Daemon:
stale_pending_threshold_sec=config.pending_stale_threshold_sec,
instance_key=config.instance_key,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
ledger_skip_store=self.ledger_skip_store,
balance_reader=(
self.account_snapshot_service.holdings_for_sell
if self.account_snapshot_service is not None