trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden skipped ledger resolution paths

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-08-24 13:38:04 +00:00
committed by GitHub
co-authored by genichin
parent b88b21b772
commit 48dfc3e76f
8 changed files with 451 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/adapters/persistence/ledger_skip_store.py
+116 -61
View File
@@ -40,63 +40,72 @@ class PostgreSQLLedgerSkipStore:
)
now = datetime.now(tz=UTC)
notify = False
with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
cur.execute(
"SELECT status FROM ledger_skip WHERE skip_key = %s FOR UPDATE",
(skip_key,),
)
row = cur.fetchone()
if row is None:
notify = True
cur.execute(
"INSERT INTO ledger_skip "
"(skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
" execution_id, side, qty, price, executed_at, order_venue, "
" execution_venue, skip_reason, status, created_at, updated_at) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, "
" 'unresolved', %s, %s)",
(
skip_key,
instance_key,
slot_id,
portfolio_id,
symbol,
order_id,
execution_id,
side,
qty,
price,
executed_at,
order_venue or "unknown",
execution_venue or "unknown",
skip_reason,
now,
now,
),
)
else:
notify = row[0] == "resolved"
cur.execute(
"UPDATE ledger_skip SET portfolio_id=%s, symbol=%s, order_id=%s, "
"execution_id=%s, qty=%s, price=%s, executed_at=%s, "
"order_venue=%s, execution_venue=%s, status='unresolved', "
"resolution_reason=NULL, resolved_at=NULL, updated_at=%s "
"WHERE skip_key=%s",
(
portfolio_id,
symbol,
order_id,
execution_id,
qty,
price,
executed_at,
order_venue or "unknown",
execution_venue or "unknown",
now,
skip_key,
),
)
conn.commit()
for _ in range(2):
with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
try:
cur.execute(
"INSERT INTO ledger_skip "
"(skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
" execution_id, side, qty, price, executed_at, order_venue, "
" execution_venue, skip_reason, status, created_at, updated_at) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, "
" 'unresolved', %s, %s)",
(
skip_key,
instance_key,
slot_id,
portfolio_id,
symbol,
order_id,
execution_id,
side,
qty,
price,
executed_at,
order_venue or "unknown",
execution_venue or "unknown",
skip_reason,
now,
now,
),
)
conn.commit()
notify = True
break
except self._psycopg.errors.UniqueViolation:
conn.rollback()
cur.execute(
"SELECT status FROM ledger_skip WHERE skip_key = %s FOR UPDATE",
(skip_key,),
)
row = cur.fetchone()
if row is None:
continue
notify = row[0] == "resolved"
cur.execute(
"UPDATE ledger_skip SET portfolio_id=%s, symbol=%s, order_id=%s, "
"execution_id=%s, qty=%s, price=%s, executed_at=%s, "
"order_venue=%s, execution_venue=%s, status='unresolved', "
"resolution_reason=NULL, resolved_at=NULL, updated_at=%s "
"WHERE skip_key=%s",
(
portfolio_id,
symbol,
order_id,
execution_id,
qty,
price,
executed_at,
order_venue or "unknown",
execution_venue or "unknown",
now,
skip_key,
),
)
conn.commit()
break
else:
raise RuntimeError(f"failed to persist ledger_skip row for key {skip_key!r}")
record = self.get(skip_key)
if record is None:
raise RuntimeError(f"ledger_skip row missing after upsert for key {skip_key!r}")
@@ -166,21 +175,24 @@ class PostgreSQLLedgerSkipStore:
if execution_id:
cur.execute(
"UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
"execution_id=COALESCE(execution_id, %s), "
"resolved_at=%s, updated_at=%s "
"WHERE status='unresolved' AND instance_key=%s AND slot_id=%s "
"AND side=%s AND execution_id=%s "
"AND side=%s AND (execution_id=%s OR (execution_id IS NULL AND order_id=%s)) "
"RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, "
"order_id, execution_id, side, qty, price, executed_at, order_venue, "
"execution_venue, skip_reason, status, resolution_reason, resolved_at, "
"created_at, updated_at",
(
resolution_reason,
execution_id,
now,
now,
instance_key,
slot_id,
side,
execution_id,
order_id,
),
)
else:
@@ -207,17 +219,60 @@ class PostgreSQLLedgerSkipStore:
conn.commit()
return [self._row_to_record(row) for row in rows]
def resolve_by_key(self, skip_key: str, *, resolution_reason: str) -> LedgerSkipRecord | None:
def resolve_for_delivery(
self,
*,
instance_key: str,
portfolio_id: str,
side: str,
order_id: str,
execution_id: str,
resolution_reason: str,
) -> list[LedgerSkipRecord]:
now = datetime.now(tz=UTC)
with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
cur.execute(
"UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
"execution_id=COALESCE(execution_id, %s), "
"resolved_at=%s, updated_at=%s "
"WHERE status='unresolved' AND instance_key=%s AND portfolio_id=%s AND side=%s "
"AND (execution_id=%s OR (execution_id IS NULL AND order_id=%s)) "
"RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
"execution_id, side, qty, price, executed_at, order_venue, execution_venue, "
"skip_reason, status, resolution_reason, resolved_at, created_at, updated_at",
(
resolution_reason,
execution_id,
now,
now,
instance_key,
portfolio_id,
side,
execution_id,
order_id,
),
)
rows = cur.fetchall()
conn.commit()
return [self._row_to_record(row) for row in rows]
def resolve_by_key(
self,
skip_key: str,
*,
instance_key: str,
resolution_reason: str,
) -> LedgerSkipRecord | None:
now = datetime.now(tz=UTC)
with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
cur.execute(
"UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
"resolved_at=%s, updated_at=%s "
"WHERE skip_key=%s AND status='unresolved' "
"WHERE skip_key=%s AND instance_key=%s AND status='unresolved' "
"RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
"execution_id, side, qty, price, executed_at, order_venue, execution_venue, "
"skip_reason, status, resolution_reason, resolved_at, created_at, updated_at",
(resolution_reason, now, now, skip_key),
(resolution_reason, now, now, skip_key, instance_key),
)
row = cur.fetchone()
conn.commit()
trex_core/application/execution.py
-21
View File
@@ -831,7 +831,6 @@ class ExecutionService:
)
self._ledger_outbox.enqueue(trade)
r.ledger_enqueued += 1
self._resolve_ledger_skip(slot, side, broker_order_id, str(fs.execution_id))
def _persist_ledger_skip(
self,
@@ -868,26 +867,6 @@ class ExecutionService:
if should_notify:
publish_ledger_skip_alarm(self._alarm_emitter, record, resolved=False)
def _resolve_ledger_skip(
self,
slot,
side: str,
broker_order_id: str,
execution_id: str | None,
) -> None:
if self._ledger_skip_store is None:
return
records = self._ledger_skip_store.resolve_for_fill(
instance_key=self._instance_key,
slot_id=slot.slot_id,
side=side,
order_id=str(broker_order_id) if broker_order_id else None,
execution_id=execution_id,
resolution_reason="ledger_enqueued",
)
for record in records:
publish_ledger_skip_alarm(self._alarm_emitter, record, resolved=True)
def cancel_pending_slot(self, slot_id) -> str:
"""Operator backstop: send cancel_order for a pending slot (dcy-022/REQ-071).
trex_core/application/ledger_outbox.py
+29 -1
View File
@@ -25,10 +25,11 @@ from __future__ import annotations
import logging
import random
from collections.abc import Callable
from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Callable, Optional, Protocol
from typing import Optional, Protocol
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.ledger_port import (
@@ -39,6 +40,7 @@ from trex_core.application.ledger_port import (
LedgerTrade,
LedgerWriteResult,
)
from trex_core.application.ledger_skip import publish_ledger_skip_alarm
_log = logging.getLogger(__name__)
@@ -171,6 +173,8 @@ class LedgerWriter:
clock: Optional[Callable[[], datetime]] = None,
rng: Optional[Callable[[], float]] = None,
alarm_emitter: Optional[AlarmEmitter] = None,
ledger_skip_store=None,
instance_key: str | None = None,
) -> None:
self._outbox = outbox
self._ledger = ledger
@@ -185,6 +189,8 @@ class LedgerWriter:
self._rng = rng or random.random # jitter source; inject for determinism
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
self._ledger_skip_store = ledger_skip_store
self._instance_key = instance_key
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
@@ -301,10 +307,32 @@ class LedgerWriter:
# execution_id either way (REQ-019 AC-1, AC-4).
if outcome.result is LedgerWriteResult.DUPLICATE_OK:
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED)
self._resolve_ledger_skip(item.trade)
return OutboxOutcome.DUPLICATE_COMPLETED
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.COMPLETED)
self._resolve_ledger_skip(item.trade)
return OutboxOutcome.COMPLETED
def _resolve_ledger_skip(self, trade: LedgerTrade) -> None:
if (
self._ledger_skip_store is None
or self._instance_key is None
or not trade.asset_id
or not trade.order_id
or not trade.execution_id
):
return
records = self._ledger_skip_store.resolve_for_delivery(
instance_key=self._instance_key,
portfolio_id=trade.asset_id,
side=trade.side,
order_id=trade.order_id,
execution_id=trade.execution_id,
resolution_reason="ledger_recorded",
)
for record in records:
publish_ledger_skip_alarm(self._alarm_emitter, record, resolved=True)
def _writeback(self, trade: LedgerTrade, outcome) -> None:
"""Option D cash/quantity write-back (dcy-018). best-effort; skips when
no SnapshotStore is wired or the response omits authoritative values."""
trex_core/runtime/cli.py
+82 -4
View File
@@ -60,6 +60,7 @@ from trex_core.runtime.config import (
_DEFAULT_TIMEOUT = 10.0
_DEFAULT_LEDGER_BACKFILL_AUDIT_DIR = "artifacts/validation/ledger_backfill"
_DEFAULT_LEDGER_SKIP_RESOLVE_AUDIT_DIR = "artifacts/validation/ledger_skip_resolve"
_log = logging.getLogger(__name__)
@@ -406,6 +407,24 @@ def _write_ledger_backfill_audit(args, report: dict) -> str:
return str(path)
def _ledger_skip_resolve_audit_path(args, skip_key: str) -> Path:
base = Path(getattr(args, "audit_dir", _DEFAULT_LEDGER_SKIP_RESOLVE_AUDIT_DIR))
stamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ")
name = (
f"{stamp}--{_sanitize_path_component(_resolve_instance(args))}"
f"--{_sanitize_path_component(skip_key)}.json"
)
return base.resolve() / name
def _write_ledger_skip_resolve_audit(args, report: dict) -> str:
path = _ledger_skip_resolve_audit_path(args, report["request"]["skip_key"])
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(_json.dumps(report, ensure_ascii=False, indent=2, default=str) + "\n",
encoding="utf-8")
return str(path)
def _lookup_ledger_backfill_evidence(cfg: DaemonConfig, *, trading_date: str, execution_id: str):
from trex_core.runtime.broker_provider import build_broker_provider
@@ -661,6 +680,14 @@ def _build_parser() -> argparse.ArgumentParser:
)
p_skips_resolve.add_argument("--skip-key", required=True)
p_skips_resolve.add_argument("--reason", default="operator_reconciled")
p_skips_resolve.add_argument("--yes", action="store_true", help="skip confirmation")
p_skips_resolve.add_argument("--live", action="store_true",
help="confirm a LIVE ledger skip mutation (with --yes)")
p_skips_resolve.add_argument(
"--audit-dir",
default=_DEFAULT_LEDGER_SKIP_RESOLVE_AUDIT_DIR,
help="directory for persistent JSON audit reports",
)
_add_common(p_skips_resolve)
return parser
@@ -1057,13 +1084,61 @@ def _cmd_ledger_skips_list(args, cfg) -> int:
return 0
def _cmd_ledger_skips_resolve(args, cfg) -> int:
def _cmd_ledger_skips_resolve(args, cfg, confirm) -> int:
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
instance = _resolve_instance(args)
broker_name, is_live = _broker_mode(cfg)
report = {
"request": {
"instance": instance,
"broker": broker_name,
"live_account": is_live,
"skip_key": args.skip_key,
"resolution_reason": args.reason,
},
"result": None,
"record": None,
"audit_path": None,
}
if is_live and not getattr(args, "live", False):
report["result"] = {"status": "rejected", "reason": "live account requires --live (with --yes)"}
report["audit_path"] = _write_ledger_skip_resolve_audit(args, report)
_fail("live account requires --live (with --yes)", args.json)
if args.json:
_emit(report, as_json=True)
else:
_emit_pretty_json(report)
return 2
if not getattr(args, "yes", False):
if args.json:
report["result"] = {"status": "rejected", "reason": "--json ledger skips resolve requires --yes"}
report["audit_path"] = _write_ledger_skip_resolve_audit(args, report)
_fail("--json ledger skips resolve requires --yes", True)
_emit(report, as_json=True)
return 2
prompt = (
f"Resolve skipped ledger record on instance={instance} broker={broker_name}"
f"{' LIVE' if is_live else ''}: {args.skip_key}?"
)
if not confirm(prompt):
report["result"] = {"status": "aborted", "reason": "operator declined confirmation"}
report["audit_path"] = _write_ledger_skip_resolve_audit(args, report)
print("aborted")
_emit_pretty_json(report)
return 1
store = PostgreSQLLedgerSkipStore(cfg.dsn)
record = store.resolve_by_key(args.skip_key, resolution_reason=args.reason)
record = store.resolve_by_key(
args.skip_key,
instance_key=instance,
resolution_reason=args.reason,
)
if record is None:
report["result"] = {"status": "not_found", "reason": f"unresolved ledger skip not found: {args.skip_key}"}
report["audit_path"] = _write_ledger_skip_resolve_audit(args, report)
_fail(f"unresolved ledger skip not found: {args.skip_key}", args.json)
if args.json:
_emit(report, as_json=True)
return 1
emitter, transport = _default_alarm_emitter(cfg)
try:
@@ -1071,8 +1146,11 @@ def _cmd_ledger_skips_resolve(args, cfg) -> int:
finally:
transport.close()
payload = {"result": "resolved", "record": _render_ledger_skip_record(record)}
report["result"] = {"status": "resolved"}
report["record"] = payload["record"]
report["audit_path"] = _write_ledger_skip_resolve_audit(args, report)
if args.json:
_emit(payload, as_json=True)
_emit({**payload, "audit_path": report["audit_path"]}, as_json=True)
else:
print(f"resolved {record.skip_key} reason={record.resolution_reason}")
return 0
@@ -1338,7 +1416,7 @@ def main(argv: list[str] | None = None, *, rpc=None, confirm=None) -> int:
and getattr(args, "ledger_cmd", None) == "skips"
and getattr(args, "ledger_skips_cmd", None) == "resolve"
):
return _cmd_ledger_skips_resolve(args, cfg)
return _cmd_ledger_skips_resolve(args, cfg, confirm)
# direct-DB control commands (pause/resume stay direct-DB, OQ-3).
if args.cmd in ("pause", "resume", "paused"):
trex_core/runtime/daemon.py
+2
View File
@@ -250,6 +250,8 @@ class Daemon:
retry_cap_sec=config.ledger_retry_cap_sec,
max_attempts=config.ledger_max_attempts,
alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100
ledger_skip_store=self.ledger_skip_store,
instance_key=config.instance_key,
)
self.kill_switch = KillSwitchService(
slot_repo=self.slot_repo,