feat: persist skipped ledger fills

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

trex_core/adapters/broker/kis.py

@@ -14,11 +14,11 @@ from __future__ import annotations
 import base64
 import hashlib
 import json
+from collections.abc import Callable
 from dataclasses import replace
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime, timedelta, timezone
 from decimal import Decimal, InvalidOperation
 from time import monotonic
-from typing import Callable, Optional
 
 import httpx
 
@@ -47,7 +47,7 @@ class KISEvidenceLookupError(RuntimeError):
     """Read-only daily-fill evidence lookup failed."""
 
 
-def _balance_http_error_code(status: Optional[int]) -> str:
+def _balance_http_error_code(status: int | None) -> str:
     if status == 401:
         return "authentication_failed"
     if status == 403:
@@ -69,7 +69,7 @@ def resolve_order_venue(venue: str, at: datetime) -> str:
     return normalized
 
 
-def _safe_int(v: object) -> Optional[int]:
+def _safe_int(v: object) -> int | None:
     try:
         return int(str(v).strip())
     except (TypeError, ValueError):
@@ -92,7 +92,7 @@ def _encode_submission_unknown(req: OrderRequest, submitted_at: datetime) -> str
     return f"{_SUBMISSION_UNKNOWN_PREFIX}{encoded}"
 
 
-def _decode_submission_unknown(broker_order_id: str) -> Optional[dict]:
+def _decode_submission_unknown(broker_order_id: str) -> dict | None:
     if not broker_order_id.startswith(_SUBMISSION_UNKNOWN_PREFIX):
         return None
     encoded = broker_order_id[len(_SUBMISSION_UNKNOWN_PREFIX) :]
@@ -138,15 +138,15 @@ class KISBroker:
         timeout: float = 5.0,
         allow_live: bool = False,
         token_cache=None,
-        clock: Optional[Callable[[], datetime]] = None,
+        clock: Callable[[], datetime] | None = None,
     ) -> None:
         self._account_ref = account_ref
         self._app_key = app_key
         self._app_secret = app_secret
         self._base_url = base_url.rstrip("/")
         self._timeout = timeout
-        self._token: Optional[str] = None
-        self._token_exp: datetime = datetime.now(tz=timezone.utc)
+        self._token: str | None = None
+        self._token_exp: datetime = datetime.now(tz=UTC)
         self._token_cache = token_cache  # optional TokenCache (L2, REQ-016)
         self._clock = clock or (lambda: datetime.now(tz=_KST))
         self._fill_poll_not_before: dict[str, float] = {}
@@ -197,12 +197,12 @@ class KISBroker:
         self._app_secret = credential.app_secret
         # Invalidate L1 in-memory token so next call re-issues.
         self._token = None
-        self._token_exp = datetime.min.replace(tzinfo=timezone.utc)
+        self._token_exp = datetime.min.replace(tzinfo=UTC)
 
     # --- auth ---
     def _ensure_token(self) -> str:
         # L1: in-memory
-        if self._token and datetime.now(tz=timezone.utc) < self._token_exp:
+        if self._token and datetime.now(tz=UTC) < self._token_exp:
             return self._token
         # L2: shared/persistent cache (Redis) — survives restart / multi-process (REQ-016)
         if self._token_cache is not None:
@@ -224,7 +224,7 @@ class KISBroker:
         data = resp.json()
         self._token = data["access_token"]
         ttl = int(data.get("expires_in", 3600))
-        self._token_exp = datetime.now(tz=timezone.utc) + timedelta(seconds=max(60, ttl - 300))
+        self._token_exp = datetime.now(tz=UTC) + timedelta(seconds=max(60, ttl - 300))
         if self._token_cache is not None:
             self._token_cache.store(self._account_ref, self._app_key, self._token, self._token_exp)
         return self._token
@@ -364,8 +364,8 @@ class KISBroker:
         *,
         side: str,
         symbol: str,
-        order_date: Optional[str] = None,
-        submitted_at: Optional[datetime] = None,
+        order_date: str | None = None,
+        submitted_at: datetime | None = None,
     ) -> FillStatus:
         if broker_order_id.startswith(_SUBMISSION_UNKNOWN_PREFIX):
             return self._poll_unknown_submission(broker_order_id, side=side, symbol=symbol)
@@ -424,7 +424,7 @@ class KISBroker:
         if row is None:
             visible_grace = (
                 submitted_at is None
-                or (self._clock().astimezone(timezone.utc) - submitted_at.astimezone(timezone.utc))
+                or (self._clock().astimezone(UTC) - submitted_at.astimezone(UTC))
                 .total_seconds()
                 < _SUBMISSION_LOOKUP_GRACE_SECONDS
             )
@@ -639,11 +639,21 @@ class KISBroker:
     def _executed_at_from_row(row: dict, fallback_date: str) -> datetime:
         order_date = str(row.get("ord_dt") or fallback_date).strip()
         order_time = str(row.get("ord_tmd") or "").strip()[:6]
+        year = int(order_date[0:4])
+        month = int(order_date[4:6])
+        day = int(order_date[6:8])
         if len(order_time) == 6 and order_time.isdigit():
-            executed_at = datetime.strptime(order_date + order_time, "%Y%m%d%H%M%S")
-            return executed_at.replace(tzinfo=_KST).astimezone(timezone.utc)
-        executed_at = datetime.strptime(order_date, "%Y%m%d")
-        return executed_at.replace(tzinfo=_KST).astimezone(timezone.utc)
+            executed_at = datetime(
+                year,
+                month,
+                day,
+                int(order_time[0:2]),
+                int(order_time[2:4]),
+                int(order_time[4:6]),
+                tzinfo=_KST,
+            )
+            return executed_at.astimezone(UTC)
+        return datetime(year, month, day, tzinfo=_KST).astimezone(UTC)
 
     def lookup_fill_evidence(self, *, trading_date: str, execution_id: str) -> dict:
         """Read one exact filled/partial-cancel evidence row for a historical execution."""
@@ -737,7 +747,7 @@ class KISBroker:
             if cancelled:
                 return FillStatus(state="cancelled", raw=row)  # full external cancel
             return FillStatus(state="accepted", raw=row)
-        if ccld_qty >= ord_qty and ord_qty > 0:
+        if ccld_qty >= ord_qty > 0:
             # Deterministic execution_id (dcy-004 OQ-3, orbit kis.py rule):
             #   kis-daily-ccld:{ord_dt}:{ord_gno_brno}:{odno}:{tot_ccld_qty}
             ord_dt = str(row.get("ord_dt") or today)
@@ -760,6 +770,7 @@ class KISBroker:
                 fee=fee,
                 tax=tax,
                 execution_venue=execution_venue,
+                executed_at=KISBroker._executed_at_from_row(row, ord_dt),
                 raw=row,
             )
         # 0 < ccld_qty < ord_qty
@@ -784,6 +795,7 @@ class KISBroker:
                 fee=fee,
                 tax=tax,
                 execution_venue=execution_venue,
+                executed_at=KISBroker._executed_at_from_row(row, ord_dt),
                 cancelled=True,
                 raw=row,
             )
@@ -827,7 +839,7 @@ class KISBroker:
 
         quantities: dict[str, int] = {}
         seen_continuations: set[tuple[str, str]] = set()
-        continuation: Optional[tuple[str, str]] = None
+        continuation: tuple[str, str] | None = None
         while True:
             request_headers = headers
             if continuation is not None:
@@ -944,7 +956,7 @@ class KISBroker:
             raise BrokerBalanceError("malformed_response")
         return amount
 
-    def get_current_price(self, symbol: str, market: str = "J") -> Optional[Decimal]:
+    def get_current_price(self, symbol: str, market: str = "J") -> Decimal | None:
         """Current traded price `stck_prpr` via KIS inquire-price (REQ-038, REQ-118).
 
         tr_id: `FHKST01010100` per REQ-038 AC-7 (live and mock share the same tr_id).

trex_core/adapters/broker/paper.py

@@ -8,9 +8,8 @@ execution code path that production (KIS) takes.
 from __future__ import annotations
 
 import threading
+from datetime import UTC, datetime
 from decimal import Decimal
-from datetime import datetime
-from typing import Optional
 
 from trex_core.adapters.broker.seq_store import SeqStore
 from trex_core.application.broker_port import (
@@ -52,9 +51,9 @@ class PaperBroker:
         self,
         *,
         fill_after_polls: int = 0,
-        qty_per_poll: Optional[int] = None,
-        reject_symbols: Optional[set[str]] = None,
-        seq_store: Optional[SeqStore] = None,
+        qty_per_poll: int | None = None,
+        reject_symbols: set[str] | None = None,
+        seq_store: SeqStore | None = None,
     ) -> None:
         self._fill_after = fill_after_polls
         self._qty_per_poll = qty_per_poll
@@ -90,8 +89,8 @@ class PaperBroker:
         *,
         side: str,
         symbol: str,
-        order_date: Optional[str] = None,
-        submitted_at: Optional[datetime] = None,
+        order_date: str | None = None,
+        submitted_at: datetime | None = None,
     ) -> FillStatus:
         with self._lock:
             o = self._orders.get(broker_order_id)
@@ -122,6 +121,7 @@ class PaperBroker:
                     state="filled", filled_qty=o.filled, avg_price=o.price,
                     execution_id=execution_id, fee=fee, tax=tax,
                     execution_venue=o.execution_venue,
+                    executed_at=datetime.now(tz=UTC),
                 )
             return FillStatus(
                 state="partially_filled",
@@ -172,7 +172,7 @@ class PaperBroker:
         with self._lock:
             self._prices[symbol] = Decimal(str(price))
 
-    def get_current_price(self, symbol: str, market: str = "J") -> Optional[Decimal]:
+    def get_current_price(self, symbol: str, market: str = "J") -> Decimal | None:
         """Return the programmed current price for symbol, or None if unset.
 
         market accepted for BrokerPort protocol conformance (REQ-118) but

trex_core/adapters/persistence/ledger_skip_store.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+from datetime import UTC, date, datetime, time, timedelta
+from typing import Any
+from uuid import UUID
+
+from trex_core.adapters.persistence.postgresql import _require_psycopg
+from trex_core.application.ledger_skip import LedgerSkipRecord, make_ledger_skip_key
+
+
+class PostgreSQLLedgerSkipStore:
+    def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None:
+        self._psycopg = _require_psycopg(psycopg_module)
+        self._dsn = dsn
+
+    def record_skip(
+        self,
+        *,
+        instance_key: str,
+        slot_id: UUID,
+        portfolio_id: str | None,
+        symbol: str | None,
+        order_id: str | None,
+        execution_id: str | None,
+        side: str,
+        qty: float,
+        price: float,
+        executed_at: datetime,
+        order_venue: str | None,
+        execution_venue: str | None,
+        skip_reason: str,
+    ) -> tuple[LedgerSkipRecord, bool]:
+        skip_key = make_ledger_skip_key(
+            instance_key=instance_key,
+            slot_id=slot_id,
+            side=side,
+            order_id=order_id,
+            execution_id=execution_id,
+            skip_reason=skip_reason,
+        )
+        now = datetime.now(tz=UTC)
+        notify = False
+        with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
+            cur.execute(
+                "SELECT status FROM ledger_skip WHERE skip_key = %s FOR UPDATE",
+                (skip_key,),
+            )
+            row = cur.fetchone()
+            if row is None:
+                notify = True
+                cur.execute(
+                    "INSERT INTO ledger_skip "
+                    "(skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
+                    " execution_id, side, qty, price, executed_at, order_venue, "
+                    " execution_venue, skip_reason, status, created_at, updated_at) "
+                    "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, "
+                    "        'unresolved', %s, %s)",
+                    (
+                        skip_key,
+                        instance_key,
+                        slot_id,
+                        portfolio_id,
+                        symbol,
+                        order_id,
+                        execution_id,
+                        side,
+                        qty,
+                        price,
+                        executed_at,
+                        order_venue or "unknown",
+                        execution_venue or "unknown",
+                        skip_reason,
+                        now,
+                        now,
+                    ),
+                )
+            else:
+                notify = row[0] == "resolved"
+                cur.execute(
+                    "UPDATE ledger_skip SET portfolio_id=%s, symbol=%s, order_id=%s, "
+                    "execution_id=%s, qty=%s, price=%s, executed_at=%s, "
+                    "order_venue=%s, execution_venue=%s, status='unresolved', "
+                    "resolution_reason=NULL, resolved_at=NULL, updated_at=%s "
+                    "WHERE skip_key=%s",
+                    (
+                        portfolio_id,
+                        symbol,
+                        order_id,
+                        execution_id,
+                        qty,
+                        price,
+                        executed_at,
+                        order_venue or "unknown",
+                        execution_venue or "unknown",
+                        now,
+                        skip_key,
+                    ),
+                )
+            conn.commit()
+        record = self.get(skip_key)
+        assert record is not None
+        return record, notify
+
+    def get(self, skip_key: str) -> LedgerSkipRecord | None:
+        with self._psycopg.connect(self._dsn) as conn, conn.cursor() as cur:
+            cur.execute(
+                "SELECT skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
+                "execution_id, side, qty, price, executed_at, order_venue, execution_venue, "
+                "skip_reason, status, resolution_reason, resolved_at, created_at, updated_at "
+                "FROM ledger_skip WHERE skip_key = %s",
+                (skip_key,),
+            )
+            row = cur.fetchone()
+        return self._row_to_record(row) if row else None
+
+    def list_records(
+        self,
+        *,
+        instance_key: str | None = None,
+        portfolio_id: str | None = None,
+        on_date: date | None = None,
+        status: str = "unresolved",
+        limit: int = 100,
+    ) -> list[LedgerSkipRecord]:
+        clauses: list[str] = []
+        params: list[object] = []
+        if status != "all":
+            clauses.append("status = %s")
+            params.append(status)
+        if instance_key is not None:
+            clauses.append("instance_key = %s")
+            params.append(instance_key)
+        if portfolio_id is not None:
+            clauses.append("portfolio_id = %s")
+            params.append(portfolio_id)
+        if on_date is not None:
+            start = datetime.combine(on_date, time.min, tzinfo=UTC)
+            end = start + timedelta(days=1)
+            clauses.append("created_at >= %s AND created_at < %s")
+            params.extend([start, end])
+        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+        with self._psycopg.connect(self._dsn) as conn, conn.cursor() as cur:
+            cur.execute(
+                "SELECT skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
+                "execution_id, side, qty, price, executed_at, order_venue, execution_venue, "
+                "skip_reason, status, resolution_reason, resolved_at, created_at, updated_at "
+                f"FROM ledger_skip {where} ORDER BY created_at DESC LIMIT %s",
+                (*params, limit),
+            )
+            rows = cur.fetchall()
+        return [self._row_to_record(row) for row in rows]
+
+    def resolve_for_fill(
+        self,
+        *,
+        instance_key: str,
+        slot_id: UUID,
+        side: str,
+        order_id: str | None,
+        execution_id: str | None,
+        resolution_reason: str,
+    ) -> list[LedgerSkipRecord]:
+        now = datetime.now(tz=UTC)
+        with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
+            if execution_id:
+                cur.execute(
+                    "UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
+                    "resolved_at=%s, updated_at=%s "
+                    "WHERE status='unresolved' AND instance_key=%s AND slot_id=%s "
+                    "AND side=%s AND execution_id=%s "
+                    "RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, "
+                    "order_id, execution_id, side, qty, price, executed_at, order_venue, "
+                    "execution_venue, skip_reason, status, resolution_reason, resolved_at, "
+                    "created_at, updated_at",
+                    (
+                        resolution_reason,
+                        now,
+                        now,
+                        instance_key,
+                        slot_id,
+                        side,
+                        execution_id,
+                    ),
+                )
+            else:
+                cur.execute(
+                    "UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
+                    "resolved_at=%s, updated_at=%s "
+                    "WHERE status='unresolved' AND instance_key=%s AND slot_id=%s "
+                    "AND side=%s AND order_id=%s "
+                    "RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, "
+                    "order_id, execution_id, side, qty, price, executed_at, order_venue, "
+                    "execution_venue, skip_reason, status, resolution_reason, resolved_at, "
+                    "created_at, updated_at",
+                    (
+                        resolution_reason,
+                        now,
+                        now,
+                        instance_key,
+                        slot_id,
+                        side,
+                        order_id,
+                    ),
+                )
+            rows = cur.fetchall()
+            conn.commit()
+        return [self._row_to_record(row) for row in rows]
+
+    def resolve_by_key(self, skip_key: str, *, resolution_reason: str) -> LedgerSkipRecord | None:
+        now = datetime.now(tz=UTC)
+        with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur:
+            cur.execute(
+                "UPDATE ledger_skip SET status='resolved', resolution_reason=%s, "
+                "resolved_at=%s, updated_at=%s "
+                "WHERE skip_key=%s AND status='unresolved' "
+                "RETURNING skip_key, instance_key, slot_id, portfolio_id, symbol, order_id, "
+                "execution_id, side, qty, price, executed_at, order_venue, execution_venue, "
+                "skip_reason, status, resolution_reason, resolved_at, created_at, updated_at",
+                (resolution_reason, now, now, skip_key),
+            )
+            row = cur.fetchone()
+            conn.commit()
+        return self._row_to_record(row) if row else None
+
+    @staticmethod
+    def _row_to_record(row: tuple) -> LedgerSkipRecord:
+        return LedgerSkipRecord(
+            skip_key=row[0],
+            instance_key=row[1],
+            slot_id=row[2],
+            portfolio_id=row[3],
+            symbol=row[4],
+            order_id=row[5],
+            execution_id=row[6],
+            side=row[7],
+            qty=float(row[8]),
+            price=float(row[9]),
+            executed_at=row[10],
+            order_venue=row[11],
+            execution_venue=row[12],
+            skip_reason=row[13],
+            status=row[14],
+            resolution_reason=row[15],
+            resolved_at=row[16],
+            created_at=row[17],
+            updated_at=row[18],
+        )

trex_core/adapters/persistence/sql/025_create_ledger_skip.sql

@@ -0,0 +1,30 @@
+CREATE TABLE IF NOT EXISTS ledger_skip (
+    skip_key TEXT PRIMARY KEY,
+    instance_key TEXT NOT NULL,
+    slot_id UUID NOT NULL,
+    portfolio_id TEXT NULL,
+    symbol TEXT NULL,
+    order_id TEXT NULL,
+    execution_id TEXT NULL,
+    side TEXT NOT NULL CHECK (side IN ('buy', 'sell')),
+    qty DOUBLE PRECISION NOT NULL,
+    price DOUBLE PRECISION NOT NULL,
+    executed_at TIMESTAMPTZ NOT NULL,
+    order_venue TEXT NOT NULL,
+    execution_venue TEXT NOT NULL,
+    skip_reason TEXT NOT NULL,
+    status TEXT NOT NULL CHECK (status IN ('unresolved', 'resolved')),
+    resolution_reason TEXT NULL,
+    resolved_at TIMESTAMPTZ NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_ledger_skip_status_created
+    ON ledger_skip (status, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_ledger_skip_instance_portfolio_created
+    ON ledger_skip (instance_key, portfolio_id, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_ledger_skip_execution_id
+    ON ledger_skip (execution_id);