feat: add bounded KIS account read recovery
This commit is contained in:
jordan
@@ -0,0 +1,242 @@
|
||||
"""Bounded account snapshot recovery for transient KIS read failures.
|
||||
|
||||
Holdings and cash capacity have independent capture times so a cash-read
|
||||
failure never invalidates a healthy holdings refresh (and vice versa). Fresh
|
||||
reconciliation remains live-only; snapshots are consumed solely by the
|
||||
new-buy admission guard and the pending-sell quantity path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable, Optional, Protocol
|
||||
|
||||
from trex_core.application.broker_port import BalanceItem, BrokerBalanceError
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AccountSnapshot:
|
||||
account_hash: str
|
||||
holdings: tuple[BalanceItem, ...]
|
||||
holdings_captured_at: Optional[datetime]
|
||||
available_cash: Optional[int]
|
||||
cash_captured_at: Optional[datetime]
|
||||
|
||||
|
||||
class AccountSnapshotStore(Protocol):
|
||||
def save(self, snapshot: AccountSnapshot) -> None: ...
|
||||
def load(self, account_hash: str) -> Optional[AccountSnapshot]: ...
|
||||
def refresh_guard(self, account_hash: str) -> AbstractContextManager[bool]: ...
|
||||
|
||||
|
||||
class AccountSnapshotBroker(Protocol):
|
||||
def account_scope_hash(self) -> str: ...
|
||||
def get_balance(self) -> list[BalanceItem]: ...
|
||||
def get_available_cash(self, *, price_check_code: str = "005930") -> int: ...
|
||||
|
||||
|
||||
class AccountSnapshotUnavailable(RuntimeError):
|
||||
"""A sanitized fail-closed result for snapshot consumers."""
|
||||
|
||||
_CODES = frozenset({
|
||||
"not_ready",
|
||||
"account_mismatch",
|
||||
"expired",
|
||||
"live_read_failed",
|
||||
})
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
if code not in self._CODES:
|
||||
raise ValueError(f"unsupported account snapshot error code: {code}")
|
||||
self.code = code
|
||||
super().__init__(f"account snapshot unavailable ({code})")
|
||||
|
||||
|
||||
class InMemoryAccountSnapshotStore:
|
||||
def __init__(self) -> None:
|
||||
self._items: dict[str, AccountSnapshot] = {}
|
||||
self._items_lock = threading.Lock()
|
||||
self._refresh_lock = threading.Lock()
|
||||
|
||||
def save(self, snapshot: AccountSnapshot) -> None:
|
||||
with self._items_lock:
|
||||
self._items[snapshot.account_hash] = snapshot
|
||||
|
||||
def load(self, account_hash: str) -> Optional[AccountSnapshot]:
|
||||
with self._items_lock:
|
||||
return self._items.get(account_hash)
|
||||
|
||||
@contextmanager
|
||||
def refresh_guard(self, account_hash: str):
|
||||
acquired = self._refresh_lock.acquire(blocking=False)
|
||||
try:
|
||||
yield acquired
|
||||
finally:
|
||||
if acquired:
|
||||
self._refresh_lock.release()
|
||||
|
||||
|
||||
class AccountSnapshotService:
|
||||
"""Refresh, age-check, and consume account-scoped broker snapshots."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
broker: AccountSnapshotBroker,
|
||||
store: AccountSnapshotStore,
|
||||
buy_max_age_sec: int = 600,
|
||||
sell_max_age_sec: int = 3600,
|
||||
refresh_interval_sec: float = 300.0,
|
||||
clock: Optional[Callable[[], datetime]] = None,
|
||||
rng: Optional[Callable[[], float]] = None,
|
||||
) -> None:
|
||||
if buy_max_age_sec < 0 or sell_max_age_sec < buy_max_age_sec:
|
||||
raise ValueError("invalid account snapshot age policy")
|
||||
if sell_max_age_sec > 21_600:
|
||||
raise ValueError("account snapshot age exceeds six-hour ceiling")
|
||||
if refresh_interval_sec <= 0:
|
||||
raise ValueError("account snapshot refresh interval must be positive")
|
||||
self._broker = broker
|
||||
self._store = store
|
||||
self._buy_max_age = buy_max_age_sec
|
||||
self._sell_max_age = sell_max_age_sec
|
||||
self._refresh_interval = refresh_interval_sec
|
||||
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
|
||||
self._rng = rng or random.random
|
||||
self._refresh_lock = threading.Lock()
|
||||
self._last_holdings_error: Optional[str] = "not_attempted"
|
||||
self._last_cash_error: Optional[str] = "not_attempted"
|
||||
|
||||
def refresh(self) -> bool:
|
||||
"""Refresh holdings and cash independently under one account-scoped owner."""
|
||||
account_hash = self._broker.account_scope_hash()
|
||||
if not account_hash:
|
||||
self._last_holdings_error = "ownership_mismatch"
|
||||
self._last_cash_error = "ownership_mismatch"
|
||||
return False
|
||||
with self._refresh_lock, self._store.refresh_guard(account_hash) as acquired:
|
||||
if not acquired:
|
||||
return False
|
||||
existing = self._store.load(account_hash)
|
||||
holdings = existing.holdings if existing is not None else ()
|
||||
holdings_captured_at = (
|
||||
existing.holdings_captured_at if existing is not None else None
|
||||
)
|
||||
available_cash = existing.available_cash if existing is not None else None
|
||||
cash_captured_at = existing.cash_captured_at if existing is not None else None
|
||||
now = self._clock()
|
||||
holdings_ok = cash_ok = False
|
||||
try:
|
||||
holdings = tuple(self._broker.get_balance())
|
||||
holdings_captured_at = now
|
||||
self._last_holdings_error = None
|
||||
holdings_ok = True
|
||||
except BrokerBalanceError as exc:
|
||||
self._last_holdings_error = exc.code
|
||||
_log.warning(
|
||||
"account holdings snapshot refresh failed: code=%s",
|
||||
exc.code,
|
||||
)
|
||||
try:
|
||||
available_cash = int(self._broker.get_available_cash())
|
||||
if available_cash < 0:
|
||||
raise BrokerBalanceError("malformed_response")
|
||||
cash_captured_at = now
|
||||
self._last_cash_error = None
|
||||
cash_ok = True
|
||||
except BrokerBalanceError as exc:
|
||||
self._last_cash_error = exc.code
|
||||
_log.warning(
|
||||
"account cash snapshot refresh failed: code=%s",
|
||||
exc.code,
|
||||
)
|
||||
if not account_hash or self._broker.account_scope_hash() != account_hash:
|
||||
self._last_holdings_error = "ownership_mismatch"
|
||||
self._last_cash_error = "ownership_mismatch"
|
||||
return False
|
||||
if not holdings_ok and not cash_ok:
|
||||
return False
|
||||
snapshot = AccountSnapshot(
|
||||
account_hash=account_hash,
|
||||
holdings=holdings,
|
||||
holdings_captured_at=holdings_captured_at,
|
||||
available_cash=available_cash,
|
||||
cash_captured_at=cash_captured_at,
|
||||
)
|
||||
self._store.save(snapshot)
|
||||
return holdings_ok and cash_ok
|
||||
|
||||
def run_forever(self, stop: threading.Event) -> None:
|
||||
"""Refresh continuously with bounded exponential backoff."""
|
||||
failures = 0
|
||||
while not stop.is_set():
|
||||
ok = self.refresh()
|
||||
failures = 0 if ok else min(failures + 1, 4)
|
||||
if ok:
|
||||
wait_sec = self._refresh_interval * (0.9 + 0.2 * self._rng())
|
||||
else:
|
||||
wait_sec = min(15.0 * (2 ** (failures - 1)), 300.0)
|
||||
if stop.wait(wait_sec):
|
||||
return
|
||||
|
||||
def _load_current(self) -> AccountSnapshot:
|
||||
account_hash = self._broker.account_scope_hash()
|
||||
if not account_hash:
|
||||
raise AccountSnapshotUnavailable("account_mismatch")
|
||||
snapshot = self._store.load(account_hash)
|
||||
if snapshot is None or snapshot.account_hash != account_hash:
|
||||
raise AccountSnapshotUnavailable("account_mismatch")
|
||||
return snapshot
|
||||
|
||||
def _age(self, captured_at: Optional[datetime], max_age_sec: int) -> float:
|
||||
if captured_at is None:
|
||||
raise AccountSnapshotUnavailable("not_ready")
|
||||
age = (self._clock() - captured_at).total_seconds()
|
||||
if age < 0 or age > max_age_sec:
|
||||
raise AccountSnapshotUnavailable("expired")
|
||||
return age
|
||||
|
||||
def holdings_for_sell(self) -> list[BalanceItem]:
|
||||
"""Prefer a live balance; fallback only on transient transport failure."""
|
||||
try:
|
||||
return self._broker.get_balance()
|
||||
except BrokerBalanceError as exc:
|
||||
if exc.code != "transport_failed":
|
||||
raise AccountSnapshotUnavailable("live_read_failed") from None
|
||||
snapshot = self._load_current()
|
||||
age = self._age(snapshot.holdings_captured_at, self._sell_max_age)
|
||||
_log.warning(
|
||||
"sell balance using account snapshot: age_sec=%.0f",
|
||||
age,
|
||||
)
|
||||
return list(snapshot.holdings)
|
||||
|
||||
def buy_guard(
|
||||
self,
|
||||
*,
|
||||
qty: Optional[float],
|
||||
limit_price: Optional[float],
|
||||
reserved_cash: float = 0.0,
|
||||
) -> Optional[str]:
|
||||
"""Return a stable rejection reason, or None when bounded admission passes."""
|
||||
if self._last_cash_error not in {None, "transport_failed"}:
|
||||
return "account_snapshot_unavailable"
|
||||
try:
|
||||
snapshot = self._load_current()
|
||||
self._age(snapshot.cash_captured_at, self._buy_max_age)
|
||||
except AccountSnapshotUnavailable:
|
||||
return "account_snapshot_unavailable"
|
||||
if qty is None or qty <= 0 or limit_price is None or limit_price <= 0:
|
||||
return "account_snapshot_price_required"
|
||||
requested = float(qty) * float(limit_price)
|
||||
if snapshot.available_cash is None:
|
||||
return "account_snapshot_unavailable"
|
||||
if requested + max(0.0, reserved_cash) > snapshot.available_cash:
|
||||
return "insufficient_snapshot_cash"
|
||||
return None
|
||||
@@ -30,6 +30,7 @@ from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import (
|
||||
BalanceItem,
|
||||
BrokerPort,
|
||||
CancelOutcome,
|
||||
CancelState,
|
||||
@@ -84,6 +85,7 @@ class ExecutionService:
|
||||
stale_pending_threshold_sec: Optional[int] = None,
|
||||
instance_key: str = "default",
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
balance_reader: Optional[Callable[[], list[BalanceItem]]] = None,
|
||||
) -> None:
|
||||
self._svc = slot_service
|
||||
self._repo = repo
|
||||
@@ -104,6 +106,10 @@ class ExecutionService:
|
||||
)
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter (None = no alarms sent).
|
||||
self._alarm_emitter = alarm_emitter
|
||||
# dcy-041 / GitHub #45: pending-sell quantity may use a bounded
|
||||
# account snapshot only through this injected reader. Reconciliation
|
||||
# remains wired directly to the live broker.
|
||||
self._sell_dispatch_balance_reader = balance_reader or broker.get_balance
|
||||
|
||||
# ------------------------------------------------------------
|
||||
def run_once(self) -> ExecutionResult:
|
||||
@@ -136,7 +142,7 @@ class ExecutionService:
|
||||
# surfaces via reconcile_core_holdings (REQ-062 A). No mirror
|
||||
# fallback by design.
|
||||
if sell_balance is None:
|
||||
sell_balance = self._balance_map()
|
||||
sell_balance = self._sell_dispatch_balance_map()
|
||||
bal = sell_balance.get(slot.symbol or "", 0)
|
||||
# dcy-021 / REQ-062 (D4): full sell (slot.qty is None) => broker
|
||||
# balance; partial sell => min(order qty, broker balance). Broker
|
||||
@@ -313,6 +319,13 @@ class ExecutionService:
|
||||
"""Current broker balance as {symbol: hldg_qty} (REQ-062 ground truth)."""
|
||||
return {b.symbol: int(b.hldg_qty) for b in self._broker.get_balance()}
|
||||
|
||||
def _sell_dispatch_balance_map(self) -> dict:
|
||||
"""Balance used only to size a not-yet-submitted pending sell."""
|
||||
return {
|
||||
b.symbol: int(b.hldg_qty)
|
||||
for b in self._sell_dispatch_balance_reader()
|
||||
}
|
||||
|
||||
def _resolve_fill_price(self, fs, slot) -> tuple[float | None, bool]:
|
||||
"""Resolve the fill price from FillStatus, with limit_price backfill.
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ class IntentService:
|
||||
default_order_timeout_sec: Optional[int] = None,
|
||||
warn_store=None, # Optional[ReconcileWarningStore] — BAT-050 / REQ-104
|
||||
entry_allowed_fn: Optional[Callable[[], bool]] = None,
|
||||
buy_guard_fn: Optional[Callable[[BuyIntent], Optional[str]]] = None,
|
||||
) -> None:
|
||||
self._svc = slot_service
|
||||
self._repo = slot_repo
|
||||
@@ -138,6 +139,10 @@ class IntentService:
|
||||
# When returns False, BuyIntent is rejected with reason 'day_start_load_failed'.
|
||||
# SellIntent is NOT gated by this (REQ-114 AC-5: 청산 경로 영향 없음).
|
||||
self._entry_allowed_fn: Optional[Callable[[], bool]] = entry_allowed_fn
|
||||
# dcy-041 / GitHub #45: opt-in bounded cash-capacity admission guard.
|
||||
# The callable returns a stable rejection reason or None. SellIntent is
|
||||
# deliberately outside this gate.
|
||||
self._buy_guard_fn = buy_guard_fn
|
||||
|
||||
# ------------------------------------------------------------
|
||||
def lookup(self, intent_id: str) -> Optional[IntentDecision]:
|
||||
@@ -270,6 +275,15 @@ class IntentService:
|
||||
symbol = _resolve(slot)
|
||||
if not symbol:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, "unknown_portfolio")
|
||||
if self._buy_guard_fn is not None:
|
||||
try:
|
||||
guard_reason = self._buy_guard_fn(intent)
|
||||
except Exception: # noqa: BLE001 - buy guard fails closed
|
||||
guard_reason = "account_snapshot_unavailable"
|
||||
if guard_reason is not None:
|
||||
return IntentDecision(
|
||||
intent.intent_id, "rejected", None, guard_reason
|
||||
)
|
||||
# M cap applies only to NEW activations (initial buy). An increase on
|
||||
# an already-active asset does not change the active asset count.
|
||||
if (not is_increase and self._max_active_slots is not None
|
||||
|
||||
@@ -129,7 +129,7 @@ def render_text(rep: OperatorReport) -> str:
|
||||
# REQ-110 WAC 실현손익 섹션 (portfolio-hub 통합)
|
||||
if rep.wac_unavailable:
|
||||
lines.append(
|
||||
f"=== WAC 실현 PnL: portfolio-hub unavailable — realized PnL unavailable ==="
|
||||
"=== WAC 실현 PnL: portfolio-hub unavailable — realized PnL unavailable ==="
|
||||
)
|
||||
if rep.wac_unavailable_reason:
|
||||
lines.append(f" ({rep.wac_unavailable_reason})")
|
||||
|
||||
Reference in New Issue
Block a user