feat(execution): reconcile timed-out orders safely
This commit is contained in:
@@ -307,6 +307,7 @@ class KISBroker:
|
||||
reason=f"submit_outcome_unknown:{type(exc).__name__}",
|
||||
venue=actual_venue,
|
||||
raw={"error_class": type(exc).__name__},
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
if getattr(resp, "status_code", 200) >= 500:
|
||||
return OrderAck(
|
||||
@@ -315,6 +316,7 @@ class KISBroker:
|
||||
reason=f"submit_outcome_unknown:http_{resp.status_code}",
|
||||
venue=actual_venue,
|
||||
raw={"http_status": resp.status_code},
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
if str(payload.get("rt_cd")) == "0":
|
||||
out = payload.get("output") or {}
|
||||
@@ -326,6 +328,7 @@ class KISBroker:
|
||||
broker_order_id=bid,
|
||||
venue=actual_venue,
|
||||
raw=payload,
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
return OrderAck(
|
||||
accepted=False,
|
||||
@@ -362,7 +365,15 @@ class KISBroker:
|
||||
"EXCG_ID_DVSN_CD": "ALL",
|
||||
}
|
||||
|
||||
def poll_fill(self, broker_order_id: str, *, side: str, symbol: str) -> FillStatus:
|
||||
def poll_fill(
|
||||
self,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
side: str,
|
||||
symbol: str,
|
||||
order_date: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> FillStatus:
|
||||
if broker_order_id.startswith(_SUBMISSION_UNKNOWN_PREFIX):
|
||||
return self._poll_unknown_submission(broker_order_id, side=side, symbol=symbol)
|
||||
now = monotonic()
|
||||
@@ -375,8 +386,8 @@ class KISBroker:
|
||||
# open duplicate KIS inquiries for the same order.
|
||||
self._fill_poll_not_before[broker_order_id] = now + _FILL_POLL_MIN_INTERVAL_SECONDS
|
||||
odno = broker_order_id.split(":")[-1]
|
||||
today = datetime.now(tz=_KST).strftime("%Y%m%d")
|
||||
tr_id, params = self._daily_fill_request(today, order_number=odno)
|
||||
inquiry_date = order_date or self._clock().astimezone(_KST).strftime("%Y%m%d")
|
||||
tr_id, params = self._daily_fill_request(inquiry_date, order_number=odno)
|
||||
try:
|
||||
resp = httpx.get(
|
||||
f"{self._base_url}/uapi/domestic-stock/v1/trading/inquire-daily-ccld",
|
||||
@@ -410,11 +421,36 @@ class KISBroker:
|
||||
None,
|
||||
)
|
||||
if row is None:
|
||||
return FillStatus(
|
||||
state="accepted",
|
||||
raw={"reason": "order_not_visible_in_fill_inquiry"},
|
||||
visible_grace = (
|
||||
submitted_at is None
|
||||
or (self._clock().astimezone(timezone.utc) - submitted_at.astimezone(timezone.utc))
|
||||
.total_seconds()
|
||||
< _SUBMISSION_LOOKUP_GRACE_SECONDS
|
||||
)
|
||||
return self._fill_status_from_row(row, side=side, today=today, fallback_odno=odno)
|
||||
return FillStatus(
|
||||
state="accepted" if visible_grace else "unknown",
|
||||
raw={
|
||||
"reason": (
|
||||
"order_visibility_grace"
|
||||
if visible_grace
|
||||
else "order_not_visible_unresolved"
|
||||
),
|
||||
"order_visible": False,
|
||||
"inquiry_date": inquiry_date,
|
||||
},
|
||||
)
|
||||
status = self._fill_status_from_row(
|
||||
row, side=side, today=inquiry_date, fallback_odno=odno
|
||||
)
|
||||
return replace(
|
||||
status,
|
||||
raw={
|
||||
**(status.raw if isinstance(status.raw, dict) else {}),
|
||||
"order_visible": True,
|
||||
"order_open_confirmed": status.state in {"accepted", "partially_filled"},
|
||||
"inquiry_date": inquiry_date,
|
||||
},
|
||||
)
|
||||
|
||||
def _poll_unknown_submission(
|
||||
self,
|
||||
|
||||
@@ -10,8 +10,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Optional
|
||||
|
||||
from trex_core.application.broker_port import (
|
||||
BalanceItem,
|
||||
@@ -126,8 +128,22 @@ class OwnershipAwareBroker:
|
||||
def submit_order(self, req: OrderRequest) -> OrderAck:
|
||||
return self._delegate.submit_order(req)
|
||||
|
||||
def poll_fill(self, broker_order_id: str, *, side: str, symbol: str) -> FillStatus:
|
||||
result = self._delegate.poll_fill(broker_order_id, side=side, symbol=symbol)
|
||||
def poll_fill(
|
||||
self,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
side: str,
|
||||
symbol: str,
|
||||
order_date: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> FillStatus:
|
||||
result = self._delegate.poll_fill(
|
||||
broker_order_id,
|
||||
side=side,
|
||||
symbol=symbol,
|
||||
order_date=order_date,
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
if result.execution_id and result.filled_qty > 0 and (
|
||||
result.is_filled or result.is_partial_cancel
|
||||
):
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from decimal import Decimal
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from trex_core.adapters.broker.seq_store import SeqStore
|
||||
@@ -80,7 +81,15 @@ class PaperBroker:
|
||||
self._orders[oid] = _Order(oid, req)
|
||||
return OrderAck(accepted=True, broker_order_id=oid, venue=req.venue)
|
||||
|
||||
def poll_fill(self, broker_order_id: str, *, side: str, symbol: str) -> FillStatus:
|
||||
def poll_fill(
|
||||
self,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
side: str,
|
||||
symbol: str,
|
||||
order_date: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> FillStatus:
|
||||
with self._lock:
|
||||
o = self._orders.get(broker_order_id)
|
||||
if o is None:
|
||||
@@ -91,7 +100,12 @@ class PaperBroker:
|
||||
if symbol in self._reject:
|
||||
return FillStatus(state="rejected", raw={"reason": "paper_reject"})
|
||||
if o.polls <= self._fill_after:
|
||||
return FillStatus(state="accepted", filled_qty=o.filled, avg_price=o.price)
|
||||
return FillStatus(
|
||||
state="accepted",
|
||||
filled_qty=o.filled,
|
||||
avg_price=o.price,
|
||||
raw={"order_open_confirmed": True},
|
||||
)
|
||||
step = self._qty_per_poll if self._qty_per_poll is not None else (o.qty - o.filled)
|
||||
o.filled = min(o.qty, o.filled + step)
|
||||
if o.filled >= o.qty:
|
||||
@@ -105,7 +119,12 @@ class PaperBroker:
|
||||
state="filled", filled_qty=o.filled, avg_price=o.price,
|
||||
execution_id=execution_id, fee=fee, tax=tax,
|
||||
)
|
||||
return FillStatus(state="partially_filled", filled_qty=o.filled, avg_price=o.price)
|
||||
return FillStatus(
|
||||
state="partially_filled",
|
||||
filled_qty=o.filled,
|
||||
avg_price=o.price,
|
||||
raw={"order_open_confirmed": True},
|
||||
)
|
||||
|
||||
def cancel_order(
|
||||
self,
|
||||
|
||||
@@ -14,7 +14,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Iterator, Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
@@ -38,9 +38,11 @@ _SLOT_COLUMNS = (
|
||||
"sell_strategy_id, forced_close_at, stop_loss_price, take_profit_price, "
|
||||
"created_at, updated_at, instance_key, retired_at, "
|
||||
"timeout_sec, slippage_pct, occupancy_seq, venue, "
|
||||
"cancel_state, cancel_requested_at"
|
||||
"cancel_state, cancel_requested_at, order_submitted_at, order_date, cancel_attempts"
|
||||
)
|
||||
|
||||
_KST = timezone(timedelta(hours=9))
|
||||
|
||||
|
||||
def _require_psycopg(module: Any = None) -> Any:
|
||||
if module is not None:
|
||||
@@ -218,7 +220,8 @@ class PostgreSQLSlotRepository:
|
||||
"forced_close_at=%s, stop_loss_price=%s, take_profit_price=%s, "
|
||||
"instance_key=%s, retired_at=%s, timeout_sec=%s, slippage_pct=%s, "
|
||||
"occupancy_seq=%s, venue=%s, cancel_state=%s, "
|
||||
"cancel_requested_at=%s, updated_at=%s WHERE slot_id=%s",
|
||||
"cancel_requested_at=%s, order_submitted_at=%s, order_date=%s, "
|
||||
"cancel_attempts=%s, updated_at=%s WHERE slot_id=%s",
|
||||
(
|
||||
slot.status.value,
|
||||
slot.symbol,
|
||||
@@ -241,6 +244,9 @@ class PostgreSQLSlotRepository:
|
||||
slot.venue,
|
||||
slot.cancel_state,
|
||||
slot.cancel_requested_at,
|
||||
slot.order_submitted_at,
|
||||
slot.order_date,
|
||||
slot.cancel_attempts,
|
||||
slot.updated_at or datetime.now(tz=timezone.utc),
|
||||
slot.slot_id,
|
||||
),
|
||||
@@ -295,16 +301,22 @@ class PostgreSQLSlotRepository:
|
||||
broker_order_id: str,
|
||||
*,
|
||||
venue: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> bool:
|
||||
submitted_at = submitted_at or datetime.now(tz=timezone.utc)
|
||||
order_date = submitted_at.astimezone(_KST).strftime("%Y%m%d")
|
||||
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE daily_slot SET broker_order_id=%s, "
|
||||
"venue=COALESCE(%s, venue), updated_at=%s "
|
||||
"venue=COALESCE(%s, venue), order_submitted_at=%s, order_date=%s, "
|
||||
"cancel_attempts=0, updated_at=%s "
|
||||
"WHERE slot_id=%s AND broker_order_id IS NULL",
|
||||
(
|
||||
broker_order_id,
|
||||
venue,
|
||||
submitted_at,
|
||||
order_date,
|
||||
datetime.now(tz=timezone.utc),
|
||||
slot_id,
|
||||
),
|
||||
@@ -366,6 +378,31 @@ class PostgreSQLSlotRepository:
|
||||
conn.commit()
|
||||
return ok
|
||||
|
||||
def reserve_cancel_attempt(
|
||||
self,
|
||||
slot_id: UUID,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
retry_before: Optional[datetime] = None,
|
||||
) -> bool:
|
||||
"""Atomically reserve first cancel or an authoritative backoff-gated retry."""
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE daily_slot SET cancel_state='unknown', cancel_requested_at=%s, "
|
||||
"cancel_attempts=cancel_attempts+1, updated_at=%s "
|
||||
"WHERE slot_id=%s AND broker_order_id=%s "
|
||||
"AND status IN ('pending_buy','pending_sell') "
|
||||
"AND (cancel_state IS NULL OR "
|
||||
"(%s IS NOT NULL AND cancel_state IN ('rejected','unknown') "
|
||||
"AND cancel_attempts < 4 AND cancel_requested_at <= %s))",
|
||||
(now, now, slot_id, broker_order_id, retry_before, retry_before),
|
||||
)
|
||||
ok = cur.rowcount == 1
|
||||
conn.commit()
|
||||
return ok
|
||||
|
||||
def list_dispatched_pending(self, instance_key: str = "default") -> list[Slot]:
|
||||
with self._psycopg.connect(self._dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
@@ -382,7 +419,8 @@ class PostgreSQLSlotRepository:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE daily_slot SET broker_order_id=NULL, cancel_state=NULL, "
|
||||
"cancel_requested_at=NULL, updated_at=%s WHERE slot_id=%s",
|
||||
"cancel_requested_at=NULL, order_submitted_at=NULL, order_date=NULL, "
|
||||
"cancel_attempts=0, updated_at=%s WHERE slot_id=%s",
|
||||
(datetime.now(tz=timezone.utc), slot_id),
|
||||
)
|
||||
conn.commit()
|
||||
@@ -410,7 +448,8 @@ class PostgreSQLSlotRepository:
|
||||
# created_at(14) updated_at(15) instance_key(16) retired_at(17)
|
||||
# timeout_sec(18) slippage_pct(19) [dcy-022/REQ-068]
|
||||
# occupancy_seq(20) [dcy-028/REQ-081], venue(21),
|
||||
# cancel_state(22), cancel_requested_at(23)
|
||||
# cancel_state(22), cancel_requested_at(23), order_submitted_at(24),
|
||||
# order_date(25), cancel_attempts(26)
|
||||
return Slot(
|
||||
slot_id=row[0] if isinstance(row[0], UUID) else UUID(str(row[0])),
|
||||
status=SlotState(row[1]),
|
||||
@@ -436,6 +475,9 @@ class PostgreSQLSlotRepository:
|
||||
venue=str(row[21]) if len(row) > 21 and row[21] is not None else "KRX",
|
||||
cancel_state=str(row[22]) if len(row) > 22 and row[22] is not None else None,
|
||||
cancel_requested_at=row[23] if len(row) > 23 else None,
|
||||
order_submitted_at=row[24] if len(row) > 24 else None,
|
||||
order_date=str(row[25]) if len(row) > 25 and row[25] is not None else None,
|
||||
cancel_attempts=int(row[26]) if len(row) > 26 and row[26] is not None else 0,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
-- GitHub #58: persist the effective timeout's immutable dispatch anchor and
|
||||
-- original KST order date separately from mutable row bookkeeping. Existing
|
||||
-- dispatched rows deliberately remain NULL: updated_at/created_at are mutable
|
||||
-- bookkeeping and cannot prove the original submission timestamp or order day.
|
||||
-- Runtime keeps those legacy rows fail-closed and raises a sanitized alarm.
|
||||
ALTER TABLE daily_slot
|
||||
ADD COLUMN IF NOT EXISTS order_submitted_at TIMESTAMPTZ;
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
ADD COLUMN IF NOT EXISTS order_date TEXT;
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
ADD COLUMN IF NOT EXISTS cancel_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
DROP CONSTRAINT IF EXISTS daily_slot_order_date_check;
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
ADD CONSTRAINT daily_slot_order_date_check
|
||||
CHECK (order_date IS NULL OR order_date ~ '^[0-9]{8}$');
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
DROP CONSTRAINT IF EXISTS daily_slot_cancel_attempts_check;
|
||||
|
||||
ALTER TABLE daily_slot
|
||||
ADD CONSTRAINT daily_slot_cancel_attempts_check
|
||||
CHECK (cancel_attempts >= 0);
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from typing import Any, Optional, Protocol
|
||||
@@ -75,6 +76,7 @@ class OrderAck:
|
||||
# never has to infer the original venue.
|
||||
venue: Optional[str] = None
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
submitted_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -135,7 +137,15 @@ class BrokerPort(Protocol):
|
||||
"""Submit an order. Returns accepted + broker_order_id (no synchronous fill)."""
|
||||
...
|
||||
|
||||
def poll_fill(self, broker_order_id: str, *, side: str, symbol: str) -> FillStatus:
|
||||
def poll_fill(
|
||||
self,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
side: str,
|
||||
symbol: str,
|
||||
order_date: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> FillStatus:
|
||||
"""Observe current fill status for a previously accepted order."""
|
||||
...
|
||||
|
||||
|
||||
@@ -23,9 +23,10 @@ single producer for ledger work items (NFR-3, single record subject).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
@@ -86,6 +87,7 @@ class ExecutionService:
|
||||
instance_key: str = "default",
|
||||
alarm_emitter: Optional[AlarmEmitter] = None,
|
||||
balance_reader: Optional[Callable[[], list[BalanceItem]]] = None,
|
||||
clock: Optional[Callable[[], datetime]] = None,
|
||||
) -> None:
|
||||
self._svc = slot_service
|
||||
self._repo = repo
|
||||
@@ -106,6 +108,11 @@ class ExecutionService:
|
||||
)
|
||||
# dcy-033 / REQ-100: best-effort alarm emitter (None = no alarms sent).
|
||||
self._alarm_emitter = alarm_emitter
|
||||
# One emission per durable unresolved state/attempt in this process.
|
||||
# A restart intentionally emits once again, covering a crash between a
|
||||
# durable cancel reservation and best-effort alarm publication.
|
||||
self._order_alarm_keys: set[tuple[str, str, int]] = set()
|
||||
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
|
||||
# dcy-041 / GitHub #45: pending-sell quantity may use a bounded
|
||||
# account snapshot only through this injected reader. Reconciliation
|
||||
# remains wired directly to the live broker.
|
||||
@@ -216,6 +223,7 @@ class ExecutionService:
|
||||
slot.slot_id,
|
||||
ack.broker_order_id,
|
||||
venue=ack.venue,
|
||||
submitted_at=ack.submitted_at,
|
||||
)
|
||||
r.dispatched += 1
|
||||
if ack.reason and ack.reason.startswith("submit_outcome_unknown:"):
|
||||
@@ -258,11 +266,11 @@ class ExecutionService:
|
||||
r.rolled_back += 1
|
||||
|
||||
def _poll(self, r: ExecutionResult) -> None:
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
now = self._clock().astimezone(timezone.utc)
|
||||
for slot in self._repo.list_dispatched_pending(self._instance_key):
|
||||
side = "buy" if slot.status is SlotState.PENDING_BUY else "sell"
|
||||
set_correlation_id(str(slot.slot_id)) # REQ-084: same key as submit
|
||||
fs = self._broker.poll_fill(slot.broker_order_id, side=side, symbol=slot.symbol or "")
|
||||
fs = self._poll_fill(slot, side)
|
||||
resolved_order_id = (
|
||||
fs.raw.get("resolved_broker_order_id") if isinstance(fs.raw, dict) else None
|
||||
)
|
||||
@@ -312,7 +320,18 @@ class ExecutionService:
|
||||
# tick, but surface if it has been stuck too long (FR-3), and
|
||||
# send a cancel if the slot-level timeout is exceeded (REQ-069).
|
||||
self._check_stale_pending(slot, now, r)
|
||||
self._try_timeout_cancel(slot, now, side, r)
|
||||
self._try_timeout_cancel(slot, now, side, fs, r)
|
||||
|
||||
def _poll_fill(self, slot, side: str):
|
||||
"""Use immutable order metadata when the adapter supports the new seam."""
|
||||
poll = self._broker.poll_fill
|
||||
parameters = inspect.signature(poll).parameters
|
||||
kwargs = {"side": side, "symbol": slot.symbol or ""}
|
||||
if "order_date" in parameters:
|
||||
kwargs["order_date"] = slot.order_date
|
||||
if "submitted_at" in parameters:
|
||||
kwargs["submitted_at"] = slot.order_submitted_at
|
||||
return poll(slot.broker_order_id, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------
|
||||
def _balance_map(self) -> dict:
|
||||
@@ -553,39 +572,40 @@ class ExecutionService:
|
||||
self._stale_threshold_sec,
|
||||
)
|
||||
|
||||
def _try_timeout_cancel(self, slot, now: datetime, side: str, r: ExecutionResult) -> None:
|
||||
"""Send cancel_order when the slot-level timeout is exceeded (dcy-022/REQ-069).
|
||||
|
||||
Best-effort: a failure only logs a warning — the slot is NOT changed here.
|
||||
Actual rollback happens on the next poll tick when the broker confirms the
|
||||
cancel (is_terminal_reject) or partial fill then cancel (is_partial_cancel).
|
||||
This design is safe across the cancel-fill race: if the cancel races with a
|
||||
fill, the next poll sees is_filled and follows the normal fill path (FR-4).
|
||||
"""
|
||||
def _try_timeout_cancel(self, slot, now: datetime, side: str, fs, r: ExecutionResult) -> None:
|
||||
"""Begin or safely retry cancellation after the immutable deadline."""
|
||||
if str(slot.broker_order_id or "").startswith("submission-unknown:"):
|
||||
return
|
||||
if slot.cancel_state is not None:
|
||||
if slot.timeout_sec is None:
|
||||
return
|
||||
if slot.timeout_sec is None or slot.updated_at is None:
|
||||
if slot.order_submitted_at is None or slot.order_date is None:
|
||||
self._emit_order_metadata_unknown_alarm(slot)
|
||||
return
|
||||
age = (now - slot.updated_at).total_seconds()
|
||||
age = (now - slot.order_submitted_at).total_seconds()
|
||||
if age < slot.timeout_sec:
|
||||
return
|
||||
|
||||
is_retry = slot.cancel_state in {
|
||||
CancelState.REJECTED.value,
|
||||
CancelState.UNKNOWN.value,
|
||||
}
|
||||
if slot.cancel_state is not None and not is_retry:
|
||||
return
|
||||
if is_retry:
|
||||
self._emit_unresolved_cancel_alarm(slot)
|
||||
if not (isinstance(fs.raw, dict) and fs.raw.get("order_open_confirmed") is True):
|
||||
return
|
||||
backoff_sec = min(300, 5 * (2 ** max(0, slot.cancel_attempts - 1)))
|
||||
retry_before = now - timedelta(seconds=backoff_sec)
|
||||
else:
|
||||
retry_before = None
|
||||
|
||||
qty = int(slot.qty or 0)
|
||||
# Write UNKNOWN before external I/O. If the process dies after KIS
|
||||
# receives the request but before the response is persisted, restart
|
||||
# polls the order instead of issuing a blind second cancellation.
|
||||
if not self._repo.mark_cancel_outcome(
|
||||
if not self._repo.reserve_cancel_attempt(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
CancelState.UNKNOWN.value,
|
||||
only_if_unset=True,
|
||||
retry_before=retry_before,
|
||||
):
|
||||
_log.error(
|
||||
"cancel attempt reservation conflict: slot=%s broker_order_id=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
)
|
||||
return
|
||||
try:
|
||||
cancel_kwargs = {
|
||||
@@ -610,8 +630,7 @@ class ExecutionService:
|
||||
)
|
||||
if not persisted:
|
||||
_log.error(
|
||||
"cancel outcome persistence conflict: slot=%s broker_order_id=%s "
|
||||
"outcome=%s",
|
||||
"cancel outcome persistence conflict: slot=%s broker_order_id=%s outcome=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
outcome.state.value,
|
||||
@@ -619,6 +638,14 @@ class ExecutionService:
|
||||
return
|
||||
if outcome.is_requested:
|
||||
r.timeout_cancelled += 1
|
||||
elif outcome.state in {CancelState.REJECTED, CancelState.UNKNOWN}:
|
||||
self._emit_unresolved_cancel_alarm(
|
||||
replace(
|
||||
slot,
|
||||
cancel_state=outcome.state.value,
|
||||
cancel_attempts=slot.cancel_attempts + 1,
|
||||
)
|
||||
)
|
||||
_log.warning(
|
||||
"timeout cancel result: slot=%s status=%s broker_order_id=%s "
|
||||
"age_sec=%.0f timeout_sec=%d outcome=%s",
|
||||
@@ -630,6 +657,52 @@ class ExecutionService:
|
||||
outcome.state.value,
|
||||
)
|
||||
|
||||
def _emit_unresolved_cancel_alarm(self, slot) -> None:
|
||||
key = (str(slot.slot_id), str(slot.cancel_state), slot.cancel_attempts)
|
||||
if key in self._order_alarm_keys:
|
||||
return
|
||||
self._order_alarm_keys.add(key)
|
||||
_log.critical(
|
||||
"cancel unresolved: slot=%s status=%s broker_order_id=%s outcome=%s "
|
||||
"(fail-closed pending; authoritative polling continues)",
|
||||
slot.slot_id,
|
||||
slot.status.value,
|
||||
slot.broker_order_id,
|
||||
slot.cancel_state,
|
||||
)
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"order_cancel_unresolved",
|
||||
f"주문 취소 미확정: 슬롯 pending 유지 (slot={slot.slot_id})",
|
||||
fields={
|
||||
"slot_id": str(slot.slot_id),
|
||||
"symbol": slot.symbol,
|
||||
"cancel_state": slot.cancel_state,
|
||||
"order_date": slot.order_date,
|
||||
},
|
||||
)
|
||||
|
||||
def _emit_order_metadata_unknown_alarm(self, slot) -> None:
|
||||
key = (str(slot.slot_id), "order_metadata_unknown", slot.cancel_attempts)
|
||||
if key in self._order_alarm_keys:
|
||||
return
|
||||
self._order_alarm_keys.add(key)
|
||||
_log.critical(
|
||||
"order reconciliation metadata unknown: slot=%s status=%s "
|
||||
"(fail-closed pending; no timeout cancel retry)",
|
||||
slot.slot_id,
|
||||
slot.status.value,
|
||||
)
|
||||
if self._alarm_emitter is not None:
|
||||
self._alarm_emitter.publish_alarm(
|
||||
"order_reconciliation_metadata_unknown",
|
||||
f"주문 원시각/원주문일 미확인: 슬롯 pending 유지 (slot={slot.slot_id})",
|
||||
fields={
|
||||
"slot_id": str(slot.slot_id),
|
||||
"symbol": slot.symbol,
|
||||
},
|
||||
)
|
||||
|
||||
def _emit_event(self, event_type: str, slot, routing_strategy_id: Optional[str] = None) -> None:
|
||||
if self._event_emitter is None:
|
||||
return
|
||||
@@ -734,11 +807,9 @@ class ExecutionService:
|
||||
cancel_kwargs["venue"] = slot.venue
|
||||
# Durable unknown reservation closes the crash window between the
|
||||
# outbound request and saving its immediate result.
|
||||
if not self._repo.mark_cancel_outcome(
|
||||
if not self._repo.reserve_cancel_attempt(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
CancelState.UNKNOWN.value,
|
||||
only_if_unset=True,
|
||||
):
|
||||
return "cancel_already_recorded"
|
||||
outcome = self._broker.cancel_order(slot.broker_order_id, **cancel_kwargs)
|
||||
|
||||
@@ -38,6 +38,7 @@ from trex_core.domain.slot import (
|
||||
)
|
||||
|
||||
_MAX_ADMIT_RETRY = 8
|
||||
_HARD_ORDER_TIMEOUT_SEC = 3600
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -129,9 +130,15 @@ class IntentService:
|
||||
# expected to set max_active_slots in the instance TOML — design §5).
|
||||
self._instance_key = instance_key
|
||||
self._max_active_slots = max_active_slots
|
||||
# dcy-022 / REQ-068: default timeout when intent omits timeout_sec.
|
||||
# None = OFF = indefinite wait (legacy behavior). Per-intent value overrides.
|
||||
self._default_order_timeout_sec = default_order_timeout_sec
|
||||
# Omission never disables timeout. Runtime validates configured values;
|
||||
# this seam remains fail-safe for direct application construction.
|
||||
self._default_order_timeout_sec = (
|
||||
default_order_timeout_sec
|
||||
if isinstance(default_order_timeout_sec, int)
|
||||
and not isinstance(default_order_timeout_sec, bool)
|
||||
and 1 <= default_order_timeout_sec <= _HARD_ORDER_TIMEOUT_SEC
|
||||
else _HARD_ORDER_TIMEOUT_SEC
|
||||
)
|
||||
# BAT-050 / REQ-104: reconcile-warning gate store (Optional).
|
||||
# When None, gate is skipped (backward compat, NFR-4).
|
||||
self._warn_store = warn_store
|
||||
@@ -219,6 +226,9 @@ class IntentService:
|
||||
dcy-036 / REQ-114: entry gate — if entry_allowed_fn returns False,
|
||||
BuyIntent is rejected before any slot lookup."""
|
||||
|
||||
timeout_error = self._timeout_error(intent.timeout_sec)
|
||||
if timeout_error is not None:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, timeout_error)
|
||||
if intent.venue not in {"AUTO", "KRX", "NXT", "SOR"}:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, "invalid_venue")
|
||||
|
||||
@@ -334,6 +344,9 @@ class IntentService:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, "already_active")
|
||||
|
||||
def _admit_sell(self, intent: SellIntent) -> IntentDecision:
|
||||
timeout_error = self._timeout_error(intent.timeout_sec)
|
||||
if timeout_error is not None:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, timeout_error)
|
||||
if intent.venue not in {"AUTO", "KRX", "NXT", "SOR"}:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, "invalid_venue")
|
||||
|
||||
@@ -385,3 +398,13 @@ class IntentService:
|
||||
except SlotStateMismatchError:
|
||||
return IntentDecision(intent.intent_id, "rejected", None, "slot_state_changed")
|
||||
return IntentDecision(intent.intent_id, "admitted", updated.slot_id, None)
|
||||
|
||||
@staticmethod
|
||||
def _timeout_error(timeout_sec: object) -> Optional[str]:
|
||||
if timeout_sec is None:
|
||||
return None
|
||||
if not isinstance(timeout_sec, int) or isinstance(timeout_sec, bool):
|
||||
return "bad_request:timeout_sec_must_be_integer"
|
||||
if not 1 <= timeout_sec <= _HARD_ORDER_TIMEOUT_SEC:
|
||||
return "bad_request:timeout_sec_out_of_range"
|
||||
return None
|
||||
|
||||
@@ -608,7 +608,9 @@ def _opt_float(v) -> Optional[float]:
|
||||
def _opt_int(v) -> Optional[int]:
|
||||
if v is None:
|
||||
return None
|
||||
return int(v)
|
||||
if not isinstance(v, int) or isinstance(v, bool):
|
||||
raise ValueError("integer required")
|
||||
return v
|
||||
|
||||
|
||||
def _opt_dt(v) -> Optional[datetime]:
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Iterator, Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
@@ -163,6 +163,7 @@ class InMemorySlotRepository:
|
||||
|
||||
# --- order execution queue (BAT-004 / REQ-013, option A) ---
|
||||
_PENDING = (SlotState.PENDING_BUY, SlotState.PENDING_SELL)
|
||||
_KST = timezone(timedelta(hours=9))
|
||||
|
||||
def list_undispatched_pending(
|
||||
self,
|
||||
@@ -191,15 +192,20 @@ class InMemorySlotRepository:
|
||||
broker_order_id: str,
|
||||
*,
|
||||
venue: Optional[str] = None,
|
||||
submitted_at: Optional[datetime] = None,
|
||||
) -> bool:
|
||||
"""Atomically persist dispatch identity and the actual broker venue."""
|
||||
"""Atomically persist dispatch identity and its immutable clock."""
|
||||
s = self._slots.get(slot_id)
|
||||
if s is None or s.broker_order_id is not None:
|
||||
return False
|
||||
submitted_at = submitted_at or datetime.now(tz=timezone.utc)
|
||||
self._slots[slot_id] = replace(
|
||||
s,
|
||||
broker_order_id=broker_order_id,
|
||||
venue=venue if venue is not None else s.venue,
|
||||
order_submitted_at=submitted_at,
|
||||
order_date=submitted_at.astimezone(self._KST).strftime("%Y%m%d"),
|
||||
cancel_attempts=0,
|
||||
updated_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
return True
|
||||
@@ -245,6 +251,39 @@ class InMemorySlotRepository:
|
||||
)
|
||||
return True
|
||||
|
||||
def reserve_cancel_attempt(
|
||||
self,
|
||||
slot_id: UUID,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
retry_before: Optional[datetime] = None,
|
||||
) -> bool:
|
||||
s = self._slots.get(slot_id)
|
||||
retry_allowed = (
|
||||
retry_before is not None
|
||||
and s is not None
|
||||
and s.cancel_state in {"rejected", "unknown"}
|
||||
and s.cancel_attempts < 4
|
||||
and s.cancel_requested_at is not None
|
||||
and s.cancel_requested_at <= retry_before
|
||||
)
|
||||
if (
|
||||
s is None
|
||||
or s.broker_order_id != broker_order_id
|
||||
or s.status not in self._PENDING
|
||||
or (s.cancel_state is not None and not retry_allowed)
|
||||
):
|
||||
return False
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
self._slots[slot_id] = replace(
|
||||
s,
|
||||
cancel_state="unknown",
|
||||
cancel_requested_at=now,
|
||||
cancel_attempts=s.cancel_attempts + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
return True
|
||||
|
||||
def list_dispatched_pending(self, instance_key: str = "default") -> list[Slot]:
|
||||
"""Pending slots already dispatched (broker_order_id NOT NULL) — poll targets."""
|
||||
return [
|
||||
@@ -266,6 +305,9 @@ class InMemorySlotRepository:
|
||||
broker_order_id=None,
|
||||
cancel_state=None,
|
||||
cancel_requested_at=None,
|
||||
order_submitted_at=None,
|
||||
order_date=None,
|
||||
cancel_attempts=0,
|
||||
updated_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
@@ -64,6 +64,11 @@ class Slot:
|
||||
# across ticks and restarts.
|
||||
cancel_state: Optional[str] = None
|
||||
cancel_requested_at: Optional[datetime] = None
|
||||
# Immutable dispatch clock and original KST order date. Unlike updated_at,
|
||||
# these survive polling/cancel bookkeeping and anchor deadline/recovery.
|
||||
order_submitted_at: Optional[datetime] = None
|
||||
order_date: Optional[str] = None
|
||||
cancel_attempts: int = 0
|
||||
|
||||
|
||||
# Canonical column contract for `Slot` (dcy-001 OQ-2 / FR-5).
|
||||
@@ -95,6 +100,9 @@ EXPECTED_SLOT_FIELDS: frozenset[str] = frozenset({
|
||||
"venue",
|
||||
"cancel_state",
|
||||
"cancel_requested_at",
|
||||
"order_submitted_at",
|
||||
"order_date",
|
||||
"cancel_attempts",
|
||||
})
|
||||
|
||||
|
||||
|
||||
@@ -213,6 +213,9 @@ class SlotService:
|
||||
venue="KRX",
|
||||
cancel_state=None,
|
||||
cancel_requested_at=None,
|
||||
order_submitted_at=None,
|
||||
order_date=None,
|
||||
cancel_attempts=0,
|
||||
updated_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
|
||||
@@ -287,6 +290,21 @@ class SlotService:
|
||||
venue=new_venue,
|
||||
cancel_state=None,
|
||||
cancel_requested_at=None,
|
||||
order_submitted_at=(
|
||||
slot.order_submitted_at
|
||||
if to_state in {SlotState.PENDING_BUY, SlotState.PENDING_SELL}
|
||||
else None
|
||||
),
|
||||
order_date=(
|
||||
slot.order_date
|
||||
if to_state in {SlotState.PENDING_BUY, SlotState.PENDING_SELL}
|
||||
else None
|
||||
),
|
||||
cancel_attempts=(
|
||||
slot.cancel_attempts
|
||||
if to_state in {SlotState.PENDING_BUY, SlotState.PENDING_SELL}
|
||||
else 0
|
||||
),
|
||||
updated_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
+15
-10
@@ -170,9 +170,8 @@ class DaemonInstanceConfig:
|
||||
# admission capacity cap. BAT-030: asset-centric is the only model; absent
|
||||
# means no cap (operators are expected to set it in production).
|
||||
max_active_slots: Optional[int] = None
|
||||
# dcy-022 / REQ-068: default order timeout in seconds (opt-in per instance).
|
||||
# None = OFF = indefinite wait (legacy behavior). Per-intent timeout_sec overrides.
|
||||
default_order_timeout_sec: Optional[int] = None
|
||||
# Per-instance override; omission uses the hard 3,600-second fallback.
|
||||
default_order_timeout_sec: int = 3600
|
||||
# dcy-024 / REQ-048 CHG-20260611-01: disable autonomous safety-sell worker.
|
||||
# Absent = True (safe default). False → trex-safety-sell thread not started.
|
||||
safety_sell_enabled: bool = True
|
||||
@@ -202,11 +201,17 @@ class DaemonInstanceConfig:
|
||||
raise RuntimeError(
|
||||
f"instance TOML 'max_active_slots' must be > 0, got {raw_max}: {path}"
|
||||
)
|
||||
raw_timeout = data.get("default_order_timeout_sec")
|
||||
default_timeout = int(raw_timeout) if raw_timeout is not None else None
|
||||
if default_timeout is not None and default_timeout <= 0:
|
||||
raw_timeout = data.get("default_order_timeout_sec", 3600)
|
||||
if not isinstance(raw_timeout, int) or isinstance(raw_timeout, bool):
|
||||
raise RuntimeError(
|
||||
f"instance TOML 'default_order_timeout_sec' must be > 0, got {raw_timeout}: {path}"
|
||||
f"instance TOML 'default_order_timeout_sec' must be an integer in 1..3600, "
|
||||
f"got {raw_timeout}: {path}"
|
||||
)
|
||||
default_timeout = raw_timeout
|
||||
if not 1 <= default_timeout <= 3600:
|
||||
raise RuntimeError(
|
||||
f"instance TOML 'default_order_timeout_sec' must be in 1..3600, "
|
||||
f"got {raw_timeout}: {path}"
|
||||
)
|
||||
raw_wd = data.get("watchdog_timeout_sec")
|
||||
watchdog_timeout = int(raw_wd) if raw_wd is not None else None
|
||||
@@ -258,9 +263,8 @@ class DaemonConfig:
|
||||
# threshold in seconds. 0 disables (default). When >0, dispatched-pending
|
||||
# slots older than this are surfaced (WARN + counter) without state change.
|
||||
pending_stale_threshold_sec: int = 0
|
||||
# dcy-022 / REQ-068: default order timeout (seconds). None = OFF = indefinite.
|
||||
# Per-intent timeout_sec overrides. Passed to IntentService at wiring time.
|
||||
default_order_timeout_sec: Optional[int] = None
|
||||
# Effective fallback passed to IntentService; never disabled by omission.
|
||||
default_order_timeout_sec: int = 3600
|
||||
# dcy-014 / REQ-048: operator-local timezone (CLI date display etc.). The
|
||||
# daily preallocation trigger was removed with the pool model (BAT-030).
|
||||
trading_timezone: str = "Asia/Seoul"
|
||||
@@ -417,4 +421,5 @@ class DaemonConfig:
|
||||
safety_sell_enabled=inst.safety_sell_enabled,
|
||||
safety_sell=safety_sell,
|
||||
credential_fetch_enabled=inst.credential_fetch_enabled,
|
||||
default_order_timeout_sec=inst.default_order_timeout_sec,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user