fix: persist uncertain cancel outcomes
This commit is contained in:
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from typing import Any, Optional, Protocol
|
||||
|
||||
|
||||
@@ -29,6 +30,30 @@ class BrokerBalanceError(RuntimeError):
|
||||
super().__init__(f"broker balance unavailable ({code})")
|
||||
|
||||
|
||||
class CancelState(str, Enum):
|
||||
"""Immediate result of a cancel request, not the final order state."""
|
||||
|
||||
REQUESTED = "requested"
|
||||
REJECTED = "rejected"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CancelOutcome:
|
||||
"""Sanitized cancel request outcome.
|
||||
|
||||
REQUESTED only means that the broker accepted the cancellation request.
|
||||
Final cancelled/filled state is established exclusively by poll_fill().
|
||||
"""
|
||||
|
||||
state: CancelState
|
||||
reason: Optional[str] = None
|
||||
|
||||
@property
|
||||
def is_requested(self) -> bool:
|
||||
return self.state is CancelState.REQUESTED
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OrderRequest:
|
||||
slot_id: str # originating slot (audit/correlation)
|
||||
@@ -114,12 +139,23 @@ class BrokerPort(Protocol):
|
||||
"""Observe current fill status for a previously accepted order."""
|
||||
...
|
||||
|
||||
def cancel_order(self, broker_order_id: str, *, side: str, symbol: str, qty: int, venue: str = "KRX") -> bool:
|
||||
"""Best-effort cancel. Returns True if the broker accepted the cancel.
|
||||
def cancel_order(
|
||||
self,
|
||||
broker_order_id: str,
|
||||
*,
|
||||
side: str,
|
||||
symbol: str,
|
||||
qty: int,
|
||||
venue: str = "KRX",
|
||||
) -> CancelOutcome:
|
||||
"""Request cancellation and classify the immediate broker outcome.
|
||||
|
||||
venue: exchange venue of the original order (REQ-118). Cancel must use
|
||||
the same EXCG_ID_DVSN_CD as the original submit. Default "KRX" preserves
|
||||
backward compatibility for existing callers (orb/tsl).
|
||||
|
||||
REQUESTED is not a final cancellation. poll_fill() remains the only
|
||||
authority for cancelled/filled terminal state.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -29,7 +29,12 @@ from datetime import datetime, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
from trex_core.application.alarm_emitter import AlarmEmitter
|
||||
from trex_core.application.broker_port import BrokerPort, OrderRequest
|
||||
from trex_core.application.broker_port import (
|
||||
BrokerPort,
|
||||
CancelOutcome,
|
||||
CancelState,
|
||||
OrderRequest,
|
||||
)
|
||||
from trex_core.application.ledger_outbox import LedgerOutbox
|
||||
from trex_core.application.ledger_port import LedgerTrade
|
||||
from trex_core.domain.pricing import ceil_to_tick, floor_to_tick
|
||||
@@ -546,12 +551,28 @@ class ExecutionService:
|
||||
"""
|
||||
if str(slot.broker_order_id or "").startswith("submission-unknown:"):
|
||||
return
|
||||
if slot.cancel_state is not None:
|
||||
return
|
||||
if slot.timeout_sec is None or slot.updated_at is None:
|
||||
return
|
||||
age = (now - slot.updated_at).total_seconds()
|
||||
if age < slot.timeout_sec:
|
||||
return
|
||||
qty = int(slot.qty or 0)
|
||||
# Write UNKNOWN before external I/O. If the process dies after KIS
|
||||
# receives the request but before the response is persisted, restart
|
||||
# polls the order instead of issuing a blind second cancellation.
|
||||
if not self._repo.mark_cancel_outcome(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
CancelState.UNKNOWN.value,
|
||||
):
|
||||
_log.error(
|
||||
"cancel attempt reservation conflict: slot=%s broker_order_id=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
)
|
||||
return
|
||||
try:
|
||||
cancel_kwargs = {
|
||||
"side": side,
|
||||
@@ -560,25 +581,40 @@ class ExecutionService:
|
||||
}
|
||||
if slot.venue != "KRX":
|
||||
cancel_kwargs["venue"] = slot.venue
|
||||
accepted = self._broker.cancel_order(slot.broker_order_id, **cancel_kwargs)
|
||||
outcome = self._broker.cancel_order(slot.broker_order_id, **cancel_kwargs)
|
||||
except Exception: # noqa: BLE001 - adapter failure becomes durable unknown
|
||||
outcome = CancelOutcome(CancelState.UNKNOWN, "adapter_exception")
|
||||
_log.warning(
|
||||
"timeout cancel outcome unknown: slot=%s broker_order_id=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
)
|
||||
persisted = self._repo.mark_cancel_outcome(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
outcome.state.value,
|
||||
)
|
||||
if not persisted:
|
||||
_log.error(
|
||||
"cancel outcome persistence conflict: slot=%s broker_order_id=%s "
|
||||
"outcome=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
outcome.state.value,
|
||||
)
|
||||
return
|
||||
if outcome.is_requested:
|
||||
r.timeout_cancelled += 1
|
||||
_log.warning(
|
||||
"timeout cancel sent: slot=%s status=%s broker_order_id=%s "
|
||||
"age_sec=%.0f timeout_sec=%d broker_accepted=%s",
|
||||
slot.slot_id,
|
||||
slot.status.value,
|
||||
slot.broker_order_id,
|
||||
age,
|
||||
slot.timeout_sec,
|
||||
accepted,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - best-effort; do not block the loop
|
||||
_log.warning(
|
||||
"timeout cancel failed (exception): slot=%s broker_order_id=%s "
|
||||
"(will retry next tick if still pending)",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
)
|
||||
_log.warning(
|
||||
"timeout cancel result: slot=%s status=%s broker_order_id=%s "
|
||||
"age_sec=%.0f timeout_sec=%d outcome=%s",
|
||||
slot.slot_id,
|
||||
slot.status.value,
|
||||
slot.broker_order_id,
|
||||
age,
|
||||
slot.timeout_sec,
|
||||
outcome.state.value,
|
||||
)
|
||||
|
||||
def _emit_event(self, event_type: str, slot, routing_strategy_id: Optional[str] = None) -> None:
|
||||
if self._event_emitter is None:
|
||||
@@ -655,9 +691,9 @@ class ExecutionService:
|
||||
def cancel_pending_slot(self, slot_id) -> str:
|
||||
"""Operator backstop: send cancel_order for a pending slot (dcy-022/REQ-071).
|
||||
|
||||
Returns "ok" (cancel sent), "not_pending" (slot is HOLDING/INACTIVE),
|
||||
or "slot_not_found". Does NOT transition the slot — the execution loop's
|
||||
next poll handles the rollback via the normal is_terminal_reject path.
|
||||
Returns "ok" (request accepted), "cancel_rejected", "cancel_unknown",
|
||||
"not_pending", "not_dispatched", or "slot_not_found". Does NOT
|
||||
transition the slot — fill polling alone confirms cancelled/filled.
|
||||
"""
|
||||
from uuid import UUID
|
||||
|
||||
@@ -680,21 +716,40 @@ class ExecutionService:
|
||||
}
|
||||
if slot.venue != "KRX":
|
||||
cancel_kwargs["venue"] = slot.venue
|
||||
self._broker.cancel_order(slot.broker_order_id, **cancel_kwargs)
|
||||
except Exception: # noqa: BLE001 - best-effort
|
||||
# Durable unknown reservation closes the crash window between the
|
||||
# outbound request and saving its immediate result.
|
||||
if not self._repo.mark_cancel_outcome(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
CancelState.UNKNOWN.value,
|
||||
):
|
||||
return "cancel_state_conflict"
|
||||
outcome = self._broker.cancel_order(slot.broker_order_id, **cancel_kwargs)
|
||||
except Exception: # noqa: BLE001 - adapter failure becomes durable unknown
|
||||
outcome = CancelOutcome(CancelState.UNKNOWN, "adapter_exception")
|
||||
_log.warning(
|
||||
"operator cancel failed: slot=%s broker_order_id=%s",
|
||||
"operator cancel outcome unknown: slot=%s broker_order_id=%s",
|
||||
slot.slot_id,
|
||||
slot.broker_order_id,
|
||||
)
|
||||
return "cancel_failed"
|
||||
if not self._repo.mark_cancel_outcome(
|
||||
slot.slot_id,
|
||||
str(slot.broker_order_id),
|
||||
outcome.state.value,
|
||||
):
|
||||
return "cancel_state_conflict"
|
||||
_log.warning(
|
||||
"operator cancel sent: slot=%s status=%s broker_order_id=%s",
|
||||
"operator cancel result: slot=%s status=%s broker_order_id=%s outcome=%s",
|
||||
slot.slot_id,
|
||||
slot.status.value,
|
||||
slot.broker_order_id,
|
||||
outcome.state.value,
|
||||
)
|
||||
return "ok"
|
||||
if outcome.state is CancelState.REQUESTED:
|
||||
return "ok"
|
||||
if outcome.state is CancelState.REJECTED:
|
||||
return "cancel_rejected"
|
||||
return "cancel_unknown"
|
||||
|
||||
def _rollback(self, slot, *, reason: str) -> None:
|
||||
total = self._balance_map().get(slot.symbol or "", 0)
|
||||
|
||||
Reference in New Issue
Block a user