trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: plan for review blockers follow-up

Browse Source 
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:21:10 +00:00
committed by GitHub
co-authored by genichin
parent 9c5a01ed67 19af864bf3
commit 2b5d9043b9
53 changed files with 3461 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/application/intents.py
+26 -2
View File
@@ -29,6 +29,7 @@ from typing import Any, Callable, Optional
from uuid import UUID
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.portfolio import BLOCKING_RECONCILE_CASES
from trex_core.application.ports import IntentStore
from trex_core.domain.slot import (
SlotAdmissionRejectedError,
@@ -270,6 +271,29 @@ class IntentService:
except Exception: # noqa: BLE001 - alarm path must not affect intent decision
pass
def _is_reconcile_trade_blocked(self, slot_id: str) -> bool:
if self._warn_store is None:
return False
if not self._warn_store.is_warned(slot_id):
return False
info = self._warn_store.get_warning_info(slot_id)
if info is None:
return True
raw_codes = info.get("case_codes")
if isinstance(raw_codes, list):
case_codes = {str(code) for code in raw_codes if code}
else:
case_code = str(info.get("case_code", ""))
case_codes = {case_code} if case_code else set()
if not case_codes:
return True
# The warning row is authoritative. Unknown cases are not evidence
# that trading is safe: they may come from corrupt data or a newer
# producer that this core version cannot interpret.
if not case_codes.issubset(BLOCKING_RECONCILE_CASES):
return True
return bool(case_codes & BLOCKING_RECONCILE_CASES)
# ------------------------------------------------------------
def _dedup(self, intent_id: str, intent_type: str, payload: dict) -> Optional[IntentDecision]:
"""Return prior decision if intent_id already seen; else None (and reserve it)."""
@@ -339,7 +363,7 @@ class IntentService:
# those paths call SlotService.sell() directly.
if self._warn_store is not None:
try:
if self._warn_store.is_warned(str(slot.slot_id)):
if self._is_reconcile_trade_blocked(str(slot.slot_id)):
return IntentDecision(
intent.intent_id, "rejected", None, "reconcile_warned"
)
@@ -432,7 +456,7 @@ class IntentService:
# SlotService.sell() / transit_slot() directly without IntentService (OQ-B).
if self._warn_store is not None:
try:
if self._warn_store.is_warned(str(slot.slot_id)):
if self._is_reconcile_trade_blocked(str(slot.slot_id)):
return IntentDecision(
intent.intent_id, "rejected", None, "reconcile_warned"
)
trex_core/application/ledger_outbox.py
+212 -19
View File
@@ -66,6 +66,10 @@ class WorkItemStatus(str, Enum):
DELIVERY_FAILED = "delivery_failed" # dcy-015 — permanent or dead-lettered (terminal)
class OutboxClaimLostError(RuntimeError):
"""The caller no longer owns the lease required for an outcome update."""
# Outcomes that move a row to the terminal delivery_failed state.
_DELIVERY_FAILED_OUTCOMES = frozenset(
{OutboxOutcome.PERMANENT_FAILURE, OutboxOutcome.DEAD_LETTERED}
@@ -104,11 +108,18 @@ class LedgerWorkItem:
next_retry_at: Optional[datetime] = None
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
instance_key: Optional[str] = None
claim_expires_at: Optional[datetime] = None
class LedgerOutbox(Protocol):
"""Durable work-item store backing the ledger writer."""
@property
def instance_key(self) -> str:
"""Immutable owner partition for rows handled by this outbox."""
...
def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem:
"""Enqueue a trade. Idempotent on correlation_id (returns the
existing row unchanged if it is already present)."""
@@ -118,6 +129,15 @@ class LedgerOutbox(Protocol):
"""Return pending work items in FIFO order, up to `limit`."""
...
def claim_pending(
self,
limit: int = 32,
*,
correlation_id: str | None = None,
) -> list[LedgerWorkItem]:
"""Atomically claim pending items for delivery."""
...
def pending_cash_delta(self) -> dict[str, float]:
"""Sum the estimated cash delta of ALL not-yet-settled (pending) rows,
grouped by asset_id (dcy-018 / REQ-029, OQ-4 estimate overlay).
@@ -136,6 +156,7 @@ class LedgerOutbox(Protocol):
*,
error: Optional[str] = None,
next_retry_at: Optional[datetime] = None,
expected_claim_expires_at: Optional[datetime] = None,
) -> LedgerWorkItem:
"""Record the outcome of a writer attempt and advance status.
@@ -151,6 +172,7 @@ class WriterStats:
retryable: int = 0
conflicts: int = 0
delivery_failed: int = 0 # dcy-015: permanent + dead-lettered terminals
claim_lost: int = 0
class LedgerWriter:
@@ -190,15 +212,46 @@ class LedgerWriter:
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
self._ledger_skip_store = ledger_skip_store
self._instance_key = instance_key
resolved_instance_key = outbox.instance_key if instance_key is None else instance_key
if not isinstance(resolved_instance_key, str) or not resolved_instance_key.strip():
raise ValueError("instance_key must be a non-empty string")
if resolved_instance_key != outbox.instance_key:
raise ValueError("writer instance_key must match outbox instance_key")
self._instance_key = resolved_instance_key
self.counters: dict[str, int] = {}
def _count(self, key: str) -> None:
self.counters[key] = self.counters.get(key, 0) + 1
def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem:
return self._outbox.enqueue(trade)
def run_once(self, limit: int = 32) -> WriterStats:
stats = WriterStats()
for item in self._outbox.list_pending(limit):
outcome = self._process(item)
items = self._outbox.claim_pending(limit)
claim_result = "claimed" if items else "empty"
self._count(f"claim:{claim_result}")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d",
self._instance_key,
claim_result,
len(items),
limit,
)
for item in items:
if not self._validate_instance(item):
continue
try:
outcome = self._process(item)
except OutboxClaimLostError:
self._count("claim:lost")
stats.claim_lost += 1
_log.warning(
"ledger outbox claim lost: instance_key=%s correlation_id=%s",
self._instance_key,
item.correlation_id,
)
continue
if outcome is OutboxOutcome.COMPLETED:
stats.completed += 1
elif outcome is OutboxOutcome.DUPLICATE_COMPLETED:
@@ -216,7 +269,50 @@ class LedgerWriter:
item = self._outbox.get(correlation_id)
if item is None:
return None
return self._process(item)
if not self._validate_instance(item):
return None
if item.status is WorkItemStatus.PENDING:
claimed = self._outbox.claim_pending(1, correlation_id=correlation_id)
if not claimed:
self._count("claim:unavailable")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=unavailable "
"correlation_id=%s",
self._instance_key,
correlation_id,
)
return None
item = claimed[0]
self._count("claim:claimed")
_log.info(
"ledger outbox claim: instance_key=%s claim_result=claimed "
"count=%d correlation_id=%s",
self._instance_key,
len(claimed),
correlation_id,
)
try:
return self._process(item)
except OutboxClaimLostError:
self._count("claim:lost")
_log.warning(
"ledger outbox claim lost: instance_key=%s correlation_id=%s",
self._instance_key,
correlation_id,
)
return None
def _validate_instance(self, item: LedgerWorkItem) -> bool:
if item.instance_key == self._instance_key:
return True
self._count("claim:foreign_instance")
_log.warning(
"ledger outbox claim rejected: instance_key=%s "
"claim_result=foreign_or_unowned correlation_id=%s",
self._instance_key,
item.correlation_id,
)
return False
def _backoff_delay(self, attempts: int) -> float:
"""Exponential backoff with jitter: min(base*2^(n-1), cap) + up to 25%."""
@@ -228,14 +324,18 @@ class LedgerWriter:
# outbox row. Skip the adapter call.
existing = self._outbox.get(item.correlation_id)
if existing is not None and existing.status is WorkItemStatus.COMPLETED:
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED)
self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED)
return OutboxOutcome.DUPLICATE_COMPLETED
try:
outcome = self._ledger.record_trade(item.trade)
except LedgerConflictError as exc:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)
item.correlation_id,
OutboxOutcome.CONFLICT_BLOCKED,
error=str(exc),
expected_claim_expires_at=item.claim_expires_at,
)
self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED)
# dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed.
if self._alarm_emitter is not None:
self._alarm_emitter.publish_alarm(
@@ -250,11 +350,16 @@ class LedgerWriter:
# dcy-015 / REQ-017/018 (issue #13): non-retryable client error →
# terminal delivery_failed immediately (no retry spam). Surface for ops.
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.PERMANENT_FAILURE, error=str(exc)
item.correlation_id,
OutboxOutcome.PERMANENT_FAILURE,
error=str(exc),
expected_claim_expires_at=item.claim_expires_at,
)
self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE)
_log.warning(
"ledger delivery_failed (permanent): correlation_id=%s reason=%s",
item.correlation_id, exc,
"ledger delivery_failed (permanent): instance_key=%s correlation_id=%s",
item.instance_key or self._instance_key,
item.correlation_id,
)
# dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry.
if self._alarm_emitter is not None:
@@ -271,12 +376,18 @@ class LedgerWriter:
new_attempts = item.attempts + 1
if new_attempts >= self._max_attempts:
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.DEAD_LETTERED, error=str(exc)
item.correlation_id,
OutboxOutcome.DEAD_LETTERED,
error=str(exc),
expected_claim_expires_at=item.claim_expires_at,
)
self._log_outcome(item, OutboxOutcome.DEAD_LETTERED)
_log.warning(
"ledger delivery_failed (dead-letter, attempts=%d): "
"correlation_id=%s reason=%s",
new_attempts, item.correlation_id, exc,
"instance_key=%s correlation_id=%s",
new_attempts,
item.instance_key or self._instance_key,
item.correlation_id,
)
# dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries.
if self._alarm_emitter is not None:
@@ -294,8 +405,11 @@ class LedgerWriter:
)
self._outbox.mark_outcome(
item.correlation_id, OutboxOutcome.RETRYABLE_FAILURE,
error=str(exc), next_retry_at=next_retry_at,
error=str(exc),
next_retry_at=next_retry_at,
expected_claim_expires_at=item.claim_expires_at,
)
self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE)
return OutboxOutcome.RETRYABLE_FAILURE
# dcy-018 / REQ-019/030 (option D): write back portfolio-hub authoritative
# cash/quantity to the snapshot store on new+duplicate (OQ-2 b). *_after
@@ -306,13 +420,37 @@ class LedgerWriter:
# duplicate_completed; the ledger has exactly one entry for this
# execution_id either way (REQ-019 AC-1, AC-4).
if outcome.result is LedgerWriteResult.DUPLICATE_OK:
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED)
self._outbox.mark_outcome(
item.correlation_id,
OutboxOutcome.DUPLICATE_COMPLETED,
expected_claim_expires_at=item.claim_expires_at,
)
self._resolve_ledger_skip(item.trade)
self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED)
return OutboxOutcome.DUPLICATE_COMPLETED
self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.COMPLETED)
self._outbox.mark_outcome(
item.correlation_id,
OutboxOutcome.COMPLETED,
expected_claim_expires_at=item.claim_expires_at,
)
self._resolve_ledger_skip(item.trade)
self._log_outcome(item, OutboxOutcome.COMPLETED)
return OutboxOutcome.COMPLETED
def _log_outcome(
self,
item: LedgerWorkItem,
outcome: OutboxOutcome,
) -> None:
self._count(f"outcome:{outcome.value}")
msg = "ledger outbox outcome: instance_key=%s correlation_id=%s outcome=%s"
args: tuple[object, ...] = (
item.instance_key or self._instance_key,
item.correlation_id,
outcome.value,
)
_log.info(msg, *args)
def _resolve_ledger_skip(self, trade: LedgerTrade) -> None:
if (
self._ledger_skip_store is None
@@ -349,7 +487,7 @@ class LedgerWriter:
except Exception: # noqa: BLE001 - write-back must not block the ledger path
_log.warning(
"snapshot cash/position write-back failed: asset_id=%s",
trade.asset_id, exc_info=True,
trade.asset_id,
)
@@ -358,7 +496,13 @@ class LedgerWriter:
class InMemoryLedgerOutbox:
"""Process-local outbox. Thread-safe; idempotent on correlation_id."""
def __init__(self, *, clock: Optional[Callable[[], datetime]] = None) -> None:
def __init__(
self,
*,
clock: Optional[Callable[[], datetime]] = None,
instance_key: str,
claim_ttl_sec: float = 60.0,
) -> None:
import threading
self._rows: dict[str, LedgerWorkItem] = {}
@@ -367,6 +511,16 @@ class InMemoryLedgerOutbox:
# dcy-015: shared clock lets tests align backoff (next_retry_at) with
# list_pending's "now" deterministically.
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
if not isinstance(instance_key, str) or not instance_key.strip():
raise ValueError("instance_key must be a non-empty string")
if claim_ttl_sec <= 0:
raise ValueError("claim_ttl_sec must be positive")
self._instance_key = instance_key
self._claim_ttl_sec = claim_ttl_sec
@property
def instance_key(self) -> str:
return self._instance_key
def _now(self) -> datetime:
return self._clock()
@@ -383,6 +537,7 @@ class InMemoryLedgerOutbox:
status=WorkItemStatus.PENDING,
created_at=self._now(),
updated_at=self._now(),
instance_key=self._instance_key,
)
self._rows[cid] = row
self._order.append(cid)
@@ -396,14 +551,45 @@ class InMemoryLedgerOutbox:
row = self._rows[cid]
# dcy-015 / REQ-019: skip rows still in backoff so they don't
# occupy the FIFO batch (head-of-line protection, NFR-3).
if row.status is WorkItemStatus.PENDING and (
row.next_retry_at is None or row.next_retry_at <= now
if (
row.status is WorkItemStatus.PENDING
and row.instance_key == self._instance_key
and (row.next_retry_at is None or row.next_retry_at <= now)
and (row.claim_expires_at is None or row.claim_expires_at <= now)
):
out.append(row)
if len(out) >= limit:
break
return out
def claim_pending(
self,
limit: int = 32,
*,
correlation_id: str | None = None,
) -> list[LedgerWorkItem]:
now = self._now()
lease_until = now + timedelta(seconds=self._claim_ttl_sec)
with self._lock:
claimed: list[LedgerWorkItem] = []
candidates = [correlation_id] if correlation_id is not None else self._order
for cid in candidates:
row = self._rows.get(cid)
if row is None or row.status is not WorkItemStatus.PENDING:
continue
if row.instance_key != self._instance_key:
continue
if row.next_retry_at is not None and row.next_retry_at > now:
continue
if row.claim_expires_at is not None and row.claim_expires_at > now:
continue
updated = replace(row, claim_expires_at=lease_until, updated_at=now)
self._rows[cid] = updated
claimed.append(updated)
if len(claimed) >= limit:
break
return claimed
def pending_cash_delta(self) -> dict[str, float]:
with self._lock:
out: dict[str, float] = {}
@@ -424,9 +610,15 @@ class InMemoryLedgerOutbox:
*,
error: Optional[str] = None,
next_retry_at: Optional[datetime] = None,
expected_claim_expires_at: Optional[datetime] = None,
) -> LedgerWorkItem:
with self._lock:
row = self._rows[correlation_id]
if expected_claim_expires_at is None:
if row.claim_expires_at is not None:
raise OutboxClaimLostError(correlation_id)
elif row.claim_expires_at != expected_claim_expires_at:
raise OutboxClaimLostError(correlation_id)
if outcome is OutboxOutcome.COMPLETED:
new_status = WorkItemStatus.COMPLETED
elif outcome is OutboxOutcome.CONFLICT_BLOCKED:
@@ -448,6 +640,7 @@ class InMemoryLedgerOutbox:
next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None
),
updated_at=self._now(),
claim_expires_at=None,
)
self._rows[correlation_id] = updated
return updated
trex_core/application/messaging_handlers.py
+62 -31
View File
@@ -17,6 +17,7 @@ from trex_core.application.account_snapshot import AccountSnapshotUnavailable
from trex_core.application.broker_port import BalanceItem
from trex_core.application.messaging import RpcRequest, RpcResponse
from trex_core.application.portfolio import (
BLOCKING_RECONCILE_CASES,
PauseStore,
ReconcileWarningStore,
SnapshotStore,
@@ -263,28 +264,27 @@ def make_slots_rpc_handler(
for s in holding_slots:
watchdog_store.update(str(s.slot_id))
# Build merged slot rows (REQ-074)
# BAT-050 / REQ-104: warn_store passed to slot_to_dict for reconcile_warned field.
slot_rows = [
slot_to_dict(
s,
snapshot_store=snapshot_store,
watchdog_store=watchdog_store,
pause_store=pause_store,
warn_store=warn_store,
watchdog_timeout_sec=watchdog_timeout_sec,
)
for s in slots
]
# AC-0023 exclusion must precede all optional data readers so excluded
# slots cannot expose state or make an otherwise valid response fail.
excluded_count = 0
warning_cases_by_slot: dict[str, set[str]] = {}
if warn_store is not None:
try:
c_warned_ids = warn_store.list_c_case_warned()
warned_slot_ids = warn_store.list_warned()
for sid in warned_slot_ids:
info = warn_store.get_warning_info(sid)
if info is None:
raise RuntimeError(f"missing warning info for slot_id={sid}")
raw_codes = info.get("case_codes")
if isinstance(raw_codes, list):
codes = {str(code) for code in raw_codes if code}
else:
case_code = str(info.get("case_code", ""))
codes = {case_code} if case_code else set()
if not codes:
raise RuntimeError(f"empty warning cases for slot_id={sid}")
if not codes.issubset(BLOCKING_RECONCILE_CASES):
raise RuntimeError(f"unsupported warning cases for slot_id={sid}")
warning_cases_by_slot[str(sid)] = codes
except Exception: # noqa: BLE001
_log.exception("warn_store.list_c_case_warned() failed — fail-closed")
_log.exception("warn_store warning query failed — fail-closed")
return RpcResponse(payload={
"slots": [],
"available_slots": None,
@@ -294,20 +294,48 @@ def make_slots_rpc_handler(
"reconcile warning store unavailable; "
"returning no slots (fail-closed)"
),
"excluded_slots": {"reconcile_quantity_mismatch": 0},
"excluded_slots": {
"reconcile_quantity_mismatch": 0,
"active_reconcile_warning_cases": {},
},
})
if c_warned_ids:
included = [
(slot, row)
for slot, row in zip(slots, slot_rows, strict=True)
if not (
slot.status.value == "holding"
and str(slot.slot_id) in c_warned_ids
)
]
excluded_count = len(slots) - len(included)
# Build merged slot rows (REQ-074)
slot_rows = [
slot_to_dict(
s,
snapshot_store=snapshot_store,
watchdog_store=watchdog_store,
pause_store=pause_store,
warning_cases_by_slot=warning_cases_by_slot,
watchdog_timeout_sec=watchdog_timeout_sec,
)
for s in slots
]
# Exclusion must precede all optional data readers so blocked slots cannot
# expose state or make an otherwise valid response fail.
excluded_count = 0
excluded_case_counts: dict[str, int] = {}
if warn_store is not None:
included: list[tuple[object, dict]] = []
for slot, row in zip(slots, slot_rows, strict=True):
blocked_holding = (
slot.status.value == "holding"
and bool(set(row.get("reconcile_warning_cases", [])) & BLOCKING_RECONCILE_CASES)
)
if blocked_holding:
excluded_count += 1
for code in row.get("reconcile_warning_cases", []):
excluded_case_counts[code] = excluded_case_counts.get(code, 0) + 1
continue
included.append((slot, row))
if included:
slots = [slot for slot, _ in included]
slot_rows = [row for _, row in included]
else:
slots = []
slot_rows = []
broker_qty_by_symbol: dict[str, int] = {}
broker_qty_available = False
@@ -410,7 +438,10 @@ def make_slots_rpc_handler(
"message": message,
}
if warn_store is not None:
base_payload["excluded_slots"] = {"reconcile_quantity_mismatch": excluded_count}
base_payload["excluded_slots"] = {
"reconcile_quantity_mismatch": excluded_count,
"active_reconcile_warning_cases": excluded_case_counts,
}
return RpcResponse(payload=base_payload)
return handler
trex_core/application/portfolio.py
+343 -18
View File
@@ -21,6 +21,8 @@ from datetime import UTC, datetime
from enum import Enum
from typing import Protocol
BLOCKING_RECONCILE_CASES = frozenset({"B", "C"})
# ---------------------------------------------------------------- portfolio-hub
@dataclass(frozen=True)
@@ -359,7 +361,7 @@ class InMemoryKillSwitchStore:
class ReconcileWarningStore(Protocol):
"""Durable reconcile-warning overlay — slot_id PK (REQ-102, dcy-034)."""
"""Durable reconcile-warning overlay keyed by (slot_id, case_code)."""
def set_warned(
self,
@@ -372,7 +374,7 @@ class ReconcileWarningStore(Protocol):
) -> None:
...
def clear_warned(self, slot_id: str) -> None:
def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None:
...
def is_warned(self, slot_id: str) -> bool:
@@ -382,10 +384,14 @@ class ReconcileWarningStore(Protocol):
...
def get_warning_info(self, slot_id: str) -> dict | None:
"""Return {"portfolio_id", "case_code", "actor", "reason"} or None.
"""Return warning info dict or None.
Used by verify to determine whether the warning is case_code='C'
(verify's responsibility) vs 'B' (sync's responsibility, REQ-106 OQ-C).
Returned dict includes:
{"portfolio_id", "case_code", "case_codes", "actor", "reason"}
where case_code is a backward-compatible preferred case and case_codes is
the full active case set for the slot.
Used by verify to determine whether C-case is active (REQ-106 OQ-C).
Returns None when the slot has no active warning.
"""
...
@@ -403,8 +409,8 @@ class InMemoryReconcileWarningStore:
def __init__(self) -> None:
self._lock = threading.Lock()
# slot_id -> {"portfolio_id", "case_code", "actor", "reason"}
self._warned: dict[str, dict] = {}
# slot_id -> case_code -> {"portfolio_id", "actor", "reason"}
self._warned: dict[str, dict[str, dict]] = {}
def set_warned(
self,
@@ -416,16 +422,24 @@ class InMemoryReconcileWarningStore:
reason: str = "",
) -> None:
with self._lock:
self._warned[slot_id] = {
per_case = self._warned.setdefault(slot_id, {})
per_case[case_code] = {
"portfolio_id": portfolio_id,
"case_code": case_code,
"actor": actor,
"reason": reason,
}
def clear_warned(self, slot_id: str) -> None:
def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None:
with self._lock:
self._warned.pop(slot_id, None)
if case_code is None:
self._warned.pop(slot_id, None)
return
per_case = self._warned.get(slot_id)
if not per_case:
return
per_case.pop(case_code, None)
if not per_case:
self._warned.pop(slot_id, None)
def is_warned(self, slot_id: str) -> bool:
with self._lock:
@@ -437,12 +451,23 @@ class InMemoryReconcileWarningStore:
def get_warning_info(self, slot_id: str) -> dict | None:
with self._lock:
info = self._warned.get(slot_id)
return dict(info) if info is not None else None
per_case = self._warned.get(slot_id)
if not per_case:
return None
case_codes = sorted(per_case.keys())
preferred_case = "C" if "C" in per_case else case_codes[0]
preferred = per_case[preferred_case]
return {
"portfolio_id": preferred.get("portfolio_id", ""),
"case_code": preferred_case,
"case_codes": case_codes,
"actor": preferred.get("actor", ""),
"reason": preferred.get("reason", ""),
}
def list_c_case_warned(self) -> set[str]:
with self._lock:
return {sid for sid, info in self._warned.items() if info.get("case_code") == "C"}
return {sid for sid, per_case in self._warned.items() if "C" in per_case}
class ReconcileNotificationStore(Protocol):
@@ -537,6 +562,291 @@ class InMemoryReconcileNotificationStore:
]
class UnmanagedPositionStore(Protocol):
"""Durable unmanaged-position lifecycle store keyed by (instance_key, symbol)."""
def observe(
self,
*,
instance_key: str,
symbol: str,
quantity: int,
observed_at: datetime | None = None,
actor: str = "reconcile",
) -> dict:
"""Return {"transition", "previous_quantity", "state"} for one observation."""
...
def resolve_absent(
self,
*,
instance_key: str,
observed_symbols: set[str],
observed_at: datetime | None = None,
actor: str = "reconcile",
) -> list[dict]:
"""Mark active positions absent from a successful broker read as resolved."""
...
def get_state(self, *, instance_key: str, symbol: str) -> dict | None:
...
def list_states(
self,
*,
instance_key: str,
lifecycle_state: str = "active",
symbol: str | None = None,
) -> list[dict]:
...
def record_operator_action(
self,
*,
instance_key: str,
symbol: str,
action: str,
evidence_ref: str,
actor: str = "operator",
note: str = "",
) -> dict | None:
"""Persist an explicit operator disposition and append an audit event."""
...
def list_audit(self, *, instance_key: str, symbol: str) -> list[dict]:
...
class InMemoryUnmanagedPositionStore:
"""Process-local unmanaged-position lifecycle store for tests + CI."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._states: dict[tuple[str, str], dict] = {}
self._audit: list[dict] = []
@staticmethod
def _latest_snapshot_at(state: dict) -> datetime:
resolved_at = state.get("resolved_at")
last_seen_at = state.get("last_seen_at")
if isinstance(resolved_at, datetime) and (
not isinstance(last_seen_at, datetime) or resolved_at >= last_seen_at
):
return resolved_at
if isinstance(last_seen_at, datetime):
return last_seen_at
return state.get("updated_at") or state.get("created_at") or datetime.min.replace(tzinfo=UTC)
def observe(
self,
*,
instance_key: str,
symbol: str,
quantity: int,
observed_at: datetime | None = None,
actor: str = "reconcile",
) -> dict:
now = observed_at or datetime.now(tz=UTC)
key = (instance_key, symbol)
with self._lock:
existing = self._states.get(key)
previous_quantity = existing.get("quantity") if existing is not None else None
preserved = {} if existing is None else {
"created_at": existing.get("created_at", now),
}
if existing is not None:
latest_snapshot_at = self._latest_snapshot_at(existing)
if now < latest_snapshot_at:
return {
"transition": "stale",
"previous_quantity": previous_quantity,
"state": dict(existing),
}
if existing is None or existing.get("lifecycle_state") == "resolved":
if existing is not None and now == self._latest_snapshot_at(existing):
return {
"transition": "stale",
"previous_quantity": previous_quantity,
"state": dict(existing),
}
transition = "detected"
state = {
"instance_key": instance_key,
"symbol": symbol,
"quantity": quantity,
"lifecycle_state": "active",
"first_seen_at": (
existing.get("first_seen_at", now) if existing is not None else now
),
"last_seen_at": now,
"quantity_changed_at": now,
"resolved_at": None,
"operator_action": None,
"evidence_ref": None,
"operator_note": "",
"operator_actor": None,
"operator_updated_at": None,
"recorded_at": now,
**preserved,
"updated_at": now,
}
self._states[key] = state
elif int(existing["quantity"]) != int(quantity):
if now <= existing["last_seen_at"]:
return {
"transition": "stale",
"previous_quantity": previous_quantity,
"state": dict(existing),
}
transition = "quantity_changed"
existing["quantity"] = quantity
existing["last_seen_at"] = now
existing["quantity_changed_at"] = now
existing["resolved_at"] = None
existing["lifecycle_state"] = "active"
existing["updated_at"] = now
state = existing
else:
transition = "unchanged"
if now > existing["last_seen_at"]:
existing["last_seen_at"] = now
existing["updated_at"] = now
state = existing
if transition != "unchanged":
self._audit.append(
{
"instance_key": instance_key,
"symbol": symbol,
"event_type": transition,
"action": None,
"quantity": quantity,
"lifecycle_state": state["lifecycle_state"],
"actor": actor,
"evidence_ref": state.get("evidence_ref"),
"note": (
f"previous_quantity={previous_quantity}"
if transition == "quantity_changed" and previous_quantity is not None
else ""
),
"recorded_at": now,
}
)
return {
"transition": transition,
"previous_quantity": previous_quantity,
"state": dict(state),
}
def resolve_absent(
self,
*,
instance_key: str,
observed_symbols: set[str],
observed_at: datetime | None = None,
actor: str = "reconcile",
) -> list[dict]:
now = observed_at or datetime.now(tz=UTC)
with self._lock:
resolved: list[dict] = []
for key, state in sorted(self._states.items()):
if key[0] != instance_key:
continue
if state.get("lifecycle_state") != "active":
continue
if state["symbol"] in observed_symbols:
continue
if now <= state["last_seen_at"]:
continue
state["lifecycle_state"] = "resolved"
state["resolved_at"] = now
state["updated_at"] = now
resolved.append(dict(state))
self._audit.append(
{
"instance_key": state["instance_key"],
"symbol": state["symbol"],
"event_type": "resolved",
"action": None,
"quantity": state["quantity"],
"lifecycle_state": "resolved",
"actor": actor,
"evidence_ref": state.get("evidence_ref"),
"note": "",
"recorded_at": now,
}
)
return resolved
def get_state(self, *, instance_key: str, symbol: str) -> dict | None:
with self._lock:
state = self._states.get((instance_key, symbol))
return dict(state) if state is not None else None
def list_states(
self,
*,
instance_key: str,
lifecycle_state: str = "active",
symbol: str | None = None,
) -> list[dict]:
with self._lock:
rows = []
for state in self._states.values():
if state["instance_key"] != instance_key:
continue
if lifecycle_state != "all" and state["lifecycle_state"] != lifecycle_state:
continue
if symbol is not None and state["symbol"] != symbol:
continue
rows.append(dict(state))
return sorted(rows, key=lambda item: item["symbol"])
def record_operator_action(
self,
*,
instance_key: str,
symbol: str,
action: str,
evidence_ref: str,
actor: str = "operator",
note: str = "",
) -> dict | None:
now = datetime.now(tz=UTC)
with self._lock:
state = self._states.get((instance_key, symbol))
if state is None:
return None
state["operator_action"] = action
state["evidence_ref"] = evidence_ref
state["operator_note"] = note
state["operator_actor"] = actor
state["operator_updated_at"] = now
state["updated_at"] = now
self._audit.append(
{
"instance_key": instance_key,
"symbol": symbol,
"event_type": "operator_action",
"action": action,
"quantity": state["quantity"],
"lifecycle_state": state["lifecycle_state"],
"actor": actor,
"evidence_ref": evidence_ref,
"note": note,
"recorded_at": now,
}
)
return dict(state)
def list_audit(self, *, instance_key: str, symbol: str) -> list[dict]:
with self._lock:
return [
dict(row)
for row in self._audit
if row["instance_key"] == instance_key and row["symbol"] == symbol
]
def slot_to_dict(
s,
*,
@@ -544,6 +854,7 @@ def slot_to_dict(
watchdog_store=None,
pause_store: PauseStore | None = None,
warn_store: ReconcileWarningStore | None = None,
warning_cases_by_slot: dict[str, set[str]] | None = None,
watchdog_timeout_sec: int = 120,
) -> dict:
"""Merged slot row for trex.slots.req (REQ-073/074).
@@ -618,10 +929,25 @@ def slot_to_dict(
# BAT-050 / REQ-104: reconcile_warned overlay (ReconcileWarningStore) — additive field.
# Bulk list_warned() called once per RPC (caller pre-fetches; see make_slots_rpc_handler).
# warn_store=None → reconcile_warned=False (backward compatible, NFR-4).
reconcile_warned = False
if warn_store is not None:
reconcile_warned = str(s.slot_id) in warn_store.list_warned()
warning_cases: set[str] = set()
if warning_cases_by_slot is not None:
warning_cases = set(warning_cases_by_slot.get(str(s.slot_id), set()))
elif warn_store is not None:
info = warn_store.get_warning_info(str(s.slot_id))
if info is not None:
raw_cases = info.get("case_codes")
if isinstance(raw_cases, list):
warning_cases = {str(c) for c in raw_cases if c}
else:
case_code = str(info.get("case_code", ""))
if case_code:
warning_cases = {case_code}
reconcile_warned = bool(warning_cases)
d["reconcile_warned"] = reconcile_warned
d["reconcile_warning_cases"] = sorted(warning_cases)
known_blocking = bool(warning_cases & BLOCKING_RECONCILE_CASES)
unknown_cases = bool(warning_cases - BLOCKING_RECONCILE_CASES)
d["trade_eligible"] = not (known_blocking or unknown_cases)
return d
@@ -637,4 +963,3 @@ def _sanitize(message: str) -> str:
out = out[:idx] + "<redacted>"
break
return out[:200]
trex_core/application/reconciliation.py
+145 -14
View File
@@ -29,11 +29,16 @@ import hashlib
import logging
import math
import re
from datetime import UTC, datetime
from typing import cast
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import ReconcileWarningStore, SnapshotStore
from trex_core.application.portfolio import (
ReconcileWarningStore,
SnapshotStore,
UnmanagedPositionStore,
)
from trex_core.domain.slot.state import SlotState
_log = logging.getLogger(__name__)
@@ -341,6 +346,37 @@ def _retry_pending_resolved(
)
def _notify_unmanaged_position_transition(
alarm_emitter: AlarmEmitter,
*,
event_type: str,
instance_key: str,
symbol: str,
quantity: int,
) -> None:
alarm_emitter.publish_alarm(
f"unmanaged_position_{event_type}",
f"unmanaged position {event_type}: symbol={symbol}",
fields={
"instance_key": instance_key,
"symbol": symbol,
"broker_quantity": quantity,
},
)
def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPort) -> bool:
marker = getattr(broker, "exposes_full_balance_snapshot", None)
if callable(marker):
try:
return bool(marker())
except Exception: # noqa: BLE001 - fail closed on broker self-report failure
return False
if isinstance(marker, bool):
return marker
return False
def reconcile_core_holdings(
*,
slot_repo,
@@ -350,6 +386,7 @@ def reconcile_core_holdings(
alarm_emitter: AlarmEmitter | None = None,
warn_store: ReconcileWarningStore | None = None,
notification_store=None,
unmanaged_position_store: UnmanagedPositionStore | None = None,
) -> dict[str, str]:
"""dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding
vs broker divergence and set durable reconcile-warning for B case.
@@ -371,12 +408,14 @@ def reconcile_core_holdings(
Returns {key: 'error'|'qty_warning'|'ok'|'unmanaged_position'|
'managed_non_holding_divergence'} for telemetry.
"""
snapshot_observed_at = datetime.now(tz=UTC)
broker_qty: dict[str, int] = {
item.symbol: int(item.hldg_qty) for item in broker.get_balance()
}
decisions: dict[str, str] = {}
matched: set[str] = set()
active_b_identity_keys: set[str] = set() # req-0007: track active B mismatches for retry
active_unmanaged_symbols: set[str] = set()
for slot in slot_repo.list_holding_slots(instance_key):
sym = slot.symbol
matched.add(sym)
@@ -458,10 +497,88 @@ def reconcile_core_holdings(
)
else:
decisions[f"broker:{sym}"] = "unmanaged_position"
_log.warning(
"unmanaged position: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
active_unmanaged_symbols.add(sym)
if unmanaged_position_store is None:
_log.warning(
"unmanaged position: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
else:
try:
observed = unmanaged_position_store.observe(
instance_key=instance_key,
symbol=sym,
quantity=bq,
observed_at=snapshot_observed_at,
)
except Exception: # noqa: BLE001
_log.exception(
"failed to persist unmanaged_position for symbol=%s instance=%s",
sym,
instance_key,
)
_log.warning(
"unmanaged position: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
else:
transition = observed["transition"]
if transition == "detected":
_log.warning(
"unmanaged position detected: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="detected",
instance_key=instance_key,
symbol=sym,
quantity=bq,
)
elif transition == "quantity_changed":
_log.warning(
"unmanaged position quantity changed: symbol=%s "
"previous_qty=%s qty=%d instance_key=%s",
sym, observed.get("previous_quantity"), bq, instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="changed",
instance_key=instance_key,
symbol=sym,
quantity=bq,
)
if (
unmanaged_position_store is not None
and _broker_balances_are_authoritative_for_unmanaged_resolution(broker)
):
try:
for state in unmanaged_position_store.resolve_absent(
instance_key=instance_key,
observed_symbols=active_unmanaged_symbols,
observed_at=snapshot_observed_at,
):
_log.info(
"unmanaged position resolved: symbol=%s qty=%d instance_key=%s",
state["symbol"], state["quantity"], instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="resolved",
instance_key=instance_key,
symbol=state["symbol"],
quantity=state["quantity"],
)
except Exception: # noqa: BLE001
_log.exception("reconcile_core_holdings: unmanaged_position resolve failed")
elif unmanaged_position_store is not None:
_log.debug(
"reconcile_core_holdings: unmanaged_position resolve skipped "
"(non-authoritative broker balance view)"
)
# req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched.
if alarm_emitter is not None and notification_store is not None:
try:
@@ -629,7 +746,7 @@ def sync_slot_to_broker(
Broker qty == 0 for slot → skipped with WARN (slot-side A-case guard).
Broker-only (no slot) → NOT handled here; caller filters (FR-3).
After update, if slot.qty == broker_qty: clear_warned(slot_id).
After update, if slot.qty == broker_qty: clear_warned(slot_id, case_code="B").
"""
from uuid import UUID
@@ -870,14 +987,15 @@ def sync_slot_to_broker(
after_avg = float(refreshed.avg_price) if (refreshed and refreshed.avg_price is not None) else None
after = {"qty": after_qty, "status": after_status, "avg_price": after_avg}
# Clear warning when qty now matches broker
# Clear B-case warning when qty now matches broker
warning_cleared = False
if after_qty is not None and int(after_qty) == broker_qty:
try:
warn_store.clear_warned(slot_id)
warn_store.clear_warned(slot_id, case_code="B")
warning_cleared = True
_log.info(
"sync: reconcile-warning cleared slot=%s symbol=%s broker_qty=%d actor=sync",
"sync: reconcile-warning cleared slot=%s symbol=%s case_code=B "
"broker_qty=%d actor=sync",
slot_id, sym, broker_qty,
)
except Exception: # noqa: BLE001
@@ -962,7 +1080,7 @@ def verify_slot_reconcile(
"reason": "no active reconcile-warning (already cleared or never set)",
}
# Check case_code — verify is C-only (REQ-106 OQ-C).
# Check case set — verify is C-only (REQ-106 OQ-C).
info = warn_store.get_warning_info(slot_id_str)
if info is None:
return {
@@ -974,8 +1092,13 @@ def verify_slot_reconcile(
"reason": "warning exists but get_warning_info returned None (unexpected)",
}
case_code = info.get("case_code", "")
if case_code != "C":
case_codes_raw = info.get("case_codes")
if isinstance(case_codes_raw, list):
case_codes = {str(code) for code in case_codes_raw if code}
else:
case_code = str(info.get("case_code", ""))
case_codes = {case_code} if case_code else set()
if "C" not in case_codes:
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
@@ -983,7 +1106,7 @@ def verify_slot_reconcile(
"hub_qty": None,
"broker_qty": None,
"reason": (
f"warning case_code='{case_code}' is not 'C' — "
f"warning case_codes={sorted(case_codes)} do not include 'C' — "
"use 'trex-core sync' to resolve B-case (slot↔broker qty mismatch)"
),
}
@@ -1014,9 +1137,17 @@ def verify_slot_reconcile(
continue
sid_str = str(s.slot_id)
warn_info = warn_store.get_warning_info(sid_str)
if warn_info is None or warn_info.get("case_code") != "C":
if warn_info is None:
continue
warn_store.clear_warned(sid_str)
warned_codes_raw = warn_info.get("case_codes")
if isinstance(warned_codes_raw, list):
warned_codes = {str(code) for code in warned_codes_raw if code}
else:
case_code = str(warn_info.get("case_code", ""))
warned_codes = {case_code} if case_code else set()
if "C" not in warned_codes:
continue
warn_store.clear_warned(sid_str, case_code="C")
store.set_warning(s.portfolio_id, False)
cleared_slots.append(sid_str)
_log.info(
trex_core/application/snapshot_scheduler.py
+33 -16
View File
@@ -12,8 +12,8 @@ from __future__ import annotations
import logging
import os
import threading
from datetime import date, datetime, time, timedelta, timezone
from typing import Callable, Optional
from collections.abc import Callable
from datetime import UTC, date, datetime, time, timedelta, timezone
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.balance_health import BalanceHealthTracker
@@ -27,6 +27,7 @@ from trex_core.application.portfolio import (
ReconcileWarningStore,
SnapshotStore,
TradingContext,
UnmanagedPositionStore,
)
from trex_core.application.reconciliation import reconcile_positions
@@ -44,45 +45,51 @@ class SnapshotScheduler:
read_port: PortfolioHubReadPort,
store: SnapshotStore,
broker: BrokerPort,
core_holdings_broker: BrokerPort | None = None,
schedule_kst: time = time(hour=7, minute=30),
clock: Optional[Callable[[], datetime]] = None,
clock: Callable[[], datetime] | None = None,
slot_repo=None,
slot_service=None,
instance_key: str = "default",
alarm_emitter: Optional[AlarmEmitter] = None,
warn_store: Optional[ReconcileWarningStore] = None,
notification_store: Optional[ReconcileNotificationStore] = None,
alarm_emitter: AlarmEmitter | None = None,
warn_store: ReconcileWarningStore | None = None,
notification_store: ReconcileNotificationStore | None = None,
unmanaged_position_store: UnmanagedPositionStore | None = None,
credential_fetch_enabled: bool = False,
kis_reinit_callback: Optional[Callable[[BrokerCredentialPayload], None]] = None,
credential_refresh: Optional[Callable[[], object]] = None,
kis_reinit_callback: Callable[[BrokerCredentialPayload], None] | None = None,
credential_refresh: Callable[[], object] | None = None,
) -> None:
self._read = read_port
self._store = store
self._broker = broker
self._core_holdings_broker = core_holdings_broker or broker
self._schedule_kst = schedule_kst
self._clock = clock or (lambda: datetime.now(tz=timezone.utc))
self._daemon_id: Optional[str] = None
self._last_fired_date: Optional[datetime] = None
self._clock = clock or (lambda: datetime.now(tz=UTC))
self._daemon_id: str | None = None
self._last_fired_date: datetime | None = None
# dcy-020 / REQ-058 (BAT-030): asset-centric slot reconcile after each
# universe fetch — always on when slot_repo is wired (the reconcile is
# what creates/retires the permanent asset slots).
self._slot_repo = slot_repo
self._slot_service = slot_service
self._instance_key = instance_key
self._last_universe_size: Optional[int] = None
self._last_universe_size: int | None = None
# dcy-033 / REQ-100: best-effort alarm emitter.
self._alarm_emitter = alarm_emitter
# BAT-050 / REQ-103: durable reconcile-warning store (Optional).
self._warn_store = warn_store
# req-0007 / ac-0024: durable notification deduplication store (Optional).
self._notification_store = notification_store
self._unmanaged_position_store = unmanaged_position_store
# dcy-036 / REQ-113/114/115: credential fetch opt-in + break-glass detection.
# credential_fetch_enabled=False (default) → env-only, no portfolio-hub fetch.
self._credential_fetch_enabled: bool = credential_fetch_enabled
# dcy-036 / REQ-113 AC-2 / REQ-115 AC-1: optional callback to reinitialize
# KISBroker with the fetched day-start credential. None = no reinit (paper
# broker or credential_fetch_enabled=False). Set by daemon.py assembly.
self._kis_reinit_callback: Optional[Callable[[BrokerCredentialPayload], None]] = kis_reinit_callback
self._kis_reinit_callback: Callable[[BrokerCredentialPayload], None] | None = (
kis_reinit_callback
)
# REQ-119: provider-owned refresh supersedes the legacy raw fetch hook.
self._credential_refresh = credential_refresh
# REQ-113: detect break-glass at init time (env var set by operator).
@@ -91,8 +98,8 @@ class SnapshotScheduler:
in ("1", "true", "yes")
)
# REQ-115: day-scoped credential cache. None until first successful fetch.
self._cached_credential: Optional[BrokerCredentialPayload] = None
self._credential_date: Optional[date] = None
self._cached_credential: BrokerCredentialPayload | None = None
self._credential_date: date | None = None
# REQ-114: entry gate. True = new entries allowed (default).
# credential_fetch_enabled=False keeps this always True (no gate).
self._entry_allowed: bool = True
@@ -265,7 +272,7 @@ class SnapshotScheduler:
try:
self._credential_refresh()
self._entry_allowed = True
except Exception as exc: # provider exposes only sanitized failure
except Exception as exc: # noqa: BLE001 - provider exposes only sanitized failure
_log.error("DAY_START_LOAD_FAILED: broker credential refresh failed: %s (instance=%s)",
exc.__class__.__name__, self._instance_key)
self._entry_allowed = False
@@ -336,6 +343,7 @@ class SnapshotScheduler:
try:
from trex_core.application.reconciliation import reconcile_core_holdings
# BAT-050 / REQ-103: pass warn_store for B-case durable warning.
<<<<<<< HEAD
self._run_balance_worker(
worker="core_holding_reconciliation",
stale_after_sec=24 * 60 * 60,
@@ -346,6 +354,15 @@ class SnapshotScheduler:
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
),
=======
reconcile_core_holdings(
slot_repo=self._slot_repo, slot_service=self._slot_service,
broker=self._core_holdings_broker, instance_key=self._instance_key,
alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
>>>>>>> origin/main
)
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")