Files
trex-core/trex_core/application/reconciliation.py
T

1190 lines
47 KiB
Python

"""Position reconciliation between portfolio-hub and broker (REQ-031, REQ-103/105/106).
The broker account is single (invariant B) so portfolio-hub quantities are
aggregated to `symbol` before being compared with `BrokerPort.get_balance()`.
Mismatches flip the asset's `warning` overlay; a subsequent successful match
clears it automatically (in-memory only — pause is the durable channel for
manual holds).
BAT-050 / REQ-103: B-case (slot qty != broker qty) and C-case (ledger/hub qty
!= broker qty) now also set a durable reconcile-warning via ReconcileWarningStore
(slot_id keyed). The in-memory warning overlay is kept for backward compat; the
durable store is additive and Optional (None => warn-log-only, backward compat).
REQ-103 case_code values in slot_reconcile_warning:
'B' — slot has a holding but slot.qty differs from broker qty
(set by reconcile_core_holdings)
'C' — portfolio-hub (ledger) qty differs from broker qty
(set by reconcile_positions)
BAT-051 / REQ-105: sync_slot_to_broker — operator CLI sync path.
BAT-051 / REQ-106: verify_slot_reconcile — operator CLI verify path (C-only, read-centric).
req-0007 / ac-0024: B/C-case mismatch alarms (detected, changed, resolved) are
published via AlarmEmitter with durable deduplication through ReconcileNotificationStore.
"""
from __future__ import annotations
import hashlib
import logging
import math
import re
from typing import cast
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import (
ReconcileWarningStore,
SnapshotStore,
UnmanagedPositionStore,
)
from trex_core.domain.slot.state import SlotState
_log = logging.getLogger(__name__)
_COST_BASIS_REASON_CODES = frozenset({
"missing",
"invalid",
"transport",
"invalid_json",
"invalid_payload",
"not_configured",
"diagnostic_exception",
"diagnostic_source_invalid",
})
_COST_BASIS_LEGACY_SOURCE_ALIASES = frozenset({"avg_cost", "wac", "cost_basis"})
def _sanitize_cost_basis_reason(reason: object) -> str | None:
"""Keep public diagnostics in the bounded REQ-123 reason taxonomy."""
if not isinstance(reason, str):
return None
if reason in _COST_BASIS_REASON_CODES:
return reason
if re.fullmatch(r"http_status_\d{3}", reason):
return reason
return None
def _normalize_cost_basis_diagnostics(
value: object,
source: object,
reason: object,
) -> tuple[float | None, str, str | None]:
"""Normalize diagnostic-port output before it reaches reconciliation results."""
safe_reason = _sanitize_cost_basis_reason(reason)
if source == "canonical":
normalized_source = "canonical"
elif source == "legacy" or source in _COST_BASIS_LEGACY_SOURCE_ALIASES:
normalized_source = "legacy"
elif source == "none":
normalized_source = "none"
else:
return None, "none", safe_reason or "diagnostic_source_invalid"
if value is None:
return None, normalized_source, safe_reason
if isinstance(value, bool):
return None, "none", safe_reason or "invalid_payload"
try:
normalized_value = float(cast(str | int | float, value))
except (TypeError, ValueError):
return None, "none", safe_reason or "invalid_payload"
if not math.isfinite(normalized_value) or normalized_value <= 0:
return None, "none", safe_reason or "invalid_payload"
if normalized_source == "none":
return None, "none", safe_reason or "invalid_payload"
return normalized_value, normalized_source, safe_reason
def _normalize_legacy_cost_basis(value: object) -> tuple[float | None, str, str | None]:
"""Make the diagnostic-free compatibility seam obey safe WAC semantics."""
normalized_value, _, _ = _normalize_cost_basis_diagnostics(value, "legacy", "invalid")
if normalized_value is None:
return None, "none", "invalid"
return normalized_value, "legacy", None
# ---------------------------------------------------------------------------
# req-0007 / ac-0024: Notification lifecycle helpers
# ---------------------------------------------------------------------------
def _c_identity_key(instance_key: str, symbol: str) -> str:
"""Return an injective, PostgreSQL TEXT-safe key for a (instance_key, symbol) C-case pair.
Uses a length-prefixed SHA-256 hex digest so that *any* inputs — including
those containing separator characters — produce distinct keys with no
collision risk. The prefix ``C:`` makes the key case-code-scoped and
human-scannable in the DB; the digest is 64 hex chars (256-bit).
Injectivity: the raw string fed to SHA-256 is ``"<N>:<instance_key>:<symbol>"``
where ``N = len(instance_key)``. The leading ``N:`` uniquely delimits
``instance_key`` from the rest, so two distinct ``(instance_key, symbol)``
pairs always produce different raw strings (and thus different digests).
Example: _c_identity_key("a\\x1fC", "b") != _c_identity_key("a", "C\\x1fb")
"""
raw = f"{len(instance_key)}:{instance_key}:{symbol}"
digest = hashlib.sha256(raw.encode()).hexdigest()
return f"C:{digest}"
def _build_alarm_fields(
*,
case_code: str,
instance_key: str,
symbol: str,
quantity_a: int,
quantity_b: int,
slot_id: str | None = None,
) -> dict:
"""Build sanitized alarm context fields (ac-0024 allowlist).
Excluded: account number/hash, credential, token, DSN, raw payload, exception.
"""
fields: dict = {
"instance_key": instance_key,
"symbol": symbol,
"case_code": case_code,
"broker_quantity": quantity_b,
}
if case_code == "B":
fields["slot_quantity"] = quantity_a
if slot_id is not None:
fields["slot_id"] = slot_id
else:
fields["hub_quantity"] = quantity_a
return fields
def _notify_mismatch(
alarm_emitter: AlarmEmitter,
notification_store,
*,
identity_key: str,
case_code: str,
instance_key: str,
symbol: str,
quantity_a: int,
quantity_b: int,
slot_id: str | None = None,
) -> None:
"""Publish detected or changed alarm; advance fingerprint only on publish success.
Suppresses if the identity fingerprint is unchanged from the last notified state.
Retries on every detection if publish previously failed (fingerprint not advanced).
"""
try:
state = notification_store.get_state(identity_key)
except Exception: # noqa: BLE001
_log.exception(
"reconcile_notification: get_state failed for %s — skipping notify", identity_key,
)
return
if state is not None and state.get("lifecycle_state") == "notified":
if state.get("quantity_a") == quantity_a and state.get("quantity_b") == quantity_b:
return # unchanged — suppress
event_type = "changed"
else:
# No prior state, OR pending_resolved (new mismatch after failed resolved) → re-detect
event_type = "detected"
fields = _build_alarm_fields(
case_code=case_code,
instance_key=instance_key,
symbol=symbol,
quantity_a=quantity_a,
quantity_b=quantity_b,
slot_id=slot_id,
)
published = alarm_emitter.publish_alarm(
f"reconcile_mismatch_{event_type}",
f"reconciliation mismatch {event_type}: {case_code}-case symbol={symbol}",
fields=fields,
)
if published:
try:
notification_store.set_state(
identity_key,
case_code=case_code,
instance_key=instance_key,
symbol=symbol,
quantity_a=quantity_a,
quantity_b=quantity_b,
lifecycle_state="notified",
)
except Exception: # noqa: BLE001
# Fingerprint write failure: at-least-once duplicate on next detection.
_log.exception(
"reconcile_notification: fingerprint persist failed for %s — "
"at-least-once duplicate possible on next detection",
identity_key,
)
def _notify_resolved(
alarm_emitter: AlarmEmitter,
notification_store,
*,
identity_key: str,
case_code: str,
instance_key: str,
symbol: str,
slot_id: str | None = None,
) -> None:
"""Publish resolved alarm; clear or mark pending_resolved on publish failure.
Warning clear is always preserved by the caller regardless of this function's outcome.
- Publish success: clear notification state.
- Publish failure + state writable: mark pending_resolved for retry.
- State persistence failure: log loss risk (business path unaffected).
"""
try:
state = notification_store.get_state(identity_key)
except Exception: # noqa: BLE001
_log.exception(
"reconcile_notification: get_state failed for %s — skipping resolved notify",
identity_key,
)
return
if state is None:
return # idempotent: already resolved or never detected
quantity_a = state["quantity_a"]
quantity_b = state["quantity_b"]
fields = _build_alarm_fields(
case_code=case_code,
instance_key=instance_key,
symbol=symbol,
quantity_a=quantity_a,
quantity_b=quantity_b,
slot_id=slot_id,
)
published = alarm_emitter.publish_alarm(
"reconcile_mismatch_resolved",
f"reconciliation mismatch resolved: {case_code}-case symbol={symbol}",
fields=fields,
)
if published:
try:
notification_store.clear_state(identity_key)
except Exception: # noqa: BLE001
# At-least-once: retry may produce duplicate resolved event.
_log.exception(
"reconcile_notification: clear_state failed for %s — "
"at-least-once duplicate resolved possible",
identity_key,
)
else:
# Publish failed — persist pending_resolved so the next opportunity retries.
try:
notification_store.set_state(
identity_key,
case_code=case_code,
instance_key=instance_key,
symbol=symbol,
quantity_a=quantity_a,
quantity_b=quantity_b,
lifecycle_state="pending_resolved",
)
except Exception: # noqa: BLE001
# State persistence failed: warning clear is kept; resolved event may be lost.
_log.error(
"reconcile_notification: resolved publish failed AND state persist failed "
"for %s — resolved event may be lost",
identity_key,
)
def _retry_pending_resolved(
alarm_emitter: AlarmEmitter,
notification_store,
*,
case_code: str,
instance_key: str,
active_identity_keys: set[str],
) -> None:
"""Retry pending_resolved publishes for identities no longer actively mismatched.
Called at the end of each reconcile pass to drain the pending_resolved queue
for identities that have since been cleared by operator sync/verify.
Only processes states belonging to ``instance_key`` so that a shared
PostgreSQL store used by multiple scheduler instances does not let instance A
re-publish resolved events for instance B's pending states.
"""
try:
pending = notification_store.list_states_by_lifecycle("pending_resolved")
except Exception: # noqa: BLE001
_log.exception(
"reconcile_notification: list_states_by_lifecycle failed — skipping pending retry",
)
return
for state in pending:
if state.get("case_code") != case_code:
continue
if state.get("instance_key") != instance_key:
continue # belongs to a different scheduler instance — do not touch
identity_key = state["identity_key"]
if identity_key in active_identity_keys:
continue # still mismatched — do not retry resolved yet
_notify_resolved(
alarm_emitter,
notification_store,
identity_key=identity_key,
case_code=state["case_code"],
instance_key=state["instance_key"],
symbol=state["symbol"],
slot_id=identity_key if case_code == "B" else None,
)
def _notify_unmanaged_position_transition(
alarm_emitter: AlarmEmitter,
*,
event_type: str,
instance_key: str,
symbol: str,
quantity: int,
) -> None:
alarm_emitter.publish_alarm(
f"unmanaged_position_{event_type}",
f"unmanaged position {event_type}: symbol={symbol}",
fields={
"instance_key": instance_key,
"symbol": symbol,
"broker_quantity": quantity,
},
)
def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPort) -> bool:
marker = getattr(broker, "exposes_full_balance_snapshot", None)
if callable(marker):
try:
return bool(marker())
except Exception: # noqa: BLE001 - fail closed on broker self-report failure
return False
if isinstance(marker, bool):
return marker
return False
def reconcile_core_holdings(
*,
slot_repo,
slot_service,
broker: BrokerPort,
instance_key: str,
alarm_emitter: AlarmEmitter | None = None,
warn_store: ReconcileWarningStore | None = None,
notification_store=None,
unmanaged_position_store: UnmanagedPositionStore | None = None,
) -> dict[str, str]:
"""dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding
vs broker divergence and set durable reconcile-warning for B case.
Broker balance is the reconcile ground truth. Decisions (no silent auto-fix,
no auto-liquidation — issue #15):
- A. core holding but broker flat (qty 0) -> slot to `error` + WARN + CRITICAL alarm
- C. both hold but qty differs (B in REQ-103 naming) -> WARN + durable reconcile-warning
- B. broker holds, no core holding slot -> WARN (unmanaged position)
REQ-103 case_code values (stored in slot_reconcile_warning.case_code):
- case_code='B': slot has a holding but slot.qty differs from broker qty
→ set by THIS function (reconcile_core_holdings)
- case_code='C': portfolio-hub (ledger) qty differs from broker qty
→ set by reconcile_positions
warn_store: Optional — when None, WARN-log-only (backward compat, REQ-103 AC-5).
notification_store: Optional — when None, alarm deduplication is skipped (backward compat).
Returns {key: 'error'|'qty_warning'|'ok'|'unmanaged_position'|
'managed_non_holding_divergence'} for telemetry.
"""
broker_qty: dict[str, int] = {
item.symbol: int(item.hldg_qty) for item in broker.get_balance()
}
decisions: dict[str, str] = {}
matched: set[str] = set()
active_b_identity_keys: set[str] = set() # req-0007: track active B mismatches for retry
active_unmanaged_symbols: set[str] = set()
for slot in slot_repo.list_holding_slots(instance_key):
sym = slot.symbol
matched.add(sym)
bq = broker_qty.get(sym, 0)
if bq == 0: # A — phantom holding
try:
slot_service.transit_slot(
slot.slot_id, SlotState.HOLDING, SlotState.ERROR,
reason="reconcile_mismatch:broker_flat", actor="reconcile",
)
decisions[str(slot.slot_id)] = "error"
_log.warning(
"core holding but broker flat: slot=%s symbol=%s -> error",
slot.slot_id, sym,
)
# dcy-033 / REQ-100: CRITICAL alarm — position mismatch.
if alarm_emitter is not None:
alarm_emitter.publish_alarm(
"core_holding_broker_flat",
f"슬롯 보유인데 브로커 잔고 0 (slot={slot.slot_id} symbol={sym})",
fields={"slot_id": str(slot.slot_id), "symbol": sym},
)
except Exception: # noqa: BLE001 - never let reconcile crash the loop
_log.exception("failed to error-quarantine slot %s", slot.slot_id)
elif slot.qty is not None and int(slot.qty) != bq: # B case (REQ-103) — qty mismatch
slot_id_str = str(slot.slot_id)
decisions[slot_id_str] = "qty_warning"
active_b_identity_keys.add(slot_id_str)
# REQ-107: structured WARNING log for B case.
_log.warning(
"reconcile_warning set: slot_id=%s portfolio_id=%s symbol=%s "
"case_code=B broker_qty=%d slot_qty=%s actor=reconcile",
slot_id_str, slot.portfolio_id, sym, bq, slot.qty,
)
# REQ-103 / BAT-050: durable reconcile-warning (case_code='B').
if warn_store is not None:
try:
warn_store.set_warned(
slot_id_str,
portfolio_id=slot.portfolio_id or "",
case_code="B",
actor="reconcile",
reason=f"slot_qty={slot.qty} broker_qty={bq} symbol={sym}",
)
except Exception: # noqa: BLE001 - never let reconcile crash the loop
_log.exception(
"failed to set reconcile_warning for slot %s", slot.slot_id
)
# req-0007 / ac-0024: publish B-case detected/changed alarm (deduped).
if alarm_emitter is not None and notification_store is not None:
try:
_notify_mismatch(
alarm_emitter,
notification_store,
identity_key=slot_id_str,
case_code="B",
instance_key=instance_key,
symbol=sym,
quantity_a=int(slot.qty),
quantity_b=bq,
slot_id=slot_id_str,
)
except Exception: # noqa: BLE001
_log.exception(
"reconcile_core_holdings: notify_mismatch failed for slot=%s", slot_id_str,
)
else:
decisions[str(slot.slot_id)] = "ok"
for sym, bq in broker_qty.items(): # unmatched broker position
if bq > 0 and sym not in matched:
managed = slot_repo.find_managed_slot_by_symbol(instance_key, sym)
if managed is not None:
decisions[f"broker:{sym}"] = "managed_non_holding_divergence"
_log.warning(
"managed non-holding divergence: slot_id=%s portfolio_id=%s "
"symbol=%s status=%s instance_key=%s",
managed.slot_id, managed.portfolio_id, sym,
managed.status.value, instance_key,
)
else:
decisions[f"broker:{sym}"] = "unmanaged_position"
active_unmanaged_symbols.add(sym)
if unmanaged_position_store is None:
_log.warning(
"unmanaged position: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
else:
try:
observed = unmanaged_position_store.observe(
instance_key=instance_key,
symbol=sym,
quantity=bq,
)
except Exception: # noqa: BLE001
_log.exception(
"failed to persist unmanaged_position for symbol=%s instance=%s",
sym,
instance_key,
)
_log.warning(
"unmanaged position: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
else:
transition = observed["transition"]
if transition == "detected":
_log.warning(
"unmanaged position detected: symbol=%s qty=%d instance_key=%s",
sym, bq, instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="detected",
instance_key=instance_key,
symbol=sym,
quantity=bq,
)
elif transition == "quantity_changed":
_log.warning(
"unmanaged position quantity changed: symbol=%s "
"previous_qty=%s qty=%d instance_key=%s",
sym, observed.get("previous_quantity"), bq, instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="changed",
instance_key=instance_key,
symbol=sym,
quantity=bq,
)
if (
unmanaged_position_store is not None
and _broker_balances_are_authoritative_for_unmanaged_resolution(broker)
):
try:
for state in unmanaged_position_store.resolve_absent(
instance_key=instance_key,
observed_symbols=active_unmanaged_symbols,
):
_log.info(
"unmanaged position resolved: symbol=%s qty=%d instance_key=%s",
state["symbol"], state["quantity"], instance_key,
)
if alarm_emitter is not None:
_notify_unmanaged_position_transition(
alarm_emitter,
event_type="resolved",
instance_key=instance_key,
symbol=state["symbol"],
quantity=state["quantity"],
)
except Exception: # noqa: BLE001
_log.exception("reconcile_core_holdings: unmanaged_position resolve failed")
elif unmanaged_position_store is not None:
_log.debug(
"reconcile_core_holdings: unmanaged_position resolve skipped "
"(non-authoritative broker balance view)"
)
# req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched.
if alarm_emitter is not None and notification_store is not None:
try:
_retry_pending_resolved(
alarm_emitter,
notification_store,
case_code="B",
instance_key=instance_key,
active_identity_keys=active_b_identity_keys,
)
except Exception: # noqa: BLE001
_log.exception("reconcile_core_holdings: pending_resolved retry failed")
return decisions
def reconcile_positions(
*,
store: SnapshotStore,
broker: BrokerPort,
slot_repo=None,
instance_key: str = "default",
warn_store: ReconcileWarningStore | None = None,
alarm_emitter: AlarmEmitter | None = None,
notification_store=None,
) -> dict[str, str]:
"""Run one reconciliation pass (REQ-031 + REQ-103 BAT-050).
Returns a `{portfolio_id: 'warning' | 'cleared' | 'ok'}` map for telemetry.
The map is also reflected onto `store` via `set_warning(...)`.
BAT-050 / REQ-103: when warn_store and slot_repo are provided, C-case mismatches
(ledger/hub qty vs broker) also set durable reconcile-warning on the corresponding
slot (portfolio_id -> slot lookup via slot_repo). When either is None, falls back
to in-memory warning only (backward compat, REQ-103 AC-5).
req-0007 / ac-0024: when alarm_emitter and notification_store are provided,
C-case detected/changed alarms are published per-symbol (one aggregate event
per symbol regardless of portfolio count).
"""
broker_qty: dict[str, int] = {
item.symbol: int(item.hldg_qty) for item in broker.get_balance()
}
# Group portfolio-hub quantities by symbol so we can compare to broker's
# per-symbol aggregate.
by_symbol: dict[str, list[tuple[str, float]]] = {}
assets = store.list_assets()
for a in assets:
sym = a.item.symbol
by_symbol.setdefault(sym, []).append((a.item.portfolio_id, a.item.asset_quantity))
decisions: dict[str, str] = {}
active_c_identity_keys: set[str] = set() # req-0007: track active C mismatches for retry
for symbol, entries in by_symbol.items():
hub_total = int(sum(qty for _, qty in entries))
broker_total = broker_qty.get(symbol, 0)
match = hub_total == broker_total
for pid, _ in entries:
asset = store.get(pid)
was_warned = bool(asset and asset.warning)
if not match:
if not was_warned:
_log.warning(
"position warning raised: portfolio_id=%s symbol=%s "
"hub=%d broker=%d", pid, symbol, hub_total, broker_total,
)
store.set_warning(pid, True)
decisions[pid] = "warning"
# REQ-103 / BAT-050: durable reconcile-warning for C case (case_code='C').
if warn_store is not None and slot_repo is not None:
try:
slot = slot_repo.find_slot_by_portfolio(instance_key, pid)
if slot is not None:
slot_id_str = str(slot.slot_id)
# REQ-107: structured WARNING log for C case.
_log.warning(
"reconcile_warning set: slot_id=%s portfolio_id=%s "
"symbol=%s case_code=C hub_qty=%d broker_qty=%d actor=reconcile",
slot_id_str, pid, symbol, hub_total, broker_total,
)
warn_store.set_warned(
slot_id_str,
portfolio_id=pid,
case_code="C",
actor="reconcile",
reason=(
f"hub_qty={hub_total} broker_qty={broker_total} "
f"symbol={symbol}"
),
)
except Exception: # noqa: BLE001
_log.exception(
"failed to set reconcile_warning for portfolio_id=%s", pid
)
else:
if was_warned:
store.set_warning(pid, False)
decisions[pid] = "cleared"
else:
decisions[pid] = "ok"
# req-0007 / ac-0024: publish one C-case detected/changed alarm per symbol
# (aggregate mismatch lifecycle — not per portfolio).
if alarm_emitter is not None and notification_store is not None:
c_key = _c_identity_key(instance_key, symbol)
if not match:
active_c_identity_keys.add(c_key)
try:
_notify_mismatch(
alarm_emitter,
notification_store,
identity_key=c_key,
case_code="C",
instance_key=instance_key,
symbol=symbol,
quantity_a=hub_total,
quantity_b=broker_total,
)
except Exception: # noqa: BLE001
_log.exception(
"reconcile_positions: notify_mismatch failed for symbol=%s", symbol,
)
# req-0007 / ac-0024: retry pending_resolved for C-case identities no longer mismatched.
if alarm_emitter is not None and notification_store is not None:
try:
_retry_pending_resolved(
alarm_emitter,
notification_store,
case_code="C",
instance_key=instance_key,
active_identity_keys=active_c_identity_keys,
)
except Exception: # noqa: BLE001
_log.exception("reconcile_positions: pending_resolved retry failed")
return decisions
# ---------------------------------------------------------------------------
# BAT-051 / REQ-105: operator sync command — slot qty ← broker
# ---------------------------------------------------------------------------
def sync_slot_to_broker(
*,
slot_id: str,
slot_repo,
slot_service,
broker: BrokerPort,
instance_key: str,
warn_store: ReconcileWarningStore,
cost_basis_port=None, # Optional[PortfolioHubCostBasisPort]
dry_run: bool = False,
alarm_emitter: AlarmEmitter | None = None,
notification_store=None,
) -> dict:
"""Sync one slot's qty to the broker ground truth (REQ-105, BAT-051).
Operator-only path (actor="sync"). No silent auto-fix — caller must confirm
before invoking (CLI --yes gate). Returns a summary dict for before/after output.
INACTIVE + broker qty > 0 → INACTIVE→HOLDING transition (OQ-A actor=sync).
HOLDING + qty mismatch → qty field updated to broker qty.
PENDING_* → skipped (in-flight, do not touch).
Broker qty == 0 for slot → skipped with WARN (slot-side A-case guard).
Broker-only (no slot) → NOT handled here; caller filters (FR-3).
After update, if slot.qty == broker_qty: clear_warned(slot_id).
"""
from uuid import UUID
try:
slot_uuid = UUID(slot_id)
except ValueError:
return {"slot_id": slot_id, "error": "invalid_slot_id", "skipped": True}
slot = slot_repo.fetch_slot(slot_uuid)
if slot is None:
return {"slot_id": slot_id, "error": "slot_not_found", "skipped": True}
sym = slot.symbol or ""
pid = slot.portfolio_id or ""
# Fetch broker balance for this symbol
broker_qty = 0
for item in broker.get_balance():
if item.symbol == sym:
broker_qty = int(item.hldg_qty)
break
before = {
"qty": float(slot.qty) if slot.qty is not None else None,
"status": slot.status.value,
"avg_price": float(slot.avg_price) if slot.avg_price is not None else None,
}
if broker_qty == 0:
_log.warning(
"sync skipped: broker_qty=0 for slot=%s symbol=%s (A-case guard — "
"use SlotService.transit_slot or manual review)",
slot_id, sym,
)
return {
"slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
"before": before,
"after": before,
"warning_cleared": False,
"dry_run": dry_run,
"skipped": True,
"skip_reason": "broker_qty_zero",
}
from trex_core.domain.slot.state import SlotState
if slot.status in (SlotState.PENDING_BUY, SlotState.PENDING_SELL):
_log.warning(
"sync skipped: slot=%s symbol=%s status=%s in-flight — "
"cannot sync while order is pending",
slot_id, sym, slot.status.value,
)
return {
"slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
"before": before,
"after": before,
"warning_cleared": False,
"dry_run": dry_run,
"skipped": True,
"skip_reason": f"status_in_flight:{slot.status.value}",
}
# Try to seed avg_price from portfolio-hub cost-basis port (FR-7, graceful degrade).
avg_price = slot.avg_price
seeded_avg_price = False
cost_basis_source = "none"
cost_basis_reason: str | None = None
def _set_cost_basis_unavailable_message() -> tuple[str, str | None]:
if pid:
return "cost_basis_unavailable", f"inactive slot {slot_id} portfolio_id={pid}"
return "cost_basis_unavailable", "inactive slot without portfolio_id"
if cost_basis_port is not None and pid:
diagnostics = getattr(cost_basis_port, "get_cost_basis_with_diagnostics", None)
if callable(diagnostics):
try:
result = diagnostics(pid)
if isinstance(result, dict):
raw_value = result.get("value")
raw_source = result.get("source")
raw_reason = result.get("reason")
elif isinstance(result, tuple) and len(result) >= 1:
raw_value = result[0]
raw_source = result[1] if len(result) >= 2 else None
raw_reason = result[2] if len(result) >= 3 else None
else:
raise TypeError("invalid_cost_basis_diagnostic_format")
cost_basis_value, cost_basis_source, cost_basis_reason = (
_normalize_cost_basis_diagnostics(raw_value, raw_source, raw_reason)
)
if cost_basis_value is not None:
avg_price = cost_basis_value
seeded_avg_price = True
cost_basis_reason = None
except Exception: # noqa: BLE001 — graceful degrade
cost_basis_source = "none"
cost_basis_reason = "diagnostic_exception"
_log.warning(
"sync: cost_basis_port.get_cost_basis_with_diagnostics failed for "
"portfolio_id=%s slot=%s — skipping avg_price seed (graceful degrade)",
pid, slot_id,
)
else:
# Backward compatibility path if diagnostic method is not available.
try:
legacy_value, legacy_source, legacy_reason = _normalize_legacy_cost_basis(
cost_basis_port.get_cost_basis(pid)
)
cost_basis_source = legacy_source
cost_basis_reason = legacy_reason
if legacy_value is not None:
avg_price = legacy_value
seeded_avg_price = True
except Exception: # noqa: BLE001 — graceful degrade
cost_basis_source = "none"
cost_basis_reason = "diagnostic_exception"
_log.warning(
"sync: cost_basis_port.get_cost_basis failed for portfolio_id=%s slot=%s "
"— skipping avg_price seed (graceful degrade)",
pid, slot_id,
)
elif pid:
cost_basis_reason = "not_configured"
_log.warning(
"sync: no cost_basis_port configured for slot=%s portfolio_id=%s "
"— skipping avg_price seed (graceful degrade)",
slot_id, pid,
)
inactive_no_cost_basis = slot.status is SlotState.INACTIVE and avg_price is None and broker_qty > 0
fail_closed_cost_basis = inactive_no_cost_basis and not dry_run
if inactive_no_cost_basis and dry_run:
return {
"slot_id": slot_id,
"symbol": sym,
"portfolio_id": pid,
"before": before,
"after": before,
"warning_cleared": False,
"dry_run": True,
"skipped": True,
"skip_reason": _set_cost_basis_unavailable_message()[0],
"seeded_avg_price": False,
"cost_basis_source": cost_basis_source,
"cost_basis_reason": cost_basis_reason,
}
if fail_closed_cost_basis:
_log.error(
"sync fail-closed: INACTIVE onboarding requires avg_price seed (portfolio_id=%s slot=%s) "
"— refuse live write",
pid,
slot_id,
)
return {
"slot_id": slot_id,
"symbol": sym,
"portfolio_id": pid,
"before": before,
"after": before,
"warning_cleared": False,
"dry_run": False,
"skipped": True,
"skip_reason": _set_cost_basis_unavailable_message()[0],
"seeded_avg_price": False,
"cost_basis_source": cost_basis_source,
"cost_basis_reason": cost_basis_reason,
"error": "cost_basis_unavailable",
}
if dry_run:
# Compute what *would* happen without writing anything.
new_status = SlotState.HOLDING if slot.status is SlotState.INACTIVE else slot.status
after = {
"qty": float(broker_qty),
"status": new_status.value,
"avg_price": float(avg_price) if avg_price is not None else None,
}
return {
"slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
"before": before,
"after": after,
"warning_cleared": False, # dry-run never clears
"dry_run": True,
"skipped": False,
"seeded_avg_price": seeded_avg_price,
"cost_basis_source": cost_basis_source,
"cost_basis_reason": cost_basis_reason,
}
# --- Live write ---
try:
if slot.status is SlotState.INACTIVE:
# OQ-A: INACTIVE→HOLDING transition (actor=sync, reason=onboarding).
slot_service.transit_slot(
slot_uuid,
SlotState.INACTIVE,
SlotState.HOLDING,
reason="onboarding",
actor="sync",
qty=float(broker_qty),
avg_price=avg_price,
portfolio_id=pid or None,
symbol=sym or None,
)
else:
# HOLDING: update qty (and optionally avg_price) in place.
# Use update_slot directly to avoid a state-machine transition.
from dataclasses import replace as _replace
updated = _replace(slot, qty=float(broker_qty), avg_price=avg_price)
slot_repo.update_slot(updated)
_log.info(
"sync updated slot qty: slot=%s symbol=%s qty %s→%d actor=sync",
slot_id, sym, slot.qty, broker_qty,
)
except Exception: # noqa: BLE001
_log.exception("sync_slot_to_broker: write failed for slot=%s", slot_id)
return {
"slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
"before": before,
"after": before,
"warning_cleared": False,
"dry_run": False,
"skipped": False,
"error": "write_failed",
"seeded_avg_price": seeded_avg_price,
"cost_basis_source": cost_basis_source,
"cost_basis_reason": cost_basis_reason,
}
# Refresh to confirm
refreshed = slot_repo.fetch_slot(slot_uuid)
after_qty = float(refreshed.qty) if (refreshed and refreshed.qty is not None) else None
after_status = refreshed.status.value if refreshed else slot.status.value
after_avg = float(refreshed.avg_price) if (refreshed and refreshed.avg_price is not None) else None
after = {"qty": after_qty, "status": after_status, "avg_price": after_avg}
# Clear warning when qty now matches broker
warning_cleared = False
if after_qty is not None and int(after_qty) == broker_qty:
try:
warn_store.clear_warned(slot_id)
warning_cleared = True
_log.info(
"sync: reconcile-warning cleared slot=%s symbol=%s broker_qty=%d actor=sync",
slot_id, sym, broker_qty,
)
except Exception: # noqa: BLE001
_log.exception("sync: failed to clear_warned for slot=%s", slot_id)
# req-0007 / ac-0024: publish B-case resolved alarm only after warning is cleared.
if warning_cleared and alarm_emitter is not None and notification_store is not None:
try:
_notify_resolved(
alarm_emitter,
notification_store,
identity_key=slot_id,
case_code="B",
instance_key=instance_key,
symbol=sym,
slot_id=slot_id,
)
except Exception: # noqa: BLE001
_log.exception(
"sync: resolved notify failed for slot=%s — warning clear kept", slot_id,
)
return {
"slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
"before": before,
"after": after,
"warning_cleared": warning_cleared,
"dry_run": False,
"skipped": False,
"seeded_avg_price": seeded_avg_price,
"cost_basis_source": cost_basis_source,
"cost_basis_reason": cost_basis_reason,
}
# ---------------------------------------------------------------------------
# BAT-051 / REQ-106: operator verify command — C-case re-check (read-centric)
# ---------------------------------------------------------------------------
def verify_slot_reconcile(
*,
portfolio_id: str,
store: SnapshotStore,
broker: BrokerPort,
slot_repo,
instance_key: str,
warn_store: ReconcileWarningStore,
alarm_emitter: AlarmEmitter | None = None,
notification_store=None,
) -> dict:
"""Re-check C-case reconcile-warning for a portfolio (REQ-106, BAT-051).
Read-centric: does NOT modify slot qty. Clears warning only when re-check
confirms ledger qty == broker qty (no force-clear path, OQ-C confirmed).
result values:
"cleared" — re-check passed, warning cleared.
"rejected" — re-check failed (still mismatched), warning NOT cleared.
"not_c_case" — slot is warned but not case_code='C' (use sync for B case).
"not_warned" — slot has no active warning (idempotent, exit 0).
"slot_not_found" — no slot for this portfolio_id in this instance.
"no_warning_info" — is_warned but get_warning_info returned None (unexpected).
"""
slot = slot_repo.find_slot_by_portfolio(instance_key, portfolio_id)
if slot is None:
return {
"portfolio_id": portfolio_id,
"slot_id": None,
"result": "slot_not_found",
"hub_qty": None,
"broker_qty": None,
"reason": f"no slot for portfolio_id={portfolio_id} in instance={instance_key}",
}
slot_id_str = str(slot.slot_id)
if not warn_store.is_warned(slot_id_str):
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "not_warned",
"hub_qty": None,
"broker_qty": None,
"reason": "no active reconcile-warning (already cleared or never set)",
}
# Check case_code — verify is C-only (REQ-106 OQ-C).
info = warn_store.get_warning_info(slot_id_str)
if info is None:
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "no_warning_info",
"hub_qty": None,
"broker_qty": None,
"reason": "warning exists but get_warning_info returned None (unexpected)",
}
case_code = info.get("case_code", "")
if case_code != "C":
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "not_c_case",
"hub_qty": None,
"broker_qty": None,
"reason": (
f"warning case_code='{case_code}' is not 'C' — "
"use 'trex-core sync' to resolve B-case (slot↔broker qty mismatch)"
),
}
# Re-run the C-case check: hub qty vs broker qty for this symbol.
# AC-0023: use symbol-level aggregate across all portfolios (not single portfolio).
sym = slot.symbol or ""
broker_qty_map: dict[str, int] = {
item.symbol: int(item.hldg_qty) for item in broker.get_balance()
}
broker_qty = broker_qty_map.get(sym, 0)
# Hub qty: sum asset_quantity for ALL portfolios with this symbol in the snapshot.
hub_qty: int = 0
for a in store.list_assets():
if a.item.symbol == sym:
hub_qty += int(a.item.asset_quantity)
match = hub_qty == broker_qty
if match:
# AC-0023: clear C-case warnings for ALL slots (any status) with this symbol.
# Only rows with case_code='C' are touched; B-case (or other) rows are preserved.
cleared_slots: list[str] = []
try:
for s in slot_repo.list_slots(instance_key):
if s.symbol != sym:
continue
sid_str = str(s.slot_id)
warn_info = warn_store.get_warning_info(sid_str)
if warn_info is None or warn_info.get("case_code") != "C":
continue
warn_store.clear_warned(sid_str)
store.set_warning(s.portfolio_id, False)
cleared_slots.append(sid_str)
_log.info(
"verify: reconcile-warning cleared (C-case resolved) "
"symbol=%s hub_qty=%d broker_qty=%d cleared_slots=%s",
sym, hub_qty, broker_qty, cleared_slots,
)
except Exception: # noqa: BLE001
_log.exception("verify: failed to clear C-case warned for symbol=%s", sym)
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "clear_failed",
"hub_qty": hub_qty,
"broker_qty": broker_qty,
"reason": "warn_store error during C-case clear — warnings NOT fully cleared",
}
# req-0007 / ac-0024: publish C-case resolved alarm (best-effort, warning clear kept).
if alarm_emitter is not None and notification_store is not None:
try:
_notify_resolved(
alarm_emitter,
notification_store,
identity_key=_c_identity_key(instance_key, sym),
case_code="C",
instance_key=instance_key,
symbol=sym,
)
except Exception: # noqa: BLE001
_log.exception(
"verify: resolved notify failed for symbol=%s — warning clear kept", sym,
)
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "cleared",
"hub_qty": hub_qty,
"broker_qty": broker_qty,
"reason": None,
}
else:
_log.warning(
"verify: C-case still mismatched — slot=%s portfolio_id=%s symbol=%s "
"hub_qty=%d broker_qty=%d diff=%d — warning NOT cleared",
slot_id_str, portfolio_id, sym, hub_qty, broker_qty, broker_qty - hub_qty,
)
return {
"portfolio_id": portfolio_id,
"slot_id": slot_id_str,
"result": "rejected",
"hub_qty": hub_qty,
"broker_qty": broker_qty,
"reason": (
f"hub_qty={hub_qty} broker_qty={broker_qty} "
f"diff={broker_qty - hub_qty} symbol={sym}"
),
}