"""Position reconciliation between portfolio-hub and broker (REQ-031, REQ-103/105/106). The broker account is single (invariant B) so portfolio-hub quantities are aggregated to `symbol` before being compared with `BrokerPort.get_balance()`. Mismatches flip the asset's `warning` overlay; a subsequent successful match clears it automatically (in-memory only — pause is the durable channel for manual holds). BAT-050 / REQ-103: B-case (slot qty != broker qty) and C-case (ledger/hub qty != broker qty) now also set a durable reconcile-warning via ReconcileWarningStore (slot_id keyed). The in-memory warning overlay is kept for backward compat; the durable store is additive and Optional (None => warn-log-only, backward compat). REQ-103 case_code values in slot_reconcile_warning: 'B' — slot has a holding but slot.qty differs from broker qty (set by reconcile_core_holdings) 'C' — portfolio-hub (ledger) qty differs from broker qty (set by reconcile_positions) BAT-051 / REQ-105: sync_slot_to_broker — operator CLI sync path. BAT-051 / REQ-106: verify_slot_reconcile — operator CLI verify path (C-only, read-centric). req-0007 / ac-0024: B/C-case mismatch alarms (detected, changed, resolved) are published via AlarmEmitter with durable deduplication through ReconcileNotificationStore. """ from __future__ import annotations import hashlib import logging import math import re from typing import cast from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.broker_port import BrokerPort from trex_core.application.portfolio import ( ReconcileWarningStore, SnapshotStore, UnmanagedPositionStore, ) from trex_core.domain.slot.state import SlotState _log = logging.getLogger(__name__) _COST_BASIS_REASON_CODES = frozenset({ "missing", "invalid", "transport", "invalid_json", "invalid_payload", "not_configured", "diagnostic_exception", "diagnostic_source_invalid", }) _COST_BASIS_LEGACY_SOURCE_ALIASES = frozenset({"avg_cost", "wac", "cost_basis"}) def _sanitize_cost_basis_reason(reason: object) -> str | None: """Keep public diagnostics in the bounded REQ-123 reason taxonomy.""" if not isinstance(reason, str): return None if reason in _COST_BASIS_REASON_CODES: return reason if re.fullmatch(r"http_status_\d{3}", reason): return reason return None def _normalize_cost_basis_diagnostics( value: object, source: object, reason: object, ) -> tuple[float | None, str, str | None]: """Normalize diagnostic-port output before it reaches reconciliation results.""" safe_reason = _sanitize_cost_basis_reason(reason) if source == "canonical": normalized_source = "canonical" elif source == "legacy" or source in _COST_BASIS_LEGACY_SOURCE_ALIASES: normalized_source = "legacy" elif source == "none": normalized_source = "none" else: return None, "none", safe_reason or "diagnostic_source_invalid" if value is None: return None, normalized_source, safe_reason if isinstance(value, bool): return None, "none", safe_reason or "invalid_payload" try: normalized_value = float(cast(str | int | float, value)) except (TypeError, ValueError): return None, "none", safe_reason or "invalid_payload" if not math.isfinite(normalized_value) or normalized_value <= 0: return None, "none", safe_reason or "invalid_payload" if normalized_source == "none": return None, "none", safe_reason or "invalid_payload" return normalized_value, normalized_source, safe_reason def _normalize_legacy_cost_basis(value: object) -> tuple[float | None, str, str | None]: """Make the diagnostic-free compatibility seam obey safe WAC semantics.""" normalized_value, _, _ = _normalize_cost_basis_diagnostics(value, "legacy", "invalid") if normalized_value is None: return None, "none", "invalid" return normalized_value, "legacy", None # --------------------------------------------------------------------------- # req-0007 / ac-0024: Notification lifecycle helpers # --------------------------------------------------------------------------- def _c_identity_key(instance_key: str, symbol: str) -> str: """Return an injective, PostgreSQL TEXT-safe key for a (instance_key, symbol) C-case pair. Uses a length-prefixed SHA-256 hex digest so that *any* inputs — including those containing separator characters — produce distinct keys with no collision risk. The prefix ``C:`` makes the key case-code-scoped and human-scannable in the DB; the digest is 64 hex chars (256-bit). Injectivity: the raw string fed to SHA-256 is ``"::"`` where ``N = len(instance_key)``. The leading ``N:`` uniquely delimits ``instance_key`` from the rest, so two distinct ``(instance_key, symbol)`` pairs always produce different raw strings (and thus different digests). Example: _c_identity_key("a\\x1fC", "b") != _c_identity_key("a", "C\\x1fb") """ raw = f"{len(instance_key)}:{instance_key}:{symbol}" digest = hashlib.sha256(raw.encode()).hexdigest() return f"C:{digest}" def _build_alarm_fields( *, case_code: str, instance_key: str, symbol: str, quantity_a: int, quantity_b: int, slot_id: str | None = None, ) -> dict: """Build sanitized alarm context fields (ac-0024 allowlist). Excluded: account number/hash, credential, token, DSN, raw payload, exception. """ fields: dict = { "instance_key": instance_key, "symbol": symbol, "case_code": case_code, "broker_quantity": quantity_b, } if case_code == "B": fields["slot_quantity"] = quantity_a if slot_id is not None: fields["slot_id"] = slot_id else: fields["hub_quantity"] = quantity_a return fields def _notify_mismatch( alarm_emitter: AlarmEmitter, notification_store, *, identity_key: str, case_code: str, instance_key: str, symbol: str, quantity_a: int, quantity_b: int, slot_id: str | None = None, ) -> None: """Publish detected or changed alarm; advance fingerprint only on publish success. Suppresses if the identity fingerprint is unchanged from the last notified state. Retries on every detection if publish previously failed (fingerprint not advanced). """ try: state = notification_store.get_state(identity_key) except Exception: # noqa: BLE001 _log.exception( "reconcile_notification: get_state failed for %s — skipping notify", identity_key, ) return if state is not None and state.get("lifecycle_state") == "notified": if state.get("quantity_a") == quantity_a and state.get("quantity_b") == quantity_b: return # unchanged — suppress event_type = "changed" else: # No prior state, OR pending_resolved (new mismatch after failed resolved) → re-detect event_type = "detected" fields = _build_alarm_fields( case_code=case_code, instance_key=instance_key, symbol=symbol, quantity_a=quantity_a, quantity_b=quantity_b, slot_id=slot_id, ) published = alarm_emitter.publish_alarm( f"reconcile_mismatch_{event_type}", f"reconciliation mismatch {event_type}: {case_code}-case symbol={symbol}", fields=fields, ) if published: try: notification_store.set_state( identity_key, case_code=case_code, instance_key=instance_key, symbol=symbol, quantity_a=quantity_a, quantity_b=quantity_b, lifecycle_state="notified", ) except Exception: # noqa: BLE001 # Fingerprint write failure: at-least-once duplicate on next detection. _log.exception( "reconcile_notification: fingerprint persist failed for %s — " "at-least-once duplicate possible on next detection", identity_key, ) def _notify_resolved( alarm_emitter: AlarmEmitter, notification_store, *, identity_key: str, case_code: str, instance_key: str, symbol: str, slot_id: str | None = None, ) -> None: """Publish resolved alarm; clear or mark pending_resolved on publish failure. Warning clear is always preserved by the caller regardless of this function's outcome. - Publish success: clear notification state. - Publish failure + state writable: mark pending_resolved for retry. - State persistence failure: log loss risk (business path unaffected). """ try: state = notification_store.get_state(identity_key) except Exception: # noqa: BLE001 _log.exception( "reconcile_notification: get_state failed for %s — skipping resolved notify", identity_key, ) return if state is None: return # idempotent: already resolved or never detected quantity_a = state["quantity_a"] quantity_b = state["quantity_b"] fields = _build_alarm_fields( case_code=case_code, instance_key=instance_key, symbol=symbol, quantity_a=quantity_a, quantity_b=quantity_b, slot_id=slot_id, ) published = alarm_emitter.publish_alarm( "reconcile_mismatch_resolved", f"reconciliation mismatch resolved: {case_code}-case symbol={symbol}", fields=fields, ) if published: try: notification_store.clear_state(identity_key) except Exception: # noqa: BLE001 # At-least-once: retry may produce duplicate resolved event. _log.exception( "reconcile_notification: clear_state failed for %s — " "at-least-once duplicate resolved possible", identity_key, ) else: # Publish failed — persist pending_resolved so the next opportunity retries. try: notification_store.set_state( identity_key, case_code=case_code, instance_key=instance_key, symbol=symbol, quantity_a=quantity_a, quantity_b=quantity_b, lifecycle_state="pending_resolved", ) except Exception: # noqa: BLE001 # State persistence failed: warning clear is kept; resolved event may be lost. _log.error( "reconcile_notification: resolved publish failed AND state persist failed " "for %s — resolved event may be lost", identity_key, ) def _retry_pending_resolved( alarm_emitter: AlarmEmitter, notification_store, *, case_code: str, instance_key: str, active_identity_keys: set[str], ) -> None: """Retry pending_resolved publishes for identities no longer actively mismatched. Called at the end of each reconcile pass to drain the pending_resolved queue for identities that have since been cleared by operator sync/verify. Only processes states belonging to ``instance_key`` so that a shared PostgreSQL store used by multiple scheduler instances does not let instance A re-publish resolved events for instance B's pending states. """ try: pending = notification_store.list_states_by_lifecycle("pending_resolved") except Exception: # noqa: BLE001 _log.exception( "reconcile_notification: list_states_by_lifecycle failed — skipping pending retry", ) return for state in pending: if state.get("case_code") != case_code: continue if state.get("instance_key") != instance_key: continue # belongs to a different scheduler instance — do not touch identity_key = state["identity_key"] if identity_key in active_identity_keys: continue # still mismatched — do not retry resolved yet _notify_resolved( alarm_emitter, notification_store, identity_key=identity_key, case_code=state["case_code"], instance_key=state["instance_key"], symbol=state["symbol"], slot_id=identity_key if case_code == "B" else None, ) def _notify_unmanaged_position_transition( alarm_emitter: AlarmEmitter, *, event_type: str, instance_key: str, symbol: str, quantity: int, ) -> None: alarm_emitter.publish_alarm( f"unmanaged_position_{event_type}", f"unmanaged position {event_type}: symbol={symbol}", fields={ "instance_key": instance_key, "symbol": symbol, "broker_quantity": quantity, }, ) def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPort) -> bool: marker = getattr(broker, "exposes_full_balance_snapshot", None) if callable(marker): try: return bool(marker()) except Exception: # noqa: BLE001 - fail closed on broker self-report failure return False if isinstance(marker, bool): return marker return False def reconcile_core_holdings( *, slot_repo, slot_service, broker: BrokerPort, instance_key: str, alarm_emitter: AlarmEmitter | None = None, warn_store: ReconcileWarningStore | None = None, notification_store=None, unmanaged_position_store: UnmanagedPositionStore | None = None, ) -> dict[str, str]: """dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding vs broker divergence and set durable reconcile-warning for B case. Broker balance is the reconcile ground truth. Decisions (no silent auto-fix, no auto-liquidation — issue #15): - A. core holding but broker flat (qty 0) -> slot to `error` + WARN + CRITICAL alarm - C. both hold but qty differs (B in REQ-103 naming) -> WARN + durable reconcile-warning - B. broker holds, no core holding slot -> WARN (unmanaged position) REQ-103 case_code values (stored in slot_reconcile_warning.case_code): - case_code='B': slot has a holding but slot.qty differs from broker qty → set by THIS function (reconcile_core_holdings) - case_code='C': portfolio-hub (ledger) qty differs from broker qty → set by reconcile_positions warn_store: Optional — when None, WARN-log-only (backward compat, REQ-103 AC-5). notification_store: Optional — when None, alarm deduplication is skipped (backward compat). Returns {key: 'error'|'qty_warning'|'ok'|'unmanaged_position'| 'managed_non_holding_divergence'} for telemetry. """ broker_qty: dict[str, int] = { item.symbol: int(item.hldg_qty) for item in broker.get_balance() } decisions: dict[str, str] = {} matched: set[str] = set() active_b_identity_keys: set[str] = set() # req-0007: track active B mismatches for retry active_unmanaged_symbols: set[str] = set() for slot in slot_repo.list_holding_slots(instance_key): sym = slot.symbol matched.add(sym) bq = broker_qty.get(sym, 0) if bq == 0: # A — phantom holding try: slot_service.transit_slot( slot.slot_id, SlotState.HOLDING, SlotState.ERROR, reason="reconcile_mismatch:broker_flat", actor="reconcile", ) decisions[str(slot.slot_id)] = "error" _log.warning( "core holding but broker flat: slot=%s symbol=%s -> error", slot.slot_id, sym, ) # dcy-033 / REQ-100: CRITICAL alarm — position mismatch. if alarm_emitter is not None: alarm_emitter.publish_alarm( "core_holding_broker_flat", f"슬롯 보유인데 브로커 잔고 0 (slot={slot.slot_id} symbol={sym})", fields={"slot_id": str(slot.slot_id), "symbol": sym}, ) except Exception: # noqa: BLE001 - never let reconcile crash the loop _log.exception("failed to error-quarantine slot %s", slot.slot_id) elif slot.qty is not None and int(slot.qty) != bq: # B case (REQ-103) — qty mismatch slot_id_str = str(slot.slot_id) decisions[slot_id_str] = "qty_warning" active_b_identity_keys.add(slot_id_str) # REQ-107: structured WARNING log for B case. _log.warning( "reconcile_warning set: slot_id=%s portfolio_id=%s symbol=%s " "case_code=B broker_qty=%d slot_qty=%s actor=reconcile", slot_id_str, slot.portfolio_id, sym, bq, slot.qty, ) # REQ-103 / BAT-050: durable reconcile-warning (case_code='B'). if warn_store is not None: try: warn_store.set_warned( slot_id_str, portfolio_id=slot.portfolio_id or "", case_code="B", actor="reconcile", reason=f"slot_qty={slot.qty} broker_qty={bq} symbol={sym}", ) except Exception: # noqa: BLE001 - never let reconcile crash the loop _log.exception( "failed to set reconcile_warning for slot %s", slot.slot_id ) # req-0007 / ac-0024: publish B-case detected/changed alarm (deduped). if alarm_emitter is not None and notification_store is not None: try: _notify_mismatch( alarm_emitter, notification_store, identity_key=slot_id_str, case_code="B", instance_key=instance_key, symbol=sym, quantity_a=int(slot.qty), quantity_b=bq, slot_id=slot_id_str, ) except Exception: # noqa: BLE001 _log.exception( "reconcile_core_holdings: notify_mismatch failed for slot=%s", slot_id_str, ) else: decisions[str(slot.slot_id)] = "ok" for sym, bq in broker_qty.items(): # unmatched broker position if bq > 0 and sym not in matched: managed = slot_repo.find_managed_slot_by_symbol(instance_key, sym) if managed is not None: decisions[f"broker:{sym}"] = "managed_non_holding_divergence" _log.warning( "managed non-holding divergence: slot_id=%s portfolio_id=%s " "symbol=%s status=%s instance_key=%s", managed.slot_id, managed.portfolio_id, sym, managed.status.value, instance_key, ) else: decisions[f"broker:{sym}"] = "unmanaged_position" active_unmanaged_symbols.add(sym) if unmanaged_position_store is None: _log.warning( "unmanaged position: symbol=%s qty=%d instance_key=%s", sym, bq, instance_key, ) else: try: observed = unmanaged_position_store.observe( instance_key=instance_key, symbol=sym, quantity=bq, ) except Exception: # noqa: BLE001 _log.exception( "failed to persist unmanaged_position for symbol=%s instance=%s", sym, instance_key, ) _log.warning( "unmanaged position: symbol=%s qty=%d instance_key=%s", sym, bq, instance_key, ) else: transition = observed["transition"] if transition == "detected": _log.warning( "unmanaged position detected: symbol=%s qty=%d instance_key=%s", sym, bq, instance_key, ) if alarm_emitter is not None: _notify_unmanaged_position_transition( alarm_emitter, event_type="detected", instance_key=instance_key, symbol=sym, quantity=bq, ) elif transition == "quantity_changed": _log.warning( "unmanaged position quantity changed: symbol=%s " "previous_qty=%s qty=%d instance_key=%s", sym, observed.get("previous_quantity"), bq, instance_key, ) if alarm_emitter is not None: _notify_unmanaged_position_transition( alarm_emitter, event_type="changed", instance_key=instance_key, symbol=sym, quantity=bq, ) if ( unmanaged_position_store is not None and _broker_balances_are_authoritative_for_unmanaged_resolution(broker) ): try: for state in unmanaged_position_store.resolve_absent( instance_key=instance_key, observed_symbols=active_unmanaged_symbols, ): _log.info( "unmanaged position resolved: symbol=%s qty=%d instance_key=%s", state["symbol"], state["quantity"], instance_key, ) if alarm_emitter is not None: _notify_unmanaged_position_transition( alarm_emitter, event_type="resolved", instance_key=instance_key, symbol=state["symbol"], quantity=state["quantity"], ) except Exception: # noqa: BLE001 _log.exception("reconcile_core_holdings: unmanaged_position resolve failed") elif unmanaged_position_store is not None: _log.debug( "reconcile_core_holdings: unmanaged_position resolve skipped " "(non-authoritative broker balance view)" ) # req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched. if alarm_emitter is not None and notification_store is not None: try: _retry_pending_resolved( alarm_emitter, notification_store, case_code="B", instance_key=instance_key, active_identity_keys=active_b_identity_keys, ) except Exception: # noqa: BLE001 _log.exception("reconcile_core_holdings: pending_resolved retry failed") return decisions def reconcile_positions( *, store: SnapshotStore, broker: BrokerPort, slot_repo=None, instance_key: str = "default", warn_store: ReconcileWarningStore | None = None, alarm_emitter: AlarmEmitter | None = None, notification_store=None, ) -> dict[str, str]: """Run one reconciliation pass (REQ-031 + REQ-103 BAT-050). Returns a `{portfolio_id: 'warning' | 'cleared' | 'ok'}` map for telemetry. The map is also reflected onto `store` via `set_warning(...)`. BAT-050 / REQ-103: when warn_store and slot_repo are provided, C-case mismatches (ledger/hub qty vs broker) also set durable reconcile-warning on the corresponding slot (portfolio_id -> slot lookup via slot_repo). When either is None, falls back to in-memory warning only (backward compat, REQ-103 AC-5). req-0007 / ac-0024: when alarm_emitter and notification_store are provided, C-case detected/changed alarms are published per-symbol (one aggregate event per symbol regardless of portfolio count). """ broker_qty: dict[str, int] = { item.symbol: int(item.hldg_qty) for item in broker.get_balance() } # Group portfolio-hub quantities by symbol so we can compare to broker's # per-symbol aggregate. by_symbol: dict[str, list[tuple[str, float]]] = {} assets = store.list_assets() for a in assets: sym = a.item.symbol by_symbol.setdefault(sym, []).append((a.item.portfolio_id, a.item.asset_quantity)) decisions: dict[str, str] = {} active_c_identity_keys: set[str] = set() # req-0007: track active C mismatches for retry for symbol, entries in by_symbol.items(): hub_total = int(sum(qty for _, qty in entries)) broker_total = broker_qty.get(symbol, 0) match = hub_total == broker_total for pid, _ in entries: asset = store.get(pid) was_warned = bool(asset and asset.warning) if not match: if not was_warned: _log.warning( "position warning raised: portfolio_id=%s symbol=%s " "hub=%d broker=%d", pid, symbol, hub_total, broker_total, ) store.set_warning(pid, True) decisions[pid] = "warning" # REQ-103 / BAT-050: durable reconcile-warning for C case (case_code='C'). if warn_store is not None and slot_repo is not None: try: slot = slot_repo.find_slot_by_portfolio(instance_key, pid) if slot is not None: slot_id_str = str(slot.slot_id) # REQ-107: structured WARNING log for C case. _log.warning( "reconcile_warning set: slot_id=%s portfolio_id=%s " "symbol=%s case_code=C hub_qty=%d broker_qty=%d actor=reconcile", slot_id_str, pid, symbol, hub_total, broker_total, ) warn_store.set_warned( slot_id_str, portfolio_id=pid, case_code="C", actor="reconcile", reason=( f"hub_qty={hub_total} broker_qty={broker_total} " f"symbol={symbol}" ), ) except Exception: # noqa: BLE001 _log.exception( "failed to set reconcile_warning for portfolio_id=%s", pid ) else: if was_warned: store.set_warning(pid, False) decisions[pid] = "cleared" else: decisions[pid] = "ok" # req-0007 / ac-0024: publish one C-case detected/changed alarm per symbol # (aggregate mismatch lifecycle — not per portfolio). if alarm_emitter is not None and notification_store is not None: c_key = _c_identity_key(instance_key, symbol) if not match: active_c_identity_keys.add(c_key) try: _notify_mismatch( alarm_emitter, notification_store, identity_key=c_key, case_code="C", instance_key=instance_key, symbol=symbol, quantity_a=hub_total, quantity_b=broker_total, ) except Exception: # noqa: BLE001 _log.exception( "reconcile_positions: notify_mismatch failed for symbol=%s", symbol, ) # req-0007 / ac-0024: retry pending_resolved for C-case identities no longer mismatched. if alarm_emitter is not None and notification_store is not None: try: _retry_pending_resolved( alarm_emitter, notification_store, case_code="C", instance_key=instance_key, active_identity_keys=active_c_identity_keys, ) except Exception: # noqa: BLE001 _log.exception("reconcile_positions: pending_resolved retry failed") return decisions # --------------------------------------------------------------------------- # BAT-051 / REQ-105: operator sync command — slot qty ← broker # --------------------------------------------------------------------------- def sync_slot_to_broker( *, slot_id: str, slot_repo, slot_service, broker: BrokerPort, instance_key: str, warn_store: ReconcileWarningStore, cost_basis_port=None, # Optional[PortfolioHubCostBasisPort] dry_run: bool = False, alarm_emitter: AlarmEmitter | None = None, notification_store=None, ) -> dict: """Sync one slot's qty to the broker ground truth (REQ-105, BAT-051). Operator-only path (actor="sync"). No silent auto-fix — caller must confirm before invoking (CLI --yes gate). Returns a summary dict for before/after output. INACTIVE + broker qty > 0 → INACTIVE→HOLDING transition (OQ-A actor=sync). HOLDING + qty mismatch → qty field updated to broker qty. PENDING_* → skipped (in-flight, do not touch). Broker qty == 0 for slot → skipped with WARN (slot-side A-case guard). Broker-only (no slot) → NOT handled here; caller filters (FR-3). After update, if slot.qty == broker_qty: clear_warned(slot_id, case_code="B"). """ from uuid import UUID try: slot_uuid = UUID(slot_id) except ValueError: return {"slot_id": slot_id, "error": "invalid_slot_id", "skipped": True} slot = slot_repo.fetch_slot(slot_uuid) if slot is None: return {"slot_id": slot_id, "error": "slot_not_found", "skipped": True} sym = slot.symbol or "" pid = slot.portfolio_id or "" # Fetch broker balance for this symbol broker_qty = 0 for item in broker.get_balance(): if item.symbol == sym: broker_qty = int(item.hldg_qty) break before = { "qty": float(slot.qty) if slot.qty is not None else None, "status": slot.status.value, "avg_price": float(slot.avg_price) if slot.avg_price is not None else None, } if broker_qty == 0: _log.warning( "sync skipped: broker_qty=0 for slot=%s symbol=%s (A-case guard — " "use SlotService.transit_slot or manual review)", slot_id, sym, ) return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": before, "warning_cleared": False, "dry_run": dry_run, "skipped": True, "skip_reason": "broker_qty_zero", } from trex_core.domain.slot.state import SlotState if slot.status in (SlotState.PENDING_BUY, SlotState.PENDING_SELL): _log.warning( "sync skipped: slot=%s symbol=%s status=%s in-flight — " "cannot sync while order is pending", slot_id, sym, slot.status.value, ) return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": before, "warning_cleared": False, "dry_run": dry_run, "skipped": True, "skip_reason": f"status_in_flight:{slot.status.value}", } # Try to seed avg_price from portfolio-hub cost-basis port (FR-7, graceful degrade). avg_price = slot.avg_price seeded_avg_price = False cost_basis_source = "none" cost_basis_reason: str | None = None def _set_cost_basis_unavailable_message() -> tuple[str, str | None]: if pid: return "cost_basis_unavailable", f"inactive slot {slot_id} portfolio_id={pid}" return "cost_basis_unavailable", "inactive slot without portfolio_id" if cost_basis_port is not None and pid: diagnostics = getattr(cost_basis_port, "get_cost_basis_with_diagnostics", None) if callable(diagnostics): try: result = diagnostics(pid) if isinstance(result, dict): raw_value = result.get("value") raw_source = result.get("source") raw_reason = result.get("reason") elif isinstance(result, tuple) and len(result) >= 1: raw_value = result[0] raw_source = result[1] if len(result) >= 2 else None raw_reason = result[2] if len(result) >= 3 else None else: raise TypeError("invalid_cost_basis_diagnostic_format") cost_basis_value, cost_basis_source, cost_basis_reason = ( _normalize_cost_basis_diagnostics(raw_value, raw_source, raw_reason) ) if cost_basis_value is not None: avg_price = cost_basis_value seeded_avg_price = True cost_basis_reason = None except Exception: # noqa: BLE001 — graceful degrade cost_basis_source = "none" cost_basis_reason = "diagnostic_exception" _log.warning( "sync: cost_basis_port.get_cost_basis_with_diagnostics failed for " "portfolio_id=%s slot=%s — skipping avg_price seed (graceful degrade)", pid, slot_id, ) else: # Backward compatibility path if diagnostic method is not available. try: legacy_value, legacy_source, legacy_reason = _normalize_legacy_cost_basis( cost_basis_port.get_cost_basis(pid) ) cost_basis_source = legacy_source cost_basis_reason = legacy_reason if legacy_value is not None: avg_price = legacy_value seeded_avg_price = True except Exception: # noqa: BLE001 — graceful degrade cost_basis_source = "none" cost_basis_reason = "diagnostic_exception" _log.warning( "sync: cost_basis_port.get_cost_basis failed for portfolio_id=%s slot=%s " "— skipping avg_price seed (graceful degrade)", pid, slot_id, ) elif pid: cost_basis_reason = "not_configured" _log.warning( "sync: no cost_basis_port configured for slot=%s portfolio_id=%s " "— skipping avg_price seed (graceful degrade)", slot_id, pid, ) inactive_no_cost_basis = slot.status is SlotState.INACTIVE and avg_price is None and broker_qty > 0 fail_closed_cost_basis = inactive_no_cost_basis and not dry_run if inactive_no_cost_basis and dry_run: return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": before, "warning_cleared": False, "dry_run": True, "skipped": True, "skip_reason": _set_cost_basis_unavailable_message()[0], "seeded_avg_price": False, "cost_basis_source": cost_basis_source, "cost_basis_reason": cost_basis_reason, } if fail_closed_cost_basis: _log.error( "sync fail-closed: INACTIVE onboarding requires avg_price seed (portfolio_id=%s slot=%s) " "— refuse live write", pid, slot_id, ) return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": before, "warning_cleared": False, "dry_run": False, "skipped": True, "skip_reason": _set_cost_basis_unavailable_message()[0], "seeded_avg_price": False, "cost_basis_source": cost_basis_source, "cost_basis_reason": cost_basis_reason, "error": "cost_basis_unavailable", } if dry_run: # Compute what *would* happen without writing anything. new_status = SlotState.HOLDING if slot.status is SlotState.INACTIVE else slot.status after = { "qty": float(broker_qty), "status": new_status.value, "avg_price": float(avg_price) if avg_price is not None else None, } return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": after, "warning_cleared": False, # dry-run never clears "dry_run": True, "skipped": False, "seeded_avg_price": seeded_avg_price, "cost_basis_source": cost_basis_source, "cost_basis_reason": cost_basis_reason, } # --- Live write --- try: if slot.status is SlotState.INACTIVE: # OQ-A: INACTIVE→HOLDING transition (actor=sync, reason=onboarding). slot_service.transit_slot( slot_uuid, SlotState.INACTIVE, SlotState.HOLDING, reason="onboarding", actor="sync", qty=float(broker_qty), avg_price=avg_price, portfolio_id=pid or None, symbol=sym or None, ) else: # HOLDING: update qty (and optionally avg_price) in place. # Use update_slot directly to avoid a state-machine transition. from dataclasses import replace as _replace updated = _replace(slot, qty=float(broker_qty), avg_price=avg_price) slot_repo.update_slot(updated) _log.info( "sync updated slot qty: slot=%s symbol=%s qty %s→%d actor=sync", slot_id, sym, slot.qty, broker_qty, ) except Exception: # noqa: BLE001 _log.exception("sync_slot_to_broker: write failed for slot=%s", slot_id) return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": before, "warning_cleared": False, "dry_run": False, "skipped": False, "error": "write_failed", "seeded_avg_price": seeded_avg_price, "cost_basis_source": cost_basis_source, "cost_basis_reason": cost_basis_reason, } # Refresh to confirm refreshed = slot_repo.fetch_slot(slot_uuid) after_qty = float(refreshed.qty) if (refreshed and refreshed.qty is not None) else None after_status = refreshed.status.value if refreshed else slot.status.value after_avg = float(refreshed.avg_price) if (refreshed and refreshed.avg_price is not None) else None after = {"qty": after_qty, "status": after_status, "avg_price": after_avg} # Clear B-case warning when qty now matches broker warning_cleared = False if after_qty is not None and int(after_qty) == broker_qty: try: warn_store.clear_warned(slot_id, case_code="B") warning_cleared = True _log.info( "sync: reconcile-warning cleared slot=%s symbol=%s case_code=B " "broker_qty=%d actor=sync", slot_id, sym, broker_qty, ) except Exception: # noqa: BLE001 _log.exception("sync: failed to clear_warned for slot=%s", slot_id) # req-0007 / ac-0024: publish B-case resolved alarm only after warning is cleared. if warning_cleared and alarm_emitter is not None and notification_store is not None: try: _notify_resolved( alarm_emitter, notification_store, identity_key=slot_id, case_code="B", instance_key=instance_key, symbol=sym, slot_id=slot_id, ) except Exception: # noqa: BLE001 _log.exception( "sync: resolved notify failed for slot=%s — warning clear kept", slot_id, ) return { "slot_id": slot_id, "symbol": sym, "portfolio_id": pid, "before": before, "after": after, "warning_cleared": warning_cleared, "dry_run": False, "skipped": False, "seeded_avg_price": seeded_avg_price, "cost_basis_source": cost_basis_source, "cost_basis_reason": cost_basis_reason, } # --------------------------------------------------------------------------- # BAT-051 / REQ-106: operator verify command — C-case re-check (read-centric) # --------------------------------------------------------------------------- def verify_slot_reconcile( *, portfolio_id: str, store: SnapshotStore, broker: BrokerPort, slot_repo, instance_key: str, warn_store: ReconcileWarningStore, alarm_emitter: AlarmEmitter | None = None, notification_store=None, ) -> dict: """Re-check C-case reconcile-warning for a portfolio (REQ-106, BAT-051). Read-centric: does NOT modify slot qty. Clears warning only when re-check confirms ledger qty == broker qty (no force-clear path, OQ-C confirmed). result values: "cleared" — re-check passed, warning cleared. "rejected" — re-check failed (still mismatched), warning NOT cleared. "not_c_case" — slot is warned but not case_code='C' (use sync for B case). "not_warned" — slot has no active warning (idempotent, exit 0). "slot_not_found" — no slot for this portfolio_id in this instance. "no_warning_info" — is_warned but get_warning_info returned None (unexpected). """ slot = slot_repo.find_slot_by_portfolio(instance_key, portfolio_id) if slot is None: return { "portfolio_id": portfolio_id, "slot_id": None, "result": "slot_not_found", "hub_qty": None, "broker_qty": None, "reason": f"no slot for portfolio_id={portfolio_id} in instance={instance_key}", } slot_id_str = str(slot.slot_id) if not warn_store.is_warned(slot_id_str): return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "not_warned", "hub_qty": None, "broker_qty": None, "reason": "no active reconcile-warning (already cleared or never set)", } # Check case set — verify is C-only (REQ-106 OQ-C). info = warn_store.get_warning_info(slot_id_str) if info is None: return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "no_warning_info", "hub_qty": None, "broker_qty": None, "reason": "warning exists but get_warning_info returned None (unexpected)", } case_codes_raw = info.get("case_codes") if isinstance(case_codes_raw, list): case_codes = {str(code) for code in case_codes_raw if code} else: case_code = str(info.get("case_code", "")) case_codes = {case_code} if case_code else set() if "C" not in case_codes: return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "not_c_case", "hub_qty": None, "broker_qty": None, "reason": ( f"warning case_codes={sorted(case_codes)} do not include 'C' — " "use 'trex-core sync' to resolve B-case (slot↔broker qty mismatch)" ), } # Re-run the C-case check: hub qty vs broker qty for this symbol. # AC-0023: use symbol-level aggregate across all portfolios (not single portfolio). sym = slot.symbol or "" broker_qty_map: dict[str, int] = { item.symbol: int(item.hldg_qty) for item in broker.get_balance() } broker_qty = broker_qty_map.get(sym, 0) # Hub qty: sum asset_quantity for ALL portfolios with this symbol in the snapshot. hub_qty: int = 0 for a in store.list_assets(): if a.item.symbol == sym: hub_qty += int(a.item.asset_quantity) match = hub_qty == broker_qty if match: # AC-0023: clear C-case warnings for ALL slots (any status) with this symbol. # Only rows with case_code='C' are touched; B-case (or other) rows are preserved. cleared_slots: list[str] = [] try: for s in slot_repo.list_slots(instance_key): if s.symbol != sym: continue sid_str = str(s.slot_id) warn_info = warn_store.get_warning_info(sid_str) if warn_info is None: continue warned_codes_raw = warn_info.get("case_codes") if isinstance(warned_codes_raw, list): warned_codes = {str(code) for code in warned_codes_raw if code} else: case_code = str(warn_info.get("case_code", "")) warned_codes = {case_code} if case_code else set() if "C" not in warned_codes: continue warn_store.clear_warned(sid_str, case_code="C") store.set_warning(s.portfolio_id, False) cleared_slots.append(sid_str) _log.info( "verify: reconcile-warning cleared (C-case resolved) " "symbol=%s hub_qty=%d broker_qty=%d cleared_slots=%s", sym, hub_qty, broker_qty, cleared_slots, ) except Exception: # noqa: BLE001 _log.exception("verify: failed to clear C-case warned for symbol=%s", sym) return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "clear_failed", "hub_qty": hub_qty, "broker_qty": broker_qty, "reason": "warn_store error during C-case clear — warnings NOT fully cleared", } # req-0007 / ac-0024: publish C-case resolved alarm (best-effort, warning clear kept). if alarm_emitter is not None and notification_store is not None: try: _notify_resolved( alarm_emitter, notification_store, identity_key=_c_identity_key(instance_key, sym), case_code="C", instance_key=instance_key, symbol=sym, ) except Exception: # noqa: BLE001 _log.exception( "verify: resolved notify failed for symbol=%s — warning clear kept", sym, ) return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "cleared", "hub_qty": hub_qty, "broker_qty": broker_qty, "reason": None, } else: _log.warning( "verify: C-case still mismatched — slot=%s portfolio_id=%s symbol=%s " "hub_qty=%d broker_qty=%d diff=%d — warning NOT cleared", slot_id_str, portfolio_id, sym, hub_qty, broker_qty, broker_qty - hub_qty, ) return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, "result": "rejected", "hub_qty": hub_qty, "broker_qty": broker_qty, "reason": ( f"hub_qty={hub_qty} broker_qty={broker_qty} " f"diff={broker_qty - hub_qty} symbol={sym}" ), }