feat(alerts): publish reconciliation quantity mismatches to trex-notifier (req-0007/ac-0024)

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

trex_core/application/reconciliation.py

@@ -19,13 +19,16 @@ durable store is additive and Optional (None => warn-log-only, backward compat).
 
 BAT-051 / REQ-105: sync_slot_to_broker — operator CLI sync path.
 BAT-051 / REQ-106: verify_slot_reconcile — operator CLI verify path (C-only, read-centric).
+
+req-0007 / ac-0024: B/C-case mismatch alarms (detected, changed, resolved) are
+published via AlarmEmitter with durable deduplication through ReconcileNotificationStore.
 """
 from __future__ import annotations
 
 import logging
 import math
 import re
-from typing import Optional, cast
+from typing import cast
 
 from trex_core.application.alarm_emitter import AlarmEmitter
 from trex_core.application.broker_port import BrokerPort
@@ -47,7 +50,7 @@ _COST_BASIS_REASON_CODES = frozenset({
 _COST_BASIS_LEGACY_SOURCE_ALIASES = frozenset({"avg_cost", "wac", "cost_basis"})
 
 
-def _sanitize_cost_basis_reason(reason: object) -> Optional[str]:
+def _sanitize_cost_basis_reason(reason: object) -> str | None:
     """Keep public diagnostics in the bounded REQ-123 reason taxonomy."""
     if not isinstance(reason, str):
         return None
@@ -62,7 +65,7 @@ def _normalize_cost_basis_diagnostics(
     value: object,
     source: object,
     reason: object,
-) -> tuple[Optional[float], str, Optional[str]]:
+) -> tuple[float | None, str, str | None]:
     """Normalize diagnostic-port output before it reaches reconciliation results."""
     safe_reason = _sanitize_cost_basis_reason(reason)
     if source == "canonical":
@@ -89,7 +92,7 @@ def _normalize_cost_basis_diagnostics(
     return normalized_value, normalized_source, safe_reason
 
 
-def _normalize_legacy_cost_basis(value: object) -> tuple[Optional[float], str, Optional[str]]:
+def _normalize_legacy_cost_basis(value: object) -> tuple[float | None, str, str | None]:
     """Make the diagnostic-free compatibility seam obey safe WAC semantics."""
     normalized_value, _, _ = _normalize_cost_basis_diagnostics(value, "legacy", "invalid")
     if normalized_value is None:
@@ -97,14 +100,233 @@ def _normalize_legacy_cost_basis(value: object) -> tuple[Optional[float], str, O
     return normalized_value, "legacy", None
 
 
+# ---------------------------------------------------------------------------
+# req-0007 / ac-0024: Notification lifecycle helpers
+# ---------------------------------------------------------------------------
+
+def _c_identity_key(instance_key: str, symbol: str) -> str:
+    """Stable C-case notification identity key (null-byte separated to avoid collisions)."""
+    return f"{instance_key}\x00C\x00{symbol}"
+
+
+def _build_alarm_fields(
+    *,
+    case_code: str,
+    instance_key: str,
+    symbol: str,
+    quantity_a: int,
+    quantity_b: int,
+    slot_id: str | None = None,
+) -> dict:
+    """Build sanitized alarm context fields (ac-0024 allowlist).
+
+    Excluded: account number/hash, credential, token, DSN, raw payload, exception.
+    """
+    fields: dict = {
+        "instance_key": instance_key,
+        "symbol": symbol,
+        "case_code": case_code,
+        "broker_quantity": quantity_b,
+    }
+    if case_code == "B":
+        fields["slot_quantity"] = quantity_a
+        if slot_id is not None:
+            fields["slot_id"] = slot_id
+    else:
+        fields["hub_quantity"] = quantity_a
+    return fields
+
+
+def _notify_mismatch(
+    alarm_emitter: AlarmEmitter,
+    notification_store,
+    *,
+    identity_key: str,
+    case_code: str,
+    instance_key: str,
+    symbol: str,
+    quantity_a: int,
+    quantity_b: int,
+    slot_id: str | None = None,
+) -> None:
+    """Publish detected or changed alarm; advance fingerprint only on publish success.
+
+    Suppresses if the identity fingerprint is unchanged from the last notified state.
+    Retries on every detection if publish previously failed (fingerprint not advanced).
+    """
+    try:
+        state = notification_store.get_state(identity_key)
+    except Exception:  # noqa: BLE001
+        _log.exception(
+            "reconcile_notification: get_state failed for %s — skipping notify", identity_key,
+        )
+        return
+
+    if state is not None and state.get("lifecycle_state") == "notified":
+        if state.get("quantity_a") == quantity_a and state.get("quantity_b") == quantity_b:
+            return  # unchanged — suppress
+        event_type = "changed"
+    else:
+        # No prior state, OR pending_resolved (new mismatch after failed resolved) → re-detect
+        event_type = "detected"
+
+    fields = _build_alarm_fields(
+        case_code=case_code,
+        instance_key=instance_key,
+        symbol=symbol,
+        quantity_a=quantity_a,
+        quantity_b=quantity_b,
+        slot_id=slot_id,
+    )
+    published = alarm_emitter.publish_alarm(
+        f"reconcile_mismatch_{event_type}",
+        f"reconciliation mismatch {event_type}: {case_code}-case symbol={symbol}",
+        fields=fields,
+    )
+
+    if published:
+        try:
+            notification_store.set_state(
+                identity_key,
+                case_code=case_code,
+                instance_key=instance_key,
+                symbol=symbol,
+                quantity_a=quantity_a,
+                quantity_b=quantity_b,
+                lifecycle_state="notified",
+            )
+        except Exception:  # noqa: BLE001
+            # Fingerprint write failure: at-least-once duplicate on next detection.
+            _log.exception(
+                "reconcile_notification: fingerprint persist failed for %s — "
+                "at-least-once duplicate possible on next detection",
+                identity_key,
+            )
+
+
+def _notify_resolved(
+    alarm_emitter: AlarmEmitter,
+    notification_store,
+    *,
+    identity_key: str,
+    case_code: str,
+    instance_key: str,
+    symbol: str,
+    slot_id: str | None = None,
+) -> None:
+    """Publish resolved alarm; clear or mark pending_resolved on publish failure.
+
+    Warning clear is always preserved by the caller regardless of this function's outcome.
+    - Publish success: clear notification state.
+    - Publish failure + state writable: mark pending_resolved for retry.
+    - State persistence failure: log loss risk (business path unaffected).
+    """
+    try:
+        state = notification_store.get_state(identity_key)
+    except Exception:  # noqa: BLE001
+        _log.exception(
+            "reconcile_notification: get_state failed for %s — skipping resolved notify",
+            identity_key,
+        )
+        return
+
+    if state is None:
+        return  # idempotent: already resolved or never detected
+
+    quantity_a = state["quantity_a"]
+    quantity_b = state["quantity_b"]
+
+    fields = _build_alarm_fields(
+        case_code=case_code,
+        instance_key=instance_key,
+        symbol=symbol,
+        quantity_a=quantity_a,
+        quantity_b=quantity_b,
+        slot_id=slot_id,
+    )
+    published = alarm_emitter.publish_alarm(
+        "reconcile_mismatch_resolved",
+        f"reconciliation mismatch resolved: {case_code}-case symbol={symbol}",
+        fields=fields,
+    )
+
+    if published:
+        try:
+            notification_store.clear_state(identity_key)
+        except Exception:  # noqa: BLE001
+            # At-least-once: retry may produce duplicate resolved event.
+            _log.exception(
+                "reconcile_notification: clear_state failed for %s — "
+                "at-least-once duplicate resolved possible",
+                identity_key,
+            )
+    else:
+        # Publish failed — persist pending_resolved so the next opportunity retries.
+        try:
+            notification_store.set_state(
+                identity_key,
+                case_code=case_code,
+                instance_key=instance_key,
+                symbol=symbol,
+                quantity_a=quantity_a,
+                quantity_b=quantity_b,
+                lifecycle_state="pending_resolved",
+            )
+        except Exception:  # noqa: BLE001
+            # State persistence failed: warning clear is kept; resolved event may be lost.
+            _log.error(
+                "reconcile_notification: resolved publish failed AND state persist failed "
+                "for %s — resolved event may be lost",
+                identity_key,
+            )
+
+
+def _retry_pending_resolved(
+    alarm_emitter: AlarmEmitter,
+    notification_store,
+    *,
+    case_code: str,
+    active_identity_keys: set[str],
+) -> None:
+    """Retry pending_resolved publishes for identities no longer actively mismatched.
+
+    Called at the end of each reconcile pass to drain the pending_resolved queue
+    for identities that have since been cleared by operator sync/verify.
+    """
+    try:
+        pending = notification_store.list_states_by_lifecycle("pending_resolved")
+    except Exception:  # noqa: BLE001
+        _log.exception(
+            "reconcile_notification: list_states_by_lifecycle failed — skipping pending retry",
+        )
+        return
+
+    for state in pending:
+        if state.get("case_code") != case_code:
+            continue
+        identity_key = state["identity_key"]
+        if identity_key in active_identity_keys:
+            continue  # still mismatched — do not retry resolved yet
+        _notify_resolved(
+            alarm_emitter,
+            notification_store,
+            identity_key=identity_key,
+            case_code=state["case_code"],
+            instance_key=state["instance_key"],
+            symbol=state["symbol"],
+            slot_id=identity_key if case_code == "B" else None,
+        )
+
+
 def reconcile_core_holdings(
     *,
     slot_repo,
     slot_service,
     broker: BrokerPort,
     instance_key: str,
-    alarm_emitter: Optional[AlarmEmitter] = None,
-    warn_store: Optional[ReconcileWarningStore] = None,
+    alarm_emitter: AlarmEmitter | None = None,
+    warn_store: ReconcileWarningStore | None = None,
+    notification_store=None,
 ) -> dict[str, str]:
     """dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding
     vs broker divergence and set durable reconcile-warning for B case.
@@ -122,6 +344,7 @@ def reconcile_core_holdings(
                      → set by reconcile_positions
 
     warn_store: Optional — when None, WARN-log-only (backward compat, REQ-103 AC-5).
+    notification_store: Optional — when None, alarm deduplication is skipped (backward compat).
     Returns {key: 'error'|'qty_warning'|'ok'|'unmanaged_position'|
     'managed_non_holding_divergence'} for telemetry.
     """
@@ -130,6 +353,7 @@ def reconcile_core_holdings(
     }
     decisions: dict[str, str] = {}
     matched: set[str] = set()
+    active_b_identity_keys: set[str] = set()  # req-0007: track active B mismatches for retry
     for slot in slot_repo.list_holding_slots(instance_key):
         sym = slot.symbol
         matched.add(sym)
@@ -157,6 +381,7 @@ def reconcile_core_holdings(
         elif slot.qty is not None and int(slot.qty) != bq:  # B case (REQ-103) — qty mismatch
             slot_id_str = str(slot.slot_id)
             decisions[slot_id_str] = "qty_warning"
+            active_b_identity_keys.add(slot_id_str)
             # REQ-107: structured WARNING log for B case.
             _log.warning(
                 "reconcile_warning set: slot_id=%s portfolio_id=%s symbol=%s "
@@ -177,6 +402,24 @@ def reconcile_core_holdings(
                     _log.exception(
                         "failed to set reconcile_warning for slot %s", slot.slot_id
                     )
+            # req-0007 / ac-0024: publish B-case detected/changed alarm (deduped).
+            if alarm_emitter is not None and notification_store is not None:
+                try:
+                    _notify_mismatch(
+                        alarm_emitter,
+                        notification_store,
+                        identity_key=slot_id_str,
+                        case_code="B",
+                        instance_key=instance_key,
+                        symbol=sym,
+                        quantity_a=int(slot.qty),
+                        quantity_b=bq,
+                        slot_id=slot_id_str,
+                    )
+                except Exception:  # noqa: BLE001
+                    _log.exception(
+                        "reconcile_core_holdings: notify_mismatch failed for slot=%s", slot_id_str,
+                    )
         else:
             decisions[str(slot.slot_id)] = "ok"
     for sym, bq in broker_qty.items():  # unmatched broker position
@@ -196,6 +439,17 @@ def reconcile_core_holdings(
                     "unmanaged position: symbol=%s qty=%d instance_key=%s",
                     sym, bq, instance_key,
                 )
+    # req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched.
+    if alarm_emitter is not None and notification_store is not None:
+        try:
+            _retry_pending_resolved(
+                alarm_emitter,
+                notification_store,
+                case_code="B",
+                active_identity_keys=active_b_identity_keys,
+            )
+        except Exception:  # noqa: BLE001
+            _log.exception("reconcile_core_holdings: pending_resolved retry failed")
     return decisions
 
 
@@ -205,7 +459,9 @@ def reconcile_positions(
     broker: BrokerPort,
     slot_repo=None,
     instance_key: str = "default",
-    warn_store: Optional[ReconcileWarningStore] = None,
+    warn_store: ReconcileWarningStore | None = None,
+    alarm_emitter: AlarmEmitter | None = None,
+    notification_store=None,
 ) -> dict[str, str]:
     """Run one reconciliation pass (REQ-031 + REQ-103 BAT-050).
 
@@ -216,6 +472,10 @@ def reconcile_positions(
     (ledger/hub qty vs broker) also set durable reconcile-warning on the corresponding
     slot (portfolio_id -> slot lookup via slot_repo). When either is None, falls back
     to in-memory warning only (backward compat, REQ-103 AC-5).
+
+    req-0007 / ac-0024: when alarm_emitter and notification_store are provided,
+    C-case detected/changed alarms are published per-symbol (one aggregate event
+    per symbol regardless of portfolio count).
     """
     broker_qty: dict[str, int] = {
         item.symbol: int(item.hldg_qty) for item in broker.get_balance()
@@ -230,6 +490,7 @@ def reconcile_positions(
         by_symbol.setdefault(sym, []).append((a.item.portfolio_id, a.item.asset_quantity))
 
     decisions: dict[str, str] = {}
+    active_c_identity_keys: set[str] = set()  # req-0007: track active C mismatches for retry
     for symbol, entries in by_symbol.items():
         hub_total = int(sum(qty for _, qty in entries))
         broker_total = broker_qty.get(symbol, 0)
@@ -277,6 +538,41 @@ def reconcile_positions(
                     decisions[pid] = "cleared"
                 else:
                     decisions[pid] = "ok"
+
+        # req-0007 / ac-0024: publish one C-case detected/changed alarm per symbol
+        # (aggregate mismatch lifecycle — not per portfolio).
+        if alarm_emitter is not None and notification_store is not None:
+            c_key = _c_identity_key(instance_key, symbol)
+            if not match:
+                active_c_identity_keys.add(c_key)
+                try:
+                    _notify_mismatch(
+                        alarm_emitter,
+                        notification_store,
+                        identity_key=c_key,
+                        case_code="C",
+                        instance_key=instance_key,
+                        symbol=symbol,
+                        quantity_a=hub_total,
+                        quantity_b=broker_total,
+                    )
+                except Exception:  # noqa: BLE001
+                    _log.exception(
+                        "reconcile_positions: notify_mismatch failed for symbol=%s", symbol,
+                    )
+
+    # req-0007 / ac-0024: retry pending_resolved for C-case identities no longer mismatched.
+    if alarm_emitter is not None and notification_store is not None:
+        try:
+            _retry_pending_resolved(
+                alarm_emitter,
+                notification_store,
+                case_code="C",
+                active_identity_keys=active_c_identity_keys,
+            )
+        except Exception:  # noqa: BLE001
+            _log.exception("reconcile_positions: pending_resolved retry failed")
+
     return decisions
 
 
@@ -291,9 +587,11 @@ def sync_slot_to_broker(
     slot_service,
     broker: BrokerPort,
     instance_key: str,
-    warn_store: "ReconcileWarningStore",
+    warn_store: ReconcileWarningStore,
     cost_basis_port=None,  # Optional[PortfolioHubCostBasisPort]
     dry_run: bool = False,
+    alarm_emitter: AlarmEmitter | None = None,
+    notification_store=None,
 ) -> dict:
     """Sync one slot's qty to the broker ground truth (REQ-105, BAT-051).
 
@@ -373,9 +671,9 @@ def sync_slot_to_broker(
     avg_price = slot.avg_price
     seeded_avg_price = False
     cost_basis_source = "none"
-    cost_basis_reason: Optional[str] = None
+    cost_basis_reason: str | None = None
 
-    def _set_cost_basis_unavailable_message() -> tuple[str, Optional[str]]:
+    def _set_cost_basis_unavailable_message() -> tuple[str, str | None]:
         if pid:
             return "cost_basis_unavailable", f"inactive slot {slot_id} portfolio_id={pid}"
         return "cost_basis_unavailable", "inactive slot without portfolio_id"
@@ -559,6 +857,22 @@ def sync_slot_to_broker(
             )
         except Exception:  # noqa: BLE001
             _log.exception("sync: failed to clear_warned for slot=%s", slot_id)
+        # req-0007 / ac-0024: publish B-case resolved alarm (best-effort, warning clear kept).
+        if alarm_emitter is not None and notification_store is not None:
+            try:
+                _notify_resolved(
+                    alarm_emitter,
+                    notification_store,
+                    identity_key=slot_id,
+                    case_code="B",
+                    instance_key=instance_key,
+                    symbol=sym,
+                    slot_id=slot_id,
+                )
+            except Exception:  # noqa: BLE001
+                _log.exception(
+                    "sync: resolved notify failed for slot=%s — warning clear kept", slot_id,
+                )
 
     return {
         "slot_id": slot_id, "symbol": sym, "portfolio_id": pid,
@@ -583,7 +897,9 @@ def verify_slot_reconcile(
     broker: BrokerPort,
     slot_repo,
     instance_key: str,
-    warn_store: "ReconcileWarningStore",
+    warn_store: ReconcileWarningStore,
+    alarm_emitter: AlarmEmitter | None = None,
+    notification_store=None,
 ) -> dict:
     """Re-check C-case reconcile-warning for a portfolio (REQ-106, BAT-051).
 
@@ -693,6 +1009,21 @@ def verify_slot_reconcile(
                 "broker_qty": broker_qty,
                 "reason": "warn_store error during C-case clear — warnings NOT fully cleared",
             }
+        # req-0007 / ac-0024: publish C-case resolved alarm (best-effort, warning clear kept).
+        if alarm_emitter is not None and notification_store is not None:
+            try:
+                _notify_resolved(
+                    alarm_emitter,
+                    notification_store,
+                    identity_key=_c_identity_key(instance_key, sym),
+                    case_code="C",
+                    instance_key=instance_key,
+                    symbol=sym,
+                )
+            except Exception:  # noqa: BLE001
+                _log.exception(
+                    "verify: resolved notify failed for symbol=%s — warning clear kept", sym,
+                )
         return {
             "portfolio_id": portfolio_id,
             "slot_id": slot_id_str,