trex/trex-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: auto-reconcile stale holding mirrors

Browse Source 
Closes #94
This commit is contained in:
Hermes Agent
2026-09-14 12:55:04 +09:00
parent adf131f19c
commit c70ac0fb24
19 changed files with 1181 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trex_core/adapters/persistence/auto_reconcile.py
+175
View File
@@ -0,0 +1,175 @@
"""Atomic persistence adapters for holding mirror auto-reconciliation."""
from __future__ import annotations
import copy
import json
from dataclasses import replace
from datetime import UTC, datetime
from typing import Any
from trex_core.adapters.persistence.postgresql import _require_psycopg
from trex_core.application.auto_reconcile import AutoReconcileCommand
from trex_core.application.portfolio import InMemoryReconcileWarningStore
from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository
from trex_core.domain.slot.models import Slot, SlotTransitionRecord
from trex_core.domain.slot.state import SlotState, slot_advisory_lock_id
_REASON = "hub_broker_matched_core_mirror_stale"
_ACTOR = "auto_reconcile"
def _eligible(current: Slot | None, command: AutoReconcileCommand) -> bool:
return bool(
current is not None
and current.slot_id == command.slot_id
and current.instance_key == command.instance_key
and current.portfolio_id == command.portfolio_id
and current.symbol == command.symbol
and current.status is SlotState.HOLDING
and current.retired_at is None
and current.qty == command.expected_qty
and current.avg_price == command.expected_avg_price
and current.limit_price is None
and current.broker_order_id is None
and current.cancel_state is None
and current.cancel_requested_at is None
and current.order_submitted_at is None
and current.order_date is None
and current.cancel_attempts == 0
)
def _audit_extra(command: AutoReconcileCommand) -> dict[str, object]:
return {
"portfolio_id": command.portfolio_id,
"symbol": command.symbol,
"before_qty": command.expected_qty,
"after_qty": command.target_qty,
"before_avg_price": command.expected_avg_price,
"after_avg_price": command.target_avg_price,
"hub_quantity": command.hub_quantity,
"broker_quantity": command.broker_quantity,
}
class InMemoryAutoReconcileStore:
"""In-process atomic-equivalent adapter used by tests and local runtimes."""
def __init__(
self,
slot_repo: InMemorySlotRepository,
warning_store: InMemoryReconcileWarningStore,
) -> None:
self._slot_repo = slot_repo
self._warning_store = warning_store
def apply_holding_mirror(self, command: AutoReconcileCommand) -> bool:
lock_id = slot_advisory_lock_id(command.instance_key)
with self._slot_repo.transactional_advisory_lock(lock_id):
with self._warning_store._lock: # same critical section as warning CRUD
current = self._slot_repo.fetch_slot(command.slot_id)
warnings = self._warning_store._warned.get(str(command.slot_id), {})
if current is None or not _eligible(current, command) or "B" not in warnings:
return False
previous_slot = current
previous_transitions = list(self._slot_repo._transitions)
previous_warnings = copy.deepcopy(self._warning_store._warned)
now = datetime.now(tz=UTC)
try:
self._slot_repo.update_slot(
replace(
current,
qty=command.target_qty,
avg_price=command.target_avg_price,
updated_at=now,
)
)
self._slot_repo.append_transition_log(
SlotTransitionRecord(
slot_id=command.slot_id,
from_state=SlotState.HOLDING,
to_state=SlotState.HOLDING,
ts=now,
reason=_REASON,
actor=_ACTOR,
extra=_audit_extra(command),
)
)
warnings.pop("B", None)
if not warnings:
self._warning_store._warned.pop(str(command.slot_id), None)
return True
except Exception:
self._slot_repo._slots[command.slot_id] = previous_slot
self._slot_repo._transitions[:] = previous_transitions
self._warning_store._warned = previous_warnings
raise
class PostgreSQLAutoReconcileStore:
"""One-transaction CAS update, B-only clear, and append-only audit."""
def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None:
self._psycopg = _require_psycopg(psycopg_module)
self._dsn = dsn
def apply_holding_mirror(self, command: AutoReconcileCommand) -> bool:
now = datetime.now(tz=UTC)
with self._psycopg.connect(self._dsn, autocommit=False) as conn:
try:
with conn.cursor() as cur:
cur.execute(
"SELECT pg_advisory_xact_lock(%s)",
(slot_advisory_lock_id(command.instance_key),),
)
cur.execute(
"UPDATE daily_slot SET qty=%s, avg_price=%s, updated_at=%s "
"WHERE slot_id=%s AND instance_key=%s AND portfolio_id=%s "
"AND symbol=%s AND status='holding' AND retired_at IS NULL "
"AND qty IS NOT DISTINCT FROM %s "
"AND avg_price IS NOT DISTINCT FROM %s "
"AND limit_price IS NULL AND broker_order_id IS NULL "
"AND cancel_state IS NULL AND cancel_requested_at IS NULL "
"AND order_submitted_at IS NULL AND order_date IS NULL "
"AND cancel_attempts=0",
(
command.target_qty,
command.target_avg_price,
now,
command.slot_id,
command.instance_key,
command.portfolio_id,
command.symbol,
command.expected_qty,
command.expected_avg_price,
),
)
if cur.rowcount != 1:
conn.rollback()
return False
cur.execute(
"DELETE FROM slot_reconcile_warning WHERE slot_id=%s AND case_code=%s",
(str(command.slot_id), "B"),
)
if cur.rowcount != 1:
conn.rollback()
return False
cur.execute(
"INSERT INTO slot_transition_log "
"(slot_id, from_state, to_state, ts, reason, actor, extra) "
"VALUES (%s, 'holding', 'holding', %s, %s, %s, %s)",
(
command.slot_id,
now,
_REASON,
_ACTOR,
json.dumps(_audit_extra(command)),
),
)
conn.commit()
return True
except Exception:
conn.rollback()
raise
trex_core/adapters/portfolio/hub_read.py
+52
View File
@@ -16,10 +16,12 @@ signal (REQ-028 AC-2) without callers seeing raw httpx exceptions.
from __future__ import annotations
import math
from datetime import UTC, datetime
from typing import Any, Optional
import httpx
from trex_core.application.auto_reconcile import HoldingsCostBasisEvidence
from trex_core.application.portfolio import (
BrokerCredentialPayload,
PortfolioHubUnavailableError,
@@ -268,6 +270,56 @@ class PortfolioHubReadAdapter:
return None, "invalid"
return value, "ok"
def get_cost_basis_evidence(self, portfolio_id: str) -> HoldingsCostBasisEvidence:
"""Return only canonical, complete evidence used by auto-reconcile."""
url = f"{self._base_url}/v1/portfolios/{portfolio_id}/holdings/cost-basis"
try:
resp = self._get(url)
except httpx.HTTPError as exc:
raise PortfolioHubUnavailableError(
f"cost_basis network error: {exc.__class__.__name__}"
) from exc
if not (200 <= resp.status_code < 300):
raise PortfolioHubUnavailableError(
f"cost_basis http_status_{resp.status_code}"
)
try:
body = resp.json()
except ValueError as exc:
raise PortfolioHubUnavailableError("cost_basis invalid_json") from exc
if not isinstance(body, dict):
raise PortfolioHubUnavailableError("cost_basis invalid_payload")
response_pid = body.get("portfolio_id")
symbol = body.get("symbol")
if str(response_pid or "") != portfolio_id or not isinstance(symbol, str) or not symbol:
raise PortfolioHubUnavailableError("cost_basis identity_mismatch")
def _finite_number(name: str, *, positive: bool) -> float:
raw = body.get(name)
if isinstance(raw, bool):
raise PortfolioHubUnavailableError(f"cost_basis invalid_{name}")
try:
value = float(str(raw))
except (TypeError, ValueError) as exc:
raise PortfolioHubUnavailableError(f"cost_basis invalid_{name}") from exc
if not math.isfinite(value) or (positive and value <= 0) or (not positive and value < 0):
raise PortfolioHubUnavailableError(f"cost_basis invalid_{name}")
return value
warning = body.get("data_completeness_warning")
if warning is not None and not isinstance(warning, str):
raise PortfolioHubUnavailableError("cost_basis invalid_completeness_warning")
return HoldingsCostBasisEvidence(
portfolio_id=portfolio_id,
symbol=symbol,
current_quantity=_finite_number("current_quantity", positive=True),
cost_basis_per_unit=_finite_number("cost_basis_per_unit", positive=True),
total_cost_basis=_finite_number("total_cost_basis", positive=True),
observed_at=datetime.now(tz=UTC),
data_completeness_warning=warning or None,
)
def get_cost_basis_with_diagnostics(
self,
portfolio_id: str,
trex_core/application/auto_reconcile.py
+46
View File
@@ -0,0 +1,46 @@
"""Fail-closed holding mirror auto-reconciliation contracts (issue #94)."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from uuid import UUID
@dataclass(frozen=True)
class HoldingsCostBasisEvidence:
"""Canonical portfolio-hub evidence captured by one successful HTTP read."""
portfolio_id: str
symbol: str
current_quantity: float
cost_basis_per_unit: float
total_cost_basis: float
observed_at: datetime
data_completeness_warning: str | None = None
@dataclass(frozen=True)
class AutoReconcileCommand:
"""CAS-bound command for one atomic mirror repair."""
slot_id: UUID
instance_key: str
portfolio_id: str
symbol: str
expected_qty: float
expected_avg_price: float | None
target_qty: float
target_avg_price: float
hub_quantity: float
broker_quantity: int
evidence_observed_at: datetime
class AutoReconcileStore(Protocol):
"""Atomic qty/avg/audit/B-warning persistence boundary."""
def apply_holding_mirror(self, command: AutoReconcileCommand) -> bool:
"""Apply exactly once when every CAS/eligibility condition still holds."""
...
trex_core/application/ports.py
+9
View File
@@ -8,6 +8,8 @@ from __future__ import annotations
from typing import Optional, Protocol
from uuid import UUID
from trex_core.application.auto_reconcile import HoldingsCostBasisEvidence
class IntentStore(Protocol):
"""Durable idempotency + audit store for accepted intents (dcy-002 OQ-1).
@@ -64,3 +66,10 @@ class PortfolioHubCostBasisPort(Protocol):
or any error occurs. The caller must handle None as a graceful-degrade signal.
"""
...
def get_cost_basis_evidence(self, portfolio_id: str) -> HoldingsCostBasisEvidence:
"""Return strict canonical evidence for automatic holding reconciliation.
Implementations must not fall back to legacy cost-basis fields.
"""
...
trex_core/application/reconciliation.py
+151 -1
View File
@@ -29,12 +29,18 @@ import hashlib
import logging
import math
import re
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from typing import cast
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.auto_reconcile import (
AutoReconcileCommand,
AutoReconcileStore,
HoldingsCostBasisEvidence,
)
from trex_core.application.broker_port import BrokerPort
from trex_core.application.portfolio import (
PortfolioItem,
ReconcileWarningStore,
SnapshotStore,
UnmanagedPositionStore,
@@ -377,6 +383,113 @@ def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPo
return False
_AUTO_RECONCILE_EVIDENCE_MAX_AGE = timedelta(seconds=60)
_AUTO_RECONCILE_FUTURE_TOLERANCE = timedelta(seconds=5)
def _try_auto_reconcile_holding(
*,
slot,
broker_quantity: int,
portfolio_items: list[PortfolioItem] | None,
cost_basis_port,
auto_reconcile_store: AutoReconcileStore | None,
observed_at: datetime,
) -> bool:
"""Return True only after the atomic mirror repair commits."""
if portfolio_items is None or cost_basis_port is None or auto_reconcile_store is None:
return False
if (
slot.status is not SlotState.HOLDING
or slot.retired_at is not None
or not slot.portfolio_id
or not slot.symbol
or slot.qty is None
or broker_quantity <= 0
or slot.limit_price is not None
or slot.broker_order_id is not None
or slot.cancel_state is not None
or slot.cancel_requested_at is not None
or slot.order_submitted_at is not None
or slot.order_date is not None
or slot.cancel_attempts != 0
):
return False
matches = [item for item in portfolio_items if item.portfolio_id == slot.portfolio_id]
symbol_matches = [item for item in portfolio_items if item.symbol == slot.symbol]
if len(matches) != 1 or len(symbol_matches) != 1:
return False
item = matches[0]
hub_quantity = item.asset_quantity
if (
item.symbol != slot.symbol
or not math.isfinite(hub_quantity)
or hub_quantity <= 0
or not float(hub_quantity).is_integer()
or int(hub_quantity) != broker_quantity
):
return False
getter = getattr(cost_basis_port, "get_cost_basis_evidence", None)
if not callable(getter):
return False
try:
evidence = cast(HoldingsCostBasisEvidence, getter(slot.portfolio_id))
except Exception: # noqa: BLE001 - fail closed without provider details
_log.warning(
"auto_reconcile skipped: slot_id=%s reason=cost_basis_evidence_unavailable",
slot.slot_id,
)
return False
age = observed_at - evidence.observed_at
expected_total = evidence.current_quantity * evidence.cost_basis_per_unit
if (
evidence.observed_at.tzinfo is None
or age > _AUTO_RECONCILE_EVIDENCE_MAX_AGE
or age < -_AUTO_RECONCILE_FUTURE_TOLERANCE
or evidence.data_completeness_warning is not None
or evidence.portfolio_id != slot.portfolio_id
or evidence.symbol != slot.symbol
or not math.isfinite(evidence.current_quantity)
or not math.isfinite(evidence.cost_basis_per_unit)
or not math.isfinite(evidence.total_cost_basis)
or evidence.current_quantity != hub_quantity
or evidence.cost_basis_per_unit <= 0
or evidence.total_cost_basis <= 0
or not math.isclose(
evidence.total_cost_basis,
expected_total,
rel_tol=1e-9,
abs_tol=1e-6,
)
):
return False
command = AutoReconcileCommand(
slot_id=slot.slot_id,
instance_key=slot.instance_key,
portfolio_id=slot.portfolio_id,
symbol=slot.symbol,
expected_qty=float(slot.qty),
expected_avg_price=(float(slot.avg_price) if slot.avg_price is not None else None),
target_qty=float(broker_quantity),
target_avg_price=float(evidence.cost_basis_per_unit),
hub_quantity=float(hub_quantity),
broker_quantity=broker_quantity,
evidence_observed_at=evidence.observed_at,
)
try:
return bool(auto_reconcile_store.apply_holding_mirror(command))
except Exception: # noqa: BLE001 - persistence failure keeps warning active
_log.warning(
"auto_reconcile failed: slot_id=%s reason=atomic_persistence_failure",
slot.slot_id,
)
return False
def reconcile_core_holdings(
*,
slot_repo,
@@ -387,6 +500,9 @@ def reconcile_core_holdings(
warn_store: ReconcileWarningStore | None = None,
notification_store=None,
unmanaged_position_store: UnmanagedPositionStore | None = None,
portfolio_items: list[PortfolioItem] | None = None,
cost_basis_port=None,
auto_reconcile_store: AutoReconcileStore | None = None,
) -> dict[str, str]:
"""dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding
vs broker divergence and set durable reconcile-warning for B case.
@@ -451,6 +567,7 @@ def reconcile_core_holdings(
slot_id_str, slot.portfolio_id, sym, bq, slot.qty,
)
# REQ-103 / BAT-050: durable reconcile-warning (case_code='B').
warning_persisted = False
if warn_store is not None:
try:
warn_store.set_warned(
@@ -460,6 +577,7 @@ def reconcile_core_holdings(
actor="reconcile",
reason=f"slot_qty={slot.qty} broker_qty={bq} symbol={sym}",
)
warning_persisted = True
except Exception: # noqa: BLE001 - never let reconcile crash the loop
_log.exception(
"failed to set reconcile_warning for slot %s", slot.slot_id
@@ -482,6 +600,38 @@ def reconcile_core_holdings(
_log.exception(
"reconcile_core_holdings: notify_mismatch failed for slot=%s", slot_id_str,
)
if warning_persisted and _try_auto_reconcile_holding(
slot=slot,
broker_quantity=bq,
portfolio_items=portfolio_items,
cost_basis_port=cost_basis_port,
auto_reconcile_store=auto_reconcile_store,
observed_at=snapshot_observed_at,
):
decisions[slot_id_str] = "auto_synced"
active_b_identity_keys.discard(slot_id_str)
_log.info(
"auto_reconcile committed: slot_id=%s portfolio_id=%s symbol=%s",
slot_id_str,
slot.portfolio_id,
sym,
)
if alarm_emitter is not None and notification_store is not None:
try:
_notify_resolved(
alarm_emitter,
notification_store,
identity_key=slot_id_str,
case_code="B",
instance_key=instance_key,
symbol=sym,
slot_id=slot_id_str,
)
except Exception: # noqa: BLE001 - business commit is authoritative
_log.exception(
"reconcile_core_holdings: auto resolved notify failed for slot=%s",
slot_id_str,
)
else:
decisions[str(slot.slot_id)] = "ok"
for sym, bq in broker_qty.items(): # unmatched broker position
trex_core/application/snapshot_scheduler.py
+14 -2
View File
@@ -16,6 +16,7 @@ from collections.abc import Callable
from datetime import UTC, date, datetime, time, timedelta, timezone
from trex_core.application.alarm_emitter import AlarmEmitter
from trex_core.application.auto_reconcile import AutoReconcileStore
from trex_core.application.balance_health import BalanceHealthTracker
from trex_core.application.broker_port import BrokerBalanceError, BrokerPort
from trex_core.application.portfolio import (
@@ -55,6 +56,8 @@ class SnapshotScheduler:
warn_store: ReconcileWarningStore | None = None,
notification_store: ReconcileNotificationStore | None = None,
unmanaged_position_store: UnmanagedPositionStore | None = None,
cost_basis_port=None,
auto_reconcile_store: AutoReconcileStore | None = None,
credential_fetch_enabled: bool = False,
kis_reinit_callback: Callable[[BrokerCredentialPayload], None] | None = None,
credential_refresh: Callable[[], object] | None = None,
@@ -81,6 +84,8 @@ class SnapshotScheduler:
# req-0007 / ac-0024: durable notification deduplication store (Optional).
self._notification_store = notification_store
self._unmanaged_position_store = unmanaged_position_store
self._cost_basis_port = cost_basis_port
self._auto_reconcile_store = auto_reconcile_store
# dcy-036 / REQ-113/114/115: credential fetch opt-in + break-glass detection.
# credential_fetch_enabled=False (default) → env-only, no portfolio-hub fetch.
self._credential_fetch_enabled: bool = credential_fetch_enabled
@@ -332,7 +337,7 @@ class SnapshotScheduler:
if self._slot_service is not None:
try:
# BAT-050 / REQ-103: pass warn_store for B-case durable warning.
self._run_core_holding_reconciliation_worker()
self._run_core_holding_reconciliation_worker(items)
except Exception: # noqa: BLE001
_log.exception("core-holding divergence reconcile failed")
self._last_fired_date = self._clock()
@@ -353,7 +358,7 @@ class SnapshotScheduler:
),
)
def _run_core_holding_reconciliation_worker(self) -> None:
def _run_core_holding_reconciliation_worker(self, items=None) -> None:
from trex_core.application.reconciliation import reconcile_core_holdings
self._run_balance_worker(
@@ -368,6 +373,13 @@ class SnapshotScheduler:
warn_store=self._warn_store, # BAT-050 / REQ-103
notification_store=self._notification_store, # req-0007 / ac-0024
unmanaged_position_store=self._unmanaged_position_store,
portfolio_items=(
list(items)
if items is not None
else [asset.item for asset in self._store.list_assets()]
),
cost_basis_port=self._cost_basis_port,
auto_reconcile_store=self._auto_reconcile_store,
),
)
trex_core/runtime/daemon.py
+4
View File
@@ -34,6 +34,7 @@ from trex_core.adapters.messaging import (
from trex_core.adapters.persistence.account_snapshot import (
PostgreSQLAccountSnapshotStore,
)
from trex_core.adapters.persistence.auto_reconcile import PostgreSQLAutoReconcileStore
from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore
from trex_core.adapters.persistence.migrate import apply_migrations
from trex_core.adapters.persistence.postgresql import (
@@ -202,6 +203,7 @@ class Daemon:
self.pause_store = PostgreSQLPauseStore(config.dsn)
# BAT-050 / REQ-102: durable reconcile-warning store ((slot_id, case_code) PK).
self.warn_store = PostgreSQLReconcileWarningStore(config.dsn)
self.auto_reconcile_store = PostgreSQLAutoReconcileStore(config.dsn)
# req-0007 / ac-0024: durable reconcile notification deduplication store.
self.notification_store = PostgreSQLReconcileNotificationStore(config.dsn)
self.unmanaged_position_store = PostgreSQLUnmanagedPositionStore(config.dsn)
@@ -286,6 +288,8 @@ class Daemon:
warn_store=self.warn_store, # BAT-050 / REQ-103: durable reconcile-warning
notification_store=self.notification_store, # req-0007 / ac-0024
unmanaged_position_store=self.unmanaged_position_store,
cost_basis_port=self.portfolio_read,
auto_reconcile_store=self.auto_reconcile_store,
credential_fetch_enabled=config.credential_fetch_enabled, # dcy-036 / REQ-113
credential_refresh=self.broker_provider.refresh_day_start_credential,
# dcy-036 / REQ-113 AC-2 / REQ-115 AC-1: KISBroker reinit callback.