From e47bebacaf27725e9ecdc5b7d8265a416a3c62d8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:16:43 +0000 Subject: [PATCH 01/21] Initial plan From 304a06f97c1228a2dee15ebcddee85da53507ab8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:17:22 +0000 Subject: [PATCH 02/21] Initial plan From ae2e59820976f9a531cf0457ff529dd63792e90f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:18:03 +0000 Subject: [PATCH 03/21] Initial plan From 45a5e4157deb7f2e708793b44802aff034ab820e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:22:40 +0000 Subject: [PATCH 04/21] fix: preserve multi-cause reconcile warnings and align slot blocking Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../portfolio/test_reconcile_warning_store.py | 42 ++++++++++ .../test_reconcile_notification.py | 2 +- .../test_reconcile_warning_blocking.py | 4 + .../test_reconcile_warning_store.py | 19 ++++- .../test_slots_handler_quantity_mismatch.py | 47 ++++++++++- tests/application/test_sync.py | 16 ++++ tests/application/test_verify.py | 38 ++++++++- tests/runtime/test_cli_operator.py | 2 +- .../sql/016_create_slot_reconcile_warning.sql | 4 +- .../028_preserve_reconcile_warning_cases.sql | 7 ++ .../portfolio/reconcile_warning_store.py | 51 +++++++----- trex_core/application/messaging_handlers.py | 83 ++++++++++++------- trex_core/application/portfolio.py | 72 ++++++++++++---- trex_core/application/reconciliation.py | 34 +++++--- trex_core/runtime/cli.py | 10 ++- trex_core/runtime/daemon.py | 2 +- 16 files changed, 337 insertions(+), 96 deletions(-) create mode 100644 tests/adapters/portfolio/test_reconcile_warning_store.py create mode 100644 trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql diff --git a/tests/adapters/portfolio/test_reconcile_warning_store.py b/tests/adapters/portfolio/test_reconcile_warning_store.py new file mode 100644 index 0000000..05cc113 --- /dev/null +++ b/tests/adapters/portfolio/test_reconcile_warning_store.py @@ -0,0 +1,42 @@ +"""PostgreSQLReconcileWarningStore integration tests.""" +from __future__ import annotations + +from trex_core.adapters.portfolio.reconcile_warning_store import ( + PostgreSQLReconcileWarningStore, +) + + +def test_warning_store_preserves_multiple_cases_for_same_slot(pg_clean): + store = PostgreSQLReconcileWarningStore(pg_clean) + store.set_warned("slot-1", portfolio_id="pf-A", case_code="B", reason="b") + store.set_warned("slot-1", portfolio_id="pf-A", case_code="C", reason="c") + + assert store.is_warned("slot-1") is True + info = store.get_warning_info("slot-1") + assert info is not None + assert info["case_codes"] == ["B", "C"] + + +def test_warning_store_case_specific_clear_and_full_clear(pg_clean): + store = PostgreSQLReconcileWarningStore(pg_clean) + store.set_warned("slot-2", portfolio_id="pf-A", case_code="B") + store.set_warned("slot-2", portfolio_id="pf-A", case_code="C") + + store.clear_warned("slot-2", case_code="C") + info = store.get_warning_info("slot-2") + assert info is not None + assert info["case_codes"] == ["B"] + + store.clear_warned("slot-2") + assert store.is_warned("slot-2") is False + + +def test_warning_store_is_durable_across_reinstantiation(pg_clean): + a = PostgreSQLReconcileWarningStore(pg_clean) + a.set_warned("slot-restart", portfolio_id="pf-A", case_code="B") + a.set_warned("slot-restart", portfolio_id="pf-A", case_code="C") + + b = PostgreSQLReconcileWarningStore(pg_clean) + info = b.get_warning_info("slot-restart") + assert info is not None + assert info["case_codes"] == ["B", "C"] diff --git a/tests/application/test_reconcile_notification.py b/tests/application/test_reconcile_notification.py index 1c1de0d..b95044d 100644 --- a/tests/application/test_reconcile_notification.py +++ b/tests/application/test_reconcile_notification.py @@ -896,7 +896,7 @@ def test_b_sync_does_not_publish_resolved_when_clear_warned_fails(): from trex_core.application.reconciliation import sync_slot_to_broker class FailingWarnStore(InMemoryReconcileWarningStore): - def clear_warned(self, slot_id): + def clear_warned(self, slot_id, *, case_code=None): raise RuntimeError("DB unavailable") ns = InMemoryReconcileNotificationStore() diff --git a/tests/application/test_reconcile_warning_blocking.py b/tests/application/test_reconcile_warning_blocking.py index 73487b7..8acff52 100644 --- a/tests/application/test_reconcile_warning_blocking.py +++ b/tests/application/test_reconcile_warning_blocking.py @@ -892,6 +892,8 @@ def test_slot_to_dict_reconcile_warned_false_when_no_warn_store(): row = slot_to_dict(slot, warn_store=None) assert "reconcile_warned" in row assert row["reconcile_warned"] is False + assert row["trade_eligible"] is True + assert row["reconcile_warning_cases"] == [] def test_slot_to_dict_reconcile_warned_true_when_warned(): @@ -902,6 +904,8 @@ def test_slot_to_dict_reconcile_warned_true_when_warned(): warn_store.set_warned(str(slot.slot_id), portfolio_id="pf-A", case_code="B") row = slot_to_dict(slot, warn_store=warn_store) assert row["reconcile_warned"] is True + assert row["trade_eligible"] is False + assert row["reconcile_warning_cases"] == ["B"] def test_slot_to_dict_paused_and_warned_independent(): diff --git a/tests/application/test_reconcile_warning_store.py b/tests/application/test_reconcile_warning_store.py index 97f329e..7224aa9 100644 --- a/tests/application/test_reconcile_warning_store.py +++ b/tests/application/test_reconcile_warning_store.py @@ -33,14 +33,27 @@ def test_clear_warned_returns_false(): assert store.is_warned("slot-1") is False -# AC-3 (REQ-102): duplicate set_warned on same slot_id => upsert (single entry). -def test_set_warned_upsert_same_slot(): +# AC-3 (REQ-123): same slot can carry multiple active warning causes. +def test_set_warned_preserves_multiple_cases_on_same_slot(): store = _store() store.set_warned("slot-1", portfolio_id="pf-A", case_code="B", reason="first") store.set_warned("slot-1", portfolio_id="pf-A", case_code="C", reason="second") - # After second set, is_warned still True — single entry (no duplicate row). assert store.is_warned("slot-1") is True assert store.list_warned() == {"slot-1"} + info = store.get_warning_info("slot-1") + assert info is not None + assert info["case_codes"] == ["B", "C"] + + +def test_clear_warned_single_case_preserves_other_case(): + store = _store() + store.set_warned("slot-1", portfolio_id="pf-A", case_code="B") + store.set_warned("slot-1", portfolio_id="pf-A", case_code="C") + store.clear_warned("slot-1", case_code="C") + info = store.get_warning_info("slot-1") + assert info is not None + assert info["case_codes"] == ["B"] + assert store.is_warned("slot-1") is True # AC-5 (REQ-102): list_warned returns set of warned slot_ids. diff --git a/tests/application/test_slots_handler_quantity_mismatch.py b/tests/application/test_slots_handler_quantity_mismatch.py index 124a495..d181765 100644 --- a/tests/application/test_slots_handler_quantity_mismatch.py +++ b/tests/application/test_slots_handler_quantity_mismatch.py @@ -128,6 +128,44 @@ def test_ac1_single_portfolio_c_case_excluded(): assert resp.payload["slots"] == [] assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 + assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"C": 1} + + +def test_holding_slot_with_b_case_is_excluded_for_strategy(): + repo = InMemorySlotRepository() + svc = SlotService(repo) + warn_store = InMemoryReconcileWarningStore() + + slot = _make_holding_slot(repo, svc, portfolio_id="pf-A", symbol=SYM_A, qty=10) + warn_store.set_warned(str(slot.slot_id), portfolio_id="pf-A", case_code="B") + + handler = _make_handler(repo, warn_store) + resp = handler(_req(filter="holding")) + + assert resp.payload["slots"] == [] + assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 + assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"B": 1} + + +def test_holding_slot_with_b_and_c_cases_keeps_both_and_reports_both(): + repo = InMemorySlotRepository() + svc = SlotService(repo) + warn_store = InMemoryReconcileWarningStore() + + slot = _make_holding_slot(repo, svc, portfolio_id="pf-A", symbol=SYM_A, qty=10) + slot_id = str(slot.slot_id) + warn_store.set_warned(slot_id, portfolio_id="pf-A", case_code="B") + warn_store.set_warned(slot_id, portfolio_id="pf-A", case_code="C") + + handler = _make_handler(repo, warn_store) + resp = handler(_req(filter="holding")) + + assert resp.payload["slots"] == [] + assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 + assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"B": 1, "C": 1} + info = warn_store.get_warning_info(slot_id) + assert info is not None + assert info["case_codes"] == ["B", "C"] # --------------------------------------------------------------------------- @@ -286,7 +324,8 @@ def test_ac5_excluded_slots_count_zero_when_no_warnings(): resp = handler(_req(filter="holding")) assert "excluded_slots" in resp.payload - assert resp.payload["excluded_slots"] == {"reconcile_quantity_mismatch": 0} + assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 0 + assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {} # --------------------------------------------------------------------------- @@ -312,8 +351,8 @@ def test_ac6_no_warn_store_no_filtering_no_excluded_slots_key(): # --------------------------------------------------------------------------- class BrokenWarnStore(InMemoryReconcileWarningStore): - """Simulates a warn_store that fails on list_c_case_warned.""" - def list_c_case_warned(self): + """Simulates a warn_store that fails on warning reads.""" + def list_warned(self): raise RuntimeError("db connection lost") @@ -356,7 +395,7 @@ def test_ac23_exclusion_precedes_broker_and_strategy_state_readers(): ) response = handler(_req(filter="holding")).payload assert response["slots"] == [] - assert response["excluded_slots"] == {"reconcile_quantity_mismatch": 1} + assert response["excluded_slots"]["reconcile_quantity_mismatch"] == 1 # --------------------------------------------------------------------------- diff --git a/tests/application/test_sync.py b/tests/application/test_sync.py index c09d08f..4079789 100644 --- a/tests/application/test_sync.py +++ b/tests/application/test_sync.py @@ -199,6 +199,22 @@ def test_sync_holding_qty_mismatch_updated(): assert not ws_out.is_warned(str(slot.slot_id)) +def test_sync_clears_only_b_case_when_slot_has_multiple_warning_causes(): + slot = _make_slot(SlotState.HOLDING, qty=5.0) + ws = InMemoryReconcileWarningStore() + slot_id = str(slot.slot_id) + ws.set_warned(slot_id, portfolio_id="pf-005930", case_code="B") + ws.set_warned(slot_id, portfolio_id="pf-005930", case_code="C") + + result, _repo, ws_out = _setup(slot, {"005930": 10}, warn_store=ws) + + assert result["warning_cleared"] is True + assert ws_out.is_warned(slot_id) is True + info = ws_out.get_warning_info(slot_id) + assert info is not None + assert info["case_codes"] == ["C"] + + # --------------------------------------------------------------------------- # AC-4: WAC seed from cost_basis_port when available # --------------------------------------------------------------------------- diff --git a/tests/application/test_verify.py b/tests/application/test_verify.py index 5adc88d..61f94cf 100644 --- a/tests/application/test_verify.py +++ b/tests/application/test_verify.py @@ -313,6 +313,40 @@ def test_verify_c_case_clear_preserves_b_case_warning(): assert b_info is not None and b_info["case_code"] == "B" +def test_verify_c_case_clear_on_slot_with_b_and_c_keeps_b(): + sym = "005930" + slot = _make_slot(status=SlotState.HOLDING, symbol=sym, pid="pf-c") + + repo = InMemorySlotRepository() + repo.add_slot(slot) + + ws = InMemoryReconcileWarningStore() + slot_id = str(slot.slot_id) + ws.set_warned(slot_id, portfolio_id="pf-c", case_code="B") + ws.set_warned(slot_id, portfolio_id="pf-c", case_code="C") + + store = SnapshotStore() + store.swap([ + PortfolioItem(portfolio_id="pf-c", symbol=sym, cash_balance=0.0, asset_quantity=10.0), + ]) + broker = FakeBroker({sym: 10}) + + result = verify_slot_reconcile( + portfolio_id="pf-c", + store=store, + broker=broker, + slot_repo=repo, + instance_key=INSTANCE, + warn_store=ws, + ) + + assert result["result"] == "cleared" + assert ws.is_warned(slot_id) is True + info = ws.get_warning_info(slot_id) + assert info is not None + assert info["case_codes"] == ["B"] + + def test_verify_c_case_clear_non_holding_slot(): """A non-HOLDING slot with C-case warning for the same symbol must also be cleared.""" sym = "005930" @@ -360,11 +394,11 @@ class _FailAfterNStore(InMemoryReconcileWarningStore): self._calls = 0 self._fail_after = fail_after - def clear_warned(self, slot_id: str) -> None: + def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None: self._calls += 1 if self._calls > self._fail_after: raise RuntimeError("simulated DB error") - super().clear_warned(slot_id) + super().clear_warned(slot_id, case_code=case_code) def test_verify_partial_clear_failure_returns_clear_failed(): diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index f110464..91f6d1e 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -418,7 +418,7 @@ def test_sync_fail_closed_skipped_error_exits_nonzero_without_writes( return SimpleNamespace(get_balance=lambda: []), None class _WarningStore: - def clear_warned(self, *_args): + def clear_warned(self, *_args, **_kwargs): raise AssertionError("warning clear must not be called") class _Connection: diff --git a/trex_core/adapters/persistence/sql/016_create_slot_reconcile_warning.sql b/trex_core/adapters/persistence/sql/016_create_slot_reconcile_warning.sql index e663962..98664c3 100644 --- a/trex_core/adapters/persistence/sql/016_create_slot_reconcile_warning.sql +++ b/trex_core/adapters/persistence/sql/016_create_slot_reconcile_warning.sql @@ -1,6 +1,6 @@ -- BAT-050 / REQ-102: reconcile-warning durable store (dcy-034). --- PK: slot_id (TEXT, UUID string). pause_store 동형 설계. --- ON CONFLICT DO UPDATE: 동일 slot_id 재삽입 시 최신 값으로 갱신. +-- Initial schema; migration 028 upgrades PK to (slot_id, case_code) +-- so B/C causes can coexist for the same slot. CREATE TABLE IF NOT EXISTS slot_reconcile_warning ( slot_id TEXT PRIMARY KEY, portfolio_id TEXT NOT NULL, diff --git a/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql b/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql new file mode 100644 index 0000000..11e967d --- /dev/null +++ b/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql @@ -0,0 +1,7 @@ +-- REQ-123 / issue #83: +-- Preserve simultaneous warning causes by keying warnings per (slot_id, case_code). +ALTER TABLE slot_reconcile_warning + DROP CONSTRAINT IF EXISTS slot_reconcile_warning_pkey; + +ALTER TABLE slot_reconcile_warning + ADD CONSTRAINT slot_reconcile_warning_pkey PRIMARY KEY (slot_id, case_code); diff --git a/trex_core/adapters/portfolio/reconcile_warning_store.py b/trex_core/adapters/portfolio/reconcile_warning_store.py index 731b565..91f8c7a 100644 --- a/trex_core/adapters/portfolio/reconcile_warning_store.py +++ b/trex_core/adapters/portfolio/reconcile_warning_store.py @@ -1,8 +1,4 @@ -"""PostgreSQL-backed durable reconcile-warning store (REQ-102, BAT-050). - -pause_store.py와 동형 설계. PK: slot_id. case_code: 'B' | 'C'. -Survives daemon restart (dcy-034 NFR-1). -""" +"""PostgreSQL-backed durable reconcile-warning store (REQ-102, BAT-050).""" from __future__ import annotations from datetime import datetime, timezone @@ -12,7 +8,7 @@ from trex_core.adapters.persistence.postgresql import _require_psycopg class PostgreSQLReconcileWarningStore: - """`slot_reconcile_warning` table CRUD. slot_id PK is the gate (REQ-102).""" + """`slot_reconcile_warning` table CRUD keyed by (slot_id, case_code).""" def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None: self._psycopg = _require_psycopg(psycopg_module) @@ -34,9 +30,8 @@ class PostgreSQLReconcileWarningStore: "INSERT INTO slot_reconcile_warning " "(slot_id, portfolio_id, case_code, actor, reason, created_at, updated_at) " "VALUES (%s, %s, %s, %s, %s, %s, %s) " - "ON CONFLICT (slot_id) DO UPDATE " + "ON CONFLICT (slot_id, case_code) DO UPDATE " "SET portfolio_id = EXCLUDED.portfolio_id, " - " case_code = EXCLUDED.case_code, " " actor = EXCLUDED.actor, " " reason = EXCLUDED.reason, " " updated_at = EXCLUDED.updated_at", @@ -44,13 +39,19 @@ class PostgreSQLReconcileWarningStore: ) conn.commit() - def clear_warned(self, slot_id: str) -> None: + def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None: with self._psycopg.connect(self._dsn, autocommit=False) as conn: with conn.cursor() as cur: - cur.execute( - "DELETE FROM slot_reconcile_warning WHERE slot_id = %s", - (slot_id,), - ) + if case_code is None: + cur.execute( + "DELETE FROM slot_reconcile_warning WHERE slot_id = %s", + (slot_id,), + ) + else: + cur.execute( + "DELETE FROM slot_reconcile_warning WHERE slot_id = %s AND case_code = %s", + (slot_id, case_code), + ) conn.commit() def is_warned(self, slot_id: str) -> bool: @@ -65,7 +66,7 @@ class PostgreSQLReconcileWarningStore: def list_warned(self) -> set[str]: with self._psycopg.connect(self._dsn) as conn: with conn.cursor() as cur: - cur.execute("SELECT slot_id FROM slot_reconcile_warning") + cur.execute("SELECT DISTINCT slot_id FROM slot_reconcile_warning") return {row[0] for row in cur.fetchall()} def list_c_case_warned(self) -> set[str]: @@ -73,26 +74,30 @@ class PostgreSQLReconcileWarningStore: with self._psycopg.connect(self._dsn) as conn: with conn.cursor() as cur: cur.execute( - "SELECT slot_id FROM slot_reconcile_warning WHERE case_code = %s", + "SELECT DISTINCT slot_id FROM slot_reconcile_warning WHERE case_code = %s", ("C",), ) return {row[0] for row in cur.fetchall()} def get_warning_info(self, slot_id: str) -> dict | None: - """Return {"portfolio_id", "case_code", "actor", "reason"} or None (REQ-105/106).""" + """Return warning info with full active case set, or None.""" with self._psycopg.connect(self._dsn) as conn: with conn.cursor() as cur: cur.execute( "SELECT portfolio_id, case_code, actor, reason " - "FROM slot_reconcile_warning WHERE slot_id = %s LIMIT 1", + "FROM slot_reconcile_warning WHERE slot_id = %s " + "ORDER BY CASE WHEN case_code = 'C' THEN 0 ELSE 1 END, updated_at DESC", (slot_id,), ) - row = cur.fetchone() - if row is None: + rows = cur.fetchall() + if not rows: return None + case_codes = sorted({str(row[1]) for row in rows if row[1]}) + preferred = rows[0] return { - "portfolio_id": row[0], - "case_code": row[1], - "actor": row[2], - "reason": row[3], + "portfolio_id": preferred[0], + "case_code": preferred[1], + "case_codes": case_codes, + "actor": preferred[2], + "reason": preferred[3], } diff --git a/trex_core/application/messaging_handlers.py b/trex_core/application/messaging_handlers.py index 9e5f291..f50c04a 100644 --- a/trex_core/application/messaging_handlers.py +++ b/trex_core/application/messaging_handlers.py @@ -263,28 +263,24 @@ def make_slots_rpc_handler( for s in holding_slots: watchdog_store.update(str(s.slot_id)) - # Build merged slot rows (REQ-074) - # BAT-050 / REQ-104: warn_store passed to slot_to_dict for reconcile_warned field. - slot_rows = [ - slot_to_dict( - s, - snapshot_store=snapshot_store, - watchdog_store=watchdog_store, - pause_store=pause_store, - warn_store=warn_store, - watchdog_timeout_sec=watchdog_timeout_sec, - ) - for s in slots - ] - - # AC-0023 exclusion must precede all optional data readers so excluded - # slots cannot expose state or make an otherwise valid response fail. - excluded_count = 0 + warning_cases_by_slot: dict[str, set[str]] = {} if warn_store is not None: try: - c_warned_ids = warn_store.list_c_case_warned() + warned_slot_ids = warn_store.list_warned() + for sid in warned_slot_ids: + info = warn_store.get_warning_info(sid) + if info is None: + continue + raw_codes = info.get("case_codes") + if isinstance(raw_codes, list): + codes = {str(code) for code in raw_codes if code} + else: + case_code = str(info.get("case_code", "")) + codes = {case_code} if case_code else set() + if codes: + warning_cases_by_slot[str(sid)] = codes except Exception: # noqa: BLE001 - _log.exception("warn_store.list_c_case_warned() failed — fail-closed") + _log.exception("warn_store warning query failed — fail-closed") return RpcResponse(payload={ "slots": [], "available_slots": None, @@ -296,18 +292,42 @@ def make_slots_rpc_handler( ), "excluded_slots": {"reconcile_quantity_mismatch": 0}, }) - if c_warned_ids: - included = [ - (slot, row) - for slot, row in zip(slots, slot_rows, strict=True) - if not ( - slot.status.value == "holding" - and str(slot.slot_id) in c_warned_ids - ) - ] - excluded_count = len(slots) - len(included) + + # Build merged slot rows (REQ-074) + slot_rows = [ + slot_to_dict( + s, + snapshot_store=snapshot_store, + watchdog_store=watchdog_store, + pause_store=pause_store, + warning_cases_by_slot=warning_cases_by_slot, + watchdog_timeout_sec=watchdog_timeout_sec, + ) + for s in slots + ] + + # Exclusion must precede all optional data readers so blocked slots cannot + # expose state or make an otherwise valid response fail. + excluded_count = 0 + excluded_case_counts: dict[str, int] = {} + if warn_store is not None: + included: list[tuple[object, dict]] = [] + for slot, row in zip(slots, slot_rows, strict=True): + blocked_holding = ( + slot.status.value == "holding" and bool(row.get("reconcile_warned")) + ) + if blocked_holding: + excluded_count += 1 + for code in row.get("reconcile_warning_cases", []): + excluded_case_counts[code] = excluded_case_counts.get(code, 0) + 1 + continue + included.append((slot, row)) + if included: slots = [slot for slot, _ in included] slot_rows = [row for _, row in included] + else: + slots = [] + slot_rows = [] broker_qty_by_symbol: dict[str, int] = {} broker_qty_available = False @@ -410,7 +430,10 @@ def make_slots_rpc_handler( "message": message, } if warn_store is not None: - base_payload["excluded_slots"] = {"reconcile_quantity_mismatch": excluded_count} + base_payload["excluded_slots"] = { + "reconcile_quantity_mismatch": excluded_count, + "reconcile_warning_cases": excluded_case_counts, + } return RpcResponse(payload=base_payload) return handler diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index abe519b..ad2b7c0 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -359,7 +359,7 @@ class InMemoryKillSwitchStore: class ReconcileWarningStore(Protocol): - """Durable reconcile-warning overlay — slot_id PK (REQ-102, dcy-034).""" + """Durable reconcile-warning overlay keyed by (slot_id, case_code).""" def set_warned( self, @@ -372,7 +372,7 @@ class ReconcileWarningStore(Protocol): ) -> None: ... - def clear_warned(self, slot_id: str) -> None: + def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None: ... def is_warned(self, slot_id: str) -> bool: @@ -382,10 +382,14 @@ class ReconcileWarningStore(Protocol): ... def get_warning_info(self, slot_id: str) -> dict | None: - """Return {"portfolio_id", "case_code", "actor", "reason"} or None. + """Return warning info dict or None. - Used by verify to determine whether the warning is case_code='C' - (verify's responsibility) vs 'B' (sync's responsibility, REQ-106 OQ-C). + Returned dict includes: + {"portfolio_id", "case_code", "case_codes", "actor", "reason"} + where case_code is a backward-compatible preferred case and case_codes is + the full active case set for the slot. + + Used by verify to determine whether C-case is active (REQ-106 OQ-C). Returns None when the slot has no active warning. """ ... @@ -403,8 +407,8 @@ class InMemoryReconcileWarningStore: def __init__(self) -> None: self._lock = threading.Lock() - # slot_id -> {"portfolio_id", "case_code", "actor", "reason"} - self._warned: dict[str, dict] = {} + # slot_id -> case_code -> {"portfolio_id", "actor", "reason"} + self._warned: dict[str, dict[str, dict]] = {} def set_warned( self, @@ -416,16 +420,24 @@ class InMemoryReconcileWarningStore: reason: str = "", ) -> None: with self._lock: - self._warned[slot_id] = { + per_case = self._warned.setdefault(slot_id, {}) + per_case[case_code] = { "portfolio_id": portfolio_id, - "case_code": case_code, "actor": actor, "reason": reason, } - def clear_warned(self, slot_id: str) -> None: + def clear_warned(self, slot_id: str, *, case_code: str | None = None) -> None: with self._lock: - self._warned.pop(slot_id, None) + if case_code is None: + self._warned.pop(slot_id, None) + return + per_case = self._warned.get(slot_id) + if not per_case: + return + per_case.pop(case_code, None) + if not per_case: + self._warned.pop(slot_id, None) def is_warned(self, slot_id: str) -> bool: with self._lock: @@ -437,12 +449,23 @@ class InMemoryReconcileWarningStore: def get_warning_info(self, slot_id: str) -> dict | None: with self._lock: - info = self._warned.get(slot_id) - return dict(info) if info is not None else None + per_case = self._warned.get(slot_id) + if not per_case: + return None + case_codes = sorted(per_case.keys()) + preferred_case = "C" if "C" in per_case else case_codes[0] + preferred = per_case[preferred_case] + return { + "portfolio_id": preferred.get("portfolio_id", ""), + "case_code": preferred_case, + "case_codes": case_codes, + "actor": preferred.get("actor", ""), + "reason": preferred.get("reason", ""), + } def list_c_case_warned(self) -> set[str]: with self._lock: - return {sid for sid, info in self._warned.items() if info.get("case_code") == "C"} + return {sid for sid, per_case in self._warned.items() if "C" in per_case} class ReconcileNotificationStore(Protocol): @@ -544,6 +567,7 @@ def slot_to_dict( watchdog_store=None, pause_store: PauseStore | None = None, warn_store: ReconcileWarningStore | None = None, + warning_cases_by_slot: dict[str, set[str]] | None = None, watchdog_timeout_sec: int = 120, ) -> dict: """Merged slot row for trex.slots.req (REQ-073/074). @@ -618,10 +642,23 @@ def slot_to_dict( # BAT-050 / REQ-104: reconcile_warned overlay (ReconcileWarningStore) — additive field. # Bulk list_warned() called once per RPC (caller pre-fetches; see make_slots_rpc_handler). # warn_store=None → reconcile_warned=False (backward compatible, NFR-4). - reconcile_warned = False - if warn_store is not None: - reconcile_warned = str(s.slot_id) in warn_store.list_warned() + warning_cases: set[str] = set() + if warning_cases_by_slot is not None: + warning_cases = set(warning_cases_by_slot.get(str(s.slot_id), set())) + elif warn_store is not None: + info = warn_store.get_warning_info(str(s.slot_id)) + if info is not None: + raw_cases = info.get("case_codes") + if isinstance(raw_cases, list): + warning_cases = {str(c) for c in raw_cases if c} + else: + case_code = str(info.get("case_code", "")) + if case_code: + warning_cases = {case_code} + reconcile_warned = bool(warning_cases) d["reconcile_warned"] = reconcile_warned + d["reconcile_warning_cases"] = sorted(warning_cases) + d["trade_eligible"] = not reconcile_warned return d @@ -637,4 +674,3 @@ def _sanitize(message: str) -> str: out = out[:idx] + "" break return out[:200] - diff --git a/trex_core/application/reconciliation.py b/trex_core/application/reconciliation.py index aca2c5d..bf326c1 100644 --- a/trex_core/application/reconciliation.py +++ b/trex_core/application/reconciliation.py @@ -629,7 +629,7 @@ def sync_slot_to_broker( Broker qty == 0 for slot → skipped with WARN (slot-side A-case guard). Broker-only (no slot) → NOT handled here; caller filters (FR-3). - After update, if slot.qty == broker_qty: clear_warned(slot_id). + After update, if slot.qty == broker_qty: clear_warned(slot_id, case_code="B"). """ from uuid import UUID @@ -870,14 +870,15 @@ def sync_slot_to_broker( after_avg = float(refreshed.avg_price) if (refreshed and refreshed.avg_price is not None) else None after = {"qty": after_qty, "status": after_status, "avg_price": after_avg} - # Clear warning when qty now matches broker + # Clear B-case warning when qty now matches broker warning_cleared = False if after_qty is not None and int(after_qty) == broker_qty: try: - warn_store.clear_warned(slot_id) + warn_store.clear_warned(slot_id, case_code="B") warning_cleared = True _log.info( - "sync: reconcile-warning cleared slot=%s symbol=%s broker_qty=%d actor=sync", + "sync: reconcile-warning cleared slot=%s symbol=%s case_code=B " + "broker_qty=%d actor=sync", slot_id, sym, broker_qty, ) except Exception: # noqa: BLE001 @@ -962,7 +963,7 @@ def verify_slot_reconcile( "reason": "no active reconcile-warning (already cleared or never set)", } - # Check case_code — verify is C-only (REQ-106 OQ-C). + # Check case set — verify is C-only (REQ-106 OQ-C). info = warn_store.get_warning_info(slot_id_str) if info is None: return { @@ -974,8 +975,13 @@ def verify_slot_reconcile( "reason": "warning exists but get_warning_info returned None (unexpected)", } - case_code = info.get("case_code", "") - if case_code != "C": + case_codes_raw = info.get("case_codes") + if isinstance(case_codes_raw, list): + case_codes = {str(code) for code in case_codes_raw if code} + else: + case_code = str(info.get("case_code", "")) + case_codes = {case_code} if case_code else set() + if "C" not in case_codes: return { "portfolio_id": portfolio_id, "slot_id": slot_id_str, @@ -983,7 +989,7 @@ def verify_slot_reconcile( "hub_qty": None, "broker_qty": None, "reason": ( - f"warning case_code='{case_code}' is not 'C' — " + f"warning case_codes={sorted(case_codes)} do not include 'C' — " "use 'trex-core sync' to resolve B-case (slot↔broker qty mismatch)" ), } @@ -1014,9 +1020,17 @@ def verify_slot_reconcile( continue sid_str = str(s.slot_id) warn_info = warn_store.get_warning_info(sid_str) - if warn_info is None or warn_info.get("case_code") != "C": + if warn_info is None: continue - warn_store.clear_warned(sid_str) + warned_codes_raw = warn_info.get("case_codes") + if isinstance(warned_codes_raw, list): + warned_codes = {str(code) for code in warned_codes_raw if code} + else: + case_code = str(warn_info.get("case_code", "")) + warned_codes = {case_code} if case_code else set() + if "C" not in warned_codes: + continue + warn_store.clear_warned(sid_str, case_code="C") store.set_warning(s.portfolio_id, False) cleared_slots.append(sid_str) _log.info( diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index 128ef21..d985ad0 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -912,7 +912,15 @@ def _cmd_verify(args, cfg) -> int: c_slots: list[str] = [] for sid in warn_store.list_warned(): info = warn_store.get_warning_info(sid) - if info and info.get("case_code") == "C": + if not info: + continue + raw_codes = info.get("case_codes") + if isinstance(raw_codes, list): + case_codes = {str(code) for code in raw_codes if code} + else: + case_code = str(info.get("case_code", "")) + case_codes = {case_code} if case_code else set() + if "C" in case_codes: c_slots.append(info.get("portfolio_id", "")) target_pids = [p for p in c_slots if p] diff --git a/trex_core/runtime/daemon.py b/trex_core/runtime/daemon.py index d232973..8af08bb 100644 --- a/trex_core/runtime/daemon.py +++ b/trex_core/runtime/daemon.py @@ -198,7 +198,7 @@ class Daemon: # BAT-010/011: per-slot watchdog (snapshot RPC writer, safety-sell reader). self.watchdog = WatchdogStore() self.pause_store = PostgreSQLPauseStore(config.dsn) - # BAT-050 / REQ-102: durable reconcile-warning store (slot_id PK). + # BAT-050 / REQ-102: durable reconcile-warning store ((slot_id, case_code) PK). self.warn_store = PostgreSQLReconcileWarningStore(config.dsn) # req-0007 / ac-0024: durable reconcile notification deduplication store. self.notification_store = PostgreSQLReconcileNotificationStore(config.dsn) From fe51d211d9ea4f3ff0cdd3efe794bc9c398fa91d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:24:51 +0000 Subject: [PATCH 05/21] feat: persist unmanaged position lifecycle state Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_store.py | 101 ++++++ .../test_unmanaged_position_lifecycle.py | 190 +++++++++++ tests/conftest.py | 2 + tests/runtime/test_cli_operator.py | 169 ++++++++++ .../028_create_unmanaged_position_state.sql | 38 +++ trex_core/adapters/portfolio/__init__.py | 2 + .../portfolio/unmanaged_position_store.py | 309 ++++++++++++++++++ trex_core/application/portfolio.py | 254 +++++++++++++- trex_core/application/reconciliation.py | 101 +++++- trex_core/application/snapshot_scheduler.py | 4 + trex_core/runtime/cli.py | 104 ++++++ trex_core/runtime/daemon.py | 5 + 12 files changed, 1274 insertions(+), 5 deletions(-) create mode 100644 tests/adapters/portfolio/test_unmanaged_position_store.py create mode 100644 tests/application/test_unmanaged_position_lifecycle.py create mode 100644 trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql create mode 100644 trex_core/adapters/portfolio/unmanaged_position_store.py diff --git a/tests/adapters/portfolio/test_unmanaged_position_store.py b/tests/adapters/portfolio/test_unmanaged_position_store.py new file mode 100644 index 0000000..b882f76 --- /dev/null +++ b/tests/adapters/portfolio/test_unmanaged_position_store.py @@ -0,0 +1,101 @@ +"""PostgreSQLUnmanagedPositionStore integration tests.""" +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + + +@pytest.fixture +def pg_unmanaged_store(pg_clean): + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + return PostgreSQLUnmanagedPositionStore(pg_clean) + + +class TestMigration028: + def test_tables_exist_after_migration(self, pg_clean): + import psycopg + + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + cur.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'public' " + "AND table_name IN ('unmanaged_position_state', 'unmanaged_position_audit')" + ) + assert {row[0] for row in cur.fetchall()} == { + "unmanaged_position_state", + "unmanaged_position_audit", + } + + +class TestCRUDRoundTrip: + def test_observe_change_resolve_and_audit(self, pg_unmanaged_store): + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + first = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + observed_at=t1, + ) + repeat = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + observed_at=t2, + ) + changed = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=5, + observed_at=t3, + ) + recorded = pg_unmanaged_store.record_operator_action( + instance_key="inst-A", + symbol="008110", + action="approved_exception", + evidence_ref="kis:2026-09-11", + actor="cli", + note="broker-confirmed", + ) + resolved = pg_unmanaged_store.resolve_absent( + instance_key="inst-A", + observed_symbols=set(), + observed_at=t3 + timedelta(minutes=1), + ) + + assert first["transition"] == "detected" + assert repeat["transition"] == "unchanged" + assert changed["transition"] == "quantity_changed" + assert recorded is not None + assert len(resolved) == 1 + state = pg_unmanaged_store.get_state(instance_key="inst-A", symbol="008110") + assert state is not None + assert state["quantity"] == 5 + assert state["first_seen_at"] == t1 + assert state["last_seen_at"] == t3 + assert state["quantity_changed_at"] == t3 + assert state["lifecycle_state"] == "resolved" + assert state["evidence_ref"] == "kis:2026-09-11" + audit = pg_unmanaged_store.list_audit(instance_key="inst-A", symbol="008110") + assert [row["event_type"] for row in audit] == [ + "detected", + "quantity_changed", + "operator_action", + "resolved", + ] + + def test_same_symbol_isolated_by_instance(self, pg_unmanaged_store): + pg_unmanaged_store.observe(instance_key="inst-A", symbol="008110", quantity=3) + pg_unmanaged_store.observe(instance_key="inst-B", symbol="008110", quantity=7) + pg_unmanaged_store.resolve_absent(instance_key="inst-A", observed_symbols=set()) + + rows_a = pg_unmanaged_store.list_states(instance_key="inst-A", lifecycle_state="all") + rows_b = pg_unmanaged_store.list_states(instance_key="inst-B", lifecycle_state="all") + assert len(rows_a) == 1 and rows_a[0]["lifecycle_state"] == "resolved" + assert len(rows_b) == 1 and rows_b[0]["quantity"] == 7 diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py new file mode 100644 index 0000000..3344117 --- /dev/null +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from trex_core.adapters.messaging.in_memory import InMemoryTransport +from trex_core.application.alarm_emitter import AlarmEmitter +from trex_core.application.broker_port import BalanceItem, BrokerBalanceError +from trex_core.application.portfolio import InMemoryUnmanagedPositionStore +from trex_core.application.reconciliation import reconcile_core_holdings +from trex_core.domain.slot import SlotService +from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository + +INSTANCE = "inst-1" + + +class FakeBroker: + def __init__(self, balances): + self._balances = balances + + def name(self): + return "fake" + + def submit_order(self, req): ... + def poll_fill(self, *args, **kwargs): ... + def cancel_order(self, *args, **kwargs): ... + + def get_balance(self): + return [BalanceItem(symbol=symbol, hldg_qty=qty) for symbol, qty in self._balances.items()] + + +class RaisingBroker(FakeBroker): + def get_balance(self): + raise BrokerBalanceError("broker_unavailable") + + +def _wire(): + repo = InMemorySlotRepository() + return repo, SlotService(repo, instance_key=INSTANCE) + + +def _make_emitter(): + transport = InMemoryTransport() + return AlarmEmitter(transport), transport + + +def test_in_memory_store_tracks_lifecycle_and_operator_audit(): + store = InMemoryUnmanagedPositionStore() + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + first = store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t1) + repeat = store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t2) + changed = store.observe(instance_key=INSTANCE, symbol="008110", quantity=4, observed_at=t3) + recorded = store.record_operator_action( + instance_key=INSTANCE, + symbol="008110", + action="investigating", + evidence_ref="kis-balance:2026-09-07", + note="confirmed at broker", + ) + resolved = store.resolve_absent( + instance_key=INSTANCE, + observed_symbols=set(), + observed_at=t3 + timedelta(minutes=1), + ) + + assert first["transition"] == "detected" + assert repeat["transition"] == "unchanged" + assert changed["transition"] == "quantity_changed" + assert recorded is not None + assert recorded["evidence_ref"] == "kis-balance:2026-09-07" + assert len(resolved) == 1 + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["quantity"] == 4 + assert state["first_seen_at"] == t1 + assert state["last_seen_at"] == t3 + assert state["quantity_changed_at"] == t3 + assert state["lifecycle_state"] == "resolved" + assert state["resolved_at"] == t3 + timedelta(minutes=1) + audit = store.list_audit(instance_key=INSTANCE, symbol="008110") + assert [row["event_type"] for row in audit] == [ + "detected", + "quantity_changed", + "operator_action", + "resolved", + ] + + +def test_in_memory_store_isolates_same_symbol_by_instance(): + store = InMemoryUnmanagedPositionStore() + + store.observe(instance_key="inst-A", symbol="008110", quantity=3) + store.observe(instance_key="inst-B", symbol="008110", quantity=7) + store.resolve_absent(instance_key="inst-A", observed_symbols=set()) + + state_a = store.get_state(instance_key="inst-A", symbol="008110") + state_b = store.get_state(instance_key="inst-B", symbol="008110") + assert state_a is not None and state_a["lifecycle_state"] == "resolved" + assert state_b is not None and state_b["lifecycle_state"] == "active" + assert state_b["quantity"] == 7 + + +def test_reconcile_core_holdings_suppresses_repeated_unmanaged_alarm_spam(): + repo, svc = _wire() + emitter, transport = _make_emitter() + store = InMemoryUnmanagedPositionStore() + + for _ in range(3): + decisions = reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FakeBroker({"008110": 3}), + instance_key=INSTANCE, + alarm_emitter=emitter, + unmanaged_position_store=store, + ) + assert decisions == {"broker:008110": "unmanaged_position"} + + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["quantity"] == 3 + codes = [payload["code"] for _, payload in transport.alarm_events()] + assert codes == ["unmanaged_position_detected"] + audit = store.list_audit(instance_key=INSTANCE, symbol="008110") + assert [row["event_type"] for row in audit] == ["detected"] + + +def test_reconcile_core_holdings_tracks_change_and_resolution(): + repo, svc = _wire() + emitter, transport = _make_emitter() + store = InMemoryUnmanagedPositionStore() + + reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FakeBroker({"008110": 3}), + instance_key=INSTANCE, + alarm_emitter=emitter, + unmanaged_position_store=store, + ) + reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FakeBroker({"008110": 5}), + instance_key=INSTANCE, + alarm_emitter=emitter, + unmanaged_position_store=store, + ) + reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FakeBroker({}), + instance_key=INSTANCE, + alarm_emitter=emitter, + unmanaged_position_store=store, + ) + + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["quantity"] == 5 + assert state["lifecycle_state"] == "resolved" + codes = [payload["code"] for _, payload in transport.alarm_events()] + assert codes == [ + "unmanaged_position_detected", + "unmanaged_position_changed", + "unmanaged_position_resolved", + ] + + +def test_broker_failure_does_not_resolve_unmanaged_state(): + repo, svc = _wire() + store = InMemoryUnmanagedPositionStore() + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3) + + with pytest.raises(BrokerBalanceError): + reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=RaisingBroker({}), + instance_key=INSTANCE, + unmanaged_position_store=store, + ) + + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" diff --git a/tests/conftest.py b/tests/conftest.py index fc525ef..cfa8acd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,8 @@ _PG_TABLES = ( "daily_slot", "reconcile_notification_state", "slot_strategy_state", + "unmanaged_position_audit", + "unmanaged_position_state", ) diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index f110464..d87683f 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -516,6 +516,175 @@ def test_verify_no_db_no_live_flag_needed(capsys, monkeypatch): assert rc == 0 +# --------------------------------------------------------------------------- +# unmanaged parser/tests +# --------------------------------------------------------------------------- + +def test_unmanaged_parser_accepts_list_and_record(): + parser = cli._build_parser() + args = parser.parse_args(["unmanaged", "list", "--status", "all", "--symbol", "008110"]) + assert args.cmd == "unmanaged" + assert args.unmanaged_cmd == "list" + assert args.status == "all" + args = parser.parse_args( + [ + "unmanaged", + "record", + "008110", + "investigating", + "--evidence-ref", + "kis-balance:2026-09-11", + ] + ) + assert args.unmanaged_cmd == "record" + assert args.action == "investigating" + + +def test_unmanaged_list_outputs_durable_state(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def list_states(self, **kwargs): + assert kwargs["instance_key"] == "inst-1" + assert kwargs["lifecycle_state"] == "all" + return [ + { + "instance_key": "inst-1", + "symbol": "008110", + "quantity": 3, + "lifecycle_state": "active", + "first_seen_at": "2026-09-07T00:00:00+00:00", + "last_seen_at": "2026-09-11T00:00:00+00:00", + "quantity_changed_at": "2026-09-07T00:00:00+00:00", + "resolved_at": None, + "operator_action": "investigating", + "evidence_ref": "kis-balance:2026-09-11", + "operator_note": "confirmed", + "operator_actor": "operator", + "operator_updated_at": "2026-09-11T00:00:00+00:00", + "created_at": "2026-09-07T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + } + ] + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + args = SimpleNamespace( + cmd="unmanaged", + unmanaged_cmd="list", + status="all", + symbol=None, + instance="inst-1", + config=None, + config_path=None, + env_file=None, + json=True, + timeout=10.0, + ) + + rc = cli._cmd_unmanaged_list(args, SimpleNamespace(dsn="postgresql://x/y")) + + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["count"] == 1 + assert out["states"][0]["symbol"] == "008110" + assert out["states"][0]["evidence_ref"] == "kis-balance:2026-09-11" + + +def test_unmanaged_record_requires_existing_state(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def record_operator_action(self, **_kwargs): + return None + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + args = SimpleNamespace( + cmd="unmanaged", + unmanaged_cmd="record", + symbol="008110", + action="approved_exception", + evidence_ref="kis-balance:2026-09-11", + note="confirmed", + actor="operator", + instance="inst-1", + config=None, + config_path=None, + env_file=None, + json=True, + timeout=10.0, + ) + + rc = cli._cmd_unmanaged_record(args, SimpleNamespace(dsn="postgresql://x/y")) + + assert rc == 1 + err = json.loads(capsys.readouterr().err) + assert "not found" in err["error"] + + +def test_unmanaged_record_persists_operator_action(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def record_operator_action(self, **kwargs): + assert kwargs["instance_key"] == "inst-1" + assert kwargs["symbol"] == "008110" + assert kwargs["action"] == "managed_inclusion" + return { + "instance_key": "inst-1", + "symbol": "008110", + "quantity": 3, + "lifecycle_state": "active", + "first_seen_at": "2026-09-07T00:00:00+00:00", + "last_seen_at": "2026-09-11T00:00:00+00:00", + "quantity_changed_at": "2026-09-07T00:00:00+00:00", + "resolved_at": None, + "operator_action": "managed_inclusion", + "evidence_ref": kwargs["evidence_ref"], + "operator_note": kwargs["note"], + "operator_actor": kwargs["actor"], + "operator_updated_at": "2026-09-11T00:00:00+00:00", + "created_at": "2026-09-07T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + } + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + args = SimpleNamespace( + cmd="unmanaged", + unmanaged_cmd="record", + symbol="008110", + action="managed_inclusion", + evidence_ref="kis-balance:2026-09-11", + note="manual inclusion planned", + actor="operator", + instance="inst-1", + config=None, + config_path=None, + env_file=None, + json=True, + timeout=10.0, + ) + + rc = cli._cmd_unmanaged_record(args, SimpleNamespace(dsn="postgresql://x/y")) + + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["result"] == "recorded" + assert out["state"]["operator_action"] == "managed_inclusion" + assert out["state"]["evidence_ref"] == "kis-balance:2026-09-11" + + # --------------------------------------------------------------------------- # REL-010 regression: _cmd_verify broker import must not raise ImportError # --------------------------------------------------------------------------- diff --git a/trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql b/trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql new file mode 100644 index 0000000..870cd7e --- /dev/null +++ b/trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql @@ -0,0 +1,38 @@ +CREATE TABLE IF NOT EXISTS unmanaged_position_state ( + instance_key TEXT NOT NULL, + symbol TEXT NOT NULL, + quantity INTEGER NOT NULL, + lifecycle_state TEXT NOT NULL, + first_seen_at TIMESTAMPTZ NOT NULL, + last_seen_at TIMESTAMPTZ NOT NULL, + quantity_changed_at TIMESTAMPTZ NOT NULL, + resolved_at TIMESTAMPTZ NULL, + operator_action TEXT NULL, + evidence_ref TEXT NULL, + operator_note TEXT NOT NULL DEFAULT '', + operator_actor TEXT NULL, + operator_updated_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (instance_key, symbol) +); + +CREATE INDEX IF NOT EXISTS idx_unmanaged_position_state_lifecycle + ON unmanaged_position_state (instance_key, lifecycle_state, updated_at DESC); + +CREATE TABLE IF NOT EXISTS unmanaged_position_audit ( + audit_id BIGSERIAL PRIMARY KEY, + instance_key TEXT NOT NULL, + symbol TEXT NOT NULL, + event_type TEXT NOT NULL, + action TEXT NULL, + quantity INTEGER NOT NULL, + lifecycle_state TEXT NOT NULL, + actor TEXT NOT NULL, + evidence_ref TEXT NULL, + note TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_unmanaged_position_audit_lookup + ON unmanaged_position_audit (instance_key, symbol, audit_id); diff --git a/trex_core/adapters/portfolio/__init__.py b/trex_core/adapters/portfolio/__init__.py index 794288e..4476194 100644 --- a/trex_core/adapters/portfolio/__init__.py +++ b/trex_core/adapters/portfolio/__init__.py @@ -3,10 +3,12 @@ from trex_core.adapters.portfolio.hub_read import PortfolioHubReadAdapter from trex_core.adapters.portfolio.pause_store import PostgreSQLPauseStore from trex_core.adapters.portfolio.reconcile_warning_store import PostgreSQLReconcileWarningStore from trex_core.adapters.portfolio.stub_read import StubPortfolioRead +from trex_core.adapters.portfolio.unmanaged_position_store import PostgreSQLUnmanagedPositionStore __all__ = [ "PortfolioHubReadAdapter", "PostgreSQLPauseStore", "PostgreSQLReconcileWarningStore", + "PostgreSQLUnmanagedPositionStore", "StubPortfolioRead", ] diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py new file mode 100644 index 0000000..694ab4b --- /dev/null +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -0,0 +1,309 @@ +"""PostgreSQL-backed unmanaged-position lifecycle store.""" +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from trex_core.adapters.persistence.postgresql import _require_psycopg + +_STATE_COLUMNS = ( + "instance_key, symbol, quantity, lifecycle_state, " + "first_seen_at, last_seen_at, quantity_changed_at, resolved_at, " + "operator_action, evidence_ref, operator_note, operator_actor, " + "operator_updated_at, created_at, updated_at" +) + + +class PostgreSQLUnmanagedPositionStore: + """CRUD + audit for unmanaged positions keyed by (instance_key, symbol).""" + + def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None: + self._psycopg = _require_psycopg(psycopg_module) + self._dsn = dsn + + def observe( + self, + *, + instance_key: str, + symbol: str, + quantity: int, + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> dict: + now = observed_at or datetime.now(tz=UTC) + with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " + "WHERE instance_key = %s AND symbol = %s " + "FOR UPDATE", + (instance_key, symbol), + ) + row = cur.fetchone() + previous = self._row_to_state(row) if row is not None else None + previous_quantity = previous.get("quantity") if previous is not None else None + if previous is None: + transition = "detected" + cur.execute( + "INSERT INTO unmanaged_position_state " + f"({_STATE_COLUMNS}) " + "VALUES (%s, %s, %s, 'active', %s, %s, %s, NULL, " + " NULL, NULL, '', NULL, NULL, %s, %s) " + f"RETURNING {_STATE_COLUMNS}", + (instance_key, symbol, quantity, now, now, now, now, now), + ) + elif previous["lifecycle_state"] == "resolved": + transition = "detected" + cur.execute( + "UPDATE unmanaged_position_state " + "SET quantity = %s, lifecycle_state = 'active', " + " first_seen_at = %s, last_seen_at = %s, quantity_changed_at = %s, " + " resolved_at = NULL, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (quantity, now, now, now, now, instance_key, symbol), + ) + elif int(previous["quantity"]) != int(quantity): + transition = "quantity_changed" + cur.execute( + "UPDATE unmanaged_position_state " + "SET quantity = %s, last_seen_at = %s, quantity_changed_at = %s, " + " resolved_at = NULL, lifecycle_state = 'active', updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (quantity, now, now, now, instance_key, symbol), + ) + else: + transition = "unchanged" + cur.execute( + "UPDATE unmanaged_position_state " + "SET last_seen_at = %s, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (now, now, instance_key, symbol), + ) + state = self._row_to_state(cur.fetchone()) + if transition != "unchanged": + note = ( + f"previous_quantity={previous_quantity}" + if transition == "quantity_changed" and previous_quantity is not None + else "" + ) + self._insert_audit( + cur, + instance_key=instance_key, + symbol=symbol, + event_type=transition, + action=None, + quantity=quantity, + lifecycle_state=state["lifecycle_state"], + actor=actor, + evidence_ref=state.get("evidence_ref"), + note=note, + created_at=now, + ) + conn.commit() + return { + "transition": transition, + "previous_quantity": previous_quantity, + "state": state, + } + + def resolve_absent( + self, + *, + instance_key: str, + observed_symbols: set[str], + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> list[dict]: + now = observed_at or datetime.now(tz=UTC) + with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " + "WHERE instance_key = %s AND lifecycle_state = 'active' " + "ORDER BY symbol " + "FOR UPDATE", + (instance_key,), + ) + active = [self._row_to_state(row) for row in cur.fetchall()] + resolved: list[dict] = [] + for state in active: + if state["symbol"] in observed_symbols: + continue + cur.execute( + "UPDATE unmanaged_position_state " + "SET lifecycle_state = 'resolved', resolved_at = %s, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (now, now, instance_key, state["symbol"]), + ) + resolved_state = self._row_to_state(cur.fetchone()) + resolved.append(resolved_state) + self._insert_audit( + cur, + instance_key=instance_key, + symbol=state["symbol"], + event_type="resolved", + action=None, + quantity=state["quantity"], + lifecycle_state="resolved", + actor=actor, + evidence_ref=resolved_state.get("evidence_ref"), + note="", + created_at=now, + ) + conn.commit() + return resolved + + def get_state(self, *, instance_key: str, symbol: str) -> dict | None: + with self._psycopg.connect(self._dsn) as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " + "WHERE instance_key = %s AND symbol = %s " + "LIMIT 1", + (instance_key, symbol), + ) + row = cur.fetchone() + return None if row is None else self._row_to_state(row) + + def list_states( + self, + *, + instance_key: str, + lifecycle_state: str = "active", + symbol: str | None = None, + ) -> list[dict]: + clauses = ["instance_key = %s"] + params: list[object] = [instance_key] + if lifecycle_state != "all": + clauses.append("lifecycle_state = %s") + params.append(lifecycle_state) + if symbol is not None: + clauses.append("symbol = %s") + params.append(symbol) + with self._psycopg.connect(self._dsn) as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " + f"WHERE {' AND '.join(clauses)} ORDER BY symbol", + tuple(params), + ) + return [self._row_to_state(row) for row in cur.fetchall()] + + def record_operator_action( + self, + *, + instance_key: str, + symbol: str, + action: str, + evidence_ref: str, + actor: str = "operator", + note: str = "", + ) -> dict | None: + now = datetime.now(tz=UTC) + with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur: + cur.execute( + "UPDATE unmanaged_position_state " + "SET operator_action = %s, evidence_ref = %s, operator_note = %s, " + " operator_actor = %s, operator_updated_at = %s, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (action, evidence_ref, note, actor, now, now, instance_key, symbol), + ) + row = cur.fetchone() + if row is None: + conn.rollback() + return None + state = self._row_to_state(row) + self._insert_audit( + cur, + instance_key=instance_key, + symbol=symbol, + event_type="operator_action", + action=action, + quantity=state["quantity"], + lifecycle_state=state["lifecycle_state"], + actor=actor, + evidence_ref=evidence_ref, + note=note, + created_at=now, + ) + conn.commit() + return state + + def list_audit(self, *, instance_key: str, symbol: str) -> list[dict]: + with self._psycopg.connect(self._dsn) as conn, conn.cursor() as cur: + cur.execute( + "SELECT event_type, action, quantity, lifecycle_state, actor, " + " evidence_ref, note, created_at " + "FROM unmanaged_position_audit " + "WHERE instance_key = %s AND symbol = %s " + "ORDER BY audit_id", + (instance_key, symbol), + ) + return [ + { + "event_type": row[0], + "action": row[1], + "quantity": row[2], + "lifecycle_state": row[3], + "actor": row[4], + "evidence_ref": row[5], + "note": row[6], + "created_at": row[7], + } + for row in cur.fetchall() + ] + + def _insert_audit( + self, + cur, + *, + instance_key: str, + symbol: str, + event_type: str, + action: str | None, + quantity: int, + lifecycle_state: str, + actor: str, + evidence_ref: str | None, + note: str, + created_at: datetime, + ) -> None: + cur.execute( + "INSERT INTO unmanaged_position_audit " + "(instance_key, symbol, event_type, action, quantity, lifecycle_state, " + " actor, evidence_ref, note, created_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ( + instance_key, + symbol, + event_type, + action, + quantity, + lifecycle_state, + actor, + evidence_ref, + note, + created_at, + ), + ) + + @staticmethod + def _row_to_state(row) -> dict: + return { + "instance_key": row[0], + "symbol": row[1], + "quantity": row[2], + "lifecycle_state": row[3], + "first_seen_at": row[4], + "last_seen_at": row[5], + "quantity_changed_at": row[6], + "resolved_at": row[7], + "operator_action": row[8], + "evidence_ref": row[9], + "operator_note": row[10], + "operator_actor": row[11], + "operator_updated_at": row[12], + "created_at": row[13], + "updated_at": row[14], + } diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index abe519b..cf2870e 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -537,6 +537,259 @@ class InMemoryReconcileNotificationStore: ] +class UnmanagedPositionStore(Protocol): + """Durable unmanaged-position lifecycle store keyed by (instance_key, symbol).""" + + def observe( + self, + *, + instance_key: str, + symbol: str, + quantity: int, + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> dict: + """Return {"transition", "previous_quantity", "state"} for one observation.""" + ... + + def resolve_absent( + self, + *, + instance_key: str, + observed_symbols: set[str], + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> list[dict]: + """Mark active positions absent from a successful broker read as resolved.""" + ... + + def get_state(self, *, instance_key: str, symbol: str) -> dict | None: + ... + + def list_states( + self, + *, + instance_key: str, + lifecycle_state: str = "active", + symbol: str | None = None, + ) -> list[dict]: + ... + + def record_operator_action( + self, + *, + instance_key: str, + symbol: str, + action: str, + evidence_ref: str, + actor: str = "operator", + note: str = "", + ) -> dict | None: + """Persist an explicit operator disposition and append an audit event.""" + ... + + def list_audit(self, *, instance_key: str, symbol: str) -> list[dict]: + ... + + +class InMemoryUnmanagedPositionStore: + """Process-local unmanaged-position lifecycle store for tests + CI.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._states: dict[tuple[str, str], dict] = {} + self._audit: list[dict] = [] + + def observe( + self, + *, + instance_key: str, + symbol: str, + quantity: int, + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> dict: + now = observed_at or datetime.now(tz=UTC) + key = (instance_key, symbol) + with self._lock: + existing = self._states.get(key) + previous_quantity = existing.get("quantity") if existing is not None else None + preserved = {} if existing is None else { + "operator_action": existing.get("operator_action"), + "evidence_ref": existing.get("evidence_ref"), + "operator_note": existing.get("operator_note"), + "operator_actor": existing.get("operator_actor"), + "operator_updated_at": existing.get("operator_updated_at"), + "created_at": existing.get("created_at", now), + } + if existing is None or existing.get("lifecycle_state") == "resolved": + transition = "detected" + state = { + "instance_key": instance_key, + "symbol": symbol, + "quantity": quantity, + "lifecycle_state": "active", + "first_seen_at": now, + "last_seen_at": now, + "quantity_changed_at": now, + "resolved_at": None, + "operator_action": None, + "evidence_ref": None, + "operator_note": "", + "operator_actor": None, + "operator_updated_at": None, + "created_at": now, + **preserved, + "updated_at": now, + } + self._states[key] = state + elif int(existing["quantity"]) != int(quantity): + transition = "quantity_changed" + existing["quantity"] = quantity + existing["last_seen_at"] = now + existing["quantity_changed_at"] = now + existing["resolved_at"] = None + existing["lifecycle_state"] = "active" + existing["updated_at"] = now + state = existing + else: + transition = "unchanged" + existing["last_seen_at"] = now + existing["updated_at"] = now + state = existing + if transition != "unchanged": + self._audit.append( + { + "instance_key": instance_key, + "symbol": symbol, + "event_type": transition, + "action": None, + "quantity": quantity, + "lifecycle_state": state["lifecycle_state"], + "actor": actor, + "evidence_ref": state.get("evidence_ref"), + "note": ( + f"previous_quantity={previous_quantity}" + if transition == "quantity_changed" and previous_quantity is not None + else "" + ), + "created_at": now, + } + ) + return { + "transition": transition, + "previous_quantity": previous_quantity, + "state": dict(state), + } + + def resolve_absent( + self, + *, + instance_key: str, + observed_symbols: set[str], + observed_at: datetime | None = None, + actor: str = "reconcile", + ) -> list[dict]: + now = observed_at or datetime.now(tz=UTC) + with self._lock: + resolved: list[dict] = [] + for key, state in sorted(self._states.items()): + if key[0] != instance_key: + continue + if state.get("lifecycle_state") != "active": + continue + if state["symbol"] in observed_symbols: + continue + state["lifecycle_state"] = "resolved" + state["resolved_at"] = now + state["updated_at"] = now + resolved.append(dict(state)) + self._audit.append( + { + "instance_key": state["instance_key"], + "symbol": state["symbol"], + "event_type": "resolved", + "action": None, + "quantity": state["quantity"], + "lifecycle_state": "resolved", + "actor": actor, + "evidence_ref": state.get("evidence_ref"), + "note": "", + "created_at": now, + } + ) + return resolved + + def get_state(self, *, instance_key: str, symbol: str) -> dict | None: + with self._lock: + state = self._states.get((instance_key, symbol)) + return dict(state) if state is not None else None + + def list_states( + self, + *, + instance_key: str, + lifecycle_state: str = "active", + symbol: str | None = None, + ) -> list[dict]: + with self._lock: + rows = [] + for state in self._states.values(): + if state["instance_key"] != instance_key: + continue + if lifecycle_state != "all" and state["lifecycle_state"] != lifecycle_state: + continue + if symbol is not None and state["symbol"] != symbol: + continue + rows.append(dict(state)) + return sorted(rows, key=lambda item: item["symbol"]) + + def record_operator_action( + self, + *, + instance_key: str, + symbol: str, + action: str, + evidence_ref: str, + actor: str = "operator", + note: str = "", + ) -> dict | None: + now = datetime.now(tz=UTC) + with self._lock: + state = self._states.get((instance_key, symbol)) + if state is None: + return None + state["operator_action"] = action + state["evidence_ref"] = evidence_ref + state["operator_note"] = note + state["operator_actor"] = actor + state["operator_updated_at"] = now + state["updated_at"] = now + self._audit.append( + { + "instance_key": instance_key, + "symbol": symbol, + "event_type": "operator_action", + "action": action, + "quantity": state["quantity"], + "lifecycle_state": state["lifecycle_state"], + "actor": actor, + "evidence_ref": evidence_ref, + "note": note, + "created_at": now, + } + ) + return dict(state) + + def list_audit(self, *, instance_key: str, symbol: str) -> list[dict]: + with self._lock: + return [ + dict(row) + for row in self._audit + if row["instance_key"] == instance_key and row["symbol"] == symbol + ] + + def slot_to_dict( s, *, @@ -637,4 +890,3 @@ def _sanitize(message: str) -> str: out = out[:idx] + "" break return out[:200] - diff --git a/trex_core/application/reconciliation.py b/trex_core/application/reconciliation.py index aca2c5d..4c2b7b4 100644 --- a/trex_core/application/reconciliation.py +++ b/trex_core/application/reconciliation.py @@ -33,7 +33,11 @@ from typing import cast from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.broker_port import BrokerPort -from trex_core.application.portfolio import ReconcileWarningStore, SnapshotStore +from trex_core.application.portfolio import ( + ReconcileWarningStore, + SnapshotStore, + UnmanagedPositionStore, +) from trex_core.domain.slot.state import SlotState _log = logging.getLogger(__name__) @@ -341,6 +345,25 @@ def _retry_pending_resolved( ) +def _notify_unmanaged_position_transition( + alarm_emitter: AlarmEmitter, + *, + event_type: str, + instance_key: str, + symbol: str, + quantity: int, +) -> None: + alarm_emitter.publish_alarm( + f"unmanaged_position_{event_type}", + f"unmanaged position {event_type}: symbol={symbol}", + fields={ + "instance_key": instance_key, + "symbol": symbol, + "broker_quantity": quantity, + }, + ) + + def reconcile_core_holdings( *, slot_repo, @@ -350,6 +373,7 @@ def reconcile_core_holdings( alarm_emitter: AlarmEmitter | None = None, warn_store: ReconcileWarningStore | None = None, notification_store=None, + unmanaged_position_store: UnmanagedPositionStore | None = None, ) -> dict[str, str]: """dcy-020 / REQ-062 (BAT-029) + REQ-103 (BAT-050): surface core-slot holding vs broker divergence and set durable reconcile-warning for B case. @@ -377,6 +401,7 @@ def reconcile_core_holdings( decisions: dict[str, str] = {} matched: set[str] = set() active_b_identity_keys: set[str] = set() # req-0007: track active B mismatches for retry + active_unmanaged_symbols: set[str] = set() for slot in slot_repo.list_holding_slots(instance_key): sym = slot.symbol matched.add(sym) @@ -458,10 +483,78 @@ def reconcile_core_holdings( ) else: decisions[f"broker:{sym}"] = "unmanaged_position" - _log.warning( - "unmanaged position: symbol=%s qty=%d instance_key=%s", - sym, bq, instance_key, + active_unmanaged_symbols.add(sym) + if unmanaged_position_store is None: + _log.warning( + "unmanaged position: symbol=%s qty=%d instance_key=%s", + sym, bq, instance_key, + ) + else: + try: + observed = unmanaged_position_store.observe( + instance_key=instance_key, + symbol=sym, + quantity=bq, + ) + except Exception: # noqa: BLE001 + _log.exception( + "failed to persist unmanaged_position for symbol=%s instance=%s", + sym, + instance_key, + ) + _log.warning( + "unmanaged position: symbol=%s qty=%d instance_key=%s", + sym, bq, instance_key, + ) + else: + transition = observed["transition"] + if transition == "detected": + _log.warning( + "unmanaged position detected: symbol=%s qty=%d instance_key=%s", + sym, bq, instance_key, + ) + if alarm_emitter is not None: + _notify_unmanaged_position_transition( + alarm_emitter, + event_type="detected", + instance_key=instance_key, + symbol=sym, + quantity=bq, + ) + elif transition == "quantity_changed": + _log.warning( + "unmanaged position quantity changed: symbol=%s " + "previous_qty=%s qty=%d instance_key=%s", + sym, observed.get("previous_quantity"), bq, instance_key, + ) + if alarm_emitter is not None: + _notify_unmanaged_position_transition( + alarm_emitter, + event_type="changed", + instance_key=instance_key, + symbol=sym, + quantity=bq, + ) + if unmanaged_position_store is not None: + try: + for state in unmanaged_position_store.resolve_absent( + instance_key=instance_key, + observed_symbols=active_unmanaged_symbols, + ): + _log.info( + "unmanaged position resolved: symbol=%s qty=%d instance_key=%s", + state["symbol"], state["quantity"], instance_key, ) + if alarm_emitter is not None: + _notify_unmanaged_position_transition( + alarm_emitter, + event_type="resolved", + instance_key=instance_key, + symbol=state["symbol"], + quantity=state["quantity"], + ) + except Exception: # noqa: BLE001 + _log.exception("reconcile_core_holdings: unmanaged_position resolve failed") # req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched. if alarm_emitter is not None and notification_store is not None: try: diff --git a/trex_core/application/snapshot_scheduler.py b/trex_core/application/snapshot_scheduler.py index b6ef0bc..d675656 100644 --- a/trex_core/application/snapshot_scheduler.py +++ b/trex_core/application/snapshot_scheduler.py @@ -23,6 +23,7 @@ from trex_core.application.portfolio import ( PortfolioHubReadPort, PortfolioHubUnavailableError, ReconcileNotificationStore, + UnmanagedPositionStore, ReconcileWarningStore, SnapshotStore, TradingContext, @@ -51,6 +52,7 @@ class SnapshotScheduler: alarm_emitter: Optional[AlarmEmitter] = None, warn_store: Optional[ReconcileWarningStore] = None, notification_store: Optional[ReconcileNotificationStore] = None, + unmanaged_position_store: Optional[UnmanagedPositionStore] = None, credential_fetch_enabled: bool = False, kis_reinit_callback: Optional[Callable[[BrokerCredentialPayload], None]] = None, credential_refresh: Optional[Callable[[], object]] = None, @@ -75,6 +77,7 @@ class SnapshotScheduler: self._warn_store = warn_store # req-0007 / ac-0024: durable notification deduplication store (Optional). self._notification_store = notification_store + self._unmanaged_position_store = unmanaged_position_store # dcy-036 / REQ-113/114/115: credential fetch opt-in + break-glass detection. # credential_fetch_enabled=False (default) → env-only, no portfolio-hub fetch. self._credential_fetch_enabled: bool = credential_fetch_enabled @@ -336,6 +339,7 @@ class SnapshotScheduler: alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100 warn_store=self._warn_store, # BAT-050 / REQ-103 notification_store=self._notification_store, # req-0007 / ac-0024 + unmanaged_position_store=self._unmanaged_position_store, ) except Exception: # noqa: BLE001 _log.exception("core-holding divergence reconcile failed") diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index 128ef21..e4ffe68 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -11,6 +11,8 @@ Subcommands: cancel — cancel a pending order (control RPC; safeguarded) sync — sync slot qty to broker ground truth (direct-DB; REQ-105) verify — re-check C-case reconcile-warning (direct-DB; REQ-106) + unmanaged list — query durable unmanaged-position lifecycle state + unmanaged record — record explicit operator disposition + evidence reference ledger backfill — evidence-backed historical ledger recovery ledger skips list — query durable skipped-ledger records ledger skips resolve — explicitly resolve a skipped-ledger record @@ -640,6 +642,37 @@ def _build_parser() -> argparse.ArgumentParser: ) _add_common(p_verify) + p_unmanaged = sub.add_parser( + "unmanaged", + help="List or annotate durable unmanaged broker positions (direct-DB)", + ) + p_unmanaged_sub = p_unmanaged.add_subparsers(dest="unmanaged_cmd", required=True) + p_unmanaged_list = p_unmanaged_sub.add_parser( + "list", + help="List unmanaged broker positions with lifecycle state", + ) + p_unmanaged_list.add_argument( + "--status", + choices=("active", "resolved", "all"), + default="active", + ) + p_unmanaged_list.add_argument("--symbol", default=None) + _add_common(p_unmanaged_list) + + p_unmanaged_record = p_unmanaged_sub.add_parser( + "record", + help="Record an explicit operator disposition for an unmanaged position", + ) + p_unmanaged_record.add_argument("symbol") + p_unmanaged_record.add_argument( + "action", + choices=("investigating", "approved_exception", "managed_inclusion"), + ) + p_unmanaged_record.add_argument("--evidence-ref", required=True) + p_unmanaged_record.add_argument("--note", default="") + p_unmanaged_record.add_argument("--actor", default="operator") + _add_common(p_unmanaged_record) + p_ledger = sub.add_parser( "ledger", help="Ledger operator tools (direct-DB enqueue + immediate writer path)", @@ -971,6 +1004,73 @@ def _cmd_verify(args, cfg) -> int: return exit_code +def _cmd_unmanaged_list(args, cfg) -> int: + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + instance = _resolve_instance(args) + states = PostgreSQLUnmanagedPositionStore(cfg.dsn).list_states( + instance_key=instance, + lifecycle_state=args.status, + symbol=args.symbol, + ) + payload = { + "instance": instance, + "status": args.status, + "count": len(states), + "states": states, + } + if args.json: + _emit(payload, as_json=True) + else: + print( + f"instance={instance} status={args.status} count={len(states)} " + f"symbol={args.symbol or '*'}" + ) + for state in states: + print( + f" {state['symbol']}\tqty={state['quantity']}\t" + f"lifecycle={state['lifecycle_state']}\t" + f"first_seen={state['first_seen_at']}\tlast_seen={state['last_seen_at']}\t" + f"resolved_at={state['resolved_at']}\t" + f"operator_action={state.get('operator_action') or '-'}\t" + f"evidence_ref={state.get('evidence_ref') or '-'}" + ) + return 0 + + +def _cmd_unmanaged_record(args, cfg) -> int: + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + instance = _resolve_instance(args) + state = PostgreSQLUnmanagedPositionStore(cfg.dsn).record_operator_action( + instance_key=instance, + symbol=args.symbol, + action=args.action, + evidence_ref=args.evidence_ref, + actor=args.actor, + note=args.note, + ) + if state is None: + _fail( + f"unmanaged position not found: instance={instance} symbol={args.symbol}", + args.json, + ) + return 1 + payload = {"result": "recorded", "state": state} + if args.json: + _emit(payload, as_json=True) + else: + print( + f"recorded unmanaged position instance={instance} symbol={args.symbol} " + f"action={args.action} evidence_ref={args.evidence_ref}" + ) + return 0 + + def _cmd_report(args, cfg) -> int: """REQ-088/110/111: holdings / today's fills / WAC realized PnL / pending — direct-DB.""" import psycopg @@ -1434,6 +1534,10 @@ def main(argv: list[str] | None = None, *, rpc=None, confirm=None) -> int: # REQ-106 (BAT-051): verify — direct-DB, C-case re-check (read-centric). if args.cmd == "verify": return _cmd_verify(args, cfg) + if args.cmd == "unmanaged" and getattr(args, "unmanaged_cmd", None) == "list": + return _cmd_unmanaged_list(args, cfg) + if args.cmd == "unmanaged" and getattr(args, "unmanaged_cmd", None) == "record": + return _cmd_unmanaged_record(args, cfg) if args.cmd == "ledger" and getattr(args, "ledger_cmd", None) == "backfill": return _cmd_ledger_backfill(args, cfg, confirm) diff --git a/trex_core/runtime/daemon.py b/trex_core/runtime/daemon.py index d232973..b0b0f2e 100644 --- a/trex_core/runtime/daemon.py +++ b/trex_core/runtime/daemon.py @@ -52,6 +52,9 @@ from trex_core.adapters.portfolio.killswitch_store import PostgreSQLKillSwitchSt from trex_core.adapters.portfolio.reconcile_notification_store import ( PostgreSQLReconcileNotificationStore, ) +from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, +) from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.account_snapshot import AccountSnapshotService from trex_core.application.broker_port import BrokerPort @@ -202,6 +205,7 @@ class Daemon: self.warn_store = PostgreSQLReconcileWarningStore(config.dsn) # req-0007 / ac-0024: durable reconcile notification deduplication store. self.notification_store = PostgreSQLReconcileNotificationStore(config.dsn) + self.unmanaged_position_store = PostgreSQLUnmanagedPositionStore(config.dsn) # BAT-012: day-scoped kill-switch exclusion store + service. self.killswitch_store = PostgreSQLKillSwitchStore(config.dsn) self.portfolio_read = _build_portfolio_read(config) @@ -278,6 +282,7 @@ class Daemon: alarm_emitter=self.alarm_emitter, # dcy-033 / REQ-100 warn_store=self.warn_store, # BAT-050 / REQ-103: durable reconcile-warning notification_store=self.notification_store, # req-0007 / ac-0024 + unmanaged_position_store=self.unmanaged_position_store, credential_fetch_enabled=config.credential_fetch_enabled, # dcy-036 / REQ-113 credential_refresh=self.broker_provider.refresh_day_start_credential, # dcy-036 / REQ-113 AC-2 / REQ-115 AC-1: KISBroker reinit callback. From 09b1b4f4c3d78c985796d3275e2b04815ee984f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:24:58 +0000 Subject: [PATCH 06/21] fix: isolate ledger outbox delivery by instance Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../persistence/test_migration_and_drift.py | 112 +++++++++------ .../test_postgresql_ledger_outbox.py | 128 +++++++++++++++--- tests/application/test_ledger_outbox.py | 27 ++++ tests/runtime/test_cli_ledger_backfill.py | 18 ++- trex_core/adapters/persistence/postgresql.py | 91 +++++++++++-- .../sql/028_ledger_outbox_instance_claims.sql | 22 +++ trex_core/application/ledger_outbox.py | 108 ++++++++++++++- trex_core/runtime/cli.py | 2 +- trex_core/runtime/daemon.py | 2 +- 9 files changed, 429 insertions(+), 81 deletions(-) create mode 100644 trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql diff --git a/tests/adapters/persistence/test_migration_and_drift.py b/tests/adapters/persistence/test_migration_and_drift.py index 3a35cd3..592cc60 100644 --- a/tests/adapters/persistence/test_migration_and_drift.py +++ b/tests/adapters/persistence/test_migration_and_drift.py @@ -6,9 +6,10 @@ iteration: EXPECTED_SLOT_FIELDS (code contract) vs daily_slot actual columns """ from __future__ import annotations +from datetime import UTC, datetime + import psycopg import pytest -from datetime import datetime, timezone from trex_core.adapters.persistence.migrate import apply_migrations, migration_files from trex_core.adapters.persistence.postgresql import ( @@ -33,24 +34,22 @@ _EXPECTED_TABLES = { def _columns(dsn: str, table: str) -> set[str]: - with psycopg.connect(dsn) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema = 'public' AND table_name = %s", - (table,), - ) - return {r[0] for r in cur.fetchall()} + with psycopg.connect(dsn) as conn, conn.cursor() as cur: + cur.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s", + (table,), + ) + return {r[0] for r in cur.fetchall()} def test_migrations_create_expected_tables(pg_clean): assert migration_files(), "no migration files found" - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" - ) - tables = {r[0] for r in cur.fetchall()} + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + cur.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" + ) + tables = {r[0] for r in cur.fetchall()} assert _EXPECTED_TABLES <= tables @@ -61,20 +60,19 @@ def test_daily_slot_columns_match_expected_contract(pg_clean): def test_strategy_state_schema_matches_instance_map_contract(pg_clean): - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT column_name, data_type, udt_name " - "FROM information_schema.columns " - "WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'" - ) - columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()} - cur.execute( - "SELECT a.attname FROM pg_index i " - "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) " - "WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary" - ) - primary_key = {row[0] for row in cur.fetchall()} + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + cur.execute( + "SELECT column_name, data_type, udt_name " + "FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'" + ) + columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()} + cur.execute( + "SELECT a.attname FROM pg_index i " + "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) " + "WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary" + ) + primary_key = {row[0] for row in cur.fetchall()} assert columns == {"slot_id": ("uuid", "uuid"), "data": ("jsonb", "jsonb")} assert primary_key == {"slot_id"} @@ -111,13 +109,12 @@ def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean): apply_migrations(pg_clean) assert repo.fetch_slot(slot.slot_id).venue == "SOR" - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - with pytest.raises(psycopg.errors.CheckViolation): - cur.execute( - "UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s", - (slot.slot_id,), - ) + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + with pytest.raises(psycopg.errors.CheckViolation): + cur.execute( + "UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s", + (slot.slot_id,), + ) def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): @@ -129,7 +126,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): side="buy", qty=1, price=1000, - executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc), + executed_at=datetime(2026, 8, 19, tzinfo=UTC), venue="NXT", order_venue="KRX", ) @@ -140,7 +137,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): side="buy", qty=1, price=1000, - executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc), + executed_at=datetime(2026, 8, 19, tzinfo=UTC), venue="KRX", order_venue="SOR", ) @@ -172,10 +169,39 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): assert legacy_row.trade.venue is None assert legacy_row.trade.order_venue == "SOR" + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + with pytest.raises(psycopg.errors.CheckViolation): + cur.execute( + "UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s", + (normal.correlation_id,), + ) + + +def test_migration_028_fail_closes_unowned_legacy_outbox_rows(pg_clean): + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") + trade = LedgerTrade( + asset_id="PF-LEGACY", + order_id="O-LEGACY", + execution_id="E-LEGACY", + side="buy", + qty=1, + price=1000, + executed_at=datetime(2026, 9, 12, tzinfo=UTC), + ) + outbox.enqueue(trade) + with psycopg.connect(pg_clean) as conn: with conn.cursor() as cur: - with pytest.raises(psycopg.errors.CheckViolation): - cur.execute( - "UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s", - (normal.correlation_id,), - ) + cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP NOT NULL") + cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP DEFAULT") + cur.execute( + "UPDATE ledger_outbox SET instance_key=NULL WHERE correlation_id=%s", + (trade.correlation_id,), + ) + conn.commit() + + apply_migrations(pg_clean) + stored = outbox.get(trade.correlation_id) + assert stored is not None + assert stored.instance_key == "__unowned__" + assert outbox.list_pending() == [] diff --git a/tests/adapters/persistence/test_postgresql_ledger_outbox.py b/tests/adapters/persistence/test_postgresql_ledger_outbox.py index ef65a3c..6f51388 100644 --- a/tests/adapters/persistence/test_postgresql_ledger_outbox.py +++ b/tests/adapters/persistence/test_postgresql_ledger_outbox.py @@ -4,7 +4,7 @@ Skipped without TREX_POSTGRESQL_TEST_DSN (same gate as other PG suites). """ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from trex_core.adapters.ledger.in_memory import InMemoryLedger from trex_core.adapters.persistence.postgresql import PostgreSQLLedgerOutbox @@ -13,34 +13,40 @@ from trex_core.application.ledger_outbox import ( OutboxOutcome, WorkItemStatus, ) -from trex_core.application.ledger_port import LedgerTrade +from trex_core.application.ledger_port import ( + LedgerRetryableError, + LedgerTrade, + LedgerWriteOutcome, + LedgerWriteResult, +) def _trade(execution_id="paper:1:10") -> LedgerTrade: return LedgerTrade( asset_id="PF-1", order_id="ORD-1", execution_id=execution_id, side="buy", qty=10, price=1000.0, fee=1.5, tax=0.0, - executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc), + executed_at=datetime(2026, 6, 4, tzinfo=UTC), venue="NXT", order_venue="SOR", ) def test_enqueue_persists_and_is_idempotent(pg_clean): - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") a = outbox.enqueue(_trade()) b = outbox.enqueue(_trade()) assert a.correlation_id == b.correlation_id pending = outbox.list_pending() assert len(pending) == 1 assert pending[0].status is WorkItemStatus.PENDING + assert pending[0].instance_key == "inst-a" assert pending[0].trade.venue == "NXT" assert pending[0].trade.order_venue == "SOR" def test_writer_completes_persisted_item(pg_clean): - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") ledger = InMemoryLedger() - writer = LedgerWriter(outbox, ledger) + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") outbox.enqueue(_trade(execution_id="pg-1")) stats = writer.run_once() assert stats.completed == 1 and ledger.count() == 1 @@ -50,9 +56,9 @@ def test_writer_completes_persisted_item(pg_clean): def test_writer_marks_conflict_blocked_on_pg(pg_clean): - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") ledger = InMemoryLedger(conflict_executions={"pg-x"}) - writer = LedgerWriter(outbox, ledger) + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") outbox.enqueue(_trade(execution_id="pg-x")) writer.run_once() row = outbox.get("PF-1:ORD-1:pg-x") @@ -61,10 +67,10 @@ def test_writer_marks_conflict_blocked_on_pg(pg_clean): def test_writer_retries_on_pg(pg_clean): - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") ledger = InMemoryLedger(fail_n_then_succeed=1) # dcy-015: base 0 => next_retry_at == now, so the row is immediately due again. - writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0) + writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0, instance_key="inst-a") outbox.enqueue(_trade(execution_id="pg-r")) writer.run_once() row1 = outbox.get("PF-1:ORD-1:pg-r") @@ -79,9 +85,9 @@ def test_writer_retries_on_pg(pg_clean): def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean): """dcy-015 / REQ-019: 404/permanent → terminal delivery_failed persisted (migration 007 status enum + last_outcome). No retry.""" - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") ledger = InMemoryLedger(permanent_executions={"pg-perm"}) - writer = LedgerWriter(outbox, ledger) + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") outbox.enqueue(_trade(execution_id="pg-perm")) s = writer.run_once() assert s.delivery_failed == 1 @@ -94,22 +100,112 @@ def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean): def test_pending_cash_delta_on_pg(pg_clean): """dcy-018 / REQ-029: estimated cash delta of pending rows summed by asset (SQL).""" - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") outbox.enqueue(LedgerTrade(asset_id="PF-1", order_id="O-1", execution_id="cd-buy", side="buy", qty=10, price=1000.0, fee=1.0, tax=0.0, - executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc))) + executed_at=datetime(2026, 6, 4, tzinfo=UTC))) deltas = outbox.pending_cash_delta() assert deltas["PF-1"] == -(10 * 1000.0 + 1.0) # buy => cash down by notional+fee def test_backoff_row_skipped_by_pending_scan_on_pg(pg_clean): """dcy-015 / REQ-019: a future next_retry_at excludes the row from list_pending.""" - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") ledger = InMemoryLedger(fail_n_then_succeed=99) # large base => next_retry_at well in the future - writer = LedgerWriter(outbox, ledger, retry_base_sec=600.0, max_attempts=99) + writer = LedgerWriter( + outbox, ledger, retry_base_sec=600.0, max_attempts=99, instance_key="inst-a" + ) outbox.enqueue(_trade(execution_id="pg-bo")) writer.run_once() # -> retryable, next_retry_at ~ now + 600s row = outbox.get("PF-1:ORD-1:pg-bo") assert row.status is WorkItemStatus.PENDING and row.next_retry_at is not None assert outbox.list_pending() == [] # still backing off -> skipped + + +def test_pending_claims_are_isolated_per_instance(pg_clean): + ledger = InMemoryLedger() + outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") + outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b") + writer_a = LedgerWriter(outbox_a, ledger, instance_key="inst-a") + writer_b = LedgerWriter(outbox_b, ledger, instance_key="inst-b") + row_a = outbox_a.enqueue(_trade(execution_id="pg-a")) + row_b = outbox_b.enqueue(_trade(execution_id="pg-b")) + + stats_a = writer_a.run_once() + assert stats_a.completed == 1 and ledger.count() == 1 + assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.COMPLETED + assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.PENDING + + stats_b = writer_b.run_once() + assert stats_b.completed == 1 and ledger.count() == 2 + assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.COMPLETED + + +def test_claim_expiry_allows_safe_recovery_after_worker_crash(pg_clean): + now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} + outbox_a = PostgreSQLLedgerOutbox( + pg_clean, + instance_key="inst-a", + claim_ttl_sec=5.0, + worker_id="worker-a", + clock=lambda: now["t"], + ) + outbox_b = PostgreSQLLedgerOutbox( + pg_clean, + instance_key="inst-a", + claim_ttl_sec=5.0, + worker_id="worker-b", + clock=lambda: now["t"], + ) + row = outbox_a.enqueue(_trade(execution_id="pg-crash")) + + claimed_a = outbox_a.claim_pending() + assert [item.correlation_id for item in claimed_a] == [row.correlation_id] + assert outbox_b.claim_pending() == [] + + now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC) + claimed_b = outbox_b.claim_pending() + assert [item.correlation_id for item in claimed_b] == [row.correlation_id] + + +def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean): + class _ForbiddenLedger: + def __init__(self): + self.calls: list[str] = [] + + def record_trade(self, trade): + self.calls.append(trade.execution_id) + raise LedgerRetryableError("portfolio-hub 403 Forbidden") + + outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") + outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b") + writer_b = LedgerWriter( + outbox_b, _ForbiddenLedger(), retry_base_sec=0.0, instance_key="inst-b" + ) + row_a = outbox_a.enqueue(_trade(execution_id="pg-403-a")) + row_b = outbox_b.enqueue(_trade(execution_id="pg-403-b")) + + stats_b = writer_b.run_once() + + assert stats_b.retryable == 1 + assert outbox_b.get(row_b.correlation_id).last_outcome is OutboxOutcome.RETRYABLE_FAILURE + assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.PENDING + + +def test_writer_marks_duplicate_completed_from_adapter_response_on_pg(pg_clean): + class _DuplicateLedger: + def record_trade(self, _trade): + return LedgerWriteOutcome(LedgerWriteResult.DUPLICATE_OK) + + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") + writer = LedgerWriter(outbox, _DuplicateLedger(), instance_key="inst-a") + row = outbox.enqueue(_trade(execution_id="pg-dup")) + + outcome = writer.process_correlation_id(row.correlation_id) + + assert outcome is OutboxOutcome.DUPLICATE_COMPLETED + stored = outbox.get(row.correlation_id) + assert stored is not None + assert stored.status is WorkItemStatus.COMPLETED + assert stored.last_outcome is OutboxOutcome.DUPLICATE_COMPLETED diff --git a/tests/application/test_ledger_outbox.py b/tests/application/test_ledger_outbox.py index 5ce7f71..e085f30 100644 --- a/tests/application/test_ledger_outbox.py +++ b/tests/application/test_ledger_outbox.py @@ -83,3 +83,30 @@ def test_process_correlation_id_only_drains_requested_row(): assert ledger.count() == 1 assert outbox.get(first.correlation_id).status is WorkItemStatus.COMPLETED assert outbox.get(second.correlation_id).status is WorkItemStatus.PENDING + + +def test_process_correlation_id_skips_foreign_instance_row(): + outbox = InMemoryLedgerOutbox(instance_key="inst-a") + ledger = InMemoryLedger() + writer = LedgerWriter(outbox, ledger, instance_key="inst-b") + item = outbox.enqueue(_trade(execution_id="e-foreign")) + + outcome = writer.process_correlation_id(item.correlation_id) + + assert outcome is None + assert ledger.count() == 0 + assert writer.counters["claim:foreign_instance"] == 1 + assert outbox.get(item.correlation_id).status is WorkItemStatus.PENDING + + +def test_writer_exposes_claim_and_outcome_counters(): + outbox = InMemoryLedgerOutbox(instance_key="inst-a") + ledger = InMemoryLedger() + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") + outbox.enqueue(_trade(execution_id="e-count")) + + stats = writer.run_once() + + assert stats.completed == 1 + assert writer.counters["claim:claimed"] == 1 + assert writer.counters["outcome:completed"] == 1 diff --git a/tests/runtime/test_cli_ledger_backfill.py b/tests/runtime/test_cli_ledger_backfill.py index e09282f..17455ff 100644 --- a/tests/runtime/test_cli_ledger_backfill.py +++ b/tests/runtime/test_cli_ledger_backfill.py @@ -187,7 +187,11 @@ def test_ledger_backfill_existing_completed_row_is_idempotent(monkeypatch, tmp_p import trex_core.adapters.ledger.portfolio_hub as hub_mod import trex_core.adapters.persistence.postgresql as pg_mod - monkeypatch.setattr(pg_mod, "PostgreSQLLedgerOutbox", lambda dsn: _Outbox()) + monkeypatch.setattr( + pg_mod, + "PostgreSQLLedgerOutbox", + lambda dsn, *, instance_key: _Outbox(), + ) monkeypatch.setattr(hub_mod, "PortfolioHubLedger", lambda *args, **kwargs: object()) monkeypatch.setattr(cli, "LedgerWriter", _Writer) monkeypatch.setattr( @@ -246,7 +250,11 @@ def test_ledger_backfill_live_enqueue_and_processes_exact_trade(monkeypatch, tmp import trex_core.adapters.ledger.portfolio_hub as hub_mod import trex_core.adapters.persistence.postgresql as pg_mod - monkeypatch.setattr(pg_mod, "PostgreSQLLedgerOutbox", lambda dsn: outbox) + monkeypatch.setattr( + pg_mod, + "PostgreSQLLedgerOutbox", + lambda dsn, *, instance_key: outbox, + ) monkeypatch.setattr(hub_mod, "PortfolioHubLedger", lambda *args, **kwargs: object()) monkeypatch.setattr(cli, "LedgerWriter", _Writer) monkeypatch.setattr( @@ -310,7 +318,11 @@ def test_ledger_backfill_resolves_matching_skip(monkeypatch, tmp_path, capsys): import trex_core.adapters.ledger.portfolio_hub as hub_mod import trex_core.adapters.persistence.postgresql as pg_mod - monkeypatch.setattr(pg_mod, "PostgreSQLLedgerOutbox", lambda dsn: _Outbox()) + monkeypatch.setattr( + pg_mod, + "PostgreSQLLedgerOutbox", + lambda dsn, *, instance_key: _Outbox(), + ) monkeypatch.setattr(hub_mod, "PortfolioHubLedger", lambda *args, **kwargs: object()) monkeypatch.setattr(cli, "LedgerWriter", _Writer) monkeypatch.setattr( diff --git a/trex_core/adapters/persistence/postgresql.py b/trex_core/adapters/persistence/postgresql.py index 6a52b3d..64cabc7 100644 --- a/trex_core/adapters/persistence/postgresql.py +++ b/trex_core/adapters/persistence/postgresql.py @@ -13,9 +13,10 @@ domain layer (NFR-1). from __future__ import annotations import json +from collections.abc import Iterator from contextlib import contextmanager from datetime import datetime, timedelta, timezone -from typing import Any, Iterator, Optional +from typing import Any, Optional from uuid import UUID, uuid4 from trex_core.application.ledger_outbox import ( @@ -26,9 +27,9 @@ from trex_core.application.ledger_outbox import ( from trex_core.application.ledger_port import LedgerTrade from trex_core.application.ports import IntentRecord from trex_core.application.strategy_state import ( + _STATE_CAP_BYTES, SetResult, StrategyStateEntry, - _STATE_CAP_BYTES, ) from trex_core.domain.slot import Slot, SlotNotFoundError, SlotState, SlotTransitionRecord @@ -565,16 +566,32 @@ class _PgIntentRecord: _OUTBOX_COLUMNS = ( "correlation_id, asset_id, order_id, execution_id, side, qty, price, " "fee, tax, executed_at, status, attempts, last_outcome, last_error, " - "created_at, updated_at, next_retry_at, venue, order_venue" + "created_at, updated_at, next_retry_at, venue, order_venue, instance_key" ) class PostgreSQLLedgerOutbox: """ledger_outbox-backed LedgerOutbox. correlation_id PK is the dedup gate.""" - def __init__(self, dsn: str, *, psycopg_module: Any = None) -> None: + def __init__( + self, + dsn: str, + *, + instance_key: str = "default", + claim_ttl_sec: float = 60.0, + worker_id: str | None = None, + clock=None, + psycopg_module: Any = None, + ) -> None: self._psycopg = _require_psycopg(psycopg_module) self._dsn = dsn + self._instance_key = instance_key + self._claim_ttl_sec = claim_ttl_sec + self._worker_id = worker_id or f"ledger-writer-{uuid4().hex}" + self._clock = clock or (lambda: datetime.now(tz=timezone.utc)) + + def _now(self) -> datetime: + return self._clock() def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem: with self._psycopg.connect(self._dsn, autocommit=False) as conn: @@ -582,8 +599,8 @@ class PostgreSQLLedgerOutbox: cur.execute( "INSERT INTO ledger_outbox " "(correlation_id, asset_id, order_id, execution_id, side, " - " qty, price, fee, tax, executed_at, venue, order_venue) " - "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " + " qty, price, fee, tax, executed_at, venue, order_venue, instance_key) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " "ON CONFLICT (correlation_id) DO NOTHING", ( trade.correlation_id, @@ -598,6 +615,7 @@ class PostgreSQLLedgerOutbox: trade.executed_at, trade.venue, trade.order_venue, + self._instance_key, ), ) conn.commit() @@ -611,12 +629,53 @@ class PostgreSQLLedgerOutbox: cur.execute( f"SELECT {_OUTBOX_COLUMNS} FROM ledger_outbox " "WHERE status = 'pending' " + "AND instance_key = %s " "AND (next_retry_at IS NULL OR next_retry_at <= now()) " - "ORDER BY created_at LIMIT %s", - (limit,), + "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) " + "ORDER BY created_at, correlation_id LIMIT %s", + (self._instance_key, limit), ) return [self._row_to_item(r) for r in cur.fetchall()] + def claim_pending( + self, + limit: int = 32, + *, + correlation_id: str | None = None, + ) -> list[LedgerWorkItem]: + now = self._now() + lease_until = now + timedelta(seconds=self._claim_ttl_sec) + correlation_sql = "" + params: list[Any] = [self._instance_key] + if correlation_id is not None: + correlation_sql = "AND correlation_id = %s " + params.append(correlation_id) + params.extend([limit, self._worker_id, lease_until, now]) + with self._psycopg.connect(self._dsn, autocommit=False) as conn: + with conn.cursor() as cur: + cur.execute( + "WITH candidates AS (" + "SELECT correlation_id FROM ledger_outbox " + "WHERE status = 'pending' " + "AND instance_key = %s " + "AND (next_retry_at IS NULL OR next_retry_at <= now()) " + "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) " + f"{correlation_sql}" + "ORDER BY created_at, correlation_id " + "LIMIT %s " + "FOR UPDATE SKIP LOCKED" + ") " + "UPDATE ledger_outbox o " + "SET claim_owner = %s, claim_expires_at = %s, updated_at = %s " + "FROM candidates " + "WHERE o.correlation_id = candidates.correlation_id " + f"RETURNING {_OUTBOX_COLUMNS}", + params, + ) + rows = [self._row_to_item(r) for r in cur.fetchall()] + conn.commit() + return rows + def pending_cash_delta(self) -> dict[str, float]: # dcy-018 / REQ-029 (OQ-4): estimated cash delta of all pending rows by # asset_id. SUM in SQL: sell → +(qty*price - fee - tax), buy → negative. @@ -627,7 +686,10 @@ class PostgreSQLLedgerOutbox: "SUM(CASE WHEN side='sell' " " THEN (qty*price - fee - tax) " " ELSE -(qty*price + fee + tax) END) " - "FROM ledger_outbox WHERE status='pending' GROUP BY asset_id" + "FROM ledger_outbox " + "WHERE status='pending' AND instance_key=%s " + "GROUP BY asset_id", + (self._instance_key,), ) return {r[0]: float(r[1]) for r in cur.fetchall() if r[1] is not None} @@ -671,16 +733,20 @@ class PostgreSQLLedgerOutbox: nra = next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None cur.execute( "UPDATE ledger_outbox SET status=%s, attempts=attempts+%s, " - "last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s " - "WHERE correlation_id=%s", + "last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s, " + "claim_owner=NULL, claim_expires_at=NULL " + "WHERE correlation_id=%s AND instance_key=%s " + "AND (claim_owner=%s OR claim_owner IS NULL)", ( status_value, attempts_delta, outcome.value, error, nra, - datetime.now(tz=timezone.utc), + self._now(), correlation_id, + self._instance_key, + self._worker_id, ), ) conn.commit() @@ -714,6 +780,7 @@ class PostgreSQLLedgerOutbox: created_at=row[14], updated_at=row[15], next_retry_at=row[16], + instance_key=row[19], ) diff --git a/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql b/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql new file mode 100644 index 0000000..186c809 --- /dev/null +++ b/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql @@ -0,0 +1,22 @@ +-- GitHub #84: persist outbox owner instance and add claim/lease fields so +-- shared-DB daemon writers only deliver their own rows and can safely recover +-- abandoned work after lease expiry. Existing rows remain fail-closed when the +-- owner instance cannot be inferred. +ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS instance_key TEXT; +ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS claim_owner TEXT; +ALTER TABLE ledger_outbox ADD COLUMN IF NOT EXISTS claim_expires_at TIMESTAMPTZ; + +UPDATE ledger_outbox +SET instance_key='__unowned__' +WHERE instance_key IS NULL; + +ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET DEFAULT '__unowned__'; +ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_pending + ON ledger_outbox (instance_key, status, created_at) + WHERE status = 'pending'; + +CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_claim_expires + ON ledger_outbox (instance_key, claim_expires_at) + WHERE status = 'pending'; diff --git a/trex_core/application/ledger_outbox.py b/trex_core/application/ledger_outbox.py index 574fe87..79e7b32 100644 --- a/trex_core/application/ledger_outbox.py +++ b/trex_core/application/ledger_outbox.py @@ -104,6 +104,7 @@ class LedgerWorkItem: next_retry_at: Optional[datetime] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None + instance_key: Optional[str] = None class LedgerOutbox(Protocol): @@ -118,6 +119,15 @@ class LedgerOutbox(Protocol): """Return pending work items in FIFO order, up to `limit`.""" ... + def claim_pending( + self, + limit: int = 32, + *, + correlation_id: str | None = None, + ) -> list[LedgerWorkItem]: + """Atomically claim pending items for delivery.""" + ... + def pending_cash_delta(self) -> dict[str, float]: """Sum the estimated cash delta of ALL not-yet-settled (pending) rows, grouped by asset_id (dcy-018 / REQ-029, OQ-4 estimate overlay). @@ -191,13 +201,27 @@ class LedgerWriter: self._alarm_emitter = alarm_emitter self._ledger_skip_store = ledger_skip_store self._instance_key = instance_key + self.counters: dict[str, int] = {} + + def _count(self, key: str) -> None: + self.counters[key] = self.counters.get(key, 0) + 1 def enqueue_fill(self, trade: LedgerTrade) -> LedgerWorkItem: return self._outbox.enqueue(trade) def run_once(self, limit: int = 32) -> WriterStats: stats = WriterStats() - for item in self._outbox.list_pending(limit): + items = self._outbox.claim_pending(limit) + claim_result = "claimed" if items else "empty" + self._count(f"claim:{claim_result}") + _log.info( + "ledger outbox claim: instance_key=%s claim_result=%s count=%d limit=%d", + self._instance_key, + claim_result, + len(items), + limit, + ) + for item in items: outcome = self._process(item) if outcome is OutboxOutcome.COMPLETED: stats.completed += 1 @@ -216,6 +240,32 @@ class LedgerWriter: item = self._outbox.get(correlation_id) if item is None: return None + if ( + self._instance_key is not None + and item.instance_key is not None + and item.instance_key != self._instance_key + ): + self._count("claim:foreign_instance") + _log.info( + "ledger outbox claim: instance_key=%s claim_result=foreign_instance " + "correlation_id=%s owner_instance_key=%s", + self._instance_key, + correlation_id, + item.instance_key, + ) + return None + if item.status is WorkItemStatus.PENDING: + claimed = self._outbox.claim_pending(1, correlation_id=correlation_id) + if not claimed: + self._count("claim:unavailable") + _log.info( + "ledger outbox claim: instance_key=%s claim_result=unavailable " + "correlation_id=%s", + self._instance_key, + correlation_id, + ) + return None + item = claimed[0] return self._process(item) def _backoff_delay(self, attempts: int) -> float: @@ -229,6 +279,7 @@ class LedgerWriter: existing = self._outbox.get(item.correlation_id) if existing is not None and existing.status is WorkItemStatus.COMPLETED: self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED) + self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED) return OutboxOutcome.DUPLICATE_COMPLETED try: outcome = self._ledger.record_trade(item.trade) @@ -236,6 +287,7 @@ class LedgerWriter: self._outbox.mark_outcome( item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc) ) + self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)) # dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed. if self._alarm_emitter is not None: self._alarm_emitter.publish_alarm( @@ -252,8 +304,10 @@ class LedgerWriter: self._outbox.mark_outcome( item.correlation_id, OutboxOutcome.PERMANENT_FAILURE, error=str(exc) ) + self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE, error=str(exc)) _log.warning( - "ledger delivery_failed (permanent): correlation_id=%s reason=%s", + "ledger delivery_failed (permanent): instance_key=%s correlation_id=%s reason=%s", + item.instance_key or self._instance_key, item.correlation_id, exc, ) # dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry. @@ -273,10 +327,14 @@ class LedgerWriter: self._outbox.mark_outcome( item.correlation_id, OutboxOutcome.DEAD_LETTERED, error=str(exc) ) + self._log_outcome(item, OutboxOutcome.DEAD_LETTERED, error=str(exc)) _log.warning( "ledger delivery_failed (dead-letter, attempts=%d): " - "correlation_id=%s reason=%s", - new_attempts, item.correlation_id, exc, + "instance_key=%s correlation_id=%s reason=%s", + new_attempts, + item.instance_key or self._instance_key, + item.correlation_id, + exc, ) # dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries. if self._alarm_emitter is not None: @@ -296,6 +354,7 @@ class LedgerWriter: item.correlation_id, OutboxOutcome.RETRYABLE_FAILURE, error=str(exc), next_retry_at=next_retry_at, ) + self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE, error=str(exc)) return OutboxOutcome.RETRYABLE_FAILURE # dcy-018 / REQ-019/030 (option D): write back portfolio-hub authoritative # cash/quantity to the snapshot store on new+duplicate (OQ-2 b). *_after @@ -308,11 +367,32 @@ class LedgerWriter: if outcome.result is LedgerWriteResult.DUPLICATE_OK: self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED) self._resolve_ledger_skip(item.trade) + self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED) return OutboxOutcome.DUPLICATE_COMPLETED self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.COMPLETED) self._resolve_ledger_skip(item.trade) + self._log_outcome(item, OutboxOutcome.COMPLETED) return OutboxOutcome.COMPLETED + def _log_outcome( + self, + item: LedgerWorkItem, + outcome: OutboxOutcome, + *, + error: str | None = None, + ) -> None: + self._count(f"outcome:{outcome.value}") + msg = "ledger outbox outcome: instance_key=%s correlation_id=%s outcome=%s" + args: tuple[object, ...] = ( + item.instance_key or self._instance_key, + item.correlation_id, + outcome.value, + ) + if error is not None: + msg += " error=%s" + args = (*args, error) + _log.info(msg, *args) + def _resolve_ledger_skip(self, trade: LedgerTrade) -> None: if ( self._ledger_skip_store is None @@ -358,7 +438,12 @@ class LedgerWriter: class InMemoryLedgerOutbox: """Process-local outbox. Thread-safe; idempotent on correlation_id.""" - def __init__(self, *, clock: Optional[Callable[[], datetime]] = None) -> None: + def __init__( + self, + *, + clock: Optional[Callable[[], datetime]] = None, + instance_key: str = "default", + ) -> None: import threading self._rows: dict[str, LedgerWorkItem] = {} @@ -367,6 +452,7 @@ class InMemoryLedgerOutbox: # dcy-015: shared clock lets tests align backoff (next_retry_at) with # list_pending's "now" deterministically. self._clock = clock or (lambda: datetime.now(tz=timezone.utc)) + self._instance_key = instance_key def _now(self) -> datetime: return self._clock() @@ -383,6 +469,7 @@ class InMemoryLedgerOutbox: status=WorkItemStatus.PENDING, created_at=self._now(), updated_at=self._now(), + instance_key=self._instance_key, ) self._rows[cid] = row self._order.append(cid) @@ -404,6 +491,17 @@ class InMemoryLedgerOutbox: break return out + def claim_pending( + self, + limit: int = 32, + *, + correlation_id: str | None = None, + ) -> list[LedgerWorkItem]: + pending = self.list_pending(limit if correlation_id is None else max(limit, 1)) + if correlation_id is None: + return pending[:limit] + return [row for row in pending if row.correlation_id == correlation_id][:1] + def pending_cash_delta(self) -> dict[str, float]: with self._lock: out: dict[str, float] = {} diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index 128ef21..182eeea 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -1333,7 +1333,7 @@ def _cmd_ledger_backfill(args, cfg, confirm) -> int: _emit_pretty_json(report) return 1 - outbox = PostgreSQLLedgerOutbox(cfg.dsn) + outbox = PostgreSQLLedgerOutbox(cfg.dsn, instance_key=instance) if not cfg.portfolio_hub or not cfg.portfolio_hub.base_url: report["result"] = {"status": "rejected", "reason": "portfolio-hub ledger adapter is not configured"} report["audit_path"] = _write_ledger_backfill_audit(args, report) diff --git a/trex_core/runtime/daemon.py b/trex_core/runtime/daemon.py index d232973..95bcd66 100644 --- a/trex_core/runtime/daemon.py +++ b/trex_core/runtime/daemon.py @@ -190,7 +190,7 @@ class Daemon: instance_key=config.instance_key, ) - self.outbox = PostgreSQLLedgerOutbox(config.dsn) + self.outbox = PostgreSQLLedgerOutbox(config.dsn, instance_key=config.instance_key) self.ledger_skip_store = PostgreSQLLedgerSkipStore(config.dsn) # BAT-008: portfolio-hub read + snapshot store + durable pause + scheduler. # (Created before the ledger writer so option-D write-back can target it.) From f2b825e72009a31c7032bdd5db003417318fc982 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:27:12 +0000 Subject: [PATCH 07/21] test: cover unmanaged position lifecycle flow Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_lifecycle.py | 2 +- tests/conftest.py | 5 +-- tests/runtime/test_cli_operator.py | 21 +++++----- trex_core/application/snapshot_scheduler.py | 38 ++++++++++--------- trex_core/runtime/cli.py | 12 +++--- trex_core/runtime/daemon.py | 9 ++--- 6 files changed, 45 insertions(+), 42 deletions(-) diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index 3344117..eec439e 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -32,7 +32,7 @@ class FakeBroker: class RaisingBroker(FakeBroker): def get_balance(self): - raise BrokerBalanceError("broker_unavailable") + raise BrokerBalanceError("transport_failed") def _wire(): diff --git a/tests/conftest.py b/tests/conftest.py index cfa8acd..a3a4413 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -96,7 +96,6 @@ def pg_clean(pg_dsn: str): from trex_core.adapters.persistence.migrate import apply_migrations apply_migrations(pg_dsn) - with psycopg.connect(pg_dsn, autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute(f"TRUNCATE {', '.join(_PG_TABLES)}") + with psycopg.connect(pg_dsn, autocommit=True) as conn, conn.cursor() as cur: + cur.execute(f"TRUNCATE {', '.join(_PG_TABLES)}") return pg_dsn diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index d87683f..b4296ae 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -383,10 +383,13 @@ def test_sync_requires_yes_when_not_dry_run(capsys, monkeypatch): """sync without --yes (and not dry-run) should abort.""" def _fake_sync(args, cfg, confirm): # Simulate the confirmation prompt returning False (no --yes, interactive aborts) - if not getattr(args, "dry_run", False) and not getattr(args, "yes", False): - if not confirm("Sync?"): - print("aborted") - return 1 + if ( + not getattr(args, "dry_run", False) + and not getattr(args, "yes", False) + and not confirm("Sync?") + ): + print("aborted") + return 1 return 0 monkeypatch.setattr(cli, "_cmd_sync", _fake_sync) rc = cli.main(["sync", "--portfolio-id", "PF-1"], confirm=lambda _: False) @@ -398,7 +401,7 @@ def test_sync_live_requires_live_flag(capsys, monkeypatch): monkeypatch.setattr(cli, "_broker_mode", lambda cfg: ("kis", True)) def _fake_sync(args, cfg, confirm): - broker, is_live = cli._broker_mode(cfg) + _broker, is_live = cli._broker_mode(cfg) if is_live and not getattr(args, "live", False): cli._fail("live account requires --live (with --yes) to confirm a real sync", args.json) return 2 @@ -415,7 +418,7 @@ def test_sync_fail_closed_skipped_error_exits_nonzero_without_writes( """REQ-123: a fail-closed skipped live sync is a CLI failure.""" class _Provider: def get_broker_for_read(self): - return SimpleNamespace(get_balance=lambda: []), None + return SimpleNamespace(get_balance=list), None class _WarningStore: def clear_warned(self, *_args): @@ -452,7 +455,7 @@ def test_sync_fail_closed_skipped_error_exits_nonzero_without_writes( } psycopg = types.ModuleType("psycopg") - setattr(psycopg, "connect", lambda *_args, **_kwargs: _Connection()) + psycopg.connect = lambda *_args, **_kwargs: _Connection() monkeypatch.setitem(sys.modules, "psycopg", psycopg) monkeypatch.setattr("trex_core.runtime.broker_provider.build_broker_provider", lambda _cfg: _Provider()) monkeypatch.setattr("trex_core.adapters.portfolio.reconcile_warning_store.PostgreSQLReconcileWarningStore", lambda *_args: _WarningStore()) @@ -707,10 +710,10 @@ def test_verify_cmd_broker_import_does_not_raise(monkeypatch): # Patch at module level — _cmd_verify uses local `from ... import` so we # must patch the source module, not the cli namespace. + import trex_core.adapters.persistence.postgresql as pg_mod + import trex_core.adapters.portfolio.reconcile_warning_store as rws_mod from trex_core.application.portfolio import InMemoryReconcileWarningStore from trex_core.domain.slot.in_memory_repository import InMemorySlotRepository - import trex_core.adapters.portfolio.reconcile_warning_store as rws_mod - import trex_core.adapters.persistence.postgresql as pg_mod monkeypatch.setattr(rws_mod, "PostgreSQLReconcileWarningStore", lambda dsn: InMemoryReconcileWarningStore()) diff --git a/trex_core/application/snapshot_scheduler.py b/trex_core/application/snapshot_scheduler.py index d675656..611a477 100644 --- a/trex_core/application/snapshot_scheduler.py +++ b/trex_core/application/snapshot_scheduler.py @@ -12,8 +12,8 @@ from __future__ import annotations import logging import os import threading -from datetime import date, datetime, time, timedelta, timezone -from typing import Callable, Optional +from collections.abc import Callable +from datetime import UTC, date, datetime, time, timedelta, timezone from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.broker_port import BrokerPort @@ -23,10 +23,10 @@ from trex_core.application.portfolio import ( PortfolioHubReadPort, PortfolioHubUnavailableError, ReconcileNotificationStore, - UnmanagedPositionStore, ReconcileWarningStore, SnapshotStore, TradingContext, + UnmanagedPositionStore, ) from trex_core.application.reconciliation import reconcile_positions @@ -45,32 +45,32 @@ class SnapshotScheduler: store: SnapshotStore, broker: BrokerPort, schedule_kst: time = time(hour=7, minute=30), - clock: Optional[Callable[[], datetime]] = None, + clock: Callable[[], datetime] | None = None, slot_repo=None, slot_service=None, instance_key: str = "default", - alarm_emitter: Optional[AlarmEmitter] = None, - warn_store: Optional[ReconcileWarningStore] = None, - notification_store: Optional[ReconcileNotificationStore] = None, - unmanaged_position_store: Optional[UnmanagedPositionStore] = None, + alarm_emitter: AlarmEmitter | None = None, + warn_store: ReconcileWarningStore | None = None, + notification_store: ReconcileNotificationStore | None = None, + unmanaged_position_store: UnmanagedPositionStore | None = None, credential_fetch_enabled: bool = False, - kis_reinit_callback: Optional[Callable[[BrokerCredentialPayload], None]] = None, - credential_refresh: Optional[Callable[[], object]] = None, + kis_reinit_callback: Callable[[BrokerCredentialPayload], None] | None = None, + credential_refresh: Callable[[], object] | None = None, ) -> None: self._read = read_port self._store = store self._broker = broker self._schedule_kst = schedule_kst - self._clock = clock or (lambda: datetime.now(tz=timezone.utc)) - self._daemon_id: Optional[str] = None - self._last_fired_date: Optional[datetime] = None + self._clock = clock or (lambda: datetime.now(tz=UTC)) + self._daemon_id: str | None = None + self._last_fired_date: datetime | None = None # dcy-020 / REQ-058 (BAT-030): asset-centric slot reconcile after each # universe fetch — always on when slot_repo is wired (the reconcile is # what creates/retires the permanent asset slots). self._slot_repo = slot_repo self._slot_service = slot_service self._instance_key = instance_key - self._last_universe_size: Optional[int] = None + self._last_universe_size: int | None = None # dcy-033 / REQ-100: best-effort alarm emitter. self._alarm_emitter = alarm_emitter # BAT-050 / REQ-103: durable reconcile-warning store (Optional). @@ -84,7 +84,9 @@ class SnapshotScheduler: # dcy-036 / REQ-113 AC-2 / REQ-115 AC-1: optional callback to reinitialize # KISBroker with the fetched day-start credential. None = no reinit (paper # broker or credential_fetch_enabled=False). Set by daemon.py assembly. - self._kis_reinit_callback: Optional[Callable[[BrokerCredentialPayload], None]] = kis_reinit_callback + self._kis_reinit_callback: Callable[[BrokerCredentialPayload], None] | None = ( + kis_reinit_callback + ) # REQ-119: provider-owned refresh supersedes the legacy raw fetch hook. self._credential_refresh = credential_refresh # REQ-113: detect break-glass at init time (env var set by operator). @@ -93,8 +95,8 @@ class SnapshotScheduler: in ("1", "true", "yes") ) # REQ-115: day-scoped credential cache. None until first successful fetch. - self._cached_credential: Optional[BrokerCredentialPayload] = None - self._credential_date: Optional[date] = None + self._cached_credential: BrokerCredentialPayload | None = None + self._credential_date: date | None = None # REQ-114: entry gate. True = new entries allowed (default). # credential_fetch_enabled=False keeps this always True (no gate). self._entry_allowed: bool = True @@ -266,7 +268,7 @@ class SnapshotScheduler: try: self._credential_refresh() self._entry_allowed = True - except Exception as exc: # provider exposes only sanitized failure + except Exception as exc: # noqa: BLE001 - provider exposes only sanitized failure _log.error("DAY_START_LOAD_FAILED: broker credential refresh failed: %s (instance=%s)", exc.__class__.__name__, self._instance_key) self._entry_allowed = False diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index e4ffe68..7c4b950 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -737,12 +737,12 @@ def _build_parser() -> argparse.ArgumentParser: def _cmd_sync(args, cfg, confirm) -> int: """REQ-105 (dcy-034, BAT-051): sync slot qty to broker ground truth.""" - from trex_core.adapters.portfolio.reconcile_warning_store import ( - PostgreSQLReconcileWarningStore, - ) from trex_core.adapters.portfolio.reconcile_notification_store import ( PostgreSQLReconcileNotificationStore, ) + from trex_core.adapters.portfolio.reconcile_warning_store import ( + PostgreSQLReconcileWarningStore, + ) from trex_core.application.reconciliation import sync_slot_to_broker if not args.portfolio_id and not args.all: @@ -891,12 +891,12 @@ def _cmd_sync(args, cfg, confirm) -> int: def _cmd_verify(args, cfg) -> int: """REQ-106 (dcy-034, BAT-051): re-check C-case reconcile-warning (read-centric).""" from trex_core.adapters.persistence.postgresql import PostgreSQLSlotRepository - from trex_core.adapters.portfolio.reconcile_warning_store import ( - PostgreSQLReconcileWarningStore, - ) from trex_core.adapters.portfolio.reconcile_notification_store import ( PostgreSQLReconcileNotificationStore, ) + from trex_core.adapters.portfolio.reconcile_warning_store import ( + PostgreSQLReconcileWarningStore, + ) from trex_core.application.portfolio import SnapshotStore from trex_core.application.reconciliation import verify_slot_reconcile diff --git a/trex_core/runtime/daemon.py b/trex_core/runtime/daemon.py index b0b0f2e..e5f117e 100644 --- a/trex_core/runtime/daemon.py +++ b/trex_core/runtime/daemon.py @@ -31,11 +31,11 @@ from trex_core.adapters.messaging import ( RabbitMQTransport, topology, ) -from trex_core.adapters.persistence.migrate import apply_migrations from trex_core.adapters.persistence.account_snapshot import ( PostgreSQLAccountSnapshotStore, ) from trex_core.adapters.persistence.ledger_skip_store import PostgreSQLLedgerSkipStore +from trex_core.adapters.persistence.migrate import apply_migrations from trex_core.adapters.persistence.postgresql import ( PgStrategyStateStore, PostgreSQLIntentStore, @@ -55,16 +55,15 @@ from trex_core.adapters.portfolio.reconcile_notification_store import ( from trex_core.adapters.portfolio.unmanaged_position_store import ( PostgreSQLUnmanagedPositionStore, ) -from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.account_snapshot import AccountSnapshotService +from trex_core.application.alarm_emitter import AlarmEmitter from trex_core.application.broker_port import BrokerPort from trex_core.application.execution import ExecutionService -from trex_core.application.intents import IntentService -from trex_core.application.intents import BuyIntent +from trex_core.application.intents import BuyIntent, IntentService +from trex_core.application.killswitch import KillSwitchService from trex_core.application.ledger_outbox import LedgerWriter from trex_core.application.ledger_port import LedgerPort from trex_core.application.messaging import MessagingTransport -from trex_core.application.killswitch import KillSwitchService from trex_core.application.messaging_handlers import ( make_control_rpc_handler, make_intent_rpc_handler, From 31d6e5c02b50799b01ed5ff950d53c8b3c70354b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:27:27 +0000 Subject: [PATCH 08/21] test: cover stale ledger outbox lease updates Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_postgresql_ledger_outbox.py | 39 +++++++++++++++++++ tests/application/test_ledger_outbox.py | 6 ++- trex_core/adapters/persistence/postgresql.py | 37 +++++++++++------- .../sql/028_ledger_outbox_instance_claims.sql | 3 +- trex_core/application/ledger_outbox.py | 36 ++++++++++++++--- 5 files changed, 99 insertions(+), 22 deletions(-) diff --git a/tests/adapters/persistence/test_postgresql_ledger_outbox.py b/tests/adapters/persistence/test_postgresql_ledger_outbox.py index 6f51388..d553846 100644 --- a/tests/adapters/persistence/test_postgresql_ledger_outbox.py +++ b/tests/adapters/persistence/test_postgresql_ledger_outbox.py @@ -169,6 +169,45 @@ def test_claim_expiry_allows_safe_recovery_after_worker_crash(pg_clean): assert [item.correlation_id for item in claimed_b] == [row.correlation_id] +def test_stale_claim_cannot_overwrite_new_owner_outcome(pg_clean): + now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} + outbox_a = PostgreSQLLedgerOutbox( + pg_clean, + instance_key="inst-a", + claim_ttl_sec=5.0, + worker_id="worker-a", + clock=lambda: now["t"], + ) + outbox_b = PostgreSQLLedgerOutbox( + pg_clean, + instance_key="inst-a", + claim_ttl_sec=5.0, + worker_id="worker-b", + clock=lambda: now["t"], + ) + row = outbox_a.enqueue(_trade(execution_id="pg-stale")) + + [claim_a] = outbox_a.claim_pending() + now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC) + [claim_b] = outbox_b.claim_pending() + outbox_b.mark_outcome( + row.correlation_id, + OutboxOutcome.COMPLETED, + expected_claim_expires_at=claim_b.claim_expires_at, + ) + + stored = outbox_a.mark_outcome( + row.correlation_id, + OutboxOutcome.RETRYABLE_FAILURE, + error="stale owner", + next_retry_at=now["t"], + expected_claim_expires_at=claim_a.claim_expires_at, + ) + + assert stored.status is WorkItemStatus.COMPLETED + assert stored.last_outcome is OutboxOutcome.COMPLETED + + def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean): class _ForbiddenLedger: def __init__(self): diff --git a/tests/application/test_ledger_outbox.py b/tests/application/test_ledger_outbox.py index e085f30..25e747d 100644 --- a/tests/application/test_ledger_outbox.py +++ b/tests/application/test_ledger_outbox.py @@ -71,9 +71,9 @@ def test_writer_retries_until_success(): def test_process_correlation_id_only_drains_requested_row(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="inst-a") ledger = InMemoryLedger() - writer = LedgerWriter(outbox, ledger) + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") first = outbox.enqueue(_trade(execution_id="e-1")) second = outbox.enqueue(_trade(execution_id="e-2")) @@ -81,6 +81,8 @@ def test_process_correlation_id_only_drains_requested_row(): assert outcome is OutboxOutcome.COMPLETED assert ledger.count() == 1 + assert writer.counters["claim:claimed"] == 1 + assert writer.counters["outcome:completed"] == 1 assert outbox.get(first.correlation_id).status is WorkItemStatus.COMPLETED assert outbox.get(second.correlation_id).status is WorkItemStatus.PENDING diff --git a/trex_core/adapters/persistence/postgresql.py b/trex_core/adapters/persistence/postgresql.py index 64cabc7..aa5b0f0 100644 --- a/trex_core/adapters/persistence/postgresql.py +++ b/trex_core/adapters/persistence/postgresql.py @@ -566,7 +566,8 @@ class _PgIntentRecord: _OUTBOX_COLUMNS = ( "correlation_id, asset_id, order_id, execution_id, side, qty, price, " "fee, tax, executed_at, status, attempts, last_outcome, last_error, " - "created_at, updated_at, next_retry_at, venue, order_venue, instance_key" + "created_at, updated_at, next_retry_at, venue, order_venue, instance_key, " + "claim_expires_at" ) @@ -710,6 +711,7 @@ class PostgreSQLLedgerOutbox: *, error: Optional[str] = None, next_retry_at: Optional[datetime] = None, + expected_claim_expires_at: datetime | None = None, ) -> LedgerWorkItem: with self._psycopg.connect(self._dsn, autocommit=False) as conn: with conn.cursor() as cur: @@ -731,23 +733,29 @@ class PostgreSQLLedgerOutbox: attempts_delta = 1 # next_retry_at only persists for a retryable backoff; cleared otherwise. nra = next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None + params: list[Any] = [ + status_value, + attempts_delta, + outcome.value, + error, + nra, + self._now(), + correlation_id, + self._instance_key, + ] + where = "WHERE correlation_id=%s AND instance_key=%s " + if expected_claim_expires_at is None: + where += "AND status=%s AND claim_owner IS NULL AND claim_expires_at IS NULL" + params.append(WorkItemStatus.COMPLETED.value) + else: + where += "AND claim_owner=%s AND claim_expires_at=%s" + params.extend([self._worker_id, expected_claim_expires_at]) cur.execute( "UPDATE ledger_outbox SET status=%s, attempts=attempts+%s, " "last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s, " "claim_owner=NULL, claim_expires_at=NULL " - "WHERE correlation_id=%s AND instance_key=%s " - "AND (claim_owner=%s OR claim_owner IS NULL)", - ( - status_value, - attempts_delta, - outcome.value, - error, - nra, - self._now(), - correlation_id, - self._instance_key, - self._worker_id, - ), + f"{where}", + params, ) conn.commit() row = self.get(correlation_id) @@ -781,6 +789,7 @@ class PostgreSQLLedgerOutbox: updated_at=row[15], next_retry_at=row[16], instance_key=row[19], + claim_expires_at=row[20], ) diff --git a/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql b/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql index 186c809..1c037c7 100644 --- a/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql +++ b/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql @@ -13,8 +13,9 @@ WHERE instance_key IS NULL; ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET DEFAULT '__unowned__'; ALTER TABLE ledger_outbox ALTER COLUMN instance_key SET NOT NULL; +DROP INDEX IF EXISTS idx_ledger_outbox_instance_pending; CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_pending - ON ledger_outbox (instance_key, status, created_at) + ON ledger_outbox (instance_key, status, created_at, correlation_id) WHERE status = 'pending'; CREATE INDEX IF NOT EXISTS idx_ledger_outbox_instance_claim_expires diff --git a/trex_core/application/ledger_outbox.py b/trex_core/application/ledger_outbox.py index 79e7b32..e0bdd5c 100644 --- a/trex_core/application/ledger_outbox.py +++ b/trex_core/application/ledger_outbox.py @@ -105,6 +105,7 @@ class LedgerWorkItem: created_at: Optional[datetime] = None updated_at: Optional[datetime] = None instance_key: Optional[str] = None + claim_expires_at: Optional[datetime] = None class LedgerOutbox(Protocol): @@ -146,6 +147,7 @@ class LedgerOutbox(Protocol): *, error: Optional[str] = None, next_retry_at: Optional[datetime] = None, + expected_claim_expires_at: Optional[datetime] = None, ) -> LedgerWorkItem: """Record the outcome of a writer attempt and advance status. @@ -266,6 +268,14 @@ class LedgerWriter: ) return None item = claimed[0] + self._count("claim:claimed") + _log.info( + "ledger outbox claim: instance_key=%s claim_result=claimed " + "count=%d correlation_id=%s", + self._instance_key, + len(claimed), + correlation_id, + ) return self._process(item) def _backoff_delay(self, attempts: int) -> float: @@ -285,7 +295,10 @@ class LedgerWriter: outcome = self._ledger.record_trade(item.trade) except LedgerConflictError as exc: self._outbox.mark_outcome( - item.correlation_id, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc) + item.correlation_id, + OutboxOutcome.CONFLICT_BLOCKED, + error=str(exc), + expected_claim_expires_at=item.claim_expires_at, ) self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)) # dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed. @@ -302,7 +315,10 @@ class LedgerWriter: # dcy-015 / REQ-017/018 (issue #13): non-retryable client error → # terminal delivery_failed immediately (no retry spam). Surface for ops. self._outbox.mark_outcome( - item.correlation_id, OutboxOutcome.PERMANENT_FAILURE, error=str(exc) + item.correlation_id, + OutboxOutcome.PERMANENT_FAILURE, + error=str(exc), + expected_claim_expires_at=item.claim_expires_at, ) self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE, error=str(exc)) _log.warning( @@ -325,7 +341,10 @@ class LedgerWriter: new_attempts = item.attempts + 1 if new_attempts >= self._max_attempts: self._outbox.mark_outcome( - item.correlation_id, OutboxOutcome.DEAD_LETTERED, error=str(exc) + item.correlation_id, + OutboxOutcome.DEAD_LETTERED, + error=str(exc), + expected_claim_expires_at=item.claim_expires_at, ) self._log_outcome(item, OutboxOutcome.DEAD_LETTERED, error=str(exc)) _log.warning( @@ -352,7 +371,9 @@ class LedgerWriter: ) self._outbox.mark_outcome( item.correlation_id, OutboxOutcome.RETRYABLE_FAILURE, - error=str(exc), next_retry_at=next_retry_at, + error=str(exc), + next_retry_at=next_retry_at, + expected_claim_expires_at=item.claim_expires_at, ) self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE, error=str(exc)) return OutboxOutcome.RETRYABLE_FAILURE @@ -369,7 +390,11 @@ class LedgerWriter: self._resolve_ledger_skip(item.trade) self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED) return OutboxOutcome.DUPLICATE_COMPLETED - self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.COMPLETED) + self._outbox.mark_outcome( + item.correlation_id, + OutboxOutcome.COMPLETED, + expected_claim_expires_at=item.claim_expires_at, + ) self._resolve_ledger_skip(item.trade) self._log_outcome(item, OutboxOutcome.COMPLETED) return OutboxOutcome.COMPLETED @@ -522,6 +547,7 @@ class InMemoryLedgerOutbox: *, error: Optional[str] = None, next_retry_at: Optional[datetime] = None, + expected_claim_expires_at: Optional[datetime] = None, ) -> LedgerWorkItem: with self._lock: row = self._rows[correlation_id] From 310b6f6dec34236679508da85bcd1a3dd288cd34 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:28:40 +0000 Subject: [PATCH 09/21] fix: align warning-case blocking semantics across reconcile paths Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_slots_handler_quantity_mismatch.py | 8 +++---- .../028_preserve_reconcile_warning_cases.sql | 23 +++++++++++++++++++ trex_core/application/intents.py | 23 +++++++++++++++++-- trex_core/application/messaging_handlers.py | 18 ++++++++++----- trex_core/application/portfolio.py | 4 +++- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tests/application/test_slots_handler_quantity_mismatch.py b/tests/application/test_slots_handler_quantity_mismatch.py index d181765..c374ced 100644 --- a/tests/application/test_slots_handler_quantity_mismatch.py +++ b/tests/application/test_slots_handler_quantity_mismatch.py @@ -128,7 +128,7 @@ def test_ac1_single_portfolio_c_case_excluded(): assert resp.payload["slots"] == [] assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 - assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"C": 1} + assert resp.payload["excluded_slots"]["active_reconcile_warning_cases"] == {"C": 1} def test_holding_slot_with_b_case_is_excluded_for_strategy(): @@ -144,7 +144,7 @@ def test_holding_slot_with_b_case_is_excluded_for_strategy(): assert resp.payload["slots"] == [] assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 - assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"B": 1} + assert resp.payload["excluded_slots"]["active_reconcile_warning_cases"] == {"B": 1} def test_holding_slot_with_b_and_c_cases_keeps_both_and_reports_both(): @@ -162,7 +162,7 @@ def test_holding_slot_with_b_and_c_cases_keeps_both_and_reports_both(): assert resp.payload["slots"] == [] assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 1 - assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {"B": 1, "C": 1} + assert resp.payload["excluded_slots"]["active_reconcile_warning_cases"] == {"B": 1, "C": 1} info = warn_store.get_warning_info(slot_id) assert info is not None assert info["case_codes"] == ["B", "C"] @@ -325,7 +325,7 @@ def test_ac5_excluded_slots_count_zero_when_no_warnings(): assert "excluded_slots" in resp.payload assert resp.payload["excluded_slots"]["reconcile_quantity_mismatch"] == 0 - assert resp.payload["excluded_slots"]["reconcile_warning_cases"] == {} + assert resp.payload["excluded_slots"]["active_reconcile_warning_cases"] == {} # --------------------------------------------------------------------------- diff --git a/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql b/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql index 11e967d..40771bf 100644 --- a/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql +++ b/trex_core/adapters/persistence/sql/028_preserve_reconcile_warning_cases.sql @@ -1,7 +1,30 @@ -- REQ-123 / issue #83: -- Preserve simultaneous warning causes by keying warnings per (slot_id, case_code). +-- NOTE: historical overwritten causes cannot be reconstructed from the prior +-- slot_id-only schema; this migration enables forward preservation only. +UPDATE slot_reconcile_warning +SET case_code = 'B' +WHERE case_code IS NULL OR case_code = ''; + +ALTER TABLE slot_reconcile_warning + ALTER COLUMN case_code SET NOT NULL; + +DELETE FROM slot_reconcile_warning w +USING ( + SELECT ctid, ROW_NUMBER() OVER ( + PARTITION BY slot_id, case_code + ORDER BY updated_at DESC, created_at DESC, ctid DESC + ) AS rn + FROM slot_reconcile_warning +) ranked +WHERE w.ctid = ranked.ctid + AND ranked.rn > 1; + ALTER TABLE slot_reconcile_warning DROP CONSTRAINT IF EXISTS slot_reconcile_warning_pkey; ALTER TABLE slot_reconcile_warning ADD CONSTRAINT slot_reconcile_warning_pkey PRIMARY KEY (slot_id, case_code); + +CREATE INDEX IF NOT EXISTS idx_slot_reconcile_warning_slot_id + ON slot_reconcile_warning (slot_id); diff --git a/trex_core/application/intents.py b/trex_core/application/intents.py index 1f4bf2e..26afb70 100644 --- a/trex_core/application/intents.py +++ b/trex_core/application/intents.py @@ -29,6 +29,7 @@ from typing import Any, Callable, Optional from uuid import UUID from trex_core.application.alarm_emitter import AlarmEmitter +from trex_core.application.portfolio import BLOCKING_RECONCILE_CASES from trex_core.application.ports import IntentStore from trex_core.domain.slot import ( SlotAdmissionRejectedError, @@ -270,6 +271,24 @@ class IntentService: except Exception: # noqa: BLE001 - alarm path must not affect intent decision pass + def _is_reconcile_trade_blocked(self, slot_id: str) -> bool: + if self._warn_store is None: + return False + if not self._warn_store.is_warned(slot_id): + return False + info = self._warn_store.get_warning_info(slot_id) + if info is None: + return True + raw_codes = info.get("case_codes") + if isinstance(raw_codes, list): + case_codes = {str(code) for code in raw_codes if code} + else: + case_code = str(info.get("case_code", "")) + case_codes = {case_code} if case_code else set() + if not case_codes: + return True + return bool(case_codes & BLOCKING_RECONCILE_CASES) + # ------------------------------------------------------------ def _dedup(self, intent_id: str, intent_type: str, payload: dict) -> Optional[IntentDecision]: """Return prior decision if intent_id already seen; else None (and reserve it).""" @@ -339,7 +358,7 @@ class IntentService: # those paths call SlotService.sell() directly. if self._warn_store is not None: try: - if self._warn_store.is_warned(str(slot.slot_id)): + if self._is_reconcile_trade_blocked(str(slot.slot_id)): return IntentDecision( intent.intent_id, "rejected", None, "reconcile_warned" ) @@ -432,7 +451,7 @@ class IntentService: # SlotService.sell() / transit_slot() directly without IntentService (OQ-B). if self._warn_store is not None: try: - if self._warn_store.is_warned(str(slot.slot_id)): + if self._is_reconcile_trade_blocked(str(slot.slot_id)): return IntentDecision( intent.intent_id, "rejected", None, "reconcile_warned" ) diff --git a/trex_core/application/messaging_handlers.py b/trex_core/application/messaging_handlers.py index f50c04a..527dbb1 100644 --- a/trex_core/application/messaging_handlers.py +++ b/trex_core/application/messaging_handlers.py @@ -17,6 +17,7 @@ from trex_core.application.account_snapshot import AccountSnapshotUnavailable from trex_core.application.broker_port import BalanceItem from trex_core.application.messaging import RpcRequest, RpcResponse from trex_core.application.portfolio import ( + BLOCKING_RECONCILE_CASES, PauseStore, ReconcileWarningStore, SnapshotStore, @@ -270,15 +271,16 @@ def make_slots_rpc_handler( for sid in warned_slot_ids: info = warn_store.get_warning_info(sid) if info is None: - continue + raise RuntimeError(f"missing warning info for slot_id={sid}") raw_codes = info.get("case_codes") if isinstance(raw_codes, list): codes = {str(code) for code in raw_codes if code} else: case_code = str(info.get("case_code", "")) codes = {case_code} if case_code else set() - if codes: - warning_cases_by_slot[str(sid)] = codes + if not codes: + raise RuntimeError(f"empty warning cases for slot_id={sid}") + warning_cases_by_slot[str(sid)] = codes except Exception: # noqa: BLE001 _log.exception("warn_store warning query failed — fail-closed") return RpcResponse(payload={ @@ -290,7 +292,10 @@ def make_slots_rpc_handler( "reconcile warning store unavailable; " "returning no slots (fail-closed)" ), - "excluded_slots": {"reconcile_quantity_mismatch": 0}, + "excluded_slots": { + "reconcile_quantity_mismatch": 0, + "active_reconcile_warning_cases": {}, + }, }) # Build merged slot rows (REQ-074) @@ -314,7 +319,8 @@ def make_slots_rpc_handler( included: list[tuple[object, dict]] = [] for slot, row in zip(slots, slot_rows, strict=True): blocked_holding = ( - slot.status.value == "holding" and bool(row.get("reconcile_warned")) + slot.status.value == "holding" + and bool(set(row.get("reconcile_warning_cases", [])) & BLOCKING_RECONCILE_CASES) ) if blocked_holding: excluded_count += 1 @@ -432,7 +438,7 @@ def make_slots_rpc_handler( if warn_store is not None: base_payload["excluded_slots"] = { "reconcile_quantity_mismatch": excluded_count, - "reconcile_warning_cases": excluded_case_counts, + "active_reconcile_warning_cases": excluded_case_counts, } return RpcResponse(payload=base_payload) diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index ad2b7c0..ba11a89 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -21,6 +21,8 @@ from datetime import UTC, datetime from enum import Enum from typing import Protocol +BLOCKING_RECONCILE_CASES = frozenset({"B", "C"}) + # ---------------------------------------------------------------- portfolio-hub @dataclass(frozen=True) @@ -658,7 +660,7 @@ def slot_to_dict( reconcile_warned = bool(warning_cases) d["reconcile_warned"] = reconcile_warned d["reconcile_warning_cases"] = sorted(warning_cases) - d["trade_eligible"] = not reconcile_warned + d["trade_eligible"] = not bool(warning_cases & BLOCKING_RECONCILE_CASES) return d From 0b5b4660e57c17a0f584973fd850234b171b6bdd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:29:05 +0000 Subject: [PATCH 10/21] fix: guard ledger outbox outcomes by lease Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- trex_core/adapters/persistence/postgresql.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/trex_core/adapters/persistence/postgresql.py b/trex_core/adapters/persistence/postgresql.py index aa5b0f0..5864957 100644 --- a/trex_core/adapters/persistence/postgresql.py +++ b/trex_core/adapters/persistence/postgresql.py @@ -745,8 +745,7 @@ class PostgreSQLLedgerOutbox: ] where = "WHERE correlation_id=%s AND instance_key=%s " if expected_claim_expires_at is None: - where += "AND status=%s AND claim_owner IS NULL AND claim_expires_at IS NULL" - params.append(WorkItemStatus.COMPLETED.value) + where += "AND claim_owner IS NULL AND claim_expires_at IS NULL" else: where += "AND claim_owner=%s AND claim_expires_at=%s" params.extend([self._worker_id, expected_claim_expires_at]) From 5a13c0494b43112f41237459a35805e866522b99 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:29:45 +0000 Subject: [PATCH 11/21] fix: guard unmanaged resolution on authoritative balances Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_lifecycle.py | 24 +++++++++ tests/runtime/test_cli_operator.py | 53 +++++++++++++++++++ trex_core/adapters/broker/ownership.py | 9 ++-- trex_core/application/reconciliation.py | 22 +++++++- trex_core/runtime/cli.py | 2 +- 5 files changed, 105 insertions(+), 5 deletions(-) diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index eec439e..bd91c6b 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -35,6 +35,11 @@ class RaisingBroker(FakeBroker): raise BrokerBalanceError("transport_failed") +class FilteredBroker(FakeBroker): + def exposes_full_balance_snapshot(self) -> bool: + return False + + def _wire(): repo = InMemorySlotRepository() return repo, SlotService(repo, instance_key=INSTANCE) @@ -188,3 +193,22 @@ def test_broker_failure_does_not_resolve_unmanaged_state(): state = store.get_state(instance_key=INSTANCE, symbol="008110") assert state is not None assert state["lifecycle_state"] == "active" + + +def test_non_authoritative_broker_view_does_not_resolve_unmanaged_state(): + repo, svc = _wire() + store = InMemoryUnmanagedPositionStore() + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3) + + decisions = reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FilteredBroker({}), + instance_key=INSTANCE, + unmanaged_position_store=store, + ) + + assert decisions == {} + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index b4296ae..51c3610 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -593,6 +593,7 @@ def test_unmanaged_list_outputs_durable_state(monkeypatch, capsys): assert rc == 0 out = json.loads(capsys.readouterr().out) assert out["count"] == 1 + assert out["instance_key"] == "inst-1" assert out["states"][0]["symbol"] == "008110" assert out["states"][0]["evidence_ref"] == "kis-balance:2026-09-11" @@ -688,6 +689,58 @@ def test_unmanaged_record_persists_operator_action(monkeypatch, capsys): assert out["state"]["evidence_ref"] == "kis-balance:2026-09-11" +def test_unmanaged_record_text_output(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def record_operator_action(self, **kwargs): + return { + "instance_key": "inst-1", + "symbol": kwargs["symbol"], + "quantity": 3, + "lifecycle_state": "active", + "first_seen_at": "2026-09-07T00:00:00+00:00", + "last_seen_at": "2026-09-11T00:00:00+00:00", + "quantity_changed_at": "2026-09-07T00:00:00+00:00", + "resolved_at": None, + "operator_action": kwargs["action"], + "evidence_ref": kwargs["evidence_ref"], + "operator_note": kwargs["note"], + "operator_actor": kwargs["actor"], + "operator_updated_at": "2026-09-11T00:00:00+00:00", + "created_at": "2026-09-07T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + } + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + args = SimpleNamespace( + cmd="unmanaged", + unmanaged_cmd="record", + symbol="008110", + action="approved_exception", + evidence_ref="kis-balance:2026-09-11", + note="confirmed", + actor="operator", + instance="inst-1", + config=None, + config_path=None, + env_file=None, + json=False, + timeout=10.0, + ) + + rc = cli._cmd_unmanaged_record(args, SimpleNamespace(dsn="postgresql://x/y")) + + assert rc == 0 + out = capsys.readouterr().out + assert "recorded unmanaged position instance=inst-1 symbol=008110" in out + assert "evidence_ref=kis-balance:2026-09-11" in out + + # --------------------------------------------------------------------------- # REL-010 regression: _cmd_verify broker import must not raise ImportError # --------------------------------------------------------------------------- diff --git a/trex_core/adapters/broker/ownership.py b/trex_core/adapters/broker/ownership.py index 63a3ffa..ab3cc0d 100644 --- a/trex_core/adapters/broker/ownership.py +++ b/trex_core/adapters/broker/ownership.py @@ -13,7 +13,6 @@ import os from datetime import datetime from pathlib import Path from threading import Lock -from typing import Optional from trex_core.application.broker_port import ( BalanceItem, @@ -134,8 +133,8 @@ class OwnershipAwareBroker: *, side: str, symbol: str, - order_date: Optional[str] = None, - submitted_at: Optional[datetime] = None, + order_date: str | None = None, + submitted_at: datetime | None = None, ) -> FillStatus: result = self._delegate.poll_fill( broker_order_id, @@ -175,6 +174,10 @@ class OwnershipAwareBroker: def get_balance(self) -> list[BalanceItem]: return self._store.project(self._delegate.get_balance()) + def exposes_full_balance_snapshot(self) -> bool: + """False: projected balances omit operator-owned/unmanaged broker positions.""" + return False + def account_scope_hash(self) -> str: return self._delegate.account_scope_hash() diff --git a/trex_core/application/reconciliation.py b/trex_core/application/reconciliation.py index 4c2b7b4..29be749 100644 --- a/trex_core/application/reconciliation.py +++ b/trex_core/application/reconciliation.py @@ -364,6 +364,18 @@ def _notify_unmanaged_position_transition( ) +def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPort) -> bool: + marker = getattr(broker, "exposes_full_balance_snapshot", None) + if callable(marker): + try: + return bool(marker()) + except Exception: # noqa: BLE001 - fail closed on broker self-report failure + return False + if isinstance(marker, bool): + return marker + return True + + def reconcile_core_holdings( *, slot_repo, @@ -535,7 +547,10 @@ def reconcile_core_holdings( symbol=sym, quantity=bq, ) - if unmanaged_position_store is not None: + if ( + unmanaged_position_store is not None + and _broker_balances_are_authoritative_for_unmanaged_resolution(broker) + ): try: for state in unmanaged_position_store.resolve_absent( instance_key=instance_key, @@ -555,6 +570,11 @@ def reconcile_core_holdings( ) except Exception: # noqa: BLE001 _log.exception("reconcile_core_holdings: unmanaged_position resolve failed") + elif unmanaged_position_store is not None: + _log.debug( + "reconcile_core_holdings: unmanaged_position resolve skipped " + "(non-authoritative broker balance view)" + ) # req-0007 / ac-0024: retry pending_resolved for B-case identities no longer mismatched. if alarm_emitter is not None and notification_store is not None: try: diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index 7c4b950..bcbe6bc 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -1016,7 +1016,7 @@ def _cmd_unmanaged_list(args, cfg) -> int: symbol=args.symbol, ) payload = { - "instance": instance, + "instance_key": instance, "status": args.status, "count": len(states), "states": states, From f801c53e0c30edb31250e536e6ac0e8d8d5c861e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:30:50 +0000 Subject: [PATCH 12/21] test: cover targeted and expired ledger claims Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_postgresql_ledger_outbox.py | 23 +++++++++++++++++++ tests/application/test_ledger_outbox.py | 6 ++--- trex_core/adapters/persistence/postgresql.py | 7 ++++-- trex_core/application/ledger_outbox.py | 12 +++++++--- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/tests/adapters/persistence/test_postgresql_ledger_outbox.py b/tests/adapters/persistence/test_postgresql_ledger_outbox.py index d553846..d9d465d 100644 --- a/tests/adapters/persistence/test_postgresql_ledger_outbox.py +++ b/tests/adapters/persistence/test_postgresql_ledger_outbox.py @@ -208,6 +208,29 @@ def test_stale_claim_cannot_overwrite_new_owner_outcome(pg_clean): assert stored.last_outcome is OutboxOutcome.COMPLETED +def test_original_owner_can_finalize_expired_unreclaimed_claim(pg_clean): + now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} + outbox = PostgreSQLLedgerOutbox( + pg_clean, + instance_key="inst-a", + claim_ttl_sec=5.0, + worker_id="worker-a", + clock=lambda: now["t"], + ) + row = outbox.enqueue(_trade(execution_id="pg-expired")) + + [claim] = outbox.claim_pending() + now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC) + stored = outbox.mark_outcome( + row.correlation_id, + OutboxOutcome.COMPLETED, + expected_claim_expires_at=claim.claim_expires_at, + ) + + assert stored.status is WorkItemStatus.COMPLETED + assert stored.last_outcome is OutboxOutcome.COMPLETED + + def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean): class _ForbiddenLedger: def __init__(self): diff --git a/tests/application/test_ledger_outbox.py b/tests/application/test_ledger_outbox.py index 25e747d..651efc6 100644 --- a/tests/application/test_ledger_outbox.py +++ b/tests/application/test_ledger_outbox.py @@ -77,14 +77,14 @@ def test_process_correlation_id_only_drains_requested_row(): first = outbox.enqueue(_trade(execution_id="e-1")) second = outbox.enqueue(_trade(execution_id="e-2")) - outcome = writer.process_correlation_id(first.correlation_id) + outcome = writer.process_correlation_id(second.correlation_id) assert outcome is OutboxOutcome.COMPLETED assert ledger.count() == 1 assert writer.counters["claim:claimed"] == 1 assert writer.counters["outcome:completed"] == 1 - assert outbox.get(first.correlation_id).status is WorkItemStatus.COMPLETED - assert outbox.get(second.correlation_id).status is WorkItemStatus.PENDING + assert outbox.get(first.correlation_id).status is WorkItemStatus.PENDING + assert outbox.get(second.correlation_id).status is WorkItemStatus.COMPLETED def test_process_correlation_id_skips_foreign_instance_row(): diff --git a/trex_core/adapters/persistence/postgresql.py b/trex_core/adapters/persistence/postgresql.py index 5864957..3812448 100644 --- a/trex_core/adapters/persistence/postgresql.py +++ b/trex_core/adapters/persistence/postgresql.py @@ -747,8 +747,11 @@ class PostgreSQLLedgerOutbox: if expected_claim_expires_at is None: where += "AND claim_owner IS NULL AND claim_expires_at IS NULL" else: - where += "AND claim_owner=%s AND claim_expires_at=%s" - params.extend([self._worker_id, expected_claim_expires_at]) + where += ( + "AND claim_owner=%s " + "AND (claim_expires_at=%s OR (status='pending' AND claim_expires_at<=%s))" + ) + params.extend([self._worker_id, expected_claim_expires_at, self._now()]) cur.execute( "UPDATE ledger_outbox SET status=%s, attempts=attempts+%s, " "last_outcome=%s, last_error=%s, next_retry_at=%s, updated_at=%s, " diff --git a/trex_core/application/ledger_outbox.py b/trex_core/application/ledger_outbox.py index e0bdd5c..2dc7eac 100644 --- a/trex_core/application/ledger_outbox.py +++ b/trex_core/application/ledger_outbox.py @@ -522,10 +522,16 @@ class InMemoryLedgerOutbox: *, correlation_id: str | None = None, ) -> list[LedgerWorkItem]: - pending = self.list_pending(limit if correlation_id is None else max(limit, 1)) if correlation_id is None: - return pending[:limit] - return [row for row in pending if row.correlation_id == correlation_id][:1] + return self.list_pending(limit) + now = self._now() + with self._lock: + row = self._rows.get(correlation_id) + if row is None or row.status is not WorkItemStatus.PENDING: + return [] + if row.next_retry_at is not None and row.next_retry_at > now: + return [] + return [row] def pending_cash_delta(self) -> dict[str, float]: with self._lock: From be2e73a2d660dd3df48233033959d1718349267e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:31:32 +0000 Subject: [PATCH 13/21] fix: harden unmanaged position lifecycle writes Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- tests/runtime/test_cli_operator.py | 47 ++++++ .../portfolio/unmanaged_position_store.py | 159 +++++++++--------- 2 files changed, 130 insertions(+), 76 deletions(-) diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index 51c3610..91d3775 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -741,6 +741,53 @@ def test_unmanaged_record_text_output(monkeypatch, capsys): assert "evidence_ref=kis-balance:2026-09-11" in out +def test_unmanaged_record_main_dispatch(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def record_operator_action(self, **kwargs): + return { + "instance_key": "default", + "symbol": kwargs["symbol"], + "quantity": 3, + "lifecycle_state": "active", + "first_seen_at": "2026-09-07T00:00:00+00:00", + "last_seen_at": "2026-09-11T00:00:00+00:00", + "quantity_changed_at": "2026-09-07T00:00:00+00:00", + "resolved_at": None, + "operator_action": kwargs["action"], + "evidence_ref": kwargs["evidence_ref"], + "operator_note": kwargs["note"], + "operator_actor": kwargs["actor"], + "operator_updated_at": "2026-09-11T00:00:00+00:00", + "created_at": "2026-09-07T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + } + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + + rc = cli.main( + [ + "unmanaged", + "record", + "008110", + "investigating", + "--evidence-ref", + "kis-balance:2026-09-11", + "--json", + ] + ) + + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["result"] == "recorded" + assert out["state"]["symbol"] == "008110" + + # --------------------------------------------------------------------------- # REL-010 regression: _cmd_verify broker import must not raise ImportError # --------------------------------------------------------------------------- diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py index 694ab4b..6e75469 100644 --- a/trex_core/adapters/portfolio/unmanaged_position_store.py +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -31,82 +31,89 @@ class PostgreSQLUnmanagedPositionStore: actor: str = "reconcile", ) -> dict: now = observed_at or datetime.now(tz=UTC) - with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur: - cur.execute( - f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " - "WHERE instance_key = %s AND symbol = %s " - "FOR UPDATE", - (instance_key, symbol), - ) - row = cur.fetchone() - previous = self._row_to_state(row) if row is not None else None - previous_quantity = previous.get("quantity") if previous is not None else None - if previous is None: - transition = "detected" - cur.execute( - "INSERT INTO unmanaged_position_state " - f"({_STATE_COLUMNS}) " - "VALUES (%s, %s, %s, 'active', %s, %s, %s, NULL, " - " NULL, NULL, '', NULL, NULL, %s, %s) " - f"RETURNING {_STATE_COLUMNS}", - (instance_key, symbol, quantity, now, now, now, now, now), - ) - elif previous["lifecycle_state"] == "resolved": - transition = "detected" - cur.execute( - "UPDATE unmanaged_position_state " - "SET quantity = %s, lifecycle_state = 'active', " - " first_seen_at = %s, last_seen_at = %s, quantity_changed_at = %s, " - " resolved_at = NULL, updated_at = %s " - "WHERE instance_key = %s AND symbol = %s " - f"RETURNING {_STATE_COLUMNS}", - (quantity, now, now, now, now, instance_key, symbol), - ) - elif int(previous["quantity"]) != int(quantity): - transition = "quantity_changed" - cur.execute( - "UPDATE unmanaged_position_state " - "SET quantity = %s, last_seen_at = %s, quantity_changed_at = %s, " - " resolved_at = NULL, lifecycle_state = 'active', updated_at = %s " - "WHERE instance_key = %s AND symbol = %s " - f"RETURNING {_STATE_COLUMNS}", - (quantity, now, now, now, instance_key, symbol), - ) - else: - transition = "unchanged" - cur.execute( - "UPDATE unmanaged_position_state " - "SET last_seen_at = %s, updated_at = %s " - "WHERE instance_key = %s AND symbol = %s " - f"RETURNING {_STATE_COLUMNS}", - (now, now, instance_key, symbol), - ) - state = self._row_to_state(cur.fetchone()) - if transition != "unchanged": - note = ( - f"previous_quantity={previous_quantity}" - if transition == "quantity_changed" and previous_quantity is not None - else "" - ) - self._insert_audit( - cur, - instance_key=instance_key, - symbol=symbol, - event_type=transition, - action=None, - quantity=quantity, - lifecycle_state=state["lifecycle_state"], - actor=actor, - evidence_ref=state.get("evidence_ref"), - note=note, - created_at=now, - ) - conn.commit() - return { - "transition": transition, - "previous_quantity": previous_quantity, - "state": state, - } + for _attempt in range(2): + try: + with self._psycopg.connect(self._dsn, autocommit=False) as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_STATE_COLUMNS} FROM unmanaged_position_state " + "WHERE instance_key = %s AND symbol = %s " + "FOR UPDATE", + (instance_key, symbol), + ) + row = cur.fetchone() + previous = self._row_to_state(row) if row is not None else None + previous_quantity = previous.get("quantity") if previous is not None else None + if previous is None: + transition = "detected" + cur.execute( + "INSERT INTO unmanaged_position_state " + f"({_STATE_COLUMNS}) " + "VALUES (%s, %s, %s, 'active', %s, %s, %s, NULL, " + " NULL, NULL, '', NULL, NULL, %s, %s) " + f"RETURNING {_STATE_COLUMNS}", + (instance_key, symbol, quantity, now, now, now, now, now), + ) + elif previous["lifecycle_state"] == "resolved": + transition = "detected" + cur.execute( + "UPDATE unmanaged_position_state " + "SET quantity = %s, lifecycle_state = 'active', " + " first_seen_at = %s, last_seen_at = %s, quantity_changed_at = %s, " + " resolved_at = NULL, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (quantity, now, now, now, now, instance_key, symbol), + ) + elif int(previous["quantity"]) != int(quantity): + transition = "quantity_changed" + cur.execute( + "UPDATE unmanaged_position_state " + "SET quantity = %s, last_seen_at = %s, quantity_changed_at = %s, " + " resolved_at = NULL, lifecycle_state = 'active', updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (quantity, now, now, now, instance_key, symbol), + ) + else: + transition = "unchanged" + cur.execute( + "UPDATE unmanaged_position_state " + "SET last_seen_at = %s, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + f"RETURNING {_STATE_COLUMNS}", + (now, now, instance_key, symbol), + ) + state = self._row_to_state(cur.fetchone()) + if transition != "unchanged": + note = ( + f"previous_quantity={previous_quantity}" + if transition == "quantity_changed" and previous_quantity is not None + else "" + ) + self._insert_audit( + cur, + instance_key=instance_key, + symbol=symbol, + event_type=transition, + action=None, + quantity=quantity, + lifecycle_state=state["lifecycle_state"], + actor=actor, + evidence_ref=state.get("evidence_ref"), + note=note, + created_at=now, + ) + conn.commit() + return { + "transition": transition, + "previous_quantity": previous_quantity, + "state": state, + } + except self._psycopg.errors.UniqueViolation: + continue + raise RuntimeError( + "failed to persist unmanaged_position_state due to repeated concurrent insert race" + ) def resolve_absent( self, From f9284d1de8bf62ac8b05181f7917fb5a8f87eba3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:33:05 +0000 Subject: [PATCH 14/21] fix: preserve unmanaged first-seen timestamps Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_store.py | 27 +++++++++++++++++++ .../test_unmanaged_position_lifecycle.py | 16 +++++++++++ .../portfolio/unmanaged_position_store.py | 6 ++--- trex_core/application/portfolio.py | 4 ++- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/tests/adapters/portfolio/test_unmanaged_position_store.py b/tests/adapters/portfolio/test_unmanaged_position_store.py index b882f76..4639bf2 100644 --- a/tests/adapters/portfolio/test_unmanaged_position_store.py +++ b/tests/adapters/portfolio/test_unmanaged_position_store.py @@ -99,3 +99,30 @@ class TestCRUDRoundTrip: rows_b = pg_unmanaged_store.list_states(instance_key="inst-B", lifecycle_state="all") assert len(rows_a) == 1 and rows_a[0]["lifecycle_state"] == "resolved" assert len(rows_b) == 1 and rows_b[0]["quantity"] == 7 + + def test_reactivated_state_preserves_original_first_seen(self, pg_unmanaged_store): + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + observed_at=t1, + ) + pg_unmanaged_store.resolve_absent( + instance_key="inst-A", + observed_symbols=set(), + observed_at=t2, + ) + reopened = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=5, + observed_at=t3, + ) + + assert reopened["transition"] == "detected" + assert reopened["state"]["first_seen_at"] == t1 + assert reopened["state"]["last_seen_at"] == t3 diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index bd91c6b..b906e06 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -109,6 +109,22 @@ def test_in_memory_store_isolates_same_symbol_by_instance(): assert state_b["quantity"] == 7 +def test_in_memory_store_preserves_first_seen_when_resolved_symbol_reappears(): + store = InMemoryUnmanagedPositionStore() + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t1) + store.resolve_absent(instance_key=INSTANCE, observed_symbols=set(), observed_at=t2) + reopened = store.observe(instance_key=INSTANCE, symbol="008110", quantity=5, observed_at=t3) + + assert reopened["transition"] == "detected" + assert reopened["state"]["first_seen_at"] == t1 + assert reopened["state"]["last_seen_at"] == t3 + assert reopened["state"]["quantity_changed_at"] == t3 + + def test_reconcile_core_holdings_suppresses_repeated_unmanaged_alarm_spam(): repo, svc = _wire() emitter, transport = _make_emitter() diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py index 6e75469..0d15adf 100644 --- a/trex_core/adapters/portfolio/unmanaged_position_store.py +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -58,11 +58,11 @@ class PostgreSQLUnmanagedPositionStore: cur.execute( "UPDATE unmanaged_position_state " "SET quantity = %s, lifecycle_state = 'active', " - " first_seen_at = %s, last_seen_at = %s, quantity_changed_at = %s, " - " resolved_at = NULL, updated_at = %s " + " last_seen_at = %s, quantity_changed_at = %s, resolved_at = NULL, " + " updated_at = %s " "WHERE instance_key = %s AND symbol = %s " f"RETURNING {_STATE_COLUMNS}", - (quantity, now, now, now, now, instance_key, symbol), + (quantity, now, now, now, instance_key, symbol), ) elif int(previous["quantity"]) != int(quantity): transition = "quantity_changed" diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index cf2870e..ced6b87 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -629,7 +629,9 @@ class InMemoryUnmanagedPositionStore: "symbol": symbol, "quantity": quantity, "lifecycle_state": "active", - "first_seen_at": now, + "first_seen_at": ( + existing.get("first_seen_at", now) if existing is not None else now + ), "last_seen_at": now, "quantity_changed_at": now, "resolved_at": None, From 853943f6763e2d4b90903c72fb5df8c02e1acfad Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sat, 12 Sep 2026 22:58:11 +0900 Subject: [PATCH 15/21] fix: fail closed on unknown reconcile warnings --- .proofline/criteria/ac-0023.md | 12 +++-- docs/interface-contract.md | 12 ++++- .../test_reconcile_warning_blocking.py | 46 +++++++++++++++++++ .../test_slots_handler_quantity_mismatch.py | 18 ++++++++ trex_core/application/intents.py | 5 ++ trex_core/application/messaging_handlers.py | 2 + trex_core/application/portfolio.py | 4 +- 7 files changed, 92 insertions(+), 7 deletions(-) diff --git a/.proofline/criteria/ac-0023.md b/.proofline/criteria/ac-0023.md index e0e01d7..b4fe994 100644 --- a/.proofline/criteria/ac-0023.md +++ b/.proofline/criteria/ac-0023.md @@ -3,20 +3,22 @@ id: "ac-0023" status: active --- -# C-case quantity mismatch — strategy RPC exclusion and intent blocking +# B/C reconcile warning — strategy RPC exclusion and intent blocking ## Criterion -KIS 계좌 잔고(`hldg_qty`)와 같은 instance가 관리하는 동일 symbol의 portfolio-hub `asset_quantity` 합계가 일치하지 않으면(`case_code=C` reconcile-warning 활성), 해당 symbol의 모든 `holding` slot을 `trex.slots.req.` 응답에서 제외해야 한다. 수량 대조는 **instance_key + symbol 단위 합계**로 수행하며, 개별 portfolio 수량을 KIS 종목 합계와 직접 비교해서는 안 된다. 응답에는 `excluded_slots.reconcile_quantity_mismatch` 필드로 제외된 slot 수를 포함한다. warn-store를 읽지 못하면 경고 유무를 모르는 상태로 slot 목록을 반환하지 않고 `degraded=true` / `reason="warn_store_unavailable"`으로 응답해야 한다(fail-closed). 신규 buy/sell intent는 durable warning을 재확인하여 `reconcile_warned` 사유로 거부한다. C-case 경고 해제는 symbol 단위 합계가 다시 일치한 경우에만 수행하며, 해당 symbol에 걸린 모든 C-case 경고를 함께 해제한다. 이 경로는 주문 submit/cancel, ledger write, slot lifecycle mutation을 유발하지 않아야 한다. +`case_code=B`(core slot↔KIS 수량 불일치) 또는 `case_code=C`(portfolio-hub↔KIS 수량 불일치) reconcile-warning이 활성인 `holding` slot은 `trex.slots.req.` 응답에서 제외해야 한다. 동일 slot의 B/C 원인은 서로 덮어쓰지 않고 동시에 보존하며, 응답에는 제외된 slot 수와 활성 case별 count를 포함한다. 신규 buy/sell intent는 같은 durable warning 원인 집합을 재확인하여 `reconcile_warned` 사유로 거부한다. warn-store를 읽지 못하거나 warning metadata가 비어 있거나 지원하지 않는 case를 포함하면 안전한 eligibility를 입증할 수 없으므로 slots RPC는 `degraded=true` / `reason="warn_store_unavailable"`, intent는 `reconcile_warning_unavailable`(조회 실패) 또는 `reconcile_warned`(empty/unsupported metadata)로 fail-closed 처리해야 한다. `sync`는 수량 일치 확인 후 B-case만 해제하고, `verify`는 instance+symbol 단위 KIS와 portfolio-hub 합계가 다시 일치한 경우 C-case만 해제하여 다른 활성 원인을 보존한다. C-case 수량 대조는 **instance_key + symbol 단위 합계**로 수행하며, 개별 portfolio 수량을 KIS 종목 합계와 직접 비교해서는 안 된다. 이 경로는 주문 submit/cancel, ledger write, slot lifecycle mutation을 유발하지 않아야 한다. ## Verification - 성공 조건: KIS 440주, hub portfolio A 200주 + B 240주(합계 440) → A·B 모두 반환되고 `excluded_slots.reconcile_quantity_mismatch=0`. -- 불일치 조건: KIS 440주, hub A 200주 + B 235주(합계 435) → A·B 모두 제외되고 `excluded_slots.reconcile_quantity_mismatch=2`. 다른 symbol 정상 slot은 같은 응답에 포함된다. +- 불일치 조건: KIS 440주, hub A 200주 + B 235주(합계 435) → A·B 모두 제외되고 `excluded_slots.reconcile_quantity_mismatch=2`, `active_reconcile_warning_cases.C=2`. 다른 symbol 정상 slot은 같은 응답에 포함된다. +- 복합 원인: 동일 slot에 B와 C가 동시에 활성일 때 두 원인이 모두 조회·차단되며, B 또는 C 하나를 해제해도 다른 원인은 보존된다. +- B-case: B warning이 활성인 holding slot은 slots RPC에서 제외되고 신규 buy/sell intent도 거부된다. - 단일 portfolio 불일치: hub 단일 portfolio qty != KIS → 해당 symbol 전체 slot 제외. -- fail-closed (slots RPC): warn-store가 `list_c_case_warned()` 중 예외를 throw하면 `slots=[], degraded=true, reason="warn_store_unavailable"` 반환. +- fail-closed (slots RPC): warn-store 조회가 실패하거나 empty/unknown case metadata를 반환하면 `slots=[], degraded=true, reason="warn_store_unavailable"` 반환. - fail-closed (intent): intent 처리 시 `warn_store.is_warned()` 호출이 실패하면 `reconcile_warning_unavailable` 사유로 거부해야 한다 — fail-open 허용 불가. -- intent race: 마지막 slots RPC 이후 경고가 설정된 경우에도 intent 처리 시점의 durable warning을 재확인해 `reconcile_warned`로 거부. +- intent race: 마지막 slots RPC 이후 경고가 설정된 경우에도 intent 처리 시점의 durable warning을 재확인해 `reconcile_warned`로 거부. unknown case도 거래 허용 근거로 사용하지 않는다. - 해제: symbol 합계가 다시 일치하면 해당 symbol의 모든 **C-case 경고만** 함께 해제. 같은 symbol의 B-case 등 다른 case 경고는 보존해야 한다. - 해제 대상: HOLDING 뿐 아니라 모든 상태의 slot에 걸린 C-case 경고를 포함한다. - 해제 실패: 해제 도중 예외가 발생하면 `result='clear_failed'`를 반환하고 완전 성공으로 보고하지 않아야 한다(부분 성공 = 실패). diff --git a/docs/interface-contract.md b/docs/interface-contract.md index 05b00df..7f020b2 100644 --- a/docs/interface-contract.md +++ b/docs/interface-contract.md @@ -110,13 +110,20 @@ "last_seen": "ISO-8601 | null" }, "paused": "bool", + "reconcile_warned": "bool", // durable B/C warning 또는 unknown case 존재 + "reconcile_warning_cases": ["B", "C"], // 활성 case code 목록 + "trade_eligible": "bool", // known blocking/unknown case가 없을 때만 true "strategy_name": "str | null", // REQ-116 additive (BAT-055) — portfolio-hub PortfolioItem 출처 "strategy_params": "dict", // REQ-116 additive (BAT-055) — portfolio-hub opaque dict; {} if absent "markets": ["str"] // REQ-116 additive (BAT-055) — portfolio-hub markets list; [] if absent } ], + "excluded_slots": { + "reconcile_quantity_mismatch": "int", + "active_reconcile_warning_cases": {"B": "int", "C": "int"} + }, "degraded": "bool", - "reason": "portfolio_hub_unavailable | snapshot_fetch_failed | cash_position_stale | null", + "reason": "portfolio_hub_unavailable | snapshot_fetch_failed | cash_position_stale | warn_store_unavailable | null", "message": "str | null" // sanitized — no credentials/raw exceptions } ``` @@ -138,11 +145,14 @@ | `warned` | 파생 | `slot_quantity != asset_quantity` (REQ-074 AC-2) | | `watchdog` | WatchdogStore | 만료 여부 + last_seen; null=store 미연결 | | `paused` | PauseStore | durable pause overlay (REQ-032) | +| `reconcile_warned`, `reconcile_warning_cases` | ReconcileWarningStore | 동일 slot의 durable B/C 원인을 동시에 보존·노출 | +| `trade_eligible` | 파생(safety gate) | 활성 B/C 또는 지원하지 않는 warning case가 있으면 false | | `strategy_name` | hub (PortfolioItem via SnapshotStore) | REQ-116 additive. portfolio-hub 정본. hub 미제공 시 null | | `strategy_params` | hub (PortfolioItem via SnapshotStore) | REQ-116 additive. opaque dict; hub 미제공 시 {} | | `markets` | hub (PortfolioItem via SnapshotStore) | REQ-116 additive. list[str]; hub 미제공 시 [] | - **slot-origin strategy 필드 없음** (REQ-074 AC-5, REQ-116 additive 재도입 후에도 유지): `strategy_id`, `sell_strategy_id`, `active_strategy_id` 는 응답에 포함되지 않는다. `strategy_name`/`strategy_params`/`markets` 는 PortfolioItem(hub) 출처이므로 AC-5 제외 대상과 구분된다 — REQ-116 Notes 참조. +- **Reconcile warning fail-closed**: B/C warning이 활성인 holding slot은 응답에서 제외된다. warning store 조회 실패, empty metadata, unknown case는 `slots=[]`, `degraded=true`, `reason="warn_store_unavailable"`로 처리한다. intent admission도 동일 durable source를 재확인하며 unknown case를 거래 허용 근거로 사용하지 않는다. - **Breaking change (holding `qty`) rollout**: 기존 전략 클라이언트가 `qty=slot mirror`로 가정했다면 `slot_quantity`를 읽도록 전환해야 한다. 권장 경로는 (1) 클라이언트에 `qty_source`/`broker_quantity_available` 체크 추가 → (2) fail-closed (`qty is null`) 처리 배포 → (3) `qty`를 broker 정본으로 사용하도록 cutover 순서다. - `spy=false`(기본): holding 슬롯 응답 후 `WatchdogStore.update(slot_id)` 호출 — 전략 폴링 = 생존 신호. - `spy=true`: watchdog 갱신 건너뜀 — 운영자 CLI(`universe`/`slots`) 가 항상 사용. diff --git a/tests/application/test_reconcile_warning_blocking.py b/tests/application/test_reconcile_warning_blocking.py index 8acff52..ed9f761 100644 --- a/tests/application/test_reconcile_warning_blocking.py +++ b/tests/application/test_reconcile_warning_blocking.py @@ -340,6 +340,52 @@ def test_intent_service_sell_rejected_when_reconcile_warned(): assert d.reason == "reconcile_warned" +def test_unknown_warning_case_blocks_buy_and_sell_intents(): + repo = InMemorySlotRepository() + svc = SlotService(repo) + warn_store = InMemoryReconcileWarningStore() + slot = _make_holding_slot(repo, svc, qty=10) + warn_store.set_warned(str(slot.slot_id), portfolio_id="pf-A", case_code="X") + + intent_svc = IntentService( + svc, + repo, + InMemoryIntentStore(), + instance_key="default", + warn_store=warn_store, + ) + + buy = intent_svc.submit_buy(BuyIntent( + intent_id="b-unknown-case", + slot_id=slot.slot_id, + entry_price=100.0, + qty=1.0, + )) + sell = intent_svc.submit_sell(SellIntent( + intent_id="s-unknown-case", + slot_id=slot.slot_id, + reason="strategy_exit", + )) + + assert buy.status == "rejected" + assert buy.reason == "reconcile_warned" + assert sell.status == "rejected" + assert sell.reason == "reconcile_warned" + + +def test_slot_to_dict_marks_unknown_warning_case_ineligible(): + repo = InMemorySlotRepository() + svc = SlotService(repo) + warn_store = InMemoryReconcileWarningStore() + slot = _make_holding_slot(repo, svc, qty=10) + warn_store.set_warned(str(slot.slot_id), portfolio_id="pf-A", case_code="X") + + row = slot_to_dict(slot, warn_store=warn_store) + + assert row["reconcile_warning_cases"] == ["X"] + assert row["trade_eligible"] is False + + def test_reconcile_warned_buy_emits_alarm_with_allowlisted_fields_after_finalize(): repo = InMemorySlotRepository() svc = SlotService(repo) diff --git a/tests/application/test_slots_handler_quantity_mismatch.py b/tests/application/test_slots_handler_quantity_mismatch.py index c374ced..333574b 100644 --- a/tests/application/test_slots_handler_quantity_mismatch.py +++ b/tests/application/test_slots_handler_quantity_mismatch.py @@ -371,6 +371,24 @@ def test_ac7_warn_store_failure_fail_closed(): assert "excluded_slots" in resp.payload +def test_unknown_warning_case_fails_closed(): + repo = InMemorySlotRepository() + svc = SlotService(repo) + warn_store = InMemoryReconcileWarningStore() + slot = _make_holding_slot(repo, svc, portfolio_id="pf-A", symbol=SYM_A, qty=10) + warn_store.set_warned(str(slot.slot_id), portfolio_id="pf-A", case_code="X") + + resp = _make_handler(repo, warn_store)(_req()) + + assert resp.payload["slots"] == [] + assert resp.payload["degraded"] is True + assert resp.payload["reason"] == "warn_store_unavailable" + assert resp.payload["excluded_slots"] == { + "reconcile_quantity_mismatch": 0, + "active_reconcile_warning_cases": {}, + } + + def test_ac23_exclusion_precedes_broker_and_strategy_state_readers(): repo = InMemorySlotRepository() svc = SlotService(repo) diff --git a/trex_core/application/intents.py b/trex_core/application/intents.py index 26afb70..7297d06 100644 --- a/trex_core/application/intents.py +++ b/trex_core/application/intents.py @@ -287,6 +287,11 @@ class IntentService: case_codes = {case_code} if case_code else set() if not case_codes: return True + # The warning row is authoritative. Unknown cases are not evidence + # that trading is safe: they may come from corrupt data or a newer + # producer that this core version cannot interpret. + if not case_codes.issubset(BLOCKING_RECONCILE_CASES): + return True return bool(case_codes & BLOCKING_RECONCILE_CASES) # ------------------------------------------------------------ diff --git a/trex_core/application/messaging_handlers.py b/trex_core/application/messaging_handlers.py index 527dbb1..f525d95 100644 --- a/trex_core/application/messaging_handlers.py +++ b/trex_core/application/messaging_handlers.py @@ -280,6 +280,8 @@ def make_slots_rpc_handler( codes = {case_code} if case_code else set() if not codes: raise RuntimeError(f"empty warning cases for slot_id={sid}") + if not codes.issubset(BLOCKING_RECONCILE_CASES): + raise RuntimeError(f"unsupported warning cases for slot_id={sid}") warning_cases_by_slot[str(sid)] = codes except Exception: # noqa: BLE001 _log.exception("warn_store warning query failed — fail-closed") diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index ba11a89..5e9e9ea 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -660,7 +660,9 @@ def slot_to_dict( reconcile_warned = bool(warning_cases) d["reconcile_warned"] = reconcile_warned d["reconcile_warning_cases"] = sorted(warning_cases) - d["trade_eligible"] = not bool(warning_cases & BLOCKING_RECONCILE_CASES) + known_blocking = bool(warning_cases & BLOCKING_RECONCILE_CASES) + unknown_cases = bool(warning_cases - BLOCKING_RECONCILE_CASES) + d["trade_eligible"] = not (known_blocking or unknown_cases) return d From f1e718e46835c8e898f1b57e691d7d5a696f41b4 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sat, 12 Sep 2026 23:06:29 +0900 Subject: [PATCH 16/21] docs: align reconcile warning acceptance criteria --- .proofline/criteria/ac-0022.md | 4 ++-- .proofline/criteria/ac-0023.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.proofline/criteria/ac-0022.md b/.proofline/criteria/ac-0022.md index 54d5f67..d08ea22 100644 --- a/.proofline/criteria/ac-0022.md +++ b/.proofline/criteria/ac-0022.md @@ -7,9 +7,9 @@ status: active ## Criterion -`trex.slots.req.`의 **전략용 slots 응답에 포함되는** `holding` 슬롯(AC-0023에 따라 C-case 수량 불일치로 제외된 slot 제외)의 `qty`는 요청한 instance에 결합된 KIS account broker holdings(`hldg_qty`)를 authoritative source로 사용해야 한다. `daily_slot.qty`와 portfolio-hub 집계값은 각각 `slot_quantity`, `asset_quantity` 진단 필드로 분리 유지한다. broker 수량을 확보하지 못했거나 freshness 허용시간을 초과하면 core/hub mirror fallback 없이 `qty=null`, `qty_source="unavailable"`, `broker_quantity_available=false`, `broker_quantity_captured_at=null`을 반환해 전략이 fail-closed하도록 해야 한다. confirmed broker zero는 `qty=0`, `qty_source="broker"`로 unavailable과 구분되어야 한다. 이 read path는 주문 submit/cancel, ledger write, slot lifecycle mutation을 유발하지 않아야 한다. +`trex.slots.req.`의 **전략용 slots 응답에 포함되는** `holding` 슬롯(AC-0023에 따라 B 또는 C reconcile warning으로 제외된 slot 제외)의 `qty`는 요청한 instance에 결합된 KIS account broker holdings(`hldg_qty`)를 authoritative source로 사용해야 한다. `daily_slot.qty`와 portfolio-hub 집계값은 각각 `slot_quantity`, `asset_quantity` 진단 필드로 분리 유지한다. broker 수량을 확보하지 못했거나 freshness 허용시간을 초과하면 core/hub mirror fallback 없이 `qty=null`, `qty_source="unavailable"`, `broker_quantity_available=false`, `broker_quantity_captured_at=null`을 반환해 전략이 fail-closed하도록 해야 한다. confirmed broker zero는 `qty=0`, `qty_source="broker"`로 unavailable과 구분되어야 한다. 이 read path는 주문 submit/cancel, ledger write, slot lifecycle mutation을 유발하지 않아야 한다. -**Scope note:** AC-0023에 따라 C-case(`case_code=C`) 수량 불일치로 전략용 slots RPC에서 제외된 slot에는 이 AC의 broker-authoritative `qty` 응답 형식을 적용하지 않는다. AC-0023 제외가 AC-0022 broker-qty 형식보다 우선한다. +**Scope note:** AC-0023에 따라 B 또는 C reconcile warning(`case_code=B` 또는 `case_code=C`)으로 전략용 slots RPC에서 제외된 slot에는 이 AC의 broker-authoritative `qty` 응답 형식을 적용하지 않는다. AC-0023 제외가 AC-0022 broker-qty 형식보다 우선한다. ## Verification diff --git a/.proofline/criteria/ac-0023.md b/.proofline/criteria/ac-0023.md index b4fe994..1ab13c4 100644 --- a/.proofline/criteria/ac-0023.md +++ b/.proofline/criteria/ac-0023.md @@ -24,4 +24,4 @@ status: active - 해제 실패: 해제 도중 예외가 발생하면 `result='clear_failed'`를 반환하고 완전 성공으로 보고하지 않아야 한다(부분 성공 = 실패). - 시간이 지났다는 이유만으로 자동 해제하지 않음. - 비침습 조건: slots RPC 호출 전후로 slot 상태 전이, broker mutation, ledger write side effect가 없어야 한다. -- AC-0022 우선순위: C-case로 제외된 slot에는 broker-authoritative `qty` 응답 형식(AC-0022)을 적용하지 않는다. +- AC-0022 우선순위: B 또는 C reconcile warning으로 제외된 slot에는 broker-authoritative `qty` 응답 형식(AC-0022)을 적용하지 않는다. From 762a953c95574c8a2cd5e1a7665bd1a91655321d Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 13 Sep 2026 09:41:51 +0900 Subject: [PATCH 17/21] fix: fence ledger outbox claims by instance --- .proofline/criteria/ac-0029.md | 21 +++ .proofline/identities.json | 2 +- scripts/virtual_execution_final_state.py | 2 +- .../persistence/test_migration_and_drift.py | 4 +- .../test_postgresql_ledger_outbox.py | 43 ++++- tests/application/test_execution_cancel.py | 2 +- .../test_execution_fill_price_backfill.py | 4 +- tests/application/test_execution_ledger.py | 8 +- tests/application/test_ledger_option_d.py | 16 +- tests/application/test_ledger_outbox.py | 116 ++++++++++++-- .../application/test_ledger_outbox_backoff.py | 2 +- .../test_messaging_event_publisher.py | 4 +- trex_core/adapters/persistence/postgresql.py | 31 +++- ... => 029_ledger_outbox_instance_claims.sql} | 0 trex_core/application/ledger_outbox.py | 149 +++++++++++++----- 15 files changed, 316 insertions(+), 88 deletions(-) create mode 100644 .proofline/criteria/ac-0029.md rename trex_core/adapters/persistence/sql/{028_ledger_outbox_instance_claims.sql => 029_ledger_outbox_instance_claims.sql} (100%) diff --git a/.proofline/criteria/ac-0029.md b/.proofline/criteria/ac-0029.md new file mode 100644 index 0000000..04b4574 --- /dev/null +++ b/.proofline/criteria/ac-0029.md @@ -0,0 +1,21 @@ +--- +id: "ac-0029" +status: active +--- + +# Ledger outbox instance ownership and lease fencing + +## Criterion + +`trex-core`의 durable ledger outbox row는 enqueue 시점의 required non-empty `instance_key`를 immutable owner로 저장해야 하며, daemon·operator backfill·targeted processing을 포함한 모든 delivery 경로는 자기 instance의 row만 claim하고 처리해야 한다. Shared PostgreSQL에서 claim은 pending·retry-due·unclaimed 또는 lease-expired row를 원자적으로 선택해 worker별 bounded lease를 설정해야 하고, 성공·중복·retryable·conflict·permanent·dead-letter outcome은 해당 claim의 fencing 정보가 일치할 때만 반영해야 한다. 다른 instance, legacy `instance_key IS NULL`, 아직 유효한 foreign lease 또는 이미 새 worker가 reclaim한 row는 처리·상태 변경·성공 통계·ledger-skip resolution 없이 fail closed 해야 한다. + +Claim owner가 crash하면 lease 만료 후 같은 instance의 다른 worker가 row를 reclaim할 수 있어야 한다. Outcome CAS가 0건이면 claim 상실로 명시적으로 처리하고 성공 outcome으로 보고해서는 안 된다. InMemory와 PostgreSQL outbox는 instance filtering, active lease exclusion, expiry recovery와 stale-claim rejection에서 같은 의미를 가져야 한다. Claim·outcome 관측 정보에는 `instance_key`, sanitized result/reason과 correlation identity만 허용하며 credential, account identifier, token, password, DSN, provider payload 또는 exception 원문을 로그·metric·alert에 포함해서는 안 된다. + +## Verification + +- 격리 조건: 서로 다른 `instance_key`의 daemon/writer 두 개가 한 PostgreSQL을 공유할 때 각 writer는 자기 row만 claim·전송·집계한다. `list_pending`, `pending_cash_delta`, batch claim과 correlation-targeted claim 모두 foreign row와 `instance_key IS NULL` legacy row를 제외한다. 누락·빈 instance key로 outbox 또는 writer를 생성하면 외부 연결이나 row 생성 전에 거부된다. +- lease 조건: worker A가 claim한 row는 lease 동안 worker B가 claim하지 못한다. A가 outcome을 기록하면 lease가 해제되고 terminal 또는 retry 상태가 정확히 저장된다. A가 crash한 경우 expiry 전에는 B가 claim하지 못하고 expiry 뒤에는 B가 reclaim할 수 있다. Reclaim 뒤 A의 stale outcome은 typed claim-lost 결과로 거부되며 B의 row 상태, outcome, attempt count, next retry와 lease를 변경하지 않는다. +- outcome 조건: adapter의 신규 성공과 `DUPLICATE_OK`, retryable, conflict, permanent 및 dead-letter 모든 분기는 claim fencing을 전달한다. CAS 실패 시 completion/retry counter, 성공 로그, snapshot write-back 이후의 ledger-skip resolution을 실행하지 않는다. Adapter duplicate는 row를 `completed/duplicate_completed`로 만들고 lease를 즉시 해제하여 재전송되지 않는다. +- InMemory parity: 두 번의 동시 claim에서 첫 claim만 성공하고 active lease row는 pending scan에서 제외된다. clock을 이동하면 expiry recovery가 가능하며 이전 claim token으로 outcome을 기록할 수 없다. Instance filter와 targeted processing의 fail-closed 동작이 PostgreSQL과 같다. +- 보안 조건: exception에 credential·account·token·password·DSN·provider payload를 식별하는 sentinel을 넣어 claim 실패, retry, permanent, dead-letter와 write-back 오류를 실행해도 rendered log, metric key/value와 alert에 sentinel 또는 exception 원문이 나타나지 않는다. +- migration 및 회귀: migration 번호는 저장소에서 유일하고 기존 row를 자동 귀속시키지 않으며 `instance_key IS NULL`로 격리한다. 빈 DB와 기존 schema 모두에서 migration apply/replay, PostgreSQL claim round-trip, restart/expiry 회수, daemon·CLI wiring을 검증한다. 전체 repository test, Ruff, compileall, ProofLine validation과 `git diff --check`를 실행한다. diff --git a/.proofline/identities.json b/.proofline/identities.json index 34f4cf3..e90cfd1 100644 --- a/.proofline/identities.json +++ b/.proofline/identities.json @@ -1,5 +1,5 @@ { "schema_version": 1, "next_line_number": 10, - "next_ac_number": 29 + "next_ac_number": 30 } diff --git a/scripts/virtual_execution_final_state.py b/scripts/virtual_execution_final_state.py index 80eba0f..f05ea30 100644 --- a/scripts/virtual_execution_final_state.py +++ b/scripts/virtual_execution_final_state.py @@ -32,7 +32,7 @@ def main() -> int: ) repo.reconcile_universe(instance, {portfolio_id: symbol}) broker = PaperBroker(fill_after_polls=0) - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger() execution = ExecutionService( slots, diff --git a/tests/adapters/persistence/test_migration_and_drift.py b/tests/adapters/persistence/test_migration_and_drift.py index 592cc60..71c5c21 100644 --- a/tests/adapters/persistence/test_migration_and_drift.py +++ b/tests/adapters/persistence/test_migration_and_drift.py @@ -118,7 +118,7 @@ def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean): def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): - outbox = PostgreSQLLedgerOutbox(pg_clean) + outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="test-instance") normal = LedgerTrade( asset_id="PF-NORMAL", order_id="O-NORMAL", @@ -177,7 +177,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): ) -def test_migration_028_fail_closes_unowned_legacy_outbox_rows(pg_clean): +def test_migration_029_fail_closes_unowned_legacy_outbox_rows(pg_clean): outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a") trade = LedgerTrade( asset_id="PF-LEGACY", diff --git a/tests/adapters/persistence/test_postgresql_ledger_outbox.py b/tests/adapters/persistence/test_postgresql_ledger_outbox.py index d9d465d..eef6e87 100644 --- a/tests/adapters/persistence/test_postgresql_ledger_outbox.py +++ b/tests/adapters/persistence/test_postgresql_ledger_outbox.py @@ -4,12 +4,16 @@ Skipped without TREX_POSTGRESQL_TEST_DSN (same gate as other PG suites). """ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +import pytest + from trex_core.adapters.ledger.in_memory import InMemoryLedger from trex_core.adapters.persistence.postgresql import PostgreSQLLedgerOutbox from trex_core.application.ledger_outbox import ( LedgerWriter, + OutboxClaimLostError, OutboxOutcome, WorkItemStatus, ) @@ -196,18 +200,27 @@ def test_stale_claim_cannot_overwrite_new_owner_outcome(pg_clean): expected_claim_expires_at=claim_b.claim_expires_at, ) - stored = outbox_a.mark_outcome( - row.correlation_id, - OutboxOutcome.RETRYABLE_FAILURE, - error="stale owner", - next_retry_at=now["t"], - expected_claim_expires_at=claim_a.claim_expires_at, - ) + with pytest.raises(OutboxClaimLostError): + outbox_a.mark_outcome( + row.correlation_id, + OutboxOutcome.RETRYABLE_FAILURE, + error="stale owner", + next_retry_at=now["t"], + expected_claim_expires_at=claim_a.claim_expires_at, + ) + stored = outbox_a.get(row.correlation_id) + assert stored is not None assert stored.status is WorkItemStatus.COMPLETED assert stored.last_outcome is OutboxOutcome.COMPLETED +@pytest.mark.parametrize("instance_key", ["", " "]) +def test_postgresql_outbox_rejects_empty_instance_before_connect(pg_clean, instance_key): + with pytest.raises(ValueError, match="non-empty"): + PostgreSQLLedgerOutbox(pg_clean, instance_key=instance_key) + + def test_original_owner_can_finalize_expired_unreclaimed_claim(pg_clean): now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} outbox = PostgreSQLLedgerOutbox( @@ -255,6 +268,22 @@ def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean): assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.PENDING +def test_concurrent_workers_claim_each_row_at_most_once(pg_clean): + outbox_a = PostgreSQLLedgerOutbox( + pg_clean, instance_key="inst-a", worker_id="worker-a" + ) + outbox_b = PostgreSQLLedgerOutbox( + pg_clean, instance_key="inst-a", worker_id="worker-b" + ) + row = outbox_a.enqueue(_trade(execution_id="pg-concurrent")) + + with ThreadPoolExecutor(max_workers=2) as pool: + claims = list(pool.map(lambda box: box.claim_pending(), (outbox_a, outbox_b))) + + claimed_ids = [item.correlation_id for batch in claims for item in batch] + assert claimed_ids == [row.correlation_id] + + def test_writer_marks_duplicate_completed_from_adapter_response_on_pg(pg_clean): class _DuplicateLedger: def record_trade(self, _trade): diff --git a/tests/application/test_execution_cancel.py b/tests/application/test_execution_cancel.py index a88fd8c..3fa65f9 100644 --- a/tests/application/test_execution_cancel.py +++ b/tests/application/test_execution_cancel.py @@ -70,7 +70,7 @@ def _wire(broker, *, stale_sec=None): svc = SlotService(repo) intents = IntentService(svc, repo, InMemoryIntentStore(), resolve_symbol=lambda pid: "005930") - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger() execution = ExecutionService( svc, repo, broker, ledger_outbox=outbox, diff --git a/tests/application/test_execution_fill_price_backfill.py b/tests/application/test_execution_fill_price_backfill.py index 6bb6ac7..2baf2d2 100644 --- a/tests/application/test_execution_fill_price_backfill.py +++ b/tests/application/test_execution_fill_price_backfill.py @@ -120,7 +120,7 @@ def _wire(broker): resolve_symbol=lambda pid: "005930", instance_key=INST, ) - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") execution = ExecutionService(svc, repo, broker, ledger_outbox=outbox, instance_key=INST) repo.reconcile_universe(INST, {"PF-1": "005930"}) return repo, svc, intents, execution, outbox @@ -966,7 +966,7 @@ class TestPaperBrokerRegression: resolve_symbol=lambda pid: "005930", instance_key=INST, ) - outbox2 = InMemoryLedgerOutbox() + outbox2 = InMemoryLedgerOutbox(instance_key="test-instance") execution2 = ExecutionService(svc2, repo2, paper, ledger_outbox=outbox2, instance_key=INST) repo2.reconcile_universe(INST, {"PF-1": "005930"}) diff --git a/tests/application/test_execution_ledger.py b/tests/application/test_execution_ledger.py index d993c2c..e1aa009 100644 --- a/tests/application/test_execution_ledger.py +++ b/tests/application/test_execution_ledger.py @@ -119,7 +119,11 @@ def _wire(broker, *, ledger=None, clock=None, alarm_emitter=None, ledger_skip_st svc = SlotService(repo) intents = IntentService(svc, repo, InMemoryIntentStore(), resolve_symbol=lambda pid: "005930") - outbox = InMemoryLedgerOutbox(clock=clock) if clock else InMemoryLedgerOutbox() + outbox = ( + InMemoryLedgerOutbox(clock=clock, instance_key="default") + if clock + else InMemoryLedgerOutbox(instance_key="default") + ) execution = ExecutionService( svc, repo, @@ -430,7 +434,7 @@ def test_duplicate_execution_id_results_in_zero_duplicate_in_ledger(): # And even if the writer somehow re-processes (simulated by a fresh outbox # row for the same execution_id, e.g. after a manual replay), the ledger # adapter dedup'd it server-side and the outcome is duplicate_completed. - replay_outbox = InMemoryLedgerOutbox() + replay_outbox = InMemoryLedgerOutbox(instance_key="test-instance") replay_writer = LedgerWriter(replay_outbox, ledger) replay_outbox.enqueue(first.trade) stats = replay_writer.run_once() diff --git a/tests/application/test_ledger_option_d.py b/tests/application/test_ledger_option_d.py index 8685773..e0e8769 100644 --- a/tests/application/test_ledger_option_d.py +++ b/tests/application/test_ledger_option_d.py @@ -39,7 +39,7 @@ def _store_with(pid="PF-1", cash=1_000_000.0, qty=0.0): # --- write-back (REQ-019/030) ---------------------------------------------- def test_writeback_updates_snapshot_on_new(): store = _store_with() - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger(cash_after=300_000.0, quantity_after=10.0) writer = LedgerWriter(outbox, ledger, snapshot_store=store) outbox.enqueue(_trade()) @@ -50,7 +50,7 @@ def test_writeback_updates_snapshot_on_new(): def test_writeback_applies_on_duplicate_too(): store = _store_with(cash=1_000_000.0) - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") # execution already recorded once → second write is duplicate, *_after still authoritative ledger = InMemoryLedger(cash_after=300_000.0, quantity_after=10.0) ledger.record_trade(_trade(execution_id="dup")) # pre-record → next is duplicate @@ -62,7 +62,7 @@ def test_writeback_applies_on_duplicate_too(): def test_writeback_skipped_when_after_missing(): store = _store_with(cash=1_000_000.0) - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger() # no cash_after/quantity_after → None writer = LedgerWriter(outbox, ledger, snapshot_store=store) outbox.enqueue(_trade()) @@ -74,7 +74,7 @@ def test_writeback_best_effort_does_not_block_completion(): class _BoomStore: def update_cash_position(self, *a, **k): raise RuntimeError("boom") - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger(cash_after=1.0, quantity_after=1.0) writer = LedgerWriter(outbox, ledger, snapshot_store=_BoomStore()) outbox.enqueue(_trade()) @@ -83,7 +83,7 @@ def test_writeback_best_effort_does_not_block_completion(): def test_no_writeback_without_store(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger(cash_after=1.0, quantity_after=1.0) writer = LedgerWriter(outbox, ledger) # no snapshot_store outbox.enqueue(_trade()) @@ -92,7 +92,7 @@ def test_no_writeback_without_store(): # --- estimate overlay (REQ-029 / OQ-4) — via make_slots_rpc_handler ---------- def test_pending_cash_delta_sums_by_asset(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") outbox.enqueue(_trade(execution_id="b", side="buy", qty=10, price=1000.0, fee=1.0)) deltas = outbox.pending_cash_delta() assert deltas["PF-1"] == trade_cash_delta(_trade(side="buy", qty=10, price=1000.0, fee=1.0)) @@ -101,7 +101,7 @@ def test_pending_cash_delta_sums_by_asset(): def test_apply_cash_estimate_overlays_pending_delta(): """Cash-estimated overlay applied through slots handler (REQ-029 OQ-4).""" - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") outbox.enqueue(_trade(execution_id="b", side="buy", qty=10, price=1000.0, fee=1.0)) store = _store_with(pid="PF-1", cash=1_000_000.0, qty=0.0) repo = InMemorySlotRepository() @@ -117,7 +117,7 @@ def test_apply_cash_estimate_overlays_pending_delta(): def test_estimate_clears_after_writeback(): """Once the ledger write completes (no longer pending), the estimate is gone.""" store = _store_with(cash=1_000_000.0) - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger(cash_after=989_999.0, quantity_after=10.0) writer = LedgerWriter(outbox, ledger, snapshot_store=store) outbox.enqueue(_trade(side="buy", qty=10, price=1000.0, fee=1.0)) diff --git a/tests/application/test_ledger_outbox.py b/tests/application/test_ledger_outbox.py index 651efc6..64fc05c 100644 --- a/tests/application/test_ledger_outbox.py +++ b/tests/application/test_ledger_outbox.py @@ -1,16 +1,20 @@ """LedgerOutbox + LedgerWriter direct unit tests (REQ-019).""" from __future__ import annotations +from dataclasses import replace from datetime import UTC, datetime, timedelta +import pytest + from trex_core.adapters.ledger.in_memory import InMemoryLedger from trex_core.application.ledger_outbox import ( InMemoryLedgerOutbox, LedgerWriter, + OutboxClaimLostError, OutboxOutcome, WorkItemStatus, ) -from trex_core.application.ledger_port import LedgerTrade +from trex_core.application.ledger_port import LedgerPermanentError, LedgerTrade def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade: @@ -22,7 +26,7 @@ def _trade(execution_id="paper:PAPER-1:10") -> LedgerTrade: def test_enqueue_is_idempotent_on_correlation_id(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") a = outbox.enqueue(_trade()) b = outbox.enqueue(_trade()) assert a.correlation_id == b.correlation_id @@ -30,7 +34,7 @@ def test_enqueue_is_idempotent_on_correlation_id(): def test_writer_run_once_drains_pending_and_marks_completed(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger() writer = LedgerWriter(outbox, ledger) outbox.enqueue(_trade(execution_id="e-1")) @@ -42,7 +46,7 @@ def test_writer_run_once_drains_pending_and_marks_completed(): def test_writer_marks_conflict_blocked_and_does_not_retry(): - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger(conflict_executions={"e-x"}) writer = LedgerWriter(outbox, ledger) outbox.enqueue(_trade(execution_id="e-x")) @@ -58,7 +62,9 @@ def test_writer_retries_until_success(): # dcy-015: retryable rows now back off; share an injected clock between the # outbox (list_pending "now") and the writer (next_retry_at) to advance past it. now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} - outbox = InMemoryLedgerOutbox(clock=lambda: now["t"]) + outbox = InMemoryLedgerOutbox( + clock=lambda: now["t"], instance_key="test-instance" + ) ledger = InMemoryLedger(fail_n_then_succeed=1) writer = LedgerWriter(outbox, ledger, clock=lambda: now["t"]) outbox.enqueue(_trade(execution_id="e-r")) @@ -90,17 +96,29 @@ def test_process_correlation_id_only_drains_requested_row(): def test_process_correlation_id_skips_foreign_instance_row(): outbox = InMemoryLedgerOutbox(instance_key="inst-a") ledger = InMemoryLedger() - writer = LedgerWriter(outbox, ledger, instance_key="inst-b") item = outbox.enqueue(_trade(execution_id="e-foreign")) - outcome = writer.process_correlation_id(item.correlation_id) - - assert outcome is None + with pytest.raises(ValueError, match="must match"): + LedgerWriter(outbox, ledger, instance_key="inst-b") assert ledger.count() == 0 - assert writer.counters["claim:foreign_instance"] == 1 assert outbox.get(item.correlation_id).status is WorkItemStatus.PENDING +def test_process_correlation_id_rejects_legacy_row_without_instance_owner(): + class _LegacyOutbox(InMemoryLedgerOutbox): + def get(self, correlation_id): + item = super().get(correlation_id) + return replace(item, instance_key=None) if item is not None else None + + outbox = _LegacyOutbox(instance_key="inst-a") + ledger = InMemoryLedger() + writer = LedgerWriter(outbox, ledger, instance_key="inst-a") + item = outbox.enqueue(_trade()) + + assert writer.process_correlation_id(item.correlation_id) is None + assert ledger.count() == 0 + + def test_writer_exposes_claim_and_outcome_counters(): outbox = InMemoryLedgerOutbox(instance_key="inst-a") ledger = InMemoryLedger() @@ -112,3 +130,81 @@ def test_writer_exposes_claim_and_outcome_counters(): assert stats.completed == 1 assert writer.counters["claim:claimed"] == 1 assert writer.counters["outcome:completed"] == 1 + + +@pytest.mark.parametrize("instance_key", ["", " "]) +def test_writer_rejects_explicit_empty_instance_key(instance_key): + outbox = InMemoryLedgerOutbox(instance_key="inst-a") + with pytest.raises(ValueError, match="non-empty"): + LedgerWriter(outbox, InMemoryLedger(), instance_key=instance_key) + + +@pytest.mark.parametrize("instance_key", ["", " "]) +def test_outbox_rejects_empty_instance_key(instance_key): + with pytest.raises(ValueError, match="non-empty"): + InMemoryLedgerOutbox(instance_key=instance_key) + + +def test_in_memory_claim_is_exclusive_and_stale_lease_is_fenced(): + now = {"t": datetime(2026, 6, 4, tzinfo=UTC)} + outbox = InMemoryLedgerOutbox( + instance_key="inst-a", + claim_ttl_sec=5.0, + clock=lambda: now["t"], + ) + item = outbox.enqueue(_trade(execution_id="e-lease")) + + [claim_a] = outbox.claim_pending() + assert outbox.claim_pending() == [] + assert outbox.list_pending() == [] + + now["t"] += timedelta(seconds=6) + [claim_b] = outbox.claim_pending() + assert claim_b.claim_expires_at != claim_a.claim_expires_at + with pytest.raises(OutboxClaimLostError): + outbox.mark_outcome( + item.correlation_id, + OutboxOutcome.COMPLETED, + expected_claim_expires_at=claim_a.claim_expires_at, + ) + + stored = outbox.mark_outcome( + item.correlation_id, + OutboxOutcome.COMPLETED, + expected_claim_expires_at=claim_b.claim_expires_at, + ) + assert stored.status is WorkItemStatus.COMPLETED + assert stored.claim_expires_at is None + + +def test_outcome_logs_do_not_render_exception_text(caplog): + sentinel = "TOKEN_SENTINEL_ACCOUNT_DSN_PAYLOAD" + + class _FailingLedger: + def record_trade(self, _trade): + raise LedgerPermanentError(sentinel) + + outbox = InMemoryLedgerOutbox(instance_key="inst-a") + writer = LedgerWriter(outbox, _FailingLedger(), instance_key="inst-a") + outbox.enqueue(_trade(execution_id="e-sensitive")) + + with caplog.at_level("INFO"): + stats = writer.run_once() + + assert stats.delivery_failed == 1 + assert sentinel not in caplog.text + + +def test_lost_claim_is_counted_without_false_completion(): + class _ClaimLosingOutbox(InMemoryLedgerOutbox): + def mark_outcome(self, correlation_id, outcome, **kwargs): + raise OutboxClaimLostError(correlation_id) + + outbox = _ClaimLosingOutbox(instance_key="inst-a") + outbox.enqueue(_trade()) + writer = LedgerWriter(outbox, InMemoryLedger(), instance_key="inst-a") + + stats = writer.run_once() + + assert stats.claim_lost == 1 + assert stats.completed == 0 diff --git a/tests/application/test_ledger_outbox_backoff.py b/tests/application/test_ledger_outbox_backoff.py index 5c5cd5d..577c2bd 100644 --- a/tests/application/test_ledger_outbox_backoff.py +++ b/tests/application/test_ledger_outbox_backoff.py @@ -47,7 +47,7 @@ class _ProgrammableLedger: def _wire(behavior, *, clock, **writer_kw): - outbox = InMemoryLedgerOutbox(clock=clock) + outbox = InMemoryLedgerOutbox(clock=clock, instance_key="test-instance") ledger = _ProgrammableLedger(behavior) writer = LedgerWriter(outbox, ledger, clock=clock, rng=lambda: 0.0, **writer_kw) return outbox, ledger, writer diff --git a/tests/application/test_messaging_event_publisher.py b/tests/application/test_messaging_event_publisher.py index 03a3bd5..e612a1c 100644 --- a/tests/application/test_messaging_event_publisher.py +++ b/tests/application/test_messaging_event_publisher.py @@ -22,7 +22,7 @@ def _wire(transport): svc = SlotService(repo) intents = IntentService(svc, repo, InMemoryIntentStore(), resolve_symbol=lambda pid: "005930") - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") ledger = InMemoryLedger() writer = LedgerWriter(outbox, ledger) repo.reconcile_universe("default", {"PF-1": "SYM"}) @@ -203,7 +203,7 @@ def test_sell_fill_dlq_sentinel_absent_when_strategy_id_set(): intents = IntentService(svc, repo, InMemoryIntentStore(), resolve_symbol=lambda pid: "005930") from trex_core.application.ledger_outbox import InMemoryLedgerOutbox - outbox = InMemoryLedgerOutbox() + outbox = InMemoryLedgerOutbox(instance_key="test-instance") execution = ExecutionService(svc, repo, PaperBroker(fill_after_polls=0), ledger_outbox=outbox, event_emitter=emitter) repo.reconcile_universe("default", {"PF-1": "SYM"}) diff --git a/trex_core/adapters/persistence/postgresql.py b/trex_core/adapters/persistence/postgresql.py index 3812448..8439926 100644 --- a/trex_core/adapters/persistence/postgresql.py +++ b/trex_core/adapters/persistence/postgresql.py @@ -21,6 +21,7 @@ from uuid import UUID, uuid4 from trex_core.application.ledger_outbox import ( LedgerWorkItem, + OutboxClaimLostError, OutboxOutcome, WorkItemStatus, ) @@ -569,6 +570,9 @@ _OUTBOX_COLUMNS = ( "created_at, updated_at, next_retry_at, venue, order_venue, instance_key, " "claim_expires_at" ) +_OUTBOX_RETURN_COLUMNS = ", ".join( + f"o.{column.strip()}" for column in _OUTBOX_COLUMNS.split(",") +) class PostgreSQLLedgerOutbox: @@ -578,12 +582,16 @@ class PostgreSQLLedgerOutbox: self, dsn: str, *, - instance_key: str = "default", + instance_key: str, claim_ttl_sec: float = 60.0, worker_id: str | None = None, clock=None, psycopg_module: Any = None, ) -> None: + if not isinstance(instance_key, str) or not instance_key.strip(): + raise ValueError("instance_key must be a non-empty string") + if claim_ttl_sec <= 0: + raise ValueError("claim_ttl_sec must be positive") self._psycopg = _require_psycopg(psycopg_module) self._dsn = dsn self._instance_key = instance_key @@ -594,6 +602,10 @@ class PostgreSQLLedgerOutbox: def _now(self) -> datetime: return self._clock() + @property + def instance_key(self) -> str: + return self._instance_key + def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem: with self._psycopg.connect(self._dsn, autocommit=False) as conn: with conn.cursor() as cur: @@ -625,16 +637,17 @@ class PostgreSQLLedgerOutbox: return existing def list_pending(self, limit: int = 32) -> list[LedgerWorkItem]: + now = self._now() with self._psycopg.connect(self._dsn) as conn: with conn.cursor() as cur: cur.execute( f"SELECT {_OUTBOX_COLUMNS} FROM ledger_outbox " "WHERE status = 'pending' " "AND instance_key = %s " - "AND (next_retry_at IS NULL OR next_retry_at <= now()) " - "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) " + "AND (next_retry_at IS NULL OR next_retry_at <= %s) " + "AND (claim_expires_at IS NULL OR claim_expires_at <= %s) " "ORDER BY created_at, correlation_id LIMIT %s", - (self._instance_key, limit), + (self._instance_key, now, now, limit), ) return [self._row_to_item(r) for r in cur.fetchall()] @@ -647,7 +660,7 @@ class PostgreSQLLedgerOutbox: now = self._now() lease_until = now + timedelta(seconds=self._claim_ttl_sec) correlation_sql = "" - params: list[Any] = [self._instance_key] + params: list[Any] = [self._instance_key, now, now] if correlation_id is not None: correlation_sql = "AND correlation_id = %s " params.append(correlation_id) @@ -659,8 +672,8 @@ class PostgreSQLLedgerOutbox: "SELECT correlation_id FROM ledger_outbox " "WHERE status = 'pending' " "AND instance_key = %s " - "AND (next_retry_at IS NULL OR next_retry_at <= now()) " - "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) " + "AND (next_retry_at IS NULL OR next_retry_at <= %s) " + "AND (claim_expires_at IS NULL OR claim_expires_at <= %s) " f"{correlation_sql}" "ORDER BY created_at, correlation_id " "LIMIT %s " @@ -670,7 +683,7 @@ class PostgreSQLLedgerOutbox: "SET claim_owner = %s, claim_expires_at = %s, updated_at = %s " "FROM candidates " "WHERE o.correlation_id = candidates.correlation_id " - f"RETURNING {_OUTBOX_COLUMNS}", + f"RETURNING {_OUTBOX_RETURN_COLUMNS}", params, ) rows = [self._row_to_item(r) for r in cur.fetchall()] @@ -759,6 +772,8 @@ class PostgreSQLLedgerOutbox: f"{where}", params, ) + if cur.rowcount != 1: + raise OutboxClaimLostError(correlation_id) conn.commit() row = self.get(correlation_id) assert row is not None diff --git a/trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql b/trex_core/adapters/persistence/sql/029_ledger_outbox_instance_claims.sql similarity index 100% rename from trex_core/adapters/persistence/sql/028_ledger_outbox_instance_claims.sql rename to trex_core/adapters/persistence/sql/029_ledger_outbox_instance_claims.sql diff --git a/trex_core/application/ledger_outbox.py b/trex_core/application/ledger_outbox.py index 2dc7eac..cf043ed 100644 --- a/trex_core/application/ledger_outbox.py +++ b/trex_core/application/ledger_outbox.py @@ -66,6 +66,10 @@ class WorkItemStatus(str, Enum): DELIVERY_FAILED = "delivery_failed" # dcy-015 — permanent or dead-lettered (terminal) +class OutboxClaimLostError(RuntimeError): + """The caller no longer owns the lease required for an outcome update.""" + + # Outcomes that move a row to the terminal delivery_failed state. _DELIVERY_FAILED_OUTCOMES = frozenset( {OutboxOutcome.PERMANENT_FAILURE, OutboxOutcome.DEAD_LETTERED} @@ -111,6 +115,11 @@ class LedgerWorkItem: class LedgerOutbox(Protocol): """Durable work-item store backing the ledger writer.""" + @property + def instance_key(self) -> str: + """Immutable owner partition for rows handled by this outbox.""" + ... + def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem: """Enqueue a trade. Idempotent on correlation_id (returns the existing row unchanged if it is already present).""" @@ -163,6 +172,7 @@ class WriterStats: retryable: int = 0 conflicts: int = 0 delivery_failed: int = 0 # dcy-015: permanent + dead-lettered terminals + claim_lost: int = 0 class LedgerWriter: @@ -202,7 +212,12 @@ class LedgerWriter: # dcy-033 / REQ-100: best-effort alarm emitter. self._alarm_emitter = alarm_emitter self._ledger_skip_store = ledger_skip_store - self._instance_key = instance_key + resolved_instance_key = outbox.instance_key if instance_key is None else instance_key + if not isinstance(resolved_instance_key, str) or not resolved_instance_key.strip(): + raise ValueError("instance_key must be a non-empty string") + if resolved_instance_key != outbox.instance_key: + raise ValueError("writer instance_key must match outbox instance_key") + self._instance_key = resolved_instance_key self.counters: dict[str, int] = {} def _count(self, key: str) -> None: @@ -224,7 +239,19 @@ class LedgerWriter: limit, ) for item in items: - outcome = self._process(item) + if not self._validate_instance(item): + continue + try: + outcome = self._process(item) + except OutboxClaimLostError: + self._count("claim:lost") + stats.claim_lost += 1 + _log.warning( + "ledger outbox claim lost: instance_key=%s correlation_id=%s", + self._instance_key, + item.correlation_id, + ) + continue if outcome is OutboxOutcome.COMPLETED: stats.completed += 1 elif outcome is OutboxOutcome.DUPLICATE_COMPLETED: @@ -242,19 +269,7 @@ class LedgerWriter: item = self._outbox.get(correlation_id) if item is None: return None - if ( - self._instance_key is not None - and item.instance_key is not None - and item.instance_key != self._instance_key - ): - self._count("claim:foreign_instance") - _log.info( - "ledger outbox claim: instance_key=%s claim_result=foreign_instance " - "correlation_id=%s owner_instance_key=%s", - self._instance_key, - correlation_id, - item.instance_key, - ) + if not self._validate_instance(item): return None if item.status is WorkItemStatus.PENDING: claimed = self._outbox.claim_pending(1, correlation_id=correlation_id) @@ -276,7 +291,28 @@ class LedgerWriter: len(claimed), correlation_id, ) - return self._process(item) + try: + return self._process(item) + except OutboxClaimLostError: + self._count("claim:lost") + _log.warning( + "ledger outbox claim lost: instance_key=%s correlation_id=%s", + self._instance_key, + correlation_id, + ) + return None + + def _validate_instance(self, item: LedgerWorkItem) -> bool: + if item.instance_key == self._instance_key: + return True + self._count("claim:foreign_instance") + _log.warning( + "ledger outbox claim rejected: instance_key=%s " + "claim_result=foreign_or_unowned correlation_id=%s", + self._instance_key, + item.correlation_id, + ) + return False def _backoff_delay(self, attempts: int) -> float: """Exponential backoff with jitter: min(base*2^(n-1), cap) + up to 25%.""" @@ -288,7 +324,6 @@ class LedgerWriter: # outbox row. Skip the adapter call. existing = self._outbox.get(item.correlation_id) if existing is not None and existing.status is WorkItemStatus.COMPLETED: - self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED) self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED) return OutboxOutcome.DUPLICATE_COMPLETED try: @@ -300,7 +335,7 @@ class LedgerWriter: error=str(exc), expected_claim_expires_at=item.claim_expires_at, ) - self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED, error=str(exc)) + self._log_outcome(item, OutboxOutcome.CONFLICT_BLOCKED) # dcy-033 / REQ-100: CRITICAL alarm — 409 conflict, operator resolution needed. if self._alarm_emitter is not None: self._alarm_emitter.publish_alarm( @@ -320,11 +355,11 @@ class LedgerWriter: error=str(exc), expected_claim_expires_at=item.claim_expires_at, ) - self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE, error=str(exc)) + self._log_outcome(item, OutboxOutcome.PERMANENT_FAILURE) _log.warning( - "ledger delivery_failed (permanent): instance_key=%s correlation_id=%s reason=%s", + "ledger delivery_failed (permanent): instance_key=%s correlation_id=%s", item.instance_key or self._instance_key, - item.correlation_id, exc, + item.correlation_id, ) # dcy-033 / REQ-100: CRITICAL alarm — permanent failure, no retry. if self._alarm_emitter is not None: @@ -346,14 +381,13 @@ class LedgerWriter: error=str(exc), expected_claim_expires_at=item.claim_expires_at, ) - self._log_outcome(item, OutboxOutcome.DEAD_LETTERED, error=str(exc)) + self._log_outcome(item, OutboxOutcome.DEAD_LETTERED) _log.warning( "ledger delivery_failed (dead-letter, attempts=%d): " - "instance_key=%s correlation_id=%s reason=%s", + "instance_key=%s correlation_id=%s", new_attempts, item.instance_key or self._instance_key, item.correlation_id, - exc, ) # dcy-033 / REQ-100: CRITICAL alarm — dead-lettered after max retries. if self._alarm_emitter is not None: @@ -375,7 +409,7 @@ class LedgerWriter: next_retry_at=next_retry_at, expected_claim_expires_at=item.claim_expires_at, ) - self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE, error=str(exc)) + self._log_outcome(item, OutboxOutcome.RETRYABLE_FAILURE) return OutboxOutcome.RETRYABLE_FAILURE # dcy-018 / REQ-019/030 (option D): write back portfolio-hub authoritative # cash/quantity to the snapshot store on new+duplicate (OQ-2 b). *_after @@ -386,7 +420,11 @@ class LedgerWriter: # duplicate_completed; the ledger has exactly one entry for this # execution_id either way (REQ-019 AC-1, AC-4). if outcome.result is LedgerWriteResult.DUPLICATE_OK: - self._outbox.mark_outcome(item.correlation_id, OutboxOutcome.DUPLICATE_COMPLETED) + self._outbox.mark_outcome( + item.correlation_id, + OutboxOutcome.DUPLICATE_COMPLETED, + expected_claim_expires_at=item.claim_expires_at, + ) self._resolve_ledger_skip(item.trade) self._log_outcome(item, OutboxOutcome.DUPLICATE_COMPLETED) return OutboxOutcome.DUPLICATE_COMPLETED @@ -403,8 +441,6 @@ class LedgerWriter: self, item: LedgerWorkItem, outcome: OutboxOutcome, - *, - error: str | None = None, ) -> None: self._count(f"outcome:{outcome.value}") msg = "ledger outbox outcome: instance_key=%s correlation_id=%s outcome=%s" @@ -413,9 +449,6 @@ class LedgerWriter: item.correlation_id, outcome.value, ) - if error is not None: - msg += " error=%s" - args = (*args, error) _log.info(msg, *args) def _resolve_ledger_skip(self, trade: LedgerTrade) -> None: @@ -454,7 +487,7 @@ class LedgerWriter: except Exception: # noqa: BLE001 - write-back must not block the ledger path _log.warning( "snapshot cash/position write-back failed: asset_id=%s", - trade.asset_id, exc_info=True, + trade.asset_id, ) @@ -467,7 +500,8 @@ class InMemoryLedgerOutbox: self, *, clock: Optional[Callable[[], datetime]] = None, - instance_key: str = "default", + instance_key: str, + claim_ttl_sec: float = 60.0, ) -> None: import threading @@ -477,7 +511,16 @@ class InMemoryLedgerOutbox: # dcy-015: shared clock lets tests align backoff (next_retry_at) with # list_pending's "now" deterministically. self._clock = clock or (lambda: datetime.now(tz=timezone.utc)) + if not isinstance(instance_key, str) or not instance_key.strip(): + raise ValueError("instance_key must be a non-empty string") + if claim_ttl_sec <= 0: + raise ValueError("claim_ttl_sec must be positive") self._instance_key = instance_key + self._claim_ttl_sec = claim_ttl_sec + + @property + def instance_key(self) -> str: + return self._instance_key def _now(self) -> datetime: return self._clock() @@ -508,8 +551,11 @@ class InMemoryLedgerOutbox: row = self._rows[cid] # dcy-015 / REQ-019: skip rows still in backoff so they don't # occupy the FIFO batch (head-of-line protection, NFR-3). - if row.status is WorkItemStatus.PENDING and ( - row.next_retry_at is None or row.next_retry_at <= now + if ( + row.status is WorkItemStatus.PENDING + and row.instance_key == self._instance_key + and (row.next_retry_at is None or row.next_retry_at <= now) + and (row.claim_expires_at is None or row.claim_expires_at <= now) ): out.append(row) if len(out) >= limit: @@ -522,16 +568,27 @@ class InMemoryLedgerOutbox: *, correlation_id: str | None = None, ) -> list[LedgerWorkItem]: - if correlation_id is None: - return self.list_pending(limit) now = self._now() + lease_until = now + timedelta(seconds=self._claim_ttl_sec) with self._lock: - row = self._rows.get(correlation_id) - if row is None or row.status is not WorkItemStatus.PENDING: - return [] - if row.next_retry_at is not None and row.next_retry_at > now: - return [] - return [row] + claimed: list[LedgerWorkItem] = [] + candidates = [correlation_id] if correlation_id is not None else self._order + for cid in candidates: + row = self._rows.get(cid) + if row is None or row.status is not WorkItemStatus.PENDING: + continue + if row.instance_key != self._instance_key: + continue + if row.next_retry_at is not None and row.next_retry_at > now: + continue + if row.claim_expires_at is not None and row.claim_expires_at > now: + continue + updated = replace(row, claim_expires_at=lease_until, updated_at=now) + self._rows[cid] = updated + claimed.append(updated) + if len(claimed) >= limit: + break + return claimed def pending_cash_delta(self) -> dict[str, float]: with self._lock: @@ -557,6 +614,11 @@ class InMemoryLedgerOutbox: ) -> LedgerWorkItem: with self._lock: row = self._rows[correlation_id] + if expected_claim_expires_at is None: + if row.claim_expires_at is not None: + raise OutboxClaimLostError(correlation_id) + elif row.claim_expires_at != expected_claim_expires_at: + raise OutboxClaimLostError(correlation_id) if outcome is OutboxOutcome.COMPLETED: new_status = WorkItemStatus.COMPLETED elif outcome is OutboxOutcome.CONFLICT_BLOCKED: @@ -578,6 +640,7 @@ class InMemoryLedgerOutbox: next_retry_at if outcome is OutboxOutcome.RETRYABLE_FAILURE else None ), updated_at=self._now(), + claim_expires_at=None, ) self._rows[correlation_id] = updated return updated From 591db7adb77304e66a0dd8ecaef52340da222866 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 13 Sep 2026 02:21:37 +0000 Subject: [PATCH 18/21] Fix unmanaged reconciliation review blockers Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .proofline/criteria/ac-0029.md | 18 ++++ .proofline/identities.json | 4 +- .proofline/lines/line-0010/dcy-0010.md | 26 +++++ .proofline/lines/line-0010/line-0010.md | 3 + .proofline/lines/line-0010/req-0010.md | 40 +++++++ docs/interface-contract.md | 7 +- docs/operations.md | 1 + docs/runtime-flows.md | 3 +- .../persistence/test_migration_and_drift.py | 102 ++++++++++-------- .../test_unmanaged_position_store.py | 12 ++- tests/application/test_snapshot_scheduler.py | 45 ++++++-- .../test_unmanaged_position_lifecycle.py | 39 ++++++- tests/runtime/test_broker_provider.py | 21 ++++ tests/runtime/test_cli_operator.py | 88 ++++++++++++++- tests/runtime/test_daemon_resolve_symbol.py | 41 +++++++ trex_core/adapters/broker/kis.py | 7 +- trex_core/adapters/broker/ownership.py | 4 + trex_core/adapters/broker/paper.py | 3 + ...> 030_create_unmanaged_position_state.sql} | 0 .../portfolio/unmanaged_position_store.py | 2 + trex_core/application/portfolio.py | 5 - trex_core/application/reconciliation.py | 2 +- trex_core/application/snapshot_scheduler.py | 4 +- trex_core/runtime/broker_provider.py | 16 ++- trex_core/runtime/cli.py | 60 ++++++++++- trex_core/runtime/daemon.py | 18 ++-- 26 files changed, 484 insertions(+), 87 deletions(-) create mode 100644 .proofline/criteria/ac-0029.md create mode 100644 .proofline/lines/line-0010/dcy-0010.md create mode 100644 .proofline/lines/line-0010/line-0010.md create mode 100644 .proofline/lines/line-0010/req-0010.md rename trex_core/adapters/persistence/sql/{028_create_unmanaged_position_state.sql => 030_create_unmanaged_position_state.sql} (100%) diff --git a/.proofline/criteria/ac-0029.md b/.proofline/criteria/ac-0029.md new file mode 100644 index 0000000..af4cfa9 --- /dev/null +++ b/.proofline/criteria/ac-0029.md @@ -0,0 +1,18 @@ +--- +id: "ac-0029" +status: active +--- + +# Durable unmanaged-position lifecycle, authority boundary, and operator audit + +## Criterion + +`trex-core`는 broker-only unmanaged holding을 `(instance_key, symbol)` keyed durable lifecycle state로 추적해야 한다. State는 `quantity`, `first_seen_at`, `last_seen_at`, `quantity_changed_at`, `resolved_at`, lifecycle state와 latest non-secret operator disposition/evidence를 제공해야 한다. `reconcile_core_holdings()`는 authoritative full-balance snapshot에서만 unresolved active rows의 absence를 `resolved`로 전이할 수 있으며, marker가 없거나 projected/partial balance인 broker reader는 resolve authority가 없다. 동일 active lifecycle에서 같은 수량 반복 관측은 새 lifecycle alarm/audit를 만들지 않고 `last_seen_at`만 갱신해야 한다. 운영자 disposition은 explicit evidence-backed command로만 기록되고 audit trail에 append되어야 하며, current state 조회와 audit 조회 모두 inspectable해야 한다. 이 기능은 broker 주문, slot/ledger mutation, warning 자동 삭제 또는 manual-holding cause inference를 수행해서는 안 된다. + +## Verification + +- Identity/lifecycle: 같은 symbol을 서로 다른 `instance_key`에서 독립 row로 저장하고, first detect → same-qty repeat → quantity change → authoritative absence resolved → re-detect reopen 순서를 InMemory/PostgreSQL parity tests로 검증한다. +- Authority boundary: missing marker, `False`, exception-throwing marker, ownership/projected broker, broker read failure 각각에서 active unmanaged row가 resolved 되지 않음을 검증한다. Explicit authoritative broker만 absence resolution을 수행해야 한다. +- Runtime wiring: production broker provider/daemon/snapshot scheduler tests에서 주문용 projected broker와 unmanaged reconciliation용 authoritative full-balance reader가 분리 주입되는지 고정한다. +- Operator audit: CLI/parser/store tests에서 `unmanaged list`, evidence-required `unmanaged record`, symbol별 `unmanaged audit`를 검증하고 audit actor가 caller-supplied arbitrary string이 아니라 trusted local identity에서 결정됨을 확인한다. +- Safety boundary: any unmanaged lifecycle path에서 broker order submit/cancel, slot create/mutate, synthetic ledger write, warning auto-clear가 발생하지 않음을 existing regression surface와 focused tests로 유지한다. diff --git a/.proofline/identities.json b/.proofline/identities.json index 34f4cf3..2c78cba 100644 --- a/.proofline/identities.json +++ b/.proofline/identities.json @@ -1,5 +1,5 @@ { "schema_version": 1, - "next_line_number": 10, - "next_ac_number": 29 + "next_line_number": 11, + "next_ac_number": 30 } diff --git a/.proofline/lines/line-0010/dcy-0010.md b/.proofline/lines/line-0010/dcy-0010.md new file mode 100644 index 0000000..861bcdb --- /dev/null +++ b/.proofline/lines/line-0010/dcy-0010.md @@ -0,0 +1,26 @@ +--- +id: "dcy-0010" +status: confirmed +--- + +# Durable unmanaged-position lifecycle + +## Problem + +현재 `reconcile_core_holdings()`는 broker 계정에만 존재하고 대상 `instance_key`의 managed slot·ledger에 없는 종목을 `unmanaged_position` warning/log로만 표면화한다. 이 상태는 매 주기 반복 관측되어도 최초 감지, 수량 변경, 해제, 운영자 triage 여부를 durable하게 구분하지 못한다. 또한 ownership projection이 켜진 KIS 경로에서는 전략용 projected balance가 manual/타 instance 보유를 제거하므로, reconciliation이 실제 broker-only unmanaged position을 아예 보지 못할 수 있다. + +## Evidence + +- GitHub issue [#85](https://github.com/fourblocks2020/trex-core/issues/85)는 unmanaged identity를 `(instance_key, symbol)`로 격리하고, 최초 감지·수량 변경·해제와 마지막 broker 관측 시각을 durable하게 보존하며, 같은 수량 반복 관측의 로그/알림 폭주를 억제할 것을 요구한다. +- Issue 본문은 운영자가 broker evidence와 함께 managed inclusion, approved exception, investigating disposition을 기록할 수 있어야 하고, broker 조회 실패 또는 partial/projected balance 부재를 quantity 0이나 resolved로 해석해서는 안 된다고 명시한다. +- 현재 ownership wrapper `get_balance()`는 strategy ownership projection만 반환하므로 unmanaged/manual holdings를 숨긴다. 따라서 production wiring은 주문용 projected broker와 별도로 authoritative full-balance reader를 reconciliation에 주입해야 한다. +- 현행 문서 `docs/interface-contract.md`와 `docs/runtime-flows.md`는 broker-only 분류가 persistence나 DB schema를 바꾸지 않는다고 적고 있어, 새 durable lifecycle contract를 소유하는 승인된 AC가 필요하다. + +## Scope + +- 이 Line은 broker-only `unmanaged_position`을 `(instance_key, symbol)` keyed durable lifecycle state로 저장하고, `quantity`, `first_seen_at`, `last_seen_at`, `quantity_changed_at`, `resolved_at`, lifecycle state 및 non-secret operator evidence를 조회 가능하게 하는 contract를 다룬다. +- Reconciliation은 authoritative full-balance snapshot에서만 부재 해제를 수행한다. Marker가 없거나 projected/partial balance인 reader는 detect/update에는 사용할 수 있어도 resolve authority를 갖지 않는다. +- Ownership-aware trading broker는 그대로 주문/전략 조회에 사용하되, unmanaged reconciliation 입력은 별도의 authoritative balance reader를 사용한다. +- 동일 active lifecycle에서 반복 같은 수량 관측은 `last_seen_at`만 갱신하고 새 detect/change alarm·audit를 만들지 않는다. Resolved 후 재탐지되면 새 active lifecycle로 reopen되며 이전 operator disposition 현재값은 carry-over 하지 않는다. +- 운영자 disposition은 explicit command와 evidence reference를 요구하고 audit trail에 append된다. CLI는 current state와 audit trail을 둘 다 조회 가능해야 한다. +- 이 Line은 broker 주문, slot 생성, synthetic ledger backfill, warning 자동 삭제 또는 manual holdings의 원인 추정을 도입하지 않는다. diff --git a/.proofline/lines/line-0010/line-0010.md b/.proofline/lines/line-0010/line-0010.md new file mode 100644 index 0000000..70331b6 --- /dev/null +++ b/.proofline/lines/line-0010/line-0010.md @@ -0,0 +1,3 @@ +--- +id: "line-0010" +--- diff --git a/.proofline/lines/line-0010/req-0010.md b/.proofline/lines/line-0010/req-0010.md new file mode 100644 index 0000000..71a8c19 --- /dev/null +++ b/.proofline/lines/line-0010/req-0010.md @@ -0,0 +1,40 @@ +--- +id: "req-0010" +status: approved +discovery: "dcy-0010" +criteria: + create: + - "ac-0029" + update: [] + retire: [] + satisfy: [] +--- + +# Durable unmanaged-position lifecycle + +## Objective + +`trex-core`는 broker 계정에 존재하지만 대상 `instance_key`의 managed slot·ledger가 소유하지 않는 종목을 stateless warning이 아니라 durable unmanaged-position lifecycle로 추적해야 한다. Lifecycle은 authoritative broker balance를 근거로 `(instance_key, symbol)`별 최초 감지, 수량 변경, 마지막 관측, 해제, 운영자 disposition과 audit를 구분해야 하며, 반복 같은 수량 관측의 로그/알림 폭주를 억제해야 한다. + +## Scope + +- Unmanaged identity는 `(instance_key, symbol)`이며 다른 instance/account의 같은 symbol과 섞이지 않는다. +- Persistent state는 최소 `symbol`, `quantity`, `first_seen_at`, `last_seen_at`, `quantity_changed_at`, `resolved_at`, lifecycle state와 latest non-secret operator evidence/disposition을 제공한다. +- `reconcile_core_holdings()`는 authoritative broker snapshot에서 broker-only holding을 처음 보면 `detected`, 수량이 바뀌면 `quantity_changed`, 이전 active row가 authoritative snapshot에서 사라지면 `resolved`로 기록한다. 같은 수량 반복 관측은 state의 `last_seen_at`만 갱신한다. +- Broker 조회 실패, missing authoritative marker, projected ownership balance, filtered/partial reader 또는 self-report failure는 resolved authority가 아니다. 이런 입력 부재를 quantity 0이나 resolved로 해석하지 않는다. +- Runtime wiring은 주문/전략용 broker projection과 unmanaged reconciliation용 authoritative full-balance reader를 분리한다. +- 운영자 disposition은 explicit command와 `evidence_ref`를 요구하고 audit trail에 append한다. Caller-supplied arbitrary actor string은 현재 계약에 포함하지 않고 trusted local CLI identity만 기록한다. +- Resolved row가 다시 나타나면 `first_seen_at`은 유지하되 새 active lifecycle로 reopen한다. Audit history는 남기고 current operator disposition/evidence는 reset한다. +- Query surface는 current state list와 symbol별 audit trail을 모두 제공한다. + +## Constraints + +- 이 기능은 slot 생성, slot 상태 변경, warning 자동 삭제, synthetic BUY ledger, broker order submit/cancel 또는 manual holdings 원인 추정을 수행하지 않는다. +- `managed_non_holding_divergence`와 기존 reconcile warning lifecycle·clear authority는 유지한다. +- Normative Design Documents: 없음. 이 REQ와 AC-0029가 normative specification을 소유한다. + +## Non-Goals + +- Broker statement ingestion, manual holding 원인 판정, 자동 편입/자동 정정 +- AMQP/portfolio-hub external wire 변경 +- Ledger remediation, slot lifecycle 재설계, ownership model 변경 diff --git a/docs/interface-contract.md b/docs/interface-contract.md index 05b00df..c44325b 100644 --- a/docs/interface-contract.md +++ b/docs/interface-contract.md @@ -437,8 +437,11 @@ resume_kill_switch 요청·응답 (REQ-046): - `get_balance()`는 검증된 `list[BalanceItem]`만 반환한다. KIS의 token/auth/authorization/transport/response/rejection 실패는 빈 목록으로 축소되지 않고 `BrokerBalanceError`를 raise한다. 안정 code allow-list는 `token_issuance_failed`, `authentication_failed`, `authorization_failed`, `transport_failed`, `malformed_response`, `broker_rejected`, `ownership_mismatch`다. - `sync`와 `verify` CLI는 첫 balance read를 완료한 뒤 cache한 결과를 해당 invocation에 전달한다. 이 read 또는 broker credential resolution이 실패하면 exit code `1`이며, `--json` stderr는 `{ "error": {"code": "…", "message": "broker balance unavailable"} }` 형태의 안전한 값만 낸다. upstream payload, credential, account 값은 계약상 출력하지 않는다. -- `reconcile_core_holdings()`의 broker-only 결과는 `unmanaged_position`(대상 instance의 managed slot 없음) 또는 `managed_non_holding_divergence`(동일 instance의 non-holding managed slot 존재)다. `find_managed_slot_by_symbol(instance_key, symbol)` 조회는 instance-scoped이며, 이 두 분류 자체는 slot/warning write나 자동 주문·청산·slot 생성을 하지 않는다. -- 이 변경은 AMQP 또는 portfolio-hub external wire contract와 DB schema를 바꾸지 않는다. +- `reconcile_core_holdings()`의 broker-only 결과는 `unmanaged_position`(대상 instance의 managed slot 없음) 또는 `managed_non_holding_divergence`(동일 instance의 non-holding managed slot 존재)다. `find_managed_slot_by_symbol(instance_key, symbol)` 조회는 instance-scoped이며, 두 분류 모두 slot 생성, 자동 주문·청산을 하지 않는다. +- `managed_non_holding_divergence`는 기존 reconcile warning lifecycle을 유지한다. 반면 `unmanaged_position`은 `(instance_key, symbol)` keyed durable state로 추적되며 `quantity`, `first_seen_at`, `last_seen_at`, `quantity_changed_at`, `resolved_at`, lifecycle state와 latest operator evidence/disposition을 저장한다. +- `unmanaged_position`의 absence resolution은 explicit authoritative full-balance snapshot reader에서만 허용된다. Ownership/projected broker 또는 marker가 없는 balance reader는 detect/update 입력일 수 있어도 resolved authority는 아니다. +- Operator surface는 `trex-core unmanaged list`, `trex-core unmanaged audit `, `trex-core unmanaged record --evidence-ref ...`이며 audit actor는 CLI의 trusted local identity에서 결정된다. 임의 caller-supplied actor string은 current contract가 아니다. +- 이 변경은 AMQP 또는 portfolio-hub external wire contract는 바꾸지 않지만, internal PostgreSQL persistence에는 unmanaged-position state/audit schema를 추가한다. ### Account snapshot recovery (dcy-041 / GitHub #45) diff --git a/docs/operations.md b/docs/operations.md index 7cdcd2d..c27010e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -66,6 +66,7 @@ pkill -TERM -f trex-core-daemon - `sync`와 `verify`는 broker balance를 먼저 한 번 읽어 해당 실행 안에서 재사용한다. balance/credential read 실패는 안전한 error code와 함께 exit `1`로 종료하며, 빈 broker balance를 추정하지 않는다. 성공한 빈 목록만 flat balance로 취급한다. - source evidence에는 mode(`fetched`/`env`/`break_glass`)와 비밀 없는 label만 기록한다. fetched mode는 KIS + `credential_fetch_enabled=true`에서만 사용하고, break-glass는 명시적 운영자 환경 설정으로 fetch를 건너뛴다. Paper와 env KIS 경로는 fetch하지 않는다. - reconciliation에서 `unmanaged_position` 또는 `managed_non_holding_divergence`가 보이면 자동 보정·자동 주문·자동 청산이 없다. broker, ledger, slot evidence를 교차 확인하고 warning/slot 상태를 보존한 채 triage한다. +- `unmanaged_position` triage는 `trex-core unmanaged list`로 현재 lifecycle state를, `trex-core unmanaged audit `로 lifecycle/operator audit trail을 확인한 뒤 `trex-core unmanaged record --evidence-ref ...`로만 disposition을 남긴다. - remediation은 **dry-run → broker/ledger/slot 교차 확인 → 사장님 명시 승인 → 단일 대상 `sync --live --yes` → 재-reconcile** 순서의 별도 release gate다. 이 문서는 실제 target 선택, live 실행 또는 승인을 부여하지 않는다. ### `report` — 일일 운영 리포트 (REQ-088 / dcy-029, REQ-110/111 / dcy-035) diff --git a/docs/runtime-flows.md b/docs/runtime-flows.md index 68b97cd..42c6da2 100644 --- a/docs/runtime-flows.md +++ b/docs/runtime-flows.md @@ -99,13 +99,14 @@ 1. 신규 자산(portfolio_id 에 슬롯 없음) → `inactive` 영구 슬롯 생성(멱등 — `ON CONFLICT (instance_key, portfolio_id)` partial unique). 2. universe 이탈 + inactive → soft-retire(`retired_at` set; 행 보존, 운영 뷰 `universe_detached`). 이탈 + holding/pending → tombstone(그대로 유지 — safety-sell 경로 보존). 3. 가드: universe 빈 응답이면 reconcile skip(degrade 가드); 직전 대비 ≥50% 축소면 retire 보류(`allow_retire=False`, 대량 소실 가드 — dcy-020 OQ-2 D3). -4. 이어서 `reconcile_core_holdings`(REQ-062/121): 브로커 잔고 ground truth 로 core holding 검증 — core holding/브로커 0 → slot `error` 격리+WARN, 수량 불일치 → WARN(상태 불변), 브로커 보유인데 대상 instance에 관리 slot이 없으면 `unmanaged_position`, 동일 instance에 `inactive`/`error` 등 non-holding 관리 slot이 있으면 `managed_non_holding_divergence`를 WARN으로 남긴다. 두 broker-only 분류는 상태·warning persistence·주문·청산·slot 생성의 자동 수정을 하지 않는다. +4. 이어서 `reconcile_core_holdings`(REQ-062/121, req-0010/ac-0029): 브로커 잔고 ground truth 로 core holding 검증 — core holding/브로커 0 → slot `error` 격리+WARN, 수량 불일치 → WARN(상태 불변), 브로커 보유인데 대상 instance에 관리 slot이 없으면 `unmanaged_position`, 동일 instance에 `inactive`/`error` 등 non-holding 관리 slot이 있으면 `managed_non_holding_divergence`를 WARN으로 남긴다. `managed_non_holding_divergence`는 기존 warning 경로를 유지하고, `unmanaged_position`은 `(instance_key, symbol)` durable lifecycle state/audit에 기록된다. 같은 수량 반복 관측은 `last_seen_at`만 갱신하고 새 detect/change alarm을 만들지 않는다. 어떤 broker-only 분류도 주문·청산·slot 생성·warning 자동 삭제를 하지 않는다. ### Broker provider and operator balance-read barrier (BAT-057 / REQ-119/120) 1. daemon과 `sync`/`verify` CLI는 같은 broker provider를 통해 브로커를 얻는다. `broker=kis` 및 `credential_fetch_enabled=true`일 때만 portfolio-hub의 유효한 인증정보를 fetched source로 사용한다. daemon의 day-start refresh는 이미 같은 KST 날짜에 얻은 source가 있으면 이를 재사용한다. 2. 명시적 `TREX_KIS_BROKER_BREAK_GLASS`는 fetched source를 건너뛰고 env KIS 설정을 사용하는 별도 source다. Paper와 `credential_fetch_enabled=false` KIS 경로는 env/config source이며 credential fetch를 하지 않는다. 3. `sync`/`verify`는 실행당 잔고를 한 번 먼저 검증해 cache한 뒤 그 값만 사용한다. KIS 잔고 읽기 실패는 빈 잔고로 해석하거나 후속 DB/slot write로 진행하지 않고 command boundary에서 안전하게 실패한다. 성공한 빈 잔고(`[]`)만 검증된 flat balance다. +4. Ownership projection이 켜진 mixed-account KIS 경로에서는 주문/전략 조회용 broker가 strategy-owned quantity만 보이도록 balance를 투영할 수 있다. 이때 unmanaged reconciliation은 provider가 별도로 넘기는 authoritative full-balance reader를 사용한다. Missing authority marker, projected/partial reader, self-report failure 또는 balance read failure는 absence resolution authority가 아니며 active unmanaged row를 `resolved`로 바꾸지 않는다. ## Flow: `포지션 정합 검증` (BAT-008 / REQ-031) diff --git a/tests/adapters/persistence/test_migration_and_drift.py b/tests/adapters/persistence/test_migration_and_drift.py index 3a35cd3..20f3f1d 100644 --- a/tests/adapters/persistence/test_migration_and_drift.py +++ b/tests/adapters/persistence/test_migration_and_drift.py @@ -6,9 +6,10 @@ iteration: EXPECTED_SLOT_FIELDS (code contract) vs daily_slot actual columns """ from __future__ import annotations +from datetime import UTC, datetime + import psycopg import pytest -from datetime import datetime, timezone from trex_core.adapters.persistence.migrate import apply_migrations, migration_files from trex_core.adapters.persistence.postgresql import ( @@ -29,28 +30,28 @@ _EXPECTED_TABLES = { "ledger_skip", # GitHub #68 skipped-ledger durable audit "reconcile_notification_state", # req-0007 / ac-0024 "slot_strategy_state", # req-0009 / ac-0027 + "unmanaged_position_state", # req-0010 / ac-0029 + "unmanaged_position_audit", # req-0010 / ac-0029 } def _columns(dsn: str, table: str) -> set[str]: - with psycopg.connect(dsn) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema = 'public' AND table_name = %s", - (table,), - ) - return {r[0] for r in cur.fetchall()} + with psycopg.connect(dsn) as conn, conn.cursor() as cur: + cur.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s", + (table,), + ) + return {r[0] for r in cur.fetchall()} def test_migrations_create_expected_tables(pg_clean): assert migration_files(), "no migration files found" - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" - ) - tables = {r[0] for r in cur.fetchall()} + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + cur.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" + ) + tables = {r[0] for r in cur.fetchall()} assert _EXPECTED_TABLES <= tables @@ -61,20 +62,19 @@ def test_daily_slot_columns_match_expected_contract(pg_clean): def test_strategy_state_schema_matches_instance_map_contract(pg_clean): - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT column_name, data_type, udt_name " - "FROM information_schema.columns " - "WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'" - ) - columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()} - cur.execute( - "SELECT a.attname FROM pg_index i " - "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) " - "WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary" - ) - primary_key = {row[0] for row in cur.fetchall()} + with psycopg.connect(pg_clean) as conn, conn.cursor() as cur: + cur.execute( + "SELECT column_name, data_type, udt_name " + "FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'slot_strategy_state'" + ) + columns = {row[0]: (row[1], row[2]) for row in cur.fetchall()} + cur.execute( + "SELECT a.attname FROM pg_index i " + "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) " + "WHERE i.indrelid = 'slot_strategy_state'::regclass AND i.indisprimary" + ) + primary_key = {row[0] for row in cur.fetchall()} assert columns == {"slot_id": ("uuid", "uuid"), "data": ("jsonb", "jsonb")} assert primary_key == {"slot_id"} @@ -92,6 +92,14 @@ def test_apply_migrations_is_idempotent(pg_dsn): apply_migrations(pg_dsn) +def test_migration_sequence_includes_canonical_unmanaged_position_number(): + names = [path.name for path in migration_files()] + assert "030_create_unmanaged_position_state.sql" in names + assert "028_create_unmanaged_position_state.sql" not in names + prefixes = [name.split("_", 1)[0] for name in names] + assert len(prefixes) == len(set(prefixes)) + + def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean): repo = PostgreSQLSlotRepository(pg_clean) repo.reconcile_universe("migration-auto", {"pf-auto": "005930"}) @@ -111,13 +119,15 @@ def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean): apply_migrations(pg_clean) assert repo.fetch_slot(slot.slot_id).venue == "SOR" - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - with pytest.raises(psycopg.errors.CheckViolation): - cur.execute( - "UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s", - (slot.slot_id,), - ) + with ( + psycopg.connect(pg_clean) as conn, + conn.cursor() as cur, + pytest.raises(psycopg.errors.CheckViolation), + ): + cur.execute( + "UPDATE daily_slot SET venue = 'AUTO' WHERE slot_id = %s", + (slot.slot_id,), + ) def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): @@ -129,7 +139,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): side="buy", qty=1, price=1000, - executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc), + executed_at=datetime(2026, 8, 19, tzinfo=UTC), venue="NXT", order_venue="KRX", ) @@ -140,7 +150,7 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): side="buy", qty=1, price=1000, - executed_at=datetime(2026, 8, 19, tzinfo=timezone.utc), + executed_at=datetime(2026, 8, 19, tzinfo=UTC), venue="KRX", order_venue="SOR", ) @@ -172,10 +182,12 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean): assert legacy_row.trade.venue is None assert legacy_row.trade.order_venue == "SOR" - with psycopg.connect(pg_clean) as conn: - with conn.cursor() as cur: - with pytest.raises(psycopg.errors.CheckViolation): - cur.execute( - "UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s", - (normal.correlation_id,), - ) + with ( + psycopg.connect(pg_clean) as conn, + conn.cursor() as cur, + pytest.raises(psycopg.errors.CheckViolation), + ): + cur.execute( + "UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s", + (normal.correlation_id,), + ) diff --git a/tests/adapters/portfolio/test_unmanaged_position_store.py b/tests/adapters/portfolio/test_unmanaged_position_store.py index 4639bf2..c03a2a8 100644 --- a/tests/adapters/portfolio/test_unmanaged_position_store.py +++ b/tests/adapters/portfolio/test_unmanaged_position_store.py @@ -15,7 +15,7 @@ def pg_unmanaged_store(pg_clean): return PostgreSQLUnmanagedPositionStore(pg_clean) -class TestMigration028: +class TestMigration030: def test_tables_exist_after_migration(self, pg_clean): import psycopg @@ -111,6 +111,14 @@ class TestCRUDRoundTrip: quantity=3, observed_at=t1, ) + pg_unmanaged_store.record_operator_action( + instance_key="inst-A", + symbol="008110", + action="investigating", + evidence_ref="kis:2026-09-07", + actor="cli:test", + note="before resolve", + ) pg_unmanaged_store.resolve_absent( instance_key="inst-A", observed_symbols=set(), @@ -126,3 +134,5 @@ class TestCRUDRoundTrip: assert reopened["transition"] == "detected" assert reopened["state"]["first_seen_at"] == t1 assert reopened["state"]["last_seen_at"] == t3 + assert reopened["state"]["operator_action"] is None + assert reopened["state"]["evidence_ref"] is None diff --git a/tests/application/test_snapshot_scheduler.py b/tests/application/test_snapshot_scheduler.py index 6b0c7c2..8b3faba 100644 --- a/tests/application/test_snapshot_scheduler.py +++ b/tests/application/test_snapshot_scheduler.py @@ -1,7 +1,8 @@ """Snapshot scheduler tests (REQ-030 AC-1; REQ-114 AC-1~9; REQ-115 AC-1~6).""" from __future__ import annotations -from datetime import datetime, timezone, timedelta +from datetime import UTC, datetime, timedelta, timezone +from types import SimpleNamespace from unittest.mock import MagicMock from trex_core.adapters.broker.paper import PaperBroker @@ -44,10 +45,42 @@ def test_run_once_failure_sets_degraded_signal(): store = SnapshotStore() sched = SnapshotScheduler(read_port=BadRead(), store=store, broker=PaperBroker()) assert sched.run_once() is False - reason, msg = store.degraded_state() + reason, _msg = store.degraded_state() assert reason is not None and reason.value == "portfolio_hub_unavailable" +def test_run_once_uses_authoritative_core_holdings_broker(monkeypatch): + read = StubPortfolioRead( + daemon_id="d-1", + items=[PortfolioItem(portfolio_id="pf-X", symbol="X", cash_balance=100, asset_quantity=0)], + ) + store = SnapshotStore() + projected_broker = object() + authoritative_broker = object() + captured = {} + + def _capture(**kwargs): + captured["broker"] = kwargs["broker"] + return {} + + monkeypatch.setattr( + "trex_core.application.reconciliation.reconcile_core_holdings", + _capture, + ) + sched = SnapshotScheduler( + read_port=read, + store=store, + broker=projected_broker, + core_holdings_broker=authoritative_broker, + slot_repo=object(), + slot_service=SimpleNamespace(instance_key="inst-1"), + ) + sched._reconcile_slots = lambda _items: None + + assert sched.run_once() is True + assert captured["broker"] is authoritative_broker + + def test_swap_preserves_warning_for_carried_over_portfolio(): store = SnapshotStore() store.swap([PortfolioItem(portfolio_id="pf-X", symbol="X", @@ -99,7 +132,7 @@ def _fixed_clock(kst_hour=8, kst_minute=0): """Return a clock callable fixed at a given KST time today.""" now_kst = datetime.now(tz=_KST).replace(hour=kst_hour, minute=kst_minute, second=0, microsecond=0) - return lambda: now_kst.astimezone(timezone.utc) + return lambda: now_kst.astimezone(UTC) # --------------------------------------------------------------------------- @@ -616,8 +649,8 @@ def test_kis_broker_reinitialize_preserves_base_url_and_flags(): def test_build_kis_reinit_callback_returns_none_for_paper(): """REQ-113 AC-2: broker=paper이면 callback=None.""" from trex_core.adapters.broker.paper import PaperBroker as PB - from trex_core.runtime.daemon import _build_kis_reinit_callback from trex_core.runtime.config import DaemonConfig + from trex_core.runtime.daemon import _build_kis_reinit_callback # minimal config: paper broker, credential_fetch_enabled=True cfg = DaemonConfig(dsn="x", broker="paper", execution_interval_sec=1.0, @@ -629,8 +662,8 @@ def test_build_kis_reinit_callback_returns_none_for_paper(): def test_build_kis_reinit_callback_returns_none_when_disabled(): """REQ-113 AC-1: credential_fetch_enabled=False이면 callback=None.""" from trex_core.adapters.broker.paper import PaperBroker as PB - from trex_core.runtime.daemon import _build_kis_reinit_callback from trex_core.runtime.config import DaemonConfig + from trex_core.runtime.daemon import _build_kis_reinit_callback cfg = DaemonConfig(dsn="x", broker="kis", execution_interval_sec=1.0, credential_fetch_enabled=False) @@ -642,8 +675,8 @@ def test_build_kis_reinit_callback_returns_callable_for_kis(): """REQ-113 AC-2 / REQ-115 AC-1: broker=kis + fetch enabled → callable 반환 + 실제 reinitialize 호출됨.""" from trex_core.adapters.broker.kis import KISBroker from trex_core.application.portfolio import BrokerCredentialPayload - from trex_core.runtime.daemon import _build_kis_reinit_callback from trex_core.runtime.config import DaemonConfig + from trex_core.runtime.daemon import _build_kis_reinit_callback cfg = DaemonConfig(dsn="x", broker="kis", execution_interval_sec=1.0, credential_fetch_enabled=True) diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index b906e06..ed94ed4 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -35,6 +35,11 @@ class RaisingBroker(FakeBroker): raise BrokerBalanceError("transport_failed") +class AuthoritativeBroker(FakeBroker): + def exposes_full_balance_snapshot(self) -> bool: + return True + + class FilteredBroker(FakeBroker): def exposes_full_balance_snapshot(self) -> bool: return False @@ -116,6 +121,13 @@ def test_in_memory_store_preserves_first_seen_when_resolved_symbol_reappears(): t3 = t2 + timedelta(minutes=1) store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t1) + store.record_operator_action( + instance_key=INSTANCE, + symbol="008110", + action="investigating", + evidence_ref="kis-balance:2026-09-07", + note="before resolve", + ) store.resolve_absent(instance_key=INSTANCE, observed_symbols=set(), observed_at=t2) reopened = store.observe(instance_key=INSTANCE, symbol="008110", quantity=5, observed_at=t3) @@ -123,6 +135,8 @@ def test_in_memory_store_preserves_first_seen_when_resolved_symbol_reappears(): assert reopened["state"]["first_seen_at"] == t1 assert reopened["state"]["last_seen_at"] == t3 assert reopened["state"]["quantity_changed_at"] == t3 + assert reopened["state"]["operator_action"] is None + assert reopened["state"]["evidence_ref"] is None def test_reconcile_core_holdings_suppresses_repeated_unmanaged_alarm_spam(): @@ -134,7 +148,7 @@ def test_reconcile_core_holdings_suppresses_repeated_unmanaged_alarm_spam(): decisions = reconcile_core_holdings( slot_repo=repo, slot_service=svc, - broker=FakeBroker({"008110": 3}), + broker=AuthoritativeBroker({"008110": 3}), instance_key=INSTANCE, alarm_emitter=emitter, unmanaged_position_store=store, @@ -166,7 +180,7 @@ def test_reconcile_core_holdings_tracks_change_and_resolution(): reconcile_core_holdings( slot_repo=repo, slot_service=svc, - broker=FakeBroker({"008110": 5}), + broker=AuthoritativeBroker({"008110": 5}), instance_key=INSTANCE, alarm_emitter=emitter, unmanaged_position_store=store, @@ -174,7 +188,7 @@ def test_reconcile_core_holdings_tracks_change_and_resolution(): reconcile_core_holdings( slot_repo=repo, slot_service=svc, - broker=FakeBroker({}), + broker=AuthoritativeBroker({}), instance_key=INSTANCE, alarm_emitter=emitter, unmanaged_position_store=store, @@ -211,6 +225,25 @@ def test_broker_failure_does_not_resolve_unmanaged_state(): assert state["lifecycle_state"] == "active" +def test_missing_authoritative_marker_does_not_resolve_unmanaged_state(): + repo, svc = _wire() + store = InMemoryUnmanagedPositionStore() + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3) + + decisions = reconcile_core_holdings( + slot_repo=repo, + slot_service=svc, + broker=FakeBroker({}), + instance_key=INSTANCE, + unmanaged_position_store=store, + ) + + assert decisions == {} + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" + + def test_non_authoritative_broker_view_does_not_resolve_unmanaged_state(): repo, svc = _wire() store = InMemoryUnmanagedPositionStore() diff --git a/tests/runtime/test_broker_provider.py b/tests/runtime/test_broker_provider.py index 2472ec1..2cd654f 100644 --- a/tests/runtime/test_broker_provider.py +++ b/tests/runtime/test_broker_provider.py @@ -23,6 +23,15 @@ class _Broker: def reinitialize(self, credential): self.credentials.append(credential) +class _ProjectedBroker(_Broker): + def __init__(self, authoritative): + super().__init__() + self.authoritative = authoritative + + def authoritative_balance_reader(self): + return self.authoritative + + class _Read: def __init__(self, ctx=None, error=None): self.ctx, self.error, self.calls = ctx, error, 0 def fetch_broker_credential(self): @@ -72,6 +81,18 @@ def test_enabled_paper_never_fetches_kis_credentials(monkeypatch): assert read.calls == 0 +def test_reconciliation_broker_uses_authoritative_balance_reader(monkeypatch): + authoritative = _Broker() + projected = _ProjectedBroker(authoritative) + monkeypatch.setattr("trex_core.runtime.broker_provider._make_env_broker", lambda cfg: projected) + provider = ConfigBrokerProvider(_cfg(broker="paper"), read_port=_Read(_ctx())) + + broker, source = provider.get_broker_for_reconciliation() + + assert broker is authoritative + assert source.mode == "env" + + def test_break_glass_is_distinct_and_skips_fetch(monkeypatch): broker = _Broker() monkeypatch.setenv("TREX_KIS_BROKER_BREAK_GLASS", "true") diff --git a/tests/runtime/test_cli_operator.py b/tests/runtime/test_cli_operator.py index 91d3775..643b4ab 100644 --- a/tests/runtime/test_cli_operator.py +++ b/tests/runtime/test_cli_operator.py @@ -523,12 +523,15 @@ def test_verify_no_db_no_live_flag_needed(capsys, monkeypatch): # unmanaged parser/tests # --------------------------------------------------------------------------- -def test_unmanaged_parser_accepts_list_and_record(): +def test_unmanaged_parser_accepts_list_audit_and_record(): parser = cli._build_parser() args = parser.parse_args(["unmanaged", "list", "--status", "all", "--symbol", "008110"]) assert args.cmd == "unmanaged" assert args.unmanaged_cmd == "list" assert args.status == "all" + args = parser.parse_args(["unmanaged", "audit", "008110"]) + assert args.unmanaged_cmd == "audit" + assert args.symbol == "008110" args = parser.parse_args( [ "unmanaged", @@ -617,7 +620,6 @@ def test_unmanaged_record_requires_existing_state(monkeypatch, capsys): action="approved_exception", evidence_ref="kis-balance:2026-09-11", note="confirmed", - actor="operator", instance="inst-1", config=None, config_path=None, @@ -634,6 +636,8 @@ def test_unmanaged_record_requires_existing_state(monkeypatch, capsys): def test_unmanaged_record_persists_operator_action(monkeypatch, capsys): + monkeypatch.setattr(cli, "_operator_actor", lambda: "cli:test") + class _Store: def __init__(self, *_args, **_kwargs): pass @@ -671,7 +675,6 @@ def test_unmanaged_record_persists_operator_action(monkeypatch, capsys): action="managed_inclusion", evidence_ref="kis-balance:2026-09-11", note="manual inclusion planned", - actor="operator", instance="inst-1", config=None, config_path=None, @@ -687,9 +690,12 @@ def test_unmanaged_record_persists_operator_action(monkeypatch, capsys): assert out["result"] == "recorded" assert out["state"]["operator_action"] == "managed_inclusion" assert out["state"]["evidence_ref"] == "kis-balance:2026-09-11" + assert out["state"]["operator_actor"] == "cli:test" def test_unmanaged_record_text_output(monkeypatch, capsys): + monkeypatch.setattr(cli, "_operator_actor", lambda: "cli:test") + class _Store: def __init__(self, *_args, **_kwargs): pass @@ -724,7 +730,6 @@ def test_unmanaged_record_text_output(monkeypatch, capsys): action="approved_exception", evidence_ref="kis-balance:2026-09-11", note="confirmed", - actor="operator", instance="inst-1", config=None, config_path=None, @@ -742,6 +747,8 @@ def test_unmanaged_record_text_output(monkeypatch, capsys): def test_unmanaged_record_main_dispatch(monkeypatch, capsys): + monkeypatch.setattr(cli, "_operator_actor", lambda: "cli:test") + class _Store: def __init__(self, *_args, **_kwargs): pass @@ -788,6 +795,79 @@ def test_unmanaged_record_main_dispatch(monkeypatch, capsys): assert out["state"]["symbol"] == "008110" +def test_unmanaged_audit_outputs_lifecycle_and_operator_history(monkeypatch, capsys): + class _Store: + def __init__(self, *_args, **_kwargs): + pass + + def get_state(self, **kwargs): + assert kwargs == {"instance_key": "inst-1", "symbol": "008110"} + return { + "instance_key": "inst-1", + "symbol": "008110", + "quantity": 3, + "lifecycle_state": "active", + "first_seen_at": "2026-09-07T00:00:00+00:00", + "last_seen_at": "2026-09-11T00:00:00+00:00", + "quantity_changed_at": "2026-09-07T00:00:00+00:00", + "resolved_at": None, + "operator_action": "investigating", + "evidence_ref": "kis-balance:2026-09-11", + "operator_note": "confirmed", + "operator_actor": "cli:test", + "operator_updated_at": "2026-09-11T00:00:00+00:00", + "created_at": "2026-09-07T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + } + + def list_audit(self, **kwargs): + assert kwargs == {"instance_key": "inst-1", "symbol": "008110"} + return [ + { + "event_type": "detected", + "recorded_at": "2026-09-07T00:00:00+00:00", + "quantity": 3, + "action": None, + "actor": "reconcile", + "evidence_ref": None, + "note": "", + }, + { + "event_type": "operator_action", + "recorded_at": "2026-09-11T00:00:00+00:00", + "quantity": 3, + "action": "investigating", + "actor": "cli:test", + "evidence_ref": "kis-balance:2026-09-11", + "note": "confirmed", + }, + ] + + monkeypatch.setattr( + "trex_core.adapters.portfolio.unmanaged_position_store.PostgreSQLUnmanagedPositionStore", + _Store, + ) + args = SimpleNamespace( + cmd="unmanaged", + unmanaged_cmd="audit", + symbol="008110", + instance="inst-1", + config=None, + config_path=None, + env_file=None, + json=True, + timeout=10.0, + ) + + rc = cli._cmd_unmanaged_audit(args, SimpleNamespace(dsn="postgresql://x/y")) + + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["symbol"] == "008110" + assert out["state"]["operator_actor"] == "cli:test" + assert [row["event_type"] for row in out["audit"]] == ["detected", "operator_action"] + + # --------------------------------------------------------------------------- # REL-010 regression: _cmd_verify broker import must not raise ImportError # --------------------------------------------------------------------------- diff --git a/tests/runtime/test_daemon_resolve_symbol.py b/tests/runtime/test_daemon_resolve_symbol.py index 140d417..4faba2e 100644 --- a/tests/runtime/test_daemon_resolve_symbol.py +++ b/tests/runtime/test_daemon_resolve_symbol.py @@ -73,3 +73,44 @@ def test_daemon_wires_shared_alarm_emitter_into_intent_service(): found = True break assert found, "Daemon must pass self.alarm_emitter into IntentService(alarm_emitter=...)" + + +def test_daemon_wires_reconciliation_broker_into_snapshot_scheduler(): + src = ( + Path(__file__).resolve().parents[2] + / "trex_core" + / "runtime" + / "daemon.py" + ).read_text(encoding="utf-8") + tree = ast.parse(src) + found = False + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not any( + isinstance(t, ast.Attribute) and t.attr == "snapshot_scheduler" + for t in node.targets + ): + continue + call = node.value + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id == "SnapshotScheduler" + ): + continue + for kw in call.keywords: + if kw.arg != "core_holdings_broker": + continue + if ( + isinstance(kw.value, ast.Attribute) + and kw.value.attr == "reconciliation_broker" + and isinstance(kw.value.value, ast.Name) + and kw.value.value.id == "self" + ): + found = True + break + assert found, ( + "Daemon must pass self.reconciliation_broker into " + "SnapshotScheduler(core_holdings_broker=...)" + ) diff --git a/trex_core/adapters/broker/kis.py b/trex_core/adapters/broker/kis.py index 4c2e711..c480cc2 100644 --- a/trex_core/adapters/broker/kis.py +++ b/trex_core/adapters/broker/kis.py @@ -382,9 +382,7 @@ class KISBroker: transport_failure: bool, ) -> float: """Return bounded state/age-based delay for the next authoritative poll.""" - if status.state != "unknown": - delay = _FILL_POLL_MIN_INTERVAL_SECONDS - elif age_seconds < 60: + if status.state != "unknown" or age_seconds < 60: delay = _FILL_POLL_MIN_INTERVAL_SECONDS elif age_seconds < 5 * 60: delay = 15.0 @@ -1284,6 +1282,9 @@ class KISBroker: seen_continuations.add(next_continuation) continuation = next_continuation + def exposes_full_balance_snapshot(self) -> bool: + return True + def get_available_cash(self, *, price_check_code: str = "005930") -> int: """Return KIS cash-only buy capacity (`nrcvb_buy_amt`). diff --git a/trex_core/adapters/broker/ownership.py b/trex_core/adapters/broker/ownership.py index ab3cc0d..8ede722 100644 --- a/trex_core/adapters/broker/ownership.py +++ b/trex_core/adapters/broker/ownership.py @@ -17,6 +17,7 @@ from threading import Lock from trex_core.application.broker_port import ( BalanceItem, BrokerBalanceError, + BrokerPort, CancelOutcome, FillStatus, OrderAck, @@ -174,6 +175,9 @@ class OwnershipAwareBroker: def get_balance(self) -> list[BalanceItem]: return self._store.project(self._delegate.get_balance()) + def authoritative_balance_reader(self) -> BrokerPort: + return self._delegate + def exposes_full_balance_snapshot(self) -> bool: """False: projected balances omit operator-owned/unmanaged broker positions.""" return False diff --git a/trex_core/adapters/broker/paper.py b/trex_core/adapters/broker/paper.py index bcf9683..d7ea95a 100644 --- a/trex_core/adapters/broker/paper.py +++ b/trex_core/adapters/broker/paper.py @@ -166,6 +166,9 @@ class PaperBroker: return [BalanceItem(symbol=sym, hldg_qty=qty) for sym, qty in balances.items() if qty != 0] + def exposes_full_balance_snapshot(self) -> bool: + return True + # --- REQ-038: programmable current price (safety-sell scenario double) --- def set_current_price(self, symbol: str, price: Decimal) -> None: """Program the current price returned by get_current_price(symbol).""" diff --git a/trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql b/trex_core/adapters/persistence/sql/030_create_unmanaged_position_state.sql similarity index 100% rename from trex_core/adapters/persistence/sql/028_create_unmanaged_position_state.sql rename to trex_core/adapters/persistence/sql/030_create_unmanaged_position_state.sql diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py index 0d15adf..291842b 100644 --- a/trex_core/adapters/portfolio/unmanaged_position_store.py +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -59,6 +59,8 @@ class PostgreSQLUnmanagedPositionStore: "UPDATE unmanaged_position_state " "SET quantity = %s, lifecycle_state = 'active', " " last_seen_at = %s, quantity_changed_at = %s, resolved_at = NULL, " + " operator_action = NULL, evidence_ref = NULL, operator_note = '', " + " operator_actor = NULL, operator_updated_at = NULL, " " updated_at = %s " "WHERE instance_key = %s AND symbol = %s " f"RETURNING {_STATE_COLUMNS}", diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index ced6b87..3346e7b 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -615,11 +615,6 @@ class InMemoryUnmanagedPositionStore: existing = self._states.get(key) previous_quantity = existing.get("quantity") if existing is not None else None preserved = {} if existing is None else { - "operator_action": existing.get("operator_action"), - "evidence_ref": existing.get("evidence_ref"), - "operator_note": existing.get("operator_note"), - "operator_actor": existing.get("operator_actor"), - "operator_updated_at": existing.get("operator_updated_at"), "created_at": existing.get("created_at", now), } if existing is None or existing.get("lifecycle_state") == "resolved": diff --git a/trex_core/application/reconciliation.py b/trex_core/application/reconciliation.py index 29be749..9aabe84 100644 --- a/trex_core/application/reconciliation.py +++ b/trex_core/application/reconciliation.py @@ -373,7 +373,7 @@ def _broker_balances_are_authoritative_for_unmanaged_resolution(broker: BrokerPo return False if isinstance(marker, bool): return marker - return True + return False def reconcile_core_holdings( diff --git a/trex_core/application/snapshot_scheduler.py b/trex_core/application/snapshot_scheduler.py index 611a477..da9aef8 100644 --- a/trex_core/application/snapshot_scheduler.py +++ b/trex_core/application/snapshot_scheduler.py @@ -44,6 +44,7 @@ class SnapshotScheduler: read_port: PortfolioHubReadPort, store: SnapshotStore, broker: BrokerPort, + core_holdings_broker: BrokerPort | None = None, schedule_kst: time = time(hour=7, minute=30), clock: Callable[[], datetime] | None = None, slot_repo=None, @@ -60,6 +61,7 @@ class SnapshotScheduler: self._read = read_port self._store = store self._broker = broker + self._core_holdings_broker = core_holdings_broker or broker self._schedule_kst = schedule_kst self._clock = clock or (lambda: datetime.now(tz=UTC)) self._daemon_id: str | None = None @@ -337,7 +339,7 @@ class SnapshotScheduler: # BAT-050 / REQ-103: pass warn_store for B-case durable warning. reconcile_core_holdings( slot_repo=self._slot_repo, slot_service=self._slot_service, - broker=self._broker, instance_key=self._instance_key, + broker=self._core_holdings_broker, instance_key=self._instance_key, alarm_emitter=self._alarm_emitter, # dcy-033 / REQ-100 warn_store=self._warn_store, # BAT-050 / REQ-103 notification_store=self._notification_store, # req-0007 / ac-0024 diff --git a/trex_core/runtime/broker_provider.py b/trex_core/runtime/broker_provider.py index 51a8ccc..1aa3677 100644 --- a/trex_core/runtime/broker_provider.py +++ b/trex_core/runtime/broker_provider.py @@ -3,7 +3,7 @@ from __future__ import annotations import os from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone +from datetime import UTC, date, datetime, timedelta, timezone from typing import Literal, Protocol from trex_core.adapters.broker.paper import PaperBroker @@ -33,6 +33,7 @@ class BrokerCredentialUnavailable(RuntimeError): class BrokerProvider(Protocol): def get_broker_for_read(self) -> tuple[BrokerPort, BrokerSource]: ... + def get_broker_for_reconciliation(self) -> tuple[BrokerPort, BrokerSource]: ... def refresh_day_start_credential(self) -> BrokerSource: ... @@ -80,7 +81,7 @@ class ConfigBrokerProvider: def __init__(self, config: DaemonConfig, *, read_port=None, clock=None) -> None: self._config = config self._read = read_port - self._clock = clock or (lambda: datetime.now(tz=timezone.utc)) + self._clock = clock or (lambda: datetime.now(tz=UTC)) self._broker: BrokerPort | None = None self._source: BrokerSource | None = None self._credential_date: date | None = None @@ -101,7 +102,7 @@ class ConfigBrokerProvider: raise BrokerCredentialUnavailable("portfolio_hub_unavailable") from None except BrokerCredentialUnavailable: raise - except Exception: + except Exception: # noqa: BLE001 - sanitize arbitrary read adapter failures raise BrokerCredentialUnavailable("portfolio_hub_unavailable") from None if ctx is None or getattr(ctx, "broker", None) is None: raise BrokerCredentialUnavailable("broker_absent") @@ -121,7 +122,7 @@ class ConfigBrokerProvider: self._broker = broker else: self._broker.reinitialize(credential) # type: ignore[attr-defined] - except Exception: + except Exception: # noqa: BLE001 - sanitize broker construction/reload failures raise BrokerCredentialUnavailable("broker_initialization_failed") from None self._credential_date = self._clock().astimezone(_KST).date() # Version is designated non-secret by the portfolio contract; no account/key material. @@ -147,6 +148,13 @@ class ConfigBrokerProvider: self._source = source return self._broker, source # type: ignore[return-value] + def get_broker_for_reconciliation(self) -> tuple[BrokerPort, BrokerSource]: + broker, source = self.get_broker_for_read() + authoritative_reader = getattr(broker, "authoritative_balance_reader", None) + if callable(authoritative_reader): + return authoritative_reader(), source + return broker, source + def refresh_day_start_credential(self) -> BrokerSource: # Paper/env/break-glass paths deliberately do no portfolio-hub call. if self._config.broker != "kis" or not self._config.credential_fetch_enabled or _break_glass_enabled(): diff --git a/trex_core/runtime/cli.py b/trex_core/runtime/cli.py index bcbe6bc..7caef82 100644 --- a/trex_core/runtime/cli.py +++ b/trex_core/runtime/cli.py @@ -12,6 +12,7 @@ Subcommands: sync — sync slot qty to broker ground truth (direct-DB; REQ-105) verify — re-check C-case reconcile-warning (direct-DB; REQ-106) unmanaged list — query durable unmanaged-position lifecycle state + unmanaged audit — inspect lifecycle/operator audit trail for one symbol unmanaged record — record explicit operator disposition + evidence reference ledger backfill — evidence-backed historical ledger recovery ledger skips list — query durable skipped-ledger records @@ -38,6 +39,7 @@ LedgerWriter path, and resolves matching durable skipped-ledger records. from __future__ import annotations import argparse +import getpass import json as _json import logging import re @@ -78,6 +80,14 @@ def _resolve_instance(args) -> str: return topology.DEFAULT_INSTANCE_KEY +def _operator_actor() -> str: + try: + username = getpass.getuser().strip() + except Exception: # noqa: BLE001 + username = "" + return f"cli:{username or 'unknown'}" + + # --------------------------------------------------------------------------- # # default transport-backed RPC (overridable in tests) # --------------------------------------------------------------------------- # @@ -659,6 +669,13 @@ def _build_parser() -> argparse.ArgumentParser: p_unmanaged_list.add_argument("--symbol", default=None) _add_common(p_unmanaged_list) + p_unmanaged_audit = p_unmanaged_sub.add_parser( + "audit", + help="Show lifecycle/operator audit trail for one unmanaged position", + ) + p_unmanaged_audit.add_argument("symbol") + _add_common(p_unmanaged_audit) + p_unmanaged_record = p_unmanaged_sub.add_parser( "record", help="Record an explicit operator disposition for an unmanaged position", @@ -670,7 +687,6 @@ def _build_parser() -> argparse.ArgumentParser: ) p_unmanaged_record.add_argument("--evidence-ref", required=True) p_unmanaged_record.add_argument("--note", default="") - p_unmanaged_record.add_argument("--actor", default="operator") _add_common(p_unmanaged_record) p_ledger = sub.add_parser( @@ -1040,6 +1056,44 @@ def _cmd_unmanaged_list(args, cfg) -> int: return 0 +def _cmd_unmanaged_audit(args, cfg) -> int: + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + instance = _resolve_instance(args) + store = PostgreSQLUnmanagedPositionStore(cfg.dsn) + state = store.get_state(instance_key=instance, symbol=args.symbol) + if state is None: + _fail( + f"unmanaged position not found: instance={instance} symbol={args.symbol}", + args.json, + ) + return 1 + audit = store.list_audit(instance_key=instance, symbol=args.symbol) + payload = { + "instance_key": instance, + "symbol": args.symbol, + "state": state, + "audit": audit, + } + if args.json: + _emit(payload, as_json=True) + else: + print( + f"instance={instance} symbol={args.symbol} lifecycle={state['lifecycle_state']} " + f"entries={len(audit)}" + ) + for row in audit: + print( + f" {row['event_type']}\tat={row['recorded_at']}\t" + f"quantity={row.get('quantity')}\taction={row.get('action') or '-'}\t" + f"actor={row.get('actor') or '-'}\tevidence_ref={row.get('evidence_ref') or '-'}\t" + f"note={row.get('note') or '-'}" + ) + return 0 + + def _cmd_unmanaged_record(args, cfg) -> int: from trex_core.adapters.portfolio.unmanaged_position_store import ( PostgreSQLUnmanagedPositionStore, @@ -1051,7 +1105,7 @@ def _cmd_unmanaged_record(args, cfg) -> int: symbol=args.symbol, action=args.action, evidence_ref=args.evidence_ref, - actor=args.actor, + actor=_operator_actor(), note=args.note, ) if state is None: @@ -1536,6 +1590,8 @@ def main(argv: list[str] | None = None, *, rpc=None, confirm=None) -> int: return _cmd_verify(args, cfg) if args.cmd == "unmanaged" and getattr(args, "unmanaged_cmd", None) == "list": return _cmd_unmanaged_list(args, cfg) + if args.cmd == "unmanaged" and getattr(args, "unmanaged_cmd", None) == "audit": + return _cmd_unmanaged_audit(args, cfg) if args.cmd == "unmanaged" and getattr(args, "unmanaged_cmd", None) == "record": return _cmd_unmanaged_record(args, cfg) diff --git a/trex_core/runtime/daemon.py b/trex_core/runtime/daemon.py index e5f117e..3532427 100644 --- a/trex_core/runtime/daemon.py +++ b/trex_core/runtime/daemon.py @@ -213,6 +213,9 @@ class Daemon: from trex_core.runtime.broker_provider import build_broker_provider self.broker_provider = build_broker_provider(config, read_port=self.portfolio_read) self.broker, self.broker_source = self.broker_provider.get_broker_for_read() + self.reconciliation_broker, self.reconciliation_broker_source = ( + self.broker_provider.get_broker_for_reconciliation() + ) self.account_snapshot_service: AccountSnapshotService | None = None if config.account_snapshot.enabled: if config.broker != "kis": @@ -271,6 +274,7 @@ class Daemon: read_port=self.portfolio_read, store=self.snapshot_store, broker=self.broker, + core_holdings_broker=self.reconciliation_broker, schedule_kst=_parse_kst_time(config.snapshot_schedule.kst_time), # dcy-020 / REQ-058·062 (BAT-030): universe slot reconcile (creates/ # retires the permanent asset slots) + core-holding divergence @@ -425,7 +429,7 @@ class Daemon: while not self.stop.wait(self.config.ledger_interval_sec): try: self.ledger_writer.run_once() - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S112 continue def _intent_rpc_loop(self) -> None: @@ -433,7 +437,7 @@ class Daemon: self._transports.append(transport) try: transport.declare_topology(self.config.instance_key) - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S110 pass handler = make_intent_rpc_handler( self.intent_service, @@ -448,7 +452,7 @@ class Daemon: self._transports.append(transport) try: transport.declare_topology(self.config.instance_key) - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S110 pass handler = make_slots_rpc_handler( self.slot_repo, @@ -472,7 +476,7 @@ class Daemon: self._transports.append(transport) try: transport.declare_topology(self.config.instance_key) - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S110 pass handler = make_strategy_state_set_handler( self.slot_repo, self.strategy_state_store, @@ -486,7 +490,7 @@ class Daemon: self._transports.append(transport) try: transport.declare_topology(self.config.instance_key) - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S110 pass handler = make_control_rpc_handler( self.pause_store, @@ -501,7 +505,7 @@ class Daemon: while not self.stop.wait(self.safety_sell.base_tick_sec): try: self.safety_sell.run_once() - except Exception: # noqa: BLE001 - loop must survive transient errors + except Exception: # noqa: BLE001,S112 - loop must survive transient errors continue def _snapshot_scheduler_loop(self) -> None: @@ -566,7 +570,7 @@ class Daemon: for tr in self._transports: try: tr.close() - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001,S110 pass def run(self) -> None: # pragma: no cover - process entrypoint From 84d81658a13f46b3c37b9c2efbf386ca9e556e09 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 13 Sep 2026 02:58:18 +0000 Subject: [PATCH 19/21] Resolve main merge and add unmanaged store race tests Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_store.py | 68 +++++++++++++++++++ .../test_unmanaged_position_lifecycle.py | 1 + .../portfolio/unmanaged_position_store.py | 2 +- trex_core/application/portfolio.py | 8 +-- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/tests/adapters/portfolio/test_unmanaged_position_store.py b/tests/adapters/portfolio/test_unmanaged_position_store.py index c03a2a8..91d3e92 100644 --- a/tests/adapters/portfolio/test_unmanaged_position_store.py +++ b/tests/adapters/portfolio/test_unmanaged_position_store.py @@ -1,6 +1,7 @@ """PostgreSQLUnmanagedPositionStore integration tests.""" from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta import pytest @@ -136,3 +137,70 @@ class TestCRUDRoundTrip: assert reopened["state"]["last_seen_at"] == t3 assert reopened["state"]["operator_action"] is None assert reopened["state"]["evidence_ref"] is None + + def test_concurrent_first_observe_keeps_single_detected_audit(self, pg_clean): + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + def _observe(): + return PostgreSQLUnmanagedPositionStore(pg_clean).observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _n: _observe(), range(2))) + + assert sorted(result["transition"] for result in results) == ["detected", "unchanged"] + store = PostgreSQLUnmanagedPositionStore(pg_clean) + state = store.get_state(instance_key="inst-A", symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" + audit = store.list_audit(instance_key="inst-A", symbol="008110") + assert [row["event_type"] for row in audit] == ["detected"] + + def test_concurrent_observe_and_resolve_leave_consistent_state_and_audit(self, pg_clean): + from trex_core.adapters.portfolio.unmanaged_position_store import ( + PostgreSQLUnmanagedPositionStore, + ) + + store = PostgreSQLUnmanagedPositionStore(pg_clean) + store.observe(instance_key="inst-A", symbol="008110", quantity=3) + + with ThreadPoolExecutor(max_workers=2) as pool: + observe_future = pool.submit( + PostgreSQLUnmanagedPositionStore(pg_clean).observe, + instance_key="inst-A", + symbol="008110", + quantity=5, + ) + resolve_future = pool.submit( + PostgreSQLUnmanagedPositionStore(pg_clean).resolve_absent, + instance_key="inst-A", + observed_symbols=set(), + ) + observe_result = observe_future.result() + resolve_result = resolve_future.result() + + state = store.get_state(instance_key="inst-A", symbol="008110") + assert state is not None + audit = store.list_audit(instance_key="inst-A", symbol="008110") + event_types = [row["event_type"] for row in audit] + assert event_types[0] == "detected" + assert event_types in ( + ["detected", "quantity_changed", "resolved"], + ["detected", "resolved", "detected"], + ) + if state["lifecycle_state"] == "resolved": + assert observe_result["transition"] == "quantity_changed" + assert len(resolve_result) == 1 + assert state["quantity"] == 5 + assert event_types[-1] == "resolved" + else: + assert state["lifecycle_state"] == "active" + assert observe_result["transition"] == "detected" + assert len(resolve_result) == 1 + assert state["quantity"] == 5 + assert event_types[-1] == "detected" diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index ed94ed4..a6de1fd 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -98,6 +98,7 @@ def test_in_memory_store_tracks_lifecycle_and_operator_audit(): "operator_action", "resolved", ] + assert all("recorded_at" in row for row in audit) def test_in_memory_store_isolates_same_symbol_by_instance(): diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py index 291842b..f2f2f38 100644 --- a/trex_core/adapters/portfolio/unmanaged_position_store.py +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -258,7 +258,7 @@ class PostgreSQLUnmanagedPositionStore: "actor": row[4], "evidence_ref": row[5], "note": row[6], - "created_at": row[7], + "recorded_at": row[7], } for row in cur.fetchall() ] diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index 5be2aa8..ac5c985 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -660,7 +660,7 @@ class InMemoryUnmanagedPositionStore: "operator_note": "", "operator_actor": None, "operator_updated_at": None, - "created_at": now, + "recorded_at": now, **preserved, "updated_at": now, } @@ -695,7 +695,7 @@ class InMemoryUnmanagedPositionStore: if transition == "quantity_changed" and previous_quantity is not None else "" ), - "created_at": now, + "recorded_at": now, } ) return { @@ -737,7 +737,7 @@ class InMemoryUnmanagedPositionStore: "actor": actor, "evidence_ref": state.get("evidence_ref"), "note": "", - "created_at": now, + "recorded_at": now, } ) return resolved @@ -798,7 +798,7 @@ class InMemoryUnmanagedPositionStore: "actor": actor, "evidence_ref": evidence_ref, "note": note, - "created_at": now, + "recorded_at": now, } ) return dict(state) From 8b1de8e26a303043994f2eee2ad6b5b6303b07d8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 13 Sep 2026 05:54:51 +0000 Subject: [PATCH 20/21] Fence unmanaged lifecycle by snapshot time Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .../test_unmanaged_position_store.py | 102 ++++++++++++++---- .../test_unmanaged_position_lifecycle.py | 43 ++++++++ .../portfolio/unmanaged_position_store.py | 85 ++++++++++++--- trex_core/application/portfolio.py | 39 ++++++- trex_core/application/reconciliation.py | 4 + 5 files changed, 239 insertions(+), 34 deletions(-) diff --git a/tests/adapters/portfolio/test_unmanaged_position_store.py b/tests/adapters/portfolio/test_unmanaged_position_store.py index 91d3e92..db69a38 100644 --- a/tests/adapters/portfolio/test_unmanaged_position_store.py +++ b/tests/adapters/portfolio/test_unmanaged_position_store.py @@ -161,13 +161,83 @@ class TestCRUDRoundTrip: audit = store.list_audit(instance_key="inst-A", symbol="008110") assert [row["event_type"] for row in audit] == ["detected"] - def test_concurrent_observe_and_resolve_leave_consistent_state_and_audit(self, pg_clean): + def test_older_absent_snapshot_applied_after_newer_observation_is_ignored(self, pg_unmanaged_store): + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + observed_at=t1, + ) + observed = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=5, + observed_at=t3, + ) + resolved = pg_unmanaged_store.resolve_absent( + instance_key="inst-A", + observed_symbols=set(), + observed_at=t2, + ) + + assert observed["transition"] == "quantity_changed" + assert resolved == [] + state = pg_unmanaged_store.get_state(instance_key="inst-A", symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" + assert state["quantity"] == 5 + assert [row["event_type"] for row in pg_unmanaged_store.list_audit( + instance_key="inst-A", + symbol="008110", + )] == ["detected", "quantity_changed"] + + def test_older_observation_applied_after_newer_resolution_is_ignored(self, pg_unmanaged_store): + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=3, + observed_at=t1, + ) + pg_unmanaged_store.resolve_absent( + instance_key="inst-A", + observed_symbols=set(), + observed_at=t3, + ) + stale = pg_unmanaged_store.observe( + instance_key="inst-A", + symbol="008110", + quantity=5, + observed_at=t2, + ) + + assert stale["transition"] == "stale" + state = pg_unmanaged_store.get_state(instance_key="inst-A", symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "resolved" + assert state["quantity"] == 3 + assert [row["event_type"] for row in pg_unmanaged_store.list_audit( + instance_key="inst-A", + symbol="008110", + )] == ["detected", "resolved"] + + def test_concurrent_observe_and_resolve_respects_newest_snapshot_timestamp(self, pg_clean): from trex_core.adapters.portfolio.unmanaged_position_store import ( PostgreSQLUnmanagedPositionStore, ) store = PostgreSQLUnmanagedPositionStore(pg_clean) - store.observe(instance_key="inst-A", symbol="008110", quantity=3) + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + store.observe(instance_key="inst-A", symbol="008110", quantity=3, observed_at=t1) with ThreadPoolExecutor(max_workers=2) as pool: observe_future = pool.submit( @@ -175,32 +245,24 @@ class TestCRUDRoundTrip: instance_key="inst-A", symbol="008110", quantity=5, + observed_at=t3, ) resolve_future = pool.submit( PostgreSQLUnmanagedPositionStore(pg_clean).resolve_absent, instance_key="inst-A", observed_symbols=set(), + observed_at=t2, ) observe_result = observe_future.result() resolve_result = resolve_future.result() state = store.get_state(instance_key="inst-A", symbol="008110") assert state is not None - audit = store.list_audit(instance_key="inst-A", symbol="008110") - event_types = [row["event_type"] for row in audit] - assert event_types[0] == "detected" - assert event_types in ( - ["detected", "quantity_changed", "resolved"], - ["detected", "resolved", "detected"], - ) - if state["lifecycle_state"] == "resolved": - assert observe_result["transition"] == "quantity_changed" - assert len(resolve_result) == 1 - assert state["quantity"] == 5 - assert event_types[-1] == "resolved" - else: - assert state["lifecycle_state"] == "active" - assert observe_result["transition"] == "detected" - assert len(resolve_result) == 1 - assert state["quantity"] == 5 - assert event_types[-1] == "detected" + assert observe_result["transition"] == "quantity_changed" + assert resolve_result == [] + assert state["lifecycle_state"] == "active" + assert state["quantity"] == 5 + assert [row["event_type"] for row in store.list_audit( + instance_key="inst-A", + symbol="008110", + )] == ["detected", "quantity_changed"] diff --git a/tests/application/test_unmanaged_position_lifecycle.py b/tests/application/test_unmanaged_position_lifecycle.py index a6de1fd..3fe04a0 100644 --- a/tests/application/test_unmanaged_position_lifecycle.py +++ b/tests/application/test_unmanaged_position_lifecycle.py @@ -140,6 +140,49 @@ def test_in_memory_store_preserves_first_seen_when_resolved_symbol_reappears(): assert reopened["state"]["evidence_ref"] is None +def test_in_memory_store_ignores_stale_resolve_after_newer_observation(): + store = InMemoryUnmanagedPositionStore() + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t1) + store.observe(instance_key=INSTANCE, symbol="008110", quantity=5, observed_at=t3) + resolved = store.resolve_absent( + instance_key=INSTANCE, + observed_symbols=set(), + observed_at=t2, + ) + + assert resolved == [] + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "active" + assert state["quantity"] == 5 + assert store.list_audit(instance_key=INSTANCE, symbol="008110")[-1]["event_type"] == "quantity_changed" + + +def test_in_memory_store_ignores_stale_observation_after_newer_resolution(): + store = InMemoryUnmanagedPositionStore() + t1 = datetime(2026, 9, 7, 0, 0, tzinfo=UTC) + t2 = t1 + timedelta(minutes=1) + t3 = t2 + timedelta(minutes=1) + + store.observe(instance_key=INSTANCE, symbol="008110", quantity=3, observed_at=t1) + store.resolve_absent(instance_key=INSTANCE, observed_symbols=set(), observed_at=t3) + stale = store.observe(instance_key=INSTANCE, symbol="008110", quantity=5, observed_at=t2) + + assert stale["transition"] == "stale" + state = store.get_state(instance_key=INSTANCE, symbol="008110") + assert state is not None + assert state["lifecycle_state"] == "resolved" + assert state["quantity"] == 3 + assert [row["event_type"] for row in store.list_audit(instance_key=INSTANCE, symbol="008110")] == [ + "detected", + "resolved", + ] + + def test_reconcile_core_holdings_suppresses_repeated_unmanaged_alarm_spam(): repo, svc = _wire() emitter, transport = _make_emitter() diff --git a/trex_core/adapters/portfolio/unmanaged_position_store.py b/trex_core/adapters/portfolio/unmanaged_position_store.py index f2f2f38..42ae613 100644 --- a/trex_core/adapters/portfolio/unmanaged_position_store.py +++ b/trex_core/adapters/portfolio/unmanaged_position_store.py @@ -21,6 +21,18 @@ class PostgreSQLUnmanagedPositionStore: self._psycopg = _require_psycopg(psycopg_module) self._dsn = dsn + @staticmethod + def _latest_snapshot_at(state: dict) -> datetime: + resolved_at = state.get("resolved_at") + last_seen_at = state.get("last_seen_at") + if isinstance(resolved_at, datetime) and ( + not isinstance(last_seen_at, datetime) or resolved_at >= last_seen_at + ): + return resolved_at + if isinstance(last_seen_at, datetime): + return last_seen_at + return state.get("updated_at") or state.get("created_at") or datetime.min.replace(tzinfo=UTC) + def observe( self, *, @@ -43,6 +55,13 @@ class PostgreSQLUnmanagedPositionStore: row = cur.fetchone() previous = self._row_to_state(row) if row is not None else None previous_quantity = previous.get("quantity") if previous is not None else None + if previous is not None and now < self._latest_snapshot_at(previous): + conn.rollback() + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": previous, + } if previous is None: transition = "detected" cur.execute( @@ -54,6 +73,13 @@ class PostgreSQLUnmanagedPositionStore: (instance_key, symbol, quantity, now, now, now, now, now), ) elif previous["lifecycle_state"] == "resolved": + if now <= self._latest_snapshot_at(previous): + conn.rollback() + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": previous, + } transition = "detected" cur.execute( "UPDATE unmanaged_position_state " @@ -63,29 +89,58 @@ class PostgreSQLUnmanagedPositionStore: " operator_actor = NULL, operator_updated_at = NULL, " " updated_at = %s " "WHERE instance_key = %s AND symbol = %s " + " AND lifecycle_state = 'resolved' AND resolved_at < %s " f"RETURNING {_STATE_COLUMNS}", - (quantity, now, now, now, instance_key, symbol), + (quantity, now, now, now, instance_key, symbol, now), ) elif int(previous["quantity"]) != int(quantity): + if now <= previous["last_seen_at"]: + conn.rollback() + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": previous, + } transition = "quantity_changed" cur.execute( "UPDATE unmanaged_position_state " "SET quantity = %s, last_seen_at = %s, quantity_changed_at = %s, " " resolved_at = NULL, lifecycle_state = 'active', updated_at = %s " "WHERE instance_key = %s AND symbol = %s " + " AND lifecycle_state = 'active' AND last_seen_at < %s " f"RETURNING {_STATE_COLUMNS}", - (quantity, now, now, now, instance_key, symbol), + (quantity, now, now, now, instance_key, symbol, now), ) else: transition = "unchanged" - cur.execute( - "UPDATE unmanaged_position_state " - "SET last_seen_at = %s, updated_at = %s " - "WHERE instance_key = %s AND symbol = %s " - f"RETURNING {_STATE_COLUMNS}", - (now, now, instance_key, symbol), - ) - state = self._row_to_state(cur.fetchone()) + if now > previous["last_seen_at"]: + cur.execute( + "UPDATE unmanaged_position_state " + "SET last_seen_at = %s, updated_at = %s " + "WHERE instance_key = %s AND symbol = %s " + " AND lifecycle_state = 'active' AND quantity = %s AND last_seen_at < %s " + f"RETURNING {_STATE_COLUMNS}", + (now, now, instance_key, symbol, quantity, now), + ) + fetched = cur.fetchone() + state = previous if fetched is None else self._row_to_state(fetched) + else: + conn.rollback() + return { + "transition": "unchanged", + "previous_quantity": previous_quantity, + "state": previous, + } + if transition != "unchanged": + fetched = cur.fetchone() + if fetched is None: + conn.rollback() + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": previous, + } + state = self._row_to_state(fetched) if transition != "unchanged": note = ( f"previous_quantity={previous_quantity}" @@ -139,14 +194,20 @@ class PostgreSQLUnmanagedPositionStore: for state in active: if state["symbol"] in observed_symbols: continue + if now <= state["last_seen_at"]: + continue cur.execute( "UPDATE unmanaged_position_state " "SET lifecycle_state = 'resolved', resolved_at = %s, updated_at = %s " "WHERE instance_key = %s AND symbol = %s " + " AND lifecycle_state = 'active' AND last_seen_at < %s " f"RETURNING {_STATE_COLUMNS}", - (now, now, instance_key, state["symbol"]), + (now, now, instance_key, state["symbol"], now), ) - resolved_state = self._row_to_state(cur.fetchone()) + row = cur.fetchone() + if row is None: + continue + resolved_state = self._row_to_state(row) resolved.append(resolved_state) self._insert_audit( cur, diff --git a/trex_core/application/portfolio.py b/trex_core/application/portfolio.py index ac5c985..729f651 100644 --- a/trex_core/application/portfolio.py +++ b/trex_core/application/portfolio.py @@ -625,6 +625,18 @@ class InMemoryUnmanagedPositionStore: self._states: dict[tuple[str, str], dict] = {} self._audit: list[dict] = [] + @staticmethod + def _latest_snapshot_at(state: dict) -> datetime: + resolved_at = state.get("resolved_at") + last_seen_at = state.get("last_seen_at") + if isinstance(resolved_at, datetime) and ( + not isinstance(last_seen_at, datetime) or resolved_at >= last_seen_at + ): + return resolved_at + if isinstance(last_seen_at, datetime): + return last_seen_at + return state.get("updated_at") or state.get("created_at") or datetime.min.replace(tzinfo=UTC) + def observe( self, *, @@ -642,7 +654,21 @@ class InMemoryUnmanagedPositionStore: preserved = {} if existing is None else { "created_at": existing.get("created_at", now), } + if existing is not None: + latest_snapshot_at = self._latest_snapshot_at(existing) + if now < latest_snapshot_at: + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": dict(existing), + } if existing is None or existing.get("lifecycle_state") == "resolved": + if existing is not None and now == self._latest_snapshot_at(existing): + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": dict(existing), + } transition = "detected" state = { "instance_key": instance_key, @@ -666,6 +692,12 @@ class InMemoryUnmanagedPositionStore: } self._states[key] = state elif int(existing["quantity"]) != int(quantity): + if now <= existing["last_seen_at"]: + return { + "transition": "stale", + "previous_quantity": previous_quantity, + "state": dict(existing), + } transition = "quantity_changed" existing["quantity"] = quantity existing["last_seen_at"] = now @@ -676,8 +708,9 @@ class InMemoryUnmanagedPositionStore: state = existing else: transition = "unchanged" - existing["last_seen_at"] = now - existing["updated_at"] = now + if now > existing["last_seen_at"]: + existing["last_seen_at"] = now + existing["updated_at"] = now state = existing if transition != "unchanged": self._audit.append( @@ -722,6 +755,8 @@ class InMemoryUnmanagedPositionStore: continue if state["symbol"] in observed_symbols: continue + if now <= state["last_seen_at"]: + continue state["lifecycle_state"] = "resolved" state["resolved_at"] = now state["updated_at"] = now diff --git a/trex_core/application/reconciliation.py b/trex_core/application/reconciliation.py index f117e35..e4edf8c 100644 --- a/trex_core/application/reconciliation.py +++ b/trex_core/application/reconciliation.py @@ -29,6 +29,7 @@ import hashlib import logging import math import re +from datetime import UTC, datetime from typing import cast from trex_core.application.alarm_emitter import AlarmEmitter @@ -407,6 +408,7 @@ def reconcile_core_holdings( Returns {key: 'error'|'qty_warning'|'ok'|'unmanaged_position'| 'managed_non_holding_divergence'} for telemetry. """ + snapshot_observed_at = datetime.now(tz=UTC) broker_qty: dict[str, int] = { item.symbol: int(item.hldg_qty) for item in broker.get_balance() } @@ -507,6 +509,7 @@ def reconcile_core_holdings( instance_key=instance_key, symbol=sym, quantity=bq, + observed_at=snapshot_observed_at, ) except Exception: # noqa: BLE001 _log.exception( @@ -555,6 +558,7 @@ def reconcile_core_holdings( for state in unmanaged_position_store.resolve_absent( instance_key=instance_key, observed_symbols=active_unmanaged_symbols, + observed_at=snapshot_observed_at, ): _log.info( "unmanaged position resolved: symbol=%s qty=%d instance_key=%s", From 3d46485a626341114767a8afbf1f220986ef3c59 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:06:08 +0000 Subject: [PATCH 21/21] Add AC-0030 unmanaged lifecycle criterion Co-authored-by: genichin <4282227+genichin@users.noreply.github.com> --- .proofline/criteria/ac-0030.md | 22 ++++++++++++++++++++++ .proofline/identities.json | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .proofline/criteria/ac-0030.md diff --git a/.proofline/criteria/ac-0030.md b/.proofline/criteria/ac-0030.md new file mode 100644 index 0000000..736ad6e --- /dev/null +++ b/.proofline/criteria/ac-0030.md @@ -0,0 +1,22 @@ +--- +id: "ac-0030" +status: active +--- + +# Unmanaged position durable lifecycle, authority fencing, and operator audit + +## Criterion + +`trex-core`는 broker 계정에 존재하지만 대상 instance의 managed slot/ledger가 소유하지 않는 종목을 `(instance_key, symbol)` 단위의 durable unmanaged-position lifecycle로 추적해야 한다. Lifecycle state는 instance 간에 섞이지 않아야 하며, 최소 `detected`, `quantity_changed`, `resolved`, `reopened` 전이와 `first_seen_at`, `last_seen_at`, `quantity_changed_at`, `resolved_at` 보존을 제공해야 한다. 같은 수량의 반복 authoritative 관측은 `last_seen_at`만 갱신하고 lifecycle audit/alarm을 다시 만들지 않아야 한다. + +absence resolution은 explicit authoritative full-balance snapshot에서만 허용되며 projected/partial/non-authoritative broker view 또는 broker read failure는 resolve authority가 아니다. observe와 resolve는 reconcile pass 전체에 전달된 동일 snapshot timestamp 또는 monotonic generation으로 fenced 되어야 하며 stale observe/stale resolve는 더 최신 state나 audit를 reopen, overwrite 또는 resolved 처리하지 못해야 한다. 최종 state와 append-only audit는 항상 가장 최신 authoritative snapshot이 결정해야 한다. + +운영자 disposition은 explicit evidence reference가 있는 명령으로만 기록되고 inspectable append-only audit trail에 남아야 한다. 이 기능은 broker order submit/cancel, slot 생성·상태 변경, synthetic ledger mutation 또는 warning 자동 삭제를 수행해서는 안 된다. + +## Verification + +- identity/lifecycle: 같은 symbol을 서로 다른 `instance_key`에서 독립 state로 저장하고 first detect, quantity change, resolved, resolved 뒤 reopened, first/last-seen 보존을 InMemory/PostgreSQL parity로 검증한다. +- authority boundary: missing marker, `False`, ownership/projected broker, partial reader, broker read failure에서는 absence resolution이 발생하지 않고 explicit authoritative full-balance snapshot에서만 resolved가 기록됨을 검증한다. +- temporal fencing: 더 오래된 absent snapshot이 더 최신 observe 뒤늦게 commit 되어도 resolved 되지 않고, 더 오래된 observe가 더 최신 resolved/observe를 reopen 또는 overwrite하지 않으며 final state/audit가 최신 authoritative snapshot 순서를 따른다는 PostgreSQL concurrency/interleaving 회귀를 검증한다. +- suppression/audit: 같은 수량 반복 관측은 duplicate lifecycle alarm/audit를 만들지 않고 `last_seen_at`만 갱신한다. operator `record`는 `evidence_ref`를 요구하고 `unmanaged audit`로 append-only history를 조회할 수 있어야 한다. +- safety boundary: unmanaged lifecycle path가 broker order, slot mutation, ledger write 또는 warning auto-clear를 수행하지 않음을 existing regression surface와 focused tests로 유지한다. diff --git a/.proofline/identities.json b/.proofline/identities.json index e90cfd1..eb60750 100644 --- a/.proofline/identities.json +++ b/.proofline/identities.json @@ -1,5 +1,5 @@ { "schema_version": 1, "next_line_number": 10, - "next_ac_number": 30 + "next_ac_number": 31 }