Merge remote-tracking branch 'origin/main' into copilot/feat-reconcile-unmanaged-position-durable-lifecycl

# Conflicts:
#	.proofline/criteria/ac-0029.md
#	.proofline/identities.json
#	tests/adapters/persistence/test_migration_and_drift.py

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

tests/adapters/persistence/test_migration_and_drift.py

@@ -30,8 +30,8 @@ _EXPECTED_TABLES = {
     "ledger_skip",             # GitHub #68 skipped-ledger durable audit
     "reconcile_notification_state",  # req-0007 / ac-0024
     "slot_strategy_state",  # req-0009 / ac-0027
-    "unmanaged_position_state",  # req-0010 / ac-0029
-    "unmanaged_position_audit",  # req-0010 / ac-0029
+    "unmanaged_position_state",  # GitHub #85 unmanaged lifecycle state
+    "unmanaged_position_audit",  # GitHub #85 unmanaged lifecycle audit
 }
 
 
@@ -131,7 +131,7 @@ def test_migration_replay_converts_existing_auto_venue_row_to_sor(pg_clean):
 
 
 def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="test-instance")
     normal = LedgerTrade(
         asset_id="PF-NORMAL",
         order_id="O-NORMAL",
@@ -191,3 +191,33 @@ def test_migration_024_separates_legacy_sor_and_is_replay_safe(pg_clean):
             "UPDATE ledger_outbox SET venue='SOR' WHERE correlation_id=%s",
             (normal.correlation_id,),
         )
+
+
+def test_migration_029_fail_closes_unowned_legacy_outbox_rows(pg_clean):
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
+    trade = LedgerTrade(
+        asset_id="PF-LEGACY",
+        order_id="O-LEGACY",
+        execution_id="E-LEGACY",
+        side="buy",
+        qty=1,
+        price=1000,
+        executed_at=datetime(2026, 9, 12, tzinfo=UTC),
+    )
+    outbox.enqueue(trade)
+
+    with psycopg.connect(pg_clean) as conn:
+        with conn.cursor() as cur:
+            cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP NOT NULL")
+            cur.execute("ALTER TABLE ledger_outbox ALTER COLUMN instance_key DROP DEFAULT")
+            cur.execute(
+                "UPDATE ledger_outbox SET instance_key=NULL WHERE correlation_id=%s",
+                (trade.correlation_id,),
+            )
+        conn.commit()
+
+    apply_migrations(pg_clean)
+    stored = outbox.get(trade.correlation_id)
+    assert stored is not None
+    assert stored.instance_key == "__unowned__"
+    assert outbox.list_pending() == []

tests/adapters/persistence/test_postgresql_ledger_outbox.py

@@ -4,43 +4,53 @@ Skipped without TREX_POSTGRESQL_TEST_DSN (same gate as other PG suites).
 """
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+
+import pytest
 
 from trex_core.adapters.ledger.in_memory import InMemoryLedger
 from trex_core.adapters.persistence.postgresql import PostgreSQLLedgerOutbox
 from trex_core.application.ledger_outbox import (
     LedgerWriter,
+    OutboxClaimLostError,
     OutboxOutcome,
     WorkItemStatus,
 )
-from trex_core.application.ledger_port import LedgerTrade
+from trex_core.application.ledger_port import (
+    LedgerRetryableError,
+    LedgerTrade,
+    LedgerWriteOutcome,
+    LedgerWriteResult,
+)
 
 
 def _trade(execution_id="paper:1:10") -> LedgerTrade:
     return LedgerTrade(
         asset_id="PF-1", order_id="ORD-1", execution_id=execution_id,
         side="buy", qty=10, price=1000.0, fee=1.5, tax=0.0,
-        executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc),
+        executed_at=datetime(2026, 6, 4, tzinfo=UTC),
         venue="NXT", order_venue="SOR",
     )
 
 
 def test_enqueue_persists_and_is_idempotent(pg_clean):
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     a = outbox.enqueue(_trade())
     b = outbox.enqueue(_trade())
     assert a.correlation_id == b.correlation_id
     pending = outbox.list_pending()
     assert len(pending) == 1
     assert pending[0].status is WorkItemStatus.PENDING
+    assert pending[0].instance_key == "inst-a"
     assert pending[0].trade.venue == "NXT"
     assert pending[0].trade.order_venue == "SOR"
 
 
 def test_writer_completes_persisted_item(pg_clean):
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     ledger = InMemoryLedger()
-    writer = LedgerWriter(outbox, ledger)
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
     outbox.enqueue(_trade(execution_id="pg-1"))
     stats = writer.run_once()
     assert stats.completed == 1 and ledger.count() == 1
@@ -50,9 +60,9 @@ def test_writer_completes_persisted_item(pg_clean):
 
 
 def test_writer_marks_conflict_blocked_on_pg(pg_clean):
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     ledger = InMemoryLedger(conflict_executions={"pg-x"})
-    writer = LedgerWriter(outbox, ledger)
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
     outbox.enqueue(_trade(execution_id="pg-x"))
     writer.run_once()
     row = outbox.get("PF-1:ORD-1:pg-x")
@@ -61,10 +71,10 @@ def test_writer_marks_conflict_blocked_on_pg(pg_clean):
 
 
 def test_writer_retries_on_pg(pg_clean):
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     ledger = InMemoryLedger(fail_n_then_succeed=1)
     # dcy-015: base 0 => next_retry_at == now, so the row is immediately due again.
-    writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0)
+    writer = LedgerWriter(outbox, ledger, retry_base_sec=0.0, instance_key="inst-a")
     outbox.enqueue(_trade(execution_id="pg-r"))
     writer.run_once()
     row1 = outbox.get("PF-1:ORD-1:pg-r")
@@ -79,9 +89,9 @@ def test_writer_retries_on_pg(pg_clean):
 def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
     """dcy-015 / REQ-019: 404/permanent → terminal delivery_failed persisted
     (migration 007 status enum + last_outcome). No retry."""
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     ledger = InMemoryLedger(permanent_executions={"pg-perm"})
-    writer = LedgerWriter(outbox, ledger)
+    writer = LedgerWriter(outbox, ledger, instance_key="inst-a")
     outbox.enqueue(_trade(execution_id="pg-perm"))
     s = writer.run_once()
     assert s.delivery_failed == 1
@@ -94,22 +104,199 @@ def test_writer_marks_delivery_failed_permanent_on_pg(pg_clean):
 
 def test_pending_cash_delta_on_pg(pg_clean):
     """dcy-018 / REQ-029: estimated cash delta of pending rows summed by asset (SQL)."""
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     outbox.enqueue(LedgerTrade(asset_id="PF-1", order_id="O-1", execution_id="cd-buy",
                                side="buy", qty=10, price=1000.0, fee=1.0, tax=0.0,
-                               executed_at=datetime(2026, 6, 4, tzinfo=timezone.utc)))
+                               executed_at=datetime(2026, 6, 4, tzinfo=UTC)))
     deltas = outbox.pending_cash_delta()
     assert deltas["PF-1"] == -(10 * 1000.0 + 1.0)  # buy => cash down by notional+fee
 
 
 def test_backoff_row_skipped_by_pending_scan_on_pg(pg_clean):
     """dcy-015 / REQ-019: a future next_retry_at excludes the row from list_pending."""
-    outbox = PostgreSQLLedgerOutbox(pg_clean)
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
     ledger = InMemoryLedger(fail_n_then_succeed=99)
     # large base => next_retry_at well in the future
-    writer = LedgerWriter(outbox, ledger, retry_base_sec=600.0, max_attempts=99)
+    writer = LedgerWriter(
+        outbox, ledger, retry_base_sec=600.0, max_attempts=99, instance_key="inst-a"
+    )
     outbox.enqueue(_trade(execution_id="pg-bo"))
     writer.run_once()  # -> retryable, next_retry_at ~ now + 600s
     row = outbox.get("PF-1:ORD-1:pg-bo")
     assert row.status is WorkItemStatus.PENDING and row.next_retry_at is not None
     assert outbox.list_pending() == []  # still backing off -> skipped
+
+
+def test_pending_claims_are_isolated_per_instance(pg_clean):
+    ledger = InMemoryLedger()
+    outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
+    outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b")
+    writer_a = LedgerWriter(outbox_a, ledger, instance_key="inst-a")
+    writer_b = LedgerWriter(outbox_b, ledger, instance_key="inst-b")
+    row_a = outbox_a.enqueue(_trade(execution_id="pg-a"))
+    row_b = outbox_b.enqueue(_trade(execution_id="pg-b"))
+
+    stats_a = writer_a.run_once()
+    assert stats_a.completed == 1 and ledger.count() == 1
+    assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.COMPLETED
+    assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.PENDING
+
+    stats_b = writer_b.run_once()
+    assert stats_b.completed == 1 and ledger.count() == 2
+    assert outbox_b.get(row_b.correlation_id).status is WorkItemStatus.COMPLETED
+
+
+def test_claim_expiry_allows_safe_recovery_after_worker_crash(pg_clean):
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+    outbox_a = PostgreSQLLedgerOutbox(
+        pg_clean,
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        worker_id="worker-a",
+        clock=lambda: now["t"],
+    )
+    outbox_b = PostgreSQLLedgerOutbox(
+        pg_clean,
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        worker_id="worker-b",
+        clock=lambda: now["t"],
+    )
+    row = outbox_a.enqueue(_trade(execution_id="pg-crash"))
+
+    claimed_a = outbox_a.claim_pending()
+    assert [item.correlation_id for item in claimed_a] == [row.correlation_id]
+    assert outbox_b.claim_pending() == []
+
+    now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
+    claimed_b = outbox_b.claim_pending()
+    assert [item.correlation_id for item in claimed_b] == [row.correlation_id]
+
+
+def test_stale_claim_cannot_overwrite_new_owner_outcome(pg_clean):
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+    outbox_a = PostgreSQLLedgerOutbox(
+        pg_clean,
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        worker_id="worker-a",
+        clock=lambda: now["t"],
+    )
+    outbox_b = PostgreSQLLedgerOutbox(
+        pg_clean,
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        worker_id="worker-b",
+        clock=lambda: now["t"],
+    )
+    row = outbox_a.enqueue(_trade(execution_id="pg-stale"))
+
+    [claim_a] = outbox_a.claim_pending()
+    now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
+    [claim_b] = outbox_b.claim_pending()
+    outbox_b.mark_outcome(
+        row.correlation_id,
+        OutboxOutcome.COMPLETED,
+        expected_claim_expires_at=claim_b.claim_expires_at,
+    )
+
+    with pytest.raises(OutboxClaimLostError):
+        outbox_a.mark_outcome(
+            row.correlation_id,
+            OutboxOutcome.RETRYABLE_FAILURE,
+            error="stale owner",
+            next_retry_at=now["t"],
+            expected_claim_expires_at=claim_a.claim_expires_at,
+        )
+
+    stored = outbox_a.get(row.correlation_id)
+    assert stored is not None
+    assert stored.status is WorkItemStatus.COMPLETED
+    assert stored.last_outcome is OutboxOutcome.COMPLETED
+
+
+@pytest.mark.parametrize("instance_key", ["", "   "])
+def test_postgresql_outbox_rejects_empty_instance_before_connect(pg_clean, instance_key):
+    with pytest.raises(ValueError, match="non-empty"):
+        PostgreSQLLedgerOutbox(pg_clean, instance_key=instance_key)
+
+
+def test_original_owner_can_finalize_expired_unreclaimed_claim(pg_clean):
+    now = {"t": datetime(2026, 6, 4, tzinfo=UTC)}
+    outbox = PostgreSQLLedgerOutbox(
+        pg_clean,
+        instance_key="inst-a",
+        claim_ttl_sec=5.0,
+        worker_id="worker-a",
+        clock=lambda: now["t"],
+    )
+    row = outbox.enqueue(_trade(execution_id="pg-expired"))
+
+    [claim] = outbox.claim_pending()
+    now["t"] = datetime(2026, 6, 4, 0, 0, 6, tzinfo=UTC)
+    stored = outbox.mark_outcome(
+        row.correlation_id,
+        OutboxOutcome.COMPLETED,
+        expected_claim_expires_at=claim.claim_expires_at,
+    )
+
+    assert stored.status is WorkItemStatus.COMPLETED
+    assert stored.last_outcome is OutboxOutcome.COMPLETED
+
+
+def test_writer_records_retryable_403_without_cross_instance_delivery(pg_clean):
+    class _ForbiddenLedger:
+        def __init__(self):
+            self.calls: list[str] = []
+
+        def record_trade(self, trade):
+            self.calls.append(trade.execution_id)
+            raise LedgerRetryableError("portfolio-hub 403 Forbidden")
+
+    outbox_a = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
+    outbox_b = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-b")
+    writer_b = LedgerWriter(
+        outbox_b, _ForbiddenLedger(), retry_base_sec=0.0, instance_key="inst-b"
+    )
+    row_a = outbox_a.enqueue(_trade(execution_id="pg-403-a"))
+    row_b = outbox_b.enqueue(_trade(execution_id="pg-403-b"))
+
+    stats_b = writer_b.run_once()
+
+    assert stats_b.retryable == 1
+    assert outbox_b.get(row_b.correlation_id).last_outcome is OutboxOutcome.RETRYABLE_FAILURE
+    assert outbox_a.get(row_a.correlation_id).status is WorkItemStatus.PENDING
+
+
+def test_concurrent_workers_claim_each_row_at_most_once(pg_clean):
+    outbox_a = PostgreSQLLedgerOutbox(
+        pg_clean, instance_key="inst-a", worker_id="worker-a"
+    )
+    outbox_b = PostgreSQLLedgerOutbox(
+        pg_clean, instance_key="inst-a", worker_id="worker-b"
+    )
+    row = outbox_a.enqueue(_trade(execution_id="pg-concurrent"))
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        claims = list(pool.map(lambda box: box.claim_pending(), (outbox_a, outbox_b)))
+
+    claimed_ids = [item.correlation_id for batch in claims for item in batch]
+    assert claimed_ids == [row.correlation_id]
+
+
+def test_writer_marks_duplicate_completed_from_adapter_response_on_pg(pg_clean):
+    class _DuplicateLedger:
+        def record_trade(self, _trade):
+            return LedgerWriteOutcome(LedgerWriteResult.DUPLICATE_OK)
+
+    outbox = PostgreSQLLedgerOutbox(pg_clean, instance_key="inst-a")
+    writer = LedgerWriter(outbox, _DuplicateLedger(), instance_key="inst-a")
+    row = outbox.enqueue(_trade(execution_id="pg-dup"))
+
+    outcome = writer.process_correlation_id(row.correlation_id)
+
+    assert outcome is OutboxOutcome.DUPLICATE_COMPLETED
+    stored = outbox.get(row.correlation_id)
+    assert stored is not None
+    assert stored.status is WorkItemStatus.COMPLETED
+    assert stored.last_outcome is OutboxOutcome.DUPLICATE_COMPLETED

tests/adapters/portfolio/test_reconcile_warning_store.py

@@ -0,0 +1,42 @@
+"""PostgreSQLReconcileWarningStore integration tests."""
+from __future__ import annotations
+
+from trex_core.adapters.portfolio.reconcile_warning_store import (
+    PostgreSQLReconcileWarningStore,
+)
+
+
+def test_warning_store_preserves_multiple_cases_for_same_slot(pg_clean):
+    store = PostgreSQLReconcileWarningStore(pg_clean)
+    store.set_warned("slot-1", portfolio_id="pf-A", case_code="B", reason="b")
+    store.set_warned("slot-1", portfolio_id="pf-A", case_code="C", reason="c")
+
+    assert store.is_warned("slot-1") is True
+    info = store.get_warning_info("slot-1")
+    assert info is not None
+    assert info["case_codes"] == ["B", "C"]
+
+
+def test_warning_store_case_specific_clear_and_full_clear(pg_clean):
+    store = PostgreSQLReconcileWarningStore(pg_clean)
+    store.set_warned("slot-2", portfolio_id="pf-A", case_code="B")
+    store.set_warned("slot-2", portfolio_id="pf-A", case_code="C")
+
+    store.clear_warned("slot-2", case_code="C")
+    info = store.get_warning_info("slot-2")
+    assert info is not None
+    assert info["case_codes"] == ["B"]
+
+    store.clear_warned("slot-2")
+    assert store.is_warned("slot-2") is False
+
+
+def test_warning_store_is_durable_across_reinstantiation(pg_clean):
+    a = PostgreSQLReconcileWarningStore(pg_clean)
+    a.set_warned("slot-restart", portfolio_id="pf-A", case_code="B")
+    a.set_warned("slot-restart", portfolio_id="pf-A", case_code="C")
+
+    b = PostgreSQLReconcileWarningStore(pg_clean)
+    info = b.get_warning_info("slot-restart")
+    assert info is not None
+    assert info["case_codes"] == ["B", "C"]