fix: fence ledger outbox claims by instance

trex_core/adapters/persistence/postgresql.py

@@ -21,6 +21,7 @@ from uuid import UUID, uuid4
 
 from trex_core.application.ledger_outbox import (
     LedgerWorkItem,
+    OutboxClaimLostError,
     OutboxOutcome,
     WorkItemStatus,
 )
@@ -569,6 +570,9 @@ _OUTBOX_COLUMNS = (
     "created_at, updated_at, next_retry_at, venue, order_venue, instance_key, "
     "claim_expires_at"
 )
+_OUTBOX_RETURN_COLUMNS = ", ".join(
+    f"o.{column.strip()}" for column in _OUTBOX_COLUMNS.split(",")
+)
 
 
 class PostgreSQLLedgerOutbox:
@@ -578,12 +582,16 @@ class PostgreSQLLedgerOutbox:
         self,
         dsn: str,
         *,
-        instance_key: str = "default",
+        instance_key: str,
         claim_ttl_sec: float = 60.0,
         worker_id: str | None = None,
         clock=None,
         psycopg_module: Any = None,
     ) -> None:
+        if not isinstance(instance_key, str) or not instance_key.strip():
+            raise ValueError("instance_key must be a non-empty string")
+        if claim_ttl_sec <= 0:
+            raise ValueError("claim_ttl_sec must be positive")
         self._psycopg = _require_psycopg(psycopg_module)
         self._dsn = dsn
         self._instance_key = instance_key
@@ -594,6 +602,10 @@ class PostgreSQLLedgerOutbox:
     def _now(self) -> datetime:
         return self._clock()
 
+    @property
+    def instance_key(self) -> str:
+        return self._instance_key
+
     def enqueue(self, trade: LedgerTrade) -> LedgerWorkItem:
         with self._psycopg.connect(self._dsn, autocommit=False) as conn:
             with conn.cursor() as cur:
@@ -625,16 +637,17 @@ class PostgreSQLLedgerOutbox:
         return existing
 
     def list_pending(self, limit: int = 32) -> list[LedgerWorkItem]:
+        now = self._now()
         with self._psycopg.connect(self._dsn) as conn:
             with conn.cursor() as cur:
                 cur.execute(
                     f"SELECT {_OUTBOX_COLUMNS} FROM ledger_outbox "
                     "WHERE status = 'pending' "
                     "AND instance_key = %s "
-                    "AND (next_retry_at IS NULL OR next_retry_at <= now()) "
-                    "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) "
+                    "AND (next_retry_at IS NULL OR next_retry_at <= %s) "
+                    "AND (claim_expires_at IS NULL OR claim_expires_at <= %s) "
                     "ORDER BY created_at, correlation_id LIMIT %s",
-                    (self._instance_key, limit),
+                    (self._instance_key, now, now, limit),
                 )
                 return [self._row_to_item(r) for r in cur.fetchall()]
 
@@ -647,7 +660,7 @@ class PostgreSQLLedgerOutbox:
         now = self._now()
         lease_until = now + timedelta(seconds=self._claim_ttl_sec)
         correlation_sql = ""
-        params: list[Any] = [self._instance_key]
+        params: list[Any] = [self._instance_key, now, now]
         if correlation_id is not None:
             correlation_sql = "AND correlation_id = %s "
             params.append(correlation_id)
@@ -659,8 +672,8 @@ class PostgreSQLLedgerOutbox:
                     "SELECT correlation_id FROM ledger_outbox "
                     "WHERE status = 'pending' "
                     "AND instance_key = %s "
-                    "AND (next_retry_at IS NULL OR next_retry_at <= now()) "
-                    "AND (claim_expires_at IS NULL OR claim_expires_at <= now()) "
+                    "AND (next_retry_at IS NULL OR next_retry_at <= %s) "
+                    "AND (claim_expires_at IS NULL OR claim_expires_at <= %s) "
                     f"{correlation_sql}"
                     "ORDER BY created_at, correlation_id "
                     "LIMIT %s "
@@ -670,7 +683,7 @@ class PostgreSQLLedgerOutbox:
                     "SET claim_owner = %s, claim_expires_at = %s, updated_at = %s "
                     "FROM candidates "
                     "WHERE o.correlation_id = candidates.correlation_id "
-                    f"RETURNING {_OUTBOX_COLUMNS}",
+                    f"RETURNING {_OUTBOX_RETURN_COLUMNS}",
                     params,
                 )
                 rows = [self._row_to_item(r) for r in cur.fetchall()]
@@ -759,6 +772,8 @@ class PostgreSQLLedgerOutbox:
                     f"{where}",
                     params,
                 )
+                if cur.rowcount != 1:
+                    raise OutboxClaimLostError(correlation_id)
                 conn.commit()
         row = self.get(correlation_id)
         assert row is not None