fix(broker): cap outage backoff overflow and expand worker matrix coverage

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-09-13 06:40:06 +00:00
committed by GitHub
co-authored by genichin
parent 5eef76d59e
commit 8b3e7ac753
5 changed files with 275 additions and 6 deletions
+14 -4
View File
@@ -1,6 +1,7 @@
"""Broker-balance worker health tracking with bounded backoff."""
from __future__ import annotations
import math
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from threading import Lock
@@ -64,6 +65,11 @@ class BalanceHealthTracker:
self._max_backoff_sec = max(self._base_backoff_sec, float(max_backoff_sec))
self._lock = Lock()
self._states: dict[str, BalanceWorkerHealth] = {}
if self._base_backoff_sec >= self._max_backoff_sec:
self._max_backoff_exponent = 0
else:
ratio = self._max_backoff_sec / self._base_backoff_sec
self._max_backoff_exponent = int(math.floor(math.log2(ratio)))
def _now(self) -> datetime:
return self._clock()
@@ -101,10 +107,14 @@ class BalanceHealthTracker:
prev = self._get(worker)
now = self._now()
failures = prev.consecutive_failures + 1
backoff = min(
self._base_backoff_sec * (2 ** max(0, failures - 1)),
self._max_backoff_sec,
)
exponent = max(0, failures - 1)
if exponent > self._max_backoff_exponent:
backoff = self._max_backoff_sec
else:
backoff = min(
self._base_backoff_sec * (2.0 ** exponent),
self._max_backoff_sec,
)
self._states[worker] = BalanceWorkerHealth(
worker=worker,
last_success_at=prev.last_success_at,