fix(broker): cap outage backoff overflow and expand worker matrix coverage
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
5eef76d59e
commit
8b3e7ac753
@@ -1,6 +1,7 @@
|
||||
"""Broker-balance worker health tracking with bounded backoff."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import Lock
|
||||
@@ -64,6 +65,11 @@ class BalanceHealthTracker:
|
||||
self._max_backoff_sec = max(self._base_backoff_sec, float(max_backoff_sec))
|
||||
self._lock = Lock()
|
||||
self._states: dict[str, BalanceWorkerHealth] = {}
|
||||
if self._base_backoff_sec >= self._max_backoff_sec:
|
||||
self._max_backoff_exponent = 0
|
||||
else:
|
||||
ratio = self._max_backoff_sec / self._base_backoff_sec
|
||||
self._max_backoff_exponent = int(math.floor(math.log2(ratio)))
|
||||
|
||||
def _now(self) -> datetime:
|
||||
return self._clock()
|
||||
@@ -101,10 +107,14 @@ class BalanceHealthTracker:
|
||||
prev = self._get(worker)
|
||||
now = self._now()
|
||||
failures = prev.consecutive_failures + 1
|
||||
backoff = min(
|
||||
self._base_backoff_sec * (2 ** max(0, failures - 1)),
|
||||
self._max_backoff_sec,
|
||||
)
|
||||
exponent = max(0, failures - 1)
|
||||
if exponent > self._max_backoff_exponent:
|
||||
backoff = self._max_backoff_sec
|
||||
else:
|
||||
backoff = min(
|
||||
self._base_backoff_sec * (2.0 ** exponent),
|
||||
self._max_backoff_sec,
|
||||
)
|
||||
self._states[worker] = BalanceWorkerHealth(
|
||||
worker=worker,
|
||||
last_success_at=prev.last_success_at,
|
||||
|
||||
Reference in New Issue
Block a user