fix(broker): cap outage backoff overflow and expand worker matrix coverage
Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>
This commit is contained in:
co-authored by
genichin
parent
5eef76d59e
commit
8b3e7ac753
@@ -23,6 +23,28 @@ def test_long_outage_backoff_is_capped_and_status_stays_valid():
|
||||
assert state["failure_code"] == "http_5xx"
|
||||
|
||||
|
||||
def test_long_outage_over_1100_failures_never_overflows_and_stays_capped():
|
||||
now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
|
||||
tracker = BalanceHealthTracker(clock=lambda: now["t"], base_backoff_sec=15, max_backoff_sec=300)
|
||||
|
||||
last_lifecycle = None
|
||||
for i in range(1_100):
|
||||
lifecycle, backoff = tracker.record_failure("execution", code="http_5xx")
|
||||
last_lifecycle = lifecycle
|
||||
if i >= 5:
|
||||
assert backoff == 300.0
|
||||
else:
|
||||
assert backoff <= 300.0
|
||||
now["t"] += timedelta(seconds=301)
|
||||
|
||||
state = tracker.status("execution", stale_after_sec=600)
|
||||
assert state["state"] == "unavailable"
|
||||
assert state["consecutive_failures"] == 1_100
|
||||
assert state["failure_code"] == "http_5xx"
|
||||
assert state["next_retry_at"] is not None
|
||||
assert last_lifecycle in {None, "ongoing"}
|
||||
|
||||
|
||||
def test_restart_resets_process_local_health_and_re_emits_detected():
|
||||
now = {"t": datetime(2026, 9, 12, tzinfo=UTC)}
|
||||
before_restart = BalanceHealthTracker(clock=lambda: now["t"])
|
||||
|
||||
Reference in New Issue
Block a user