feat(ops): track broker balance worker failure lifecycle and backoff

Co-authored-by: genichin <4282227+genichin@users.noreply.github.com>

trex_core/adapters/broker/kis.py

@@ -54,7 +54,11 @@ def _balance_http_error_code(status: int | None) -> str:
         return "authentication_failed"
     if status == 403:
         return "authorization_failed"
-    if status in {408, 425, 429} or (status is not None and status >= 500):
+    if status in {408, 504}:
+        return "timeout"
+    if status is not None and status >= 500:
+        return "http_5xx"
+    if status in {425, 429}:
         return "transport_failed"
     return "broker_rejected"
 
@@ -1235,6 +1239,8 @@ class KISBroker:
             except httpx.HTTPStatusError as exc:
                 status = exc.response.status_code if exc.response is not None else None
                 raise BrokerBalanceError(_balance_http_error_code(status)) from None
+            except httpx.TimeoutException:
+                raise BrokerBalanceError("timeout") from None
             except httpx.HTTPError:
                 raise BrokerBalanceError("transport_failed") from None
             except (ValueError, TypeError):
@@ -1318,6 +1324,8 @@ class KISBroker:
         except httpx.HTTPStatusError as exc:
             status = exc.response.status_code if exc.response is not None else None
             raise BrokerBalanceError(_balance_http_error_code(status)) from None
+        except httpx.TimeoutException:
+            raise BrokerBalanceError("timeout") from None
         except httpx.HTTPError:
             raise BrokerBalanceError("transport_failed") from None
         except (KeyError, TypeError, ValueError):