mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
71 lines
2.9 KiB
Python
71 lines
2.9 KiB
Python
"""Shared transient-failure retry for the HTTP source clients.
|
|
|
|
The retry *mechanism* is generic; each client owns the *policy* of what counts as
|
|
transient. A caller signals "retry this" by raising ``TransientHttpError``
|
|
(carrying any server-advised ``retry_after``); transport-level errors (read /
|
|
connect timeouts, connection resets) are always treated as transient.
|
|
``call_with_retry`` backs off exponentially between attempts — honouring
|
|
``retry_after`` when present — and re-raises the last error once attempts are
|
|
exhausted, leaving the caller to decide how to surface it (the EPC client lets it
|
|
propagate; postcodes.io soft-fails to the seed postcode).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import random
|
|
import time
|
|
from typing import Callable, Optional, TypeVar
|
|
|
|
import httpx
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
class TransientHttpError(Exception):
|
|
"""A failure worth retrying. ``retry_after`` is the server-advised delay (a
|
|
429's ``Retry-After``), used in place of the computed backoff when present."""
|
|
|
|
def __init__(self, message: str, retry_after: Optional[float] = None) -> None:
|
|
super().__init__(message)
|
|
self.retry_after = retry_after
|
|
|
|
|
|
def call_with_retry(
|
|
fn: Callable[[], T],
|
|
max_retries: int = 5,
|
|
backoff_base: float = 1.0,
|
|
backoff_multiplier: float = 2.0,
|
|
max_backoff: float = 60.0,
|
|
jitter: bool = False,
|
|
) -> T:
|
|
"""Retry ``fn`` on transient failures — ``TransientHttpError`` (e.g. a 429)
|
|
and ``httpx.TransportError`` (read/connect timeouts, connection resets) —
|
|
backing off exponentially, or by the error's ``retry_after`` when it carries
|
|
one. Non-transient failures propagate immediately; the last transient error
|
|
is re-raised once ``max_retries`` is exhausted.
|
|
|
|
``jitter`` applies **full jitter** to the *computed* backoff (a server-advised
|
|
``retry_after`` is honoured exactly): the delay becomes a uniform random draw
|
|
in ``[0, computed_backoff]``. This de-synchronises retries across many
|
|
concurrent callers so they don't re-collide in lockstep — the "synchronised
|
|
requests look like a DDoS" failure mode Google's Solar API best-practices warn
|
|
about, and the cause of the 429 storm under 32 concurrent containers."""
|
|
last_exc: Optional[Exception] = None
|
|
for attempt in range(max_retries + 1):
|
|
try:
|
|
return fn()
|
|
except (TransientHttpError, httpx.TransportError) as exc:
|
|
last_exc = exc
|
|
if attempt < max_retries:
|
|
retry_after = (
|
|
exc.retry_after if isinstance(exc, TransientHttpError) else None
|
|
)
|
|
if retry_after is not None:
|
|
delay = min(retry_after, max_backoff)
|
|
else:
|
|
delay = min(backoff_base * (backoff_multiplier**attempt), max_backoff)
|
|
if jitter:
|
|
delay = random.uniform(0.0, delay)
|
|
time.sleep(delay)
|
|
assert last_exc is not None
|
|
raise last_exc
|