mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
de7fb94ff7
Closes out the cohort-broadening work with its decision record and consolidates the retry plumbing. ADR-0034 documents broadening the EPC-Prediction cohort to the real unit postcodes nearest the target (via postcodes.io) when its own postcode holds no same-type comparable — extending ADR-0031 decision 5. Records why postcodes.io was chosen over council[] (whole-LA, no property_type in rows), a bulk Code-Point Open / ONSPD dataset, and the OS Places radius API, and the lazy / nearest-first early-stop / soft-fail policy. Broadening-specific docstrings now cite 0034. Retry consolidation: extract the EPC client's call_with_retry into a shared infrastructure/http_retry.py keyed off a generic TransientHttpError marker, so the mechanism (exponential backoff, Retry-After) is shared while each client keeps its own transient policy. EpcRateLimitError now subclasses TransientHttpError (still an EpcApiError); PostcodesIoClient routes through the same helper, raising TransientHttpError on 429/5xx and soft-failing to the seed once exhausted (the EPC client propagates instead). Direct tests for the shared helper; EPC + postcodes.io suites repointed at the shared sleep. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
60 lines
2.3 KiB
Python
60 lines
2.3 KiB
Python
"""Shared transient-failure retry for the HTTP source clients.
|
|
|
|
The retry *mechanism* is generic; each client owns the *policy* of what counts as
|
|
transient. A caller signals "retry this" by raising ``TransientHttpError``
|
|
(carrying any server-advised ``retry_after``); transport-level errors (read /
|
|
connect timeouts, connection resets) are always treated as transient.
|
|
``call_with_retry`` backs off exponentially between attempts — honouring
|
|
``retry_after`` when present — and re-raises the last error once attempts are
|
|
exhausted, leaving the caller to decide how to surface it (the EPC client lets it
|
|
propagate; postcodes.io soft-fails to the seed postcode).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Callable, Optional, TypeVar
|
|
|
|
import httpx
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
class TransientHttpError(Exception):
|
|
"""A failure worth retrying. ``retry_after`` is the server-advised delay (a
|
|
429's ``Retry-After``), used in place of the computed backoff when present."""
|
|
|
|
def __init__(self, message: str, retry_after: Optional[float] = None) -> None:
|
|
super().__init__(message)
|
|
self.retry_after = retry_after
|
|
|
|
|
|
def call_with_retry(
|
|
fn: Callable[[], T],
|
|
max_retries: int = 5,
|
|
backoff_base: float = 1.0,
|
|
backoff_multiplier: float = 2.0,
|
|
max_backoff: float = 60.0,
|
|
) -> T:
|
|
"""Retry ``fn`` on transient failures — ``TransientHttpError`` (e.g. a 429)
|
|
and ``httpx.TransportError`` (read/connect timeouts, connection resets) —
|
|
backing off exponentially, or by the error's ``retry_after`` when it carries
|
|
one. Non-transient failures propagate immediately; the last transient error
|
|
is re-raised once ``max_retries`` is exhausted."""
|
|
last_exc: Optional[Exception] = None
|
|
for attempt in range(max_retries + 1):
|
|
try:
|
|
return fn()
|
|
except (TransientHttpError, httpx.TransportError) as exc:
|
|
last_exc = exc
|
|
if attempt < max_retries:
|
|
retry_after = (
|
|
exc.retry_after if isinstance(exc, TransientHttpError) else None
|
|
)
|
|
if retry_after is not None:
|
|
delay = retry_after
|
|
else:
|
|
delay = backoff_base * (backoff_multiplier**attempt)
|
|
time.sleep(min(delay, max_backoff))
|
|
assert last_exc is not None
|
|
raise last_exc
|