Model/tests/infrastructure/test_http_retry.py
Khalim Conn-Kowlessar 88ed0c2e88 Add opt-in full-jitter backoff to de-synchronise concurrent retries 🟩
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-25 10:45:00 +00:00

166 lines
4.5 KiB
Python

"""call_with_retry — the shared transient-failure retry both HTTP source clients
use. Generic mechanism (exponential backoff, Retry-After), per-client policy
(what gets raised as transient)."""
from __future__ import annotations
from typing import Iterator
from unittest.mock import MagicMock, call, patch
import httpx
import pytest
from infrastructure.http_retry import TransientHttpError, call_with_retry
@pytest.fixture(autouse=True)
def _no_sleep() -> Iterator[MagicMock]:
with patch("infrastructure.http_retry.time.sleep") as sleep:
yield sleep
def test_returns_immediately_on_success(_no_sleep: MagicMock) -> None:
# Act
result = call_with_retry(lambda: 42)
# Assert
assert result == 42
_no_sleep.assert_not_called()
def test_retries_a_transient_error_then_succeeds(_no_sleep: MagicMock) -> None:
# Arrange — fail twice transiently, then succeed.
calls = {"n": 0}
def fn() -> str:
calls["n"] += 1
if calls["n"] < 3:
raise TransientHttpError("429")
return "ok"
# Act
result = call_with_retry(fn)
# Assert — exponential backoff between the three attempts.
assert result == "ok"
assert _no_sleep.call_args_list == [call(1.0), call(2.0)]
def test_honours_retry_after_over_the_computed_backoff(_no_sleep: MagicMock) -> None:
# Arrange
raised = {"done": False}
def fn() -> str:
if not raised["done"]:
raised["done"] = True
raise TransientHttpError("429", retry_after=7.0)
return "ok"
# Act
call_with_retry(fn)
# Assert
_no_sleep.assert_called_once_with(7.0)
def test_retries_transport_errors(_no_sleep: MagicMock) -> None:
# Arrange
attempts = {"n": 0}
def fn() -> str:
attempts["n"] += 1
if attempts["n"] == 1:
raise httpx.ReadTimeout("slow")
return "ok"
# Act
result = call_with_retry(fn)
# Assert
assert result == "ok"
assert attempts["n"] == 2
def test_reraises_the_last_transient_error_once_exhausted(
_no_sleep: MagicMock,
) -> None:
# Arrange — always transient.
def fn() -> str:
raise TransientHttpError("persistent 429")
# Act / Assert
with pytest.raises(TransientHttpError, match="persistent 429"):
call_with_retry(fn, max_retries=2)
assert _no_sleep.call_count == 2
def test_non_transient_error_propagates_without_retry(_no_sleep: MagicMock) -> None:
# Arrange
attempts = {"n": 0}
def fn() -> str:
attempts["n"] += 1
raise ValueError("not transient")
# Act / Assert
with pytest.raises(ValueError, match="not transient"):
call_with_retry(fn)
assert attempts["n"] == 1
_no_sleep.assert_not_called()
def test_jitter_full_randomizes_the_computed_backoff(_no_sleep: MagicMock) -> None:
# Arrange — fail once transiently (no server Retry-After), then succeed.
calls = {"n": 0}
def fn() -> str:
calls["n"] += 1
if calls["n"] < 2:
raise TransientHttpError("429")
return "ok"
# Act — full jitter draws the sleep uniformly from [0, computed_backoff].
with patch("infrastructure.http_retry.random.uniform", return_value=0.4) as runi:
result = call_with_retry(fn, jitter=True)
# Assert — the first computed backoff is 1.0; jitter draws over [0, 1.0]
# and the drawn value is what we sleep for (de-syncs concurrent retries).
assert result == "ok"
runi.assert_called_once_with(0.0, 1.0)
_no_sleep.assert_called_once_with(0.4)
def test_jitter_does_not_apply_to_a_server_retry_after(_no_sleep: MagicMock) -> None:
# Arrange — the server advised an exact Retry-After; jitter must not perturb it.
raised = {"done": False}
def fn() -> str:
if not raised["done"]:
raised["done"] = True
raise TransientHttpError("429", retry_after=7.0)
return "ok"
# Act
with patch("infrastructure.http_retry.random.uniform") as runi:
call_with_retry(fn, jitter=True)
# Assert — Retry-After is honoured verbatim, no random draw.
runi.assert_not_called()
_no_sleep.assert_called_once_with(7.0)
def test_retry_after_is_capped_by_max_backoff(_no_sleep: MagicMock) -> None:
# Arrange
raised = {"done": False}
def fn() -> str:
if not raised["done"]:
raised["done"] = True
raise TransientHttpError("429", retry_after=999.0)
return "ok"
# Act
call_with_retry(fn, max_backoff=60.0)
# Assert
_no_sleep.assert_called_once_with(60.0)