311 lines
10 KiB
Python
311 lines
10 KiB
Python
"""Retrying a transient failure without making an outage worse.
|
|
|
|
The engine had exactly one retry in it, and it was the wrong shape
|
|
(``ocr/mistral_hook.py``): two attempts back to back with no sleep, catching
|
|
every ``HTTPError`` alike. That retries a 401 — which will never succeed —
|
|
and, worse, retries a **429 instantly**, adding load to a service that has just
|
|
said it has too much. It then discarded the error, so a total provider outage
|
|
looked like a document with no text on it.
|
|
|
|
Three rules, and each is here because its absence causes a specific failure:
|
|
|
|
**Classify before retrying.** A 400 or a 401 is the caller's fault and will
|
|
fail identically forever; retrying costs latency and buys nothing. A 429, a
|
|
5xx, a reset connection or a timeout may well succeed on the next attempt.
|
|
Anything not known to be retryable is treated as fatal, so a new error class
|
|
fails fast rather than being hammered.
|
|
|
|
**Back off exponentially, and jitter.** Without backoff, N clients retrying a
|
|
recovering service re-saturate it the moment it comes back. Without jitter they
|
|
do it *in lockstep* — the thundering herd — because they all failed at the same
|
|
instant and all wait the same interval. Full jitter (sleep uniformly in
|
|
``[0, computed]``) is the variant that spreads them, and it is what AWS's
|
|
architecture guidance recommends.
|
|
|
|
**Obey ``Retry-After``.** When a server states how long to wait, guessing is
|
|
strictly worse than listening.
|
|
|
|
The delay budget is capped so a retry can never outlive the caller's own
|
|
deadline, and every attempt is logged with its reason, so an outage is visible
|
|
in the logs rather than only in the output.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import random
|
|
import time
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from typing import Any, TypeVar
|
|
|
|
from app.services import logs
|
|
|
|
logger = logs.get_logger(__name__)
|
|
|
|
T = TypeVar("T")
|
|
|
|
# Defaults chosen for a call already covered by a per-page timeout: the whole
|
|
# retry sequence must stay well inside it, so the ceiling is low.
|
|
DEFAULT_ATTEMPTS = 3
|
|
DEFAULT_BASE_DELAY = 0.5
|
|
DEFAULT_MAX_DELAY = 8.0
|
|
# A server-supplied Retry-After longer than this is a signal to give up rather
|
|
# than to sleep: the caller has a deadline of its own.
|
|
MAX_HONOURED_RETRY_AFTER = 30.0
|
|
|
|
# HTTP status codes worth trying again. 408 request timeout, 425 too early,
|
|
# 429 too many requests, and the 5xx family that indicates the server — not the
|
|
# request — is at fault.
|
|
RETRYABLE_STATUS = frozenset({408, 425, 429, 500, 502, 503, 504, 507, 509})
|
|
|
|
|
|
class RetryExhausted(RuntimeError):
|
|
"""Every attempt failed. Carries the last cause, which must not be lost."""
|
|
|
|
def __init__(self, label: str, attempts: int, last: BaseException | None) -> None:
|
|
super().__init__(
|
|
f"{label} failed after {attempts} attempt(s): "
|
|
f"{type(last).__name__ if last else 'unknown'}: {last}"
|
|
)
|
|
self.label = label
|
|
self.attempts = attempts
|
|
self.last = last
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RetryPolicy:
|
|
attempts: int = DEFAULT_ATTEMPTS
|
|
base_delay: float = DEFAULT_BASE_DELAY
|
|
max_delay: float = DEFAULT_MAX_DELAY
|
|
# Total wall-clock the retry sequence may spend sleeping. Bounds the worst
|
|
# case so a retry cannot silently blow a caller's timeout.
|
|
max_total_delay: float = 12.0
|
|
|
|
|
|
def backoff_delay(
|
|
attempt: int,
|
|
*,
|
|
base: float = DEFAULT_BASE_DELAY,
|
|
cap: float = DEFAULT_MAX_DELAY,
|
|
rng: random.Random | None = None,
|
|
) -> float:
|
|
"""Full-jitter exponential backoff for a zero-based *attempt* index.
|
|
|
|
``uniform(0, min(cap, base * 2**attempt))``. Returning a *range* rather
|
|
than the exponential value itself is the point: it is what decorrelates
|
|
clients that failed simultaneously.
|
|
"""
|
|
ceiling = min(cap, base * (2.0 ** max(0, attempt)))
|
|
source = rng or random
|
|
return source.uniform(0.0, ceiling)
|
|
|
|
|
|
def retry_after_seconds(value: Any) -> float | None:
|
|
"""Parse a ``Retry-After`` header value in delta-seconds form.
|
|
|
|
The HTTP-date form is deliberately not honoured: it needs a trusted clock
|
|
on both ends, and every provider this engine talks to sends seconds.
|
|
"""
|
|
if value is None:
|
|
return None
|
|
try:
|
|
seconds = float(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if seconds < 0:
|
|
return None
|
|
return min(seconds, MAX_HONOURED_RETRY_AFTER)
|
|
|
|
|
|
def status_is_retryable(status: int | None) -> bool:
|
|
return status is not None and int(status) in RETRYABLE_STATUS
|
|
|
|
|
|
def is_retryable_exception(exc: BaseException) -> bool:
|
|
"""Whether *exc* describes a transient condition.
|
|
|
|
Deliberately conservative: an unrecognised error is fatal. Retrying an
|
|
unknown failure is how a bug becomes an outage amplifier.
|
|
"""
|
|
import http.client
|
|
import socket
|
|
import urllib.error
|
|
|
|
if isinstance(exc, urllib.error.HTTPError):
|
|
return status_is_retryable(exc.code)
|
|
if isinstance(exc, urllib.error.URLError):
|
|
# DNS failure, refused connection, unreachable host — all transient
|
|
# from the caller's point of view.
|
|
return True
|
|
if isinstance(exc, (TimeoutError, socket.timeout, ConnectionError, http.client.HTTPException)):
|
|
return True
|
|
if isinstance(exc, OSError):
|
|
return True
|
|
# An httpx error, if httpx is installed, without importing it eagerly.
|
|
name = type(exc).__module__.split(".")[0]
|
|
if name == "httpx":
|
|
return type(exc).__name__ in {
|
|
"ConnectError",
|
|
"ConnectTimeout",
|
|
"ReadTimeout",
|
|
"WriteTimeout",
|
|
"PoolTimeout",
|
|
"ReadError",
|
|
"WriteError",
|
|
"RemoteProtocolError",
|
|
"TimeoutException",
|
|
}
|
|
if isinstance(exc, ValueError):
|
|
# Includes json.JSONDecodeError: a truncated response body is a
|
|
# transport symptom, not a permanent disagreement about the API.
|
|
return type(exc).__name__ == "JSONDecodeError"
|
|
return False
|
|
|
|
|
|
def _sleep_for(
|
|
attempt: int,
|
|
policy: RetryPolicy,
|
|
spent: float,
|
|
hint: float | None,
|
|
rng: random.Random | None,
|
|
) -> float:
|
|
"""How long to wait before the next attempt, respecting the total budget."""
|
|
delay = (
|
|
hint
|
|
if hint is not None
|
|
else backoff_delay(attempt, base=policy.base_delay, cap=policy.max_delay, rng=rng)
|
|
)
|
|
return max(0.0, min(delay, policy.max_total_delay - spent))
|
|
|
|
|
|
def call_with_retry(
|
|
fn: Callable[[], T],
|
|
*,
|
|
label: str,
|
|
policy: RetryPolicy | None = None,
|
|
retryable: Callable[[BaseException], bool] = is_retryable_exception,
|
|
retry_after: Callable[[BaseException], float | None] | None = None,
|
|
sleep: Callable[[float], None] = time.sleep,
|
|
rng: random.Random | None = None,
|
|
) -> T:
|
|
"""Call *fn*, retrying transient failures with backoff and jitter.
|
|
|
|
Raises :class:`RetryExhausted` carrying the final cause when every attempt
|
|
fails, and re-raises immediately on a failure classified as fatal — the
|
|
caller must be able to tell "the provider is down" from "this request is
|
|
malformed".
|
|
|
|
*sleep* and *rng* are injected so the policy can be tested without spending
|
|
the wall-clock it describes.
|
|
"""
|
|
active = policy or RetryPolicy()
|
|
last: BaseException | None = None
|
|
spent = 0.0
|
|
|
|
for attempt in range(active.attempts):
|
|
try:
|
|
return fn()
|
|
except Exception as exc:
|
|
last = exc
|
|
final = attempt >= active.attempts - 1
|
|
if not retryable(exc):
|
|
logs.warn(
|
|
logger,
|
|
"retry_fatal",
|
|
label=label,
|
|
attempt=attempt + 1,
|
|
error=type(exc).__name__,
|
|
detail=str(exc)[:200],
|
|
)
|
|
raise
|
|
if final:
|
|
break
|
|
hint = retry_after(exc) if retry_after else None
|
|
delay = _sleep_for(attempt, active, spent, hint, rng)
|
|
logs.warn(
|
|
logger,
|
|
"retry_attempt",
|
|
label=label,
|
|
attempt=attempt + 1,
|
|
of=active.attempts,
|
|
error=type(exc).__name__,
|
|
detail=str(exc)[:200],
|
|
delay_s=round(delay, 3),
|
|
honoured_retry_after=hint is not None,
|
|
)
|
|
if delay > 0:
|
|
sleep(delay)
|
|
spent += delay
|
|
|
|
logs.error(
|
|
logger,
|
|
"retry_exhausted",
|
|
label=label,
|
|
attempts=active.attempts,
|
|
error=type(last).__name__ if last else "unknown",
|
|
detail=str(last)[:200] if last else "",
|
|
)
|
|
raise RetryExhausted(label, active.attempts, last)
|
|
|
|
|
|
async def call_with_retry_async(
|
|
fn: Callable[[], Any],
|
|
*,
|
|
label: str,
|
|
policy: RetryPolicy | None = None,
|
|
retryable: Callable[[BaseException], bool] = is_retryable_exception,
|
|
retry_after: Callable[[BaseException], float | None] | None = None,
|
|
rng: random.Random | None = None,
|
|
) -> Any:
|
|
"""Async twin of :func:`call_with_retry`, for the httpx call paths."""
|
|
import asyncio
|
|
|
|
active = policy or RetryPolicy()
|
|
last: BaseException | None = None
|
|
spent = 0.0
|
|
|
|
for attempt in range(active.attempts):
|
|
try:
|
|
return await fn()
|
|
except Exception as exc:
|
|
last = exc
|
|
final = attempt >= active.attempts - 1
|
|
if not retryable(exc):
|
|
logs.warn(
|
|
logger,
|
|
"retry_fatal",
|
|
label=label,
|
|
attempt=attempt + 1,
|
|
error=type(exc).__name__,
|
|
detail=str(exc)[:200],
|
|
)
|
|
raise
|
|
if final:
|
|
break
|
|
hint = retry_after(exc) if retry_after else None
|
|
delay = _sleep_for(attempt, active, spent, hint, rng)
|
|
logs.warn(
|
|
logger,
|
|
"retry_attempt",
|
|
label=label,
|
|
attempt=attempt + 1,
|
|
of=active.attempts,
|
|
error=type(exc).__name__,
|
|
detail=str(exc)[:200],
|
|
delay_s=round(delay, 3),
|
|
honoured_retry_after=hint is not None,
|
|
)
|
|
if delay > 0:
|
|
await asyncio.sleep(delay)
|
|
spent += delay
|
|
|
|
logs.error(
|
|
logger,
|
|
"retry_exhausted",
|
|
label=label,
|
|
attempts=active.attempts,
|
|
error=type(last).__name__ if last else "unknown",
|
|
detail=str(last)[:200] if last else "",
|
|
)
|
|
raise RetryExhausted(label, active.attempts, last)
|