"""Retrying a transient failure without making an outage worse. The engine had exactly one retry in it, and it was the wrong shape (``ocr/mistral_hook.py``): two attempts back to back with no sleep, catching every ``HTTPError`` alike. That retries a 401 — which will never succeed — and, worse, retries a **429 instantly**, adding load to a service that has just said it has too much. It then discarded the error, so a total provider outage looked like a document with no text on it. Three rules, and each is here because its absence causes a specific failure: **Classify before retrying.** A 400 or a 401 is the caller's fault and will fail identically forever; retrying costs latency and buys nothing. A 429, a 5xx, a reset connection or a timeout may well succeed on the next attempt. Anything not known to be retryable is treated as fatal, so a new error class fails fast rather than being hammered. **Back off exponentially, and jitter.** Without backoff, N clients retrying a recovering service re-saturate it the moment it comes back. Without jitter they do it *in lockstep* — the thundering herd — because they all failed at the same instant and all wait the same interval. Full jitter (sleep uniformly in ``[0, computed]``) is the variant that spreads them, and it is what AWS's architecture guidance recommends. **Obey ``Retry-After``.** When a server states how long to wait, guessing is strictly worse than listening. The delay budget is capped so a retry can never outlive the caller's own deadline, and every attempt is logged with its reason, so an outage is visible in the logs rather than only in the output. """ from __future__ import annotations import random import time from collections.abc import Callable from dataclasses import dataclass from typing import Any, TypeVar from app.services import logs logger = logs.get_logger(__name__) T = TypeVar("T") # Defaults chosen for a call already covered by a per-page timeout: the whole # retry sequence must stay well inside it, so the ceiling is low. DEFAULT_ATTEMPTS = 3 DEFAULT_BASE_DELAY = 0.5 DEFAULT_MAX_DELAY = 8.0 # A server-supplied Retry-After longer than this is a signal to give up rather # than to sleep: the caller has a deadline of its own. MAX_HONOURED_RETRY_AFTER = 30.0 # HTTP status codes worth trying again. 408 request timeout, 425 too early, # 429 too many requests, and the 5xx family that indicates the server — not the # request — is at fault. RETRYABLE_STATUS = frozenset({408, 425, 429, 500, 502, 503, 504, 507, 509}) class RetryExhausted(RuntimeError): """Every attempt failed. Carries the last cause, which must not be lost.""" def __init__(self, label: str, attempts: int, last: BaseException | None) -> None: super().__init__( f"{label} failed after {attempts} attempt(s): " f"{type(last).__name__ if last else 'unknown'}: {last}" ) self.label = label self.attempts = attempts self.last = last @dataclass(frozen=True) class RetryPolicy: attempts: int = DEFAULT_ATTEMPTS base_delay: float = DEFAULT_BASE_DELAY max_delay: float = DEFAULT_MAX_DELAY # Total wall-clock the retry sequence may spend sleeping. Bounds the worst # case so a retry cannot silently blow a caller's timeout. max_total_delay: float = 12.0 def backoff_delay( attempt: int, *, base: float = DEFAULT_BASE_DELAY, cap: float = DEFAULT_MAX_DELAY, rng: random.Random | None = None, ) -> float: """Full-jitter exponential backoff for a zero-based *attempt* index. ``uniform(0, min(cap, base * 2**attempt))``. Returning a *range* rather than the exponential value itself is the point: it is what decorrelates clients that failed simultaneously. """ ceiling = min(cap, base * (2.0 ** max(0, attempt))) source = rng or random return source.uniform(0.0, ceiling) def retry_after_seconds(value: Any) -> float | None: """Parse a ``Retry-After`` header value in delta-seconds form. The HTTP-date form is deliberately not honoured: it needs a trusted clock on both ends, and every provider this engine talks to sends seconds. """ if value is None: return None try: seconds = float(str(value).strip()) except (TypeError, ValueError): return None if seconds < 0: return None return min(seconds, MAX_HONOURED_RETRY_AFTER) def status_is_retryable(status: int | None) -> bool: return status is not None and int(status) in RETRYABLE_STATUS def is_retryable_exception(exc: BaseException) -> bool: """Whether *exc* describes a transient condition. Deliberately conservative: an unrecognised error is fatal. Retrying an unknown failure is how a bug becomes an outage amplifier. """ import http.client import socket import urllib.error if isinstance(exc, urllib.error.HTTPError): return status_is_retryable(exc.code) if isinstance(exc, urllib.error.URLError): # DNS failure, refused connection, unreachable host — all transient # from the caller's point of view. return True if isinstance(exc, (TimeoutError, socket.timeout, ConnectionError, http.client.HTTPException)): return True if isinstance(exc, OSError): return True # An httpx error, if httpx is installed, without importing it eagerly. name = type(exc).__module__.split(".")[0] if name == "httpx": return type(exc).__name__ in { "ConnectError", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout", "ReadError", "WriteError", "RemoteProtocolError", "TimeoutException", } if isinstance(exc, ValueError): # Includes json.JSONDecodeError: a truncated response body is a # transport symptom, not a permanent disagreement about the API. return type(exc).__name__ == "JSONDecodeError" return False def _sleep_for( attempt: int, policy: RetryPolicy, spent: float, hint: float | None, rng: random.Random | None, ) -> float: """How long to wait before the next attempt, respecting the total budget.""" delay = ( hint if hint is not None else backoff_delay(attempt, base=policy.base_delay, cap=policy.max_delay, rng=rng) ) return max(0.0, min(delay, policy.max_total_delay - spent)) def call_with_retry( fn: Callable[[], T], *, label: str, policy: RetryPolicy | None = None, retryable: Callable[[BaseException], bool] = is_retryable_exception, retry_after: Callable[[BaseException], float | None] | None = None, sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None, ) -> T: """Call *fn*, retrying transient failures with backoff and jitter. Raises :class:`RetryExhausted` carrying the final cause when every attempt fails, and re-raises immediately on a failure classified as fatal — the caller must be able to tell "the provider is down" from "this request is malformed". *sleep* and *rng* are injected so the policy can be tested without spending the wall-clock it describes. """ active = policy or RetryPolicy() last: BaseException | None = None spent = 0.0 for attempt in range(active.attempts): try: return fn() except Exception as exc: last = exc final = attempt >= active.attempts - 1 if not retryable(exc): logs.warn( logger, "retry_fatal", label=label, attempt=attempt + 1, error=type(exc).__name__, detail=str(exc)[:200], ) raise if final: break hint = retry_after(exc) if retry_after else None delay = _sleep_for(attempt, active, spent, hint, rng) logs.warn( logger, "retry_attempt", label=label, attempt=attempt + 1, of=active.attempts, error=type(exc).__name__, detail=str(exc)[:200], delay_s=round(delay, 3), honoured_retry_after=hint is not None, ) if delay > 0: sleep(delay) spent += delay logs.error( logger, "retry_exhausted", label=label, attempts=active.attempts, error=type(last).__name__ if last else "unknown", detail=str(last)[:200] if last else "", ) raise RetryExhausted(label, active.attempts, last) async def call_with_retry_async( fn: Callable[[], Any], *, label: str, policy: RetryPolicy | None = None, retryable: Callable[[BaseException], bool] = is_retryable_exception, retry_after: Callable[[BaseException], float | None] | None = None, rng: random.Random | None = None, ) -> Any: """Async twin of :func:`call_with_retry`, for the httpx call paths.""" import asyncio active = policy or RetryPolicy() last: BaseException | None = None spent = 0.0 for attempt in range(active.attempts): try: return await fn() except Exception as exc: last = exc final = attempt >= active.attempts - 1 if not retryable(exc): logs.warn( logger, "retry_fatal", label=label, attempt=attempt + 1, error=type(exc).__name__, detail=str(exc)[:200], ) raise if final: break hint = retry_after(exc) if retry_after else None delay = _sleep_for(attempt, active, spent, hint, rng) logs.warn( logger, "retry_attempt", label=label, attempt=attempt + 1, of=active.attempts, error=type(exc).__name__, detail=str(exc)[:200], delay_s=round(delay, 3), honoured_retry_after=hint is not None, ) if delay > 0: await asyncio.sleep(delay) spent += delay logs.error( logger, "retry_exhausted", label=label, attempts=active.attempts, error=type(last).__name__ if last else "unknown", detail=str(last)[:200] if last else "", ) raise RetryExhausted(label, active.attempts, last)