Circuit Breakers and Fallbacks

Reliability patterns for AI systems: circuit breakers, fallback models, retry budgets, and graceful degradation when models fail or degrade.

Reliability Patterns

Circuit Breaker States

stateDiagram-v2
    [*] --> Closed: normal operation

    Closed --> Open: failure threshold reached
    Closed --> Closed: success (reset failure count)

    Open --> Half-Open: timeout elapsed
    Open --> Open: failure (reset timeout)

    Half-Open --> Closed: success (close circuit)
    Half-Open --> Open: failure (re-open)

    note right of Closed: Requests flow normally
    note right of Open: Requests fail fast, fallback used
    note right of Half-Open: Test if service recovered

Circuit Breaker Implementation

from enum import Enum
from dataclasses import dataclass, field
from datetime import datetime, timedelta
import time
import asyncio
import logging

logger = logging.getLogger(__name__)

class CircuitState(Enum):
    CLOSED = "closed"
    OPEN = "open"
    HALF_OPEN = "half_open"

@dataclass
class CircuitBreakerConfig:
    failure_threshold: int = 5
    recovery_timeout_seconds: int = 30
    expected_exception: type = Exception
    fallback_model: Optional[str] = None

@dataclass
class CircuitBreakerState:
    state: CircuitState = CircuitState.CLOSED
    failure_count: int = 0
    last_failure_time: Optional[datetime] = None
    half_open_calls: int = 0

class CircuitBreaker:
    def __init__(self, name: str, config: CircuitBreakerConfig):
        self.name = name
        self.config = config
        self.state = CircuitBreakerState()

    def _record_success(self):
        self.state.failure_count = 0
        self.state.state = CircuitState.CLOSED

    def _record_failure(self):
        self.state.failure_count += 1
        self.state.last_failure_time = datetime.now()
        if self.state.failure_count >= self.config.failure_threshold:
            self.state.state = CircuitState.OPEN

    def _can_attempt(self) -> bool:
        if self.state.state == CircuitState.CLOSED:
            return True
        if self.state.state == CircuitState.OPEN:
            if (self.state.last_failure_time and 
                datetime.now() - self.state.last_failure_time > 
                timedelta(seconds=self.config.recovery_timeout_seconds)):
                self.state.state = CircuitState.HALF_OPEN
                self.state.half_open_calls = 0
                return True
            return False
        if self.state.state == CircuitState.HALF_OPEN:
            self.state.half_open_calls += 1
            return self.state.half_open_calls <= 1
        return False

    async def call(self, func, *args, fallback=None, **kwargs):
        if not self._can_attempt():
            logger.warning(f"Circuit {self.name} is OPEN — using fallback")
            if fallback:
                return await fallback(*args, **kwargs)
            raise CircuitOpenError(f"Circuit {self.name} is open")

        try:
            result = await func(*args, **kwargs)
            self._record_success()
            return result
        except self.config.expected_exception as e:
            self._record_failure()
            logger.error(f"Circuit {self.name} recorded failure: {e}")
            if fallback:
                return await fallback(*args, **kwargs)
            raise

class CircuitOpenError(Exception):
    pass

# Usage
breaker = CircuitBreaker(
    "llm-inference",
    CircuitBreakerConfig(
        failure_threshold=5,
        recovery_timeout_seconds=30,
        fallback_model="smaller-fallback-model",
    ),
)

async def generate_with_fallback(prompt):
    async def primary():
        return await llm_client.generate(prompt, model="large-model")

    async def fallback():
        return await llm_client.generate(prompt, model="smaller-model")

    return await breaker.call(primary, fallback=fallback)

Fallback Model Strategy

Retry with Exponential Backoff

import asyncio
import random
from functools import wraps

def retry_with_backoff(
    max_retries: int = 3,
    base_delay: float = 1.0,
    max_delay: float = 60.0,
    exponential_base: float = 2.0,
    jitter: bool = True,
):
    def decorator(func):
        @wraps(func)
        async def wrapper(*args, **kwargs):
            delay = base_delay
            last_exception = None

            for attempt in range(max_retries + 1):
                try:
                    return await func(*args, **kwargs)
                except (RateLimitError, TimeoutError, ServiceUnavailable) as e:
                    last_exception = e
                    if attempt == max_retries:
                        break

                    sleep_time = min(delay, max_delay)
                    if jitter:
                        sleep_time *= random.uniform(0.5, 1.5)

                    logger.warning(
                        f"Attempt {attempt + 1} failed: {e}. Retrying in {sleep_time:.1f}s"
                    )
                    await asyncio.sleep(sleep_time)
                    delay *= exponential_base

            raise last_exception
        return wrapper
    return decorator

@retry_with_backoff(max_retries=3, base_delay=1.0)
async def generate_with_retry(prompt):
    return await llm_client.generate(prompt)

Error Budget

# Error budget for AI inference
error_budget:
  window: 1h
  max_error_rate: 0.01  # 1% of requests may fail
  max_latency_violation_rate: 0.05  # 5% may exceed p99 latency
  max_cost_violation_rate: 0.0  # No cost budget overruns

  # When budget exhausted
  action_on_exhausted:
    - route_to_fallback_model
    - alert_oncall
    - pause_non_critical_traffic

Anti-Patterns

Related Notes

• Guardrails And Safety — Guardrails, incident response
• Reliability — Retries, fallbacks, circuit breakers, SLOs
• Observability — Monitoring and alerting
• Canary And Rollback — Canary deployment, rollback strategies
• Observability — Monitoring AI systems for reliability

Sources

• Google SRE (circuit breakers, error budgets), Release It! (Nygard), Circuit breaker pattern (Martin Fowler), AWS Fault Injection, Azure Circuit Breaker, Hystrix/Resilience4j, AI incident response playbooks.*