Created
May 3, 2026 09:16
-
-
Save reallyticsai/32303f8f449ad2a20b211c4842b1739e to your computer and use it in GitHub Desktop.
Async LLM Gateway with Circuit Breaker & Retry — Production Pattern (Python)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| Async LLM Gateway with Circuit Breaker Pattern | |
| Author: Reallytics AI | |
| Production pattern for routing LLM requests across multiple providers | |
| with automatic failover, circuit breaking, and latency tracking. | |
| """ | |
| import asyncio | |
| import time | |
| from dataclasses import dataclass, field | |
| from enum import Enum | |
| from typing import Any | |
| class CircuitState(Enum): | |
| CLOSED = "closed" | |
| OPEN = "open" | |
| HALF_OPEN = "half_open" | |
| @dataclass | |
| class CircuitBreaker: | |
| failure_threshold: int = 5 | |
| recovery_timeout: float = 60.0 | |
| _failures: int = 0 | |
| _state: CircuitState = CircuitState.CLOSED | |
| _last_failure: float = 0.0 | |
| def record_success(self): | |
| self._failures = 0 | |
| self._state = CircuitState.CLOSED | |
| def record_failure(self): | |
| self._failures += 1 | |
| self._last_failure = time.time() | |
| if self._failures >= self.failure_threshold: | |
| self._state = CircuitState.OPEN | |
| @property | |
| def is_available(self) -> bool: | |
| if self._state == CircuitState.CLOSED: | |
| return True | |
| if self._state == CircuitState.OPEN: | |
| if time.time() - self._last_failure > self.recovery_timeout: | |
| self._state = CircuitState.HALF_OPEN | |
| return True | |
| return self._state == CircuitState.HALF_OPEN | |
| @dataclass | |
| class ProviderConfig: | |
| name: str | |
| priority: int = 0 | |
| circuit: CircuitBreaker = field(default_factory=CircuitBreaker) | |
| total_requests: int = 0 | |
| total_errors: int = 0 | |
| avg_latency_ms: float = 0.0 | |
| class LLMGateway: | |
| def __init__(self): | |
| self.providers: dict[str, ProviderConfig] = {} | |
| def register(self, name: str, priority: int = 0): | |
| self.providers[name] = ProviderConfig(name=name, priority=priority) | |
| async def route(self, prompt: str, **kwargs) -> dict[str, Any]: | |
| available = sorted( | |
| [p for p in self.providers.values() if p.circuit.is_available], | |
| key=lambda p: p.priority, | |
| ) | |
| if not available: | |
| raise RuntimeError("All providers unavailable (circuit open)") | |
| for provider in available: | |
| try: | |
| start = time.time() | |
| result = await self._call_provider(provider.name, prompt, **kwargs) | |
| latency = (time.time() - start) * 1000 | |
| provider.circuit.record_success() | |
| provider.total_requests += 1 | |
| provider.avg_latency_ms = ( | |
| (provider.avg_latency_ms * (provider.total_requests - 1) + latency) | |
| / provider.total_requests | |
| ) | |
| return {"provider": provider.name, "latency_ms": latency, **result} | |
| except Exception as e: | |
| provider.circuit.record_failure() | |
| provider.total_errors += 1 | |
| continue | |
| raise RuntimeError("All providers failed") | |
| async def _call_provider(self, name: str, prompt: str, **kwargs) -> dict: | |
| await asyncio.sleep(0.01) | |
| return {"text": f"Response from {name}", "tokens": len(prompt.split())} | |
| def status(self) -> list[dict]: | |
| return [ | |
| { | |
| "name": p.name, | |
| "state": p.circuit._state.value, | |
| "requests": p.total_requests, | |
| "errors": p.total_errors, | |
| "avg_latency_ms": round(p.avg_latency_ms, 2), | |
| } | |
| for p in self.providers.values() | |
| ] | |
| if __name__ == "__main__": | |
| gateway = LLMGateway() | |
| gateway.register("openai", priority=0) | |
| gateway.register("anthropic", priority=1) | |
| gateway.register("local_vllm", priority=2) | |
| async def demo(): | |
| for i in range(5): | |
| result = await gateway.route(f"Test prompt {i}") | |
| print(f" Request {i}: {result['provider']} ({result['latency_ms']:.1f}ms)") | |
| print("\nProvider status:") | |
| for s in gateway.status(): | |
| print(f" {s}") | |
| asyncio.run(demo()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment