Skip to content

Instantly share code, notes, and snippets.

@reallyticsai
Created May 3, 2026 09:16
Show Gist options
  • Select an option

  • Save reallyticsai/32303f8f449ad2a20b211c4842b1739e to your computer and use it in GitHub Desktop.

Select an option

Save reallyticsai/32303f8f449ad2a20b211c4842b1739e to your computer and use it in GitHub Desktop.
Async LLM Gateway with Circuit Breaker & Retry — Production Pattern (Python)
"""
Async LLM Gateway with Circuit Breaker Pattern
Author: Reallytics AI
Production pattern for routing LLM requests across multiple providers
with automatic failover, circuit breaking, and latency tracking.
"""
import asyncio
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any
class CircuitState(Enum):
CLOSED = "closed"
OPEN = "open"
HALF_OPEN = "half_open"
@dataclass
class CircuitBreaker:
failure_threshold: int = 5
recovery_timeout: float = 60.0
_failures: int = 0
_state: CircuitState = CircuitState.CLOSED
_last_failure: float = 0.0
def record_success(self):
self._failures = 0
self._state = CircuitState.CLOSED
def record_failure(self):
self._failures += 1
self._last_failure = time.time()
if self._failures >= self.failure_threshold:
self._state = CircuitState.OPEN
@property
def is_available(self) -> bool:
if self._state == CircuitState.CLOSED:
return True
if self._state == CircuitState.OPEN:
if time.time() - self._last_failure > self.recovery_timeout:
self._state = CircuitState.HALF_OPEN
return True
return self._state == CircuitState.HALF_OPEN
@dataclass
class ProviderConfig:
name: str
priority: int = 0
circuit: CircuitBreaker = field(default_factory=CircuitBreaker)
total_requests: int = 0
total_errors: int = 0
avg_latency_ms: float = 0.0
class LLMGateway:
def __init__(self):
self.providers: dict[str, ProviderConfig] = {}
def register(self, name: str, priority: int = 0):
self.providers[name] = ProviderConfig(name=name, priority=priority)
async def route(self, prompt: str, **kwargs) -> dict[str, Any]:
available = sorted(
[p for p in self.providers.values() if p.circuit.is_available],
key=lambda p: p.priority,
)
if not available:
raise RuntimeError("All providers unavailable (circuit open)")
for provider in available:
try:
start = time.time()
result = await self._call_provider(provider.name, prompt, **kwargs)
latency = (time.time() - start) * 1000
provider.circuit.record_success()
provider.total_requests += 1
provider.avg_latency_ms = (
(provider.avg_latency_ms * (provider.total_requests - 1) + latency)
/ provider.total_requests
)
return {"provider": provider.name, "latency_ms": latency, **result}
except Exception as e:
provider.circuit.record_failure()
provider.total_errors += 1
continue
raise RuntimeError("All providers failed")
async def _call_provider(self, name: str, prompt: str, **kwargs) -> dict:
await asyncio.sleep(0.01)
return {"text": f"Response from {name}", "tokens": len(prompt.split())}
def status(self) -> list[dict]:
return [
{
"name": p.name,
"state": p.circuit._state.value,
"requests": p.total_requests,
"errors": p.total_errors,
"avg_latency_ms": round(p.avg_latency_ms, 2),
}
for p in self.providers.values()
]
if __name__ == "__main__":
gateway = LLMGateway()
gateway.register("openai", priority=0)
gateway.register("anthropic", priority=1)
gateway.register("local_vllm", priority=2)
async def demo():
for i in range(5):
result = await gateway.route(f"Test prompt {i}")
print(f" Request {i}: {result['provider']} ({result['latency_ms']:.1f}ms)")
print("\nProvider status:")
for s in gateway.status():
print(f" {s}")
asyncio.run(demo())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment