Repository navigation
Expand file tree
/
Copy pathserver.py
More file actions
6284 lines (5666 loc) · 282 KB
/
Copy pathserver.py
File metadata and controls
6284 lines (5666 loc) · 282 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Headroom Proxy Server - Production Ready.
A full-featured LLM proxy with optimization, caching, rate limiting,
and observability.
Features:
- Context optimization (SmartCrusher, CacheAligner — live-zone-only after Phase B)
- Semantic caching (save costs on repeated queries)
- Rate limiting (token bucket)
- Retry with exponential backoff
- Cost tracking and budgets
- Request tagging and metadata
- Provider fallback
- Prometheus metrics
- Full request/response logging
Usage:
python -m headroom.proxy.server --port 8787
# With Claude Code:
ANTHROPIC_BASE_URL=http://localhost:8787 claude
"""
from __future__ import annotations
import argparse
import asyncio
import concurrent.futures
import contextlib
import hmac
import ipaddress
import json
import logging
import math
import os
import sys
import threading
import time
from collections import OrderedDict
from collections.abc import Callable, Mapping
from dataclasses import fields, is_dataclass, replace
from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, TypedDict, cast
from urllib.parse import urlsplit
if TYPE_CHECKING:
from ..backends.base import Backend
from ..cache.compression_cache import CompressionCache
from ..memory.tracker import MemoryTracker
from .outcome import RequestOutcome
import httpx
try:
import uvicorn
from fastapi import Depends, FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
FASTAPI_AVAILABLE = True
except ImportError:
FASTAPI_AVAILABLE = False
# Add parent to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from headroom._version import __version__
from headroom.agent_savings import DEFAULT_PROFILE, proxy_pipeline_kwargs
from headroom.cache.compression_feedback import get_compression_feedback
from headroom.cache.compression_store import format_retrieval_miss_detail, get_compression_store
from headroom.ccr import (
CCR_TOOL_NAME,
# Batch processing
CCRResponseHandler,
CCRToolInjector,
ContextTracker,
ContextTrackerConfig,
ResponseHandlerConfig,
parse_tool_call,
)
from headroom.config import (
DEFAULT_EXCLUDE_TOOLS,
CacheAlignerConfig,
ReadLifecycleConfig,
)
from headroom.dashboard import get_dashboard_html
from headroom.observability import (
LangfuseTracingConfig,
OTelMetricsConfig,
configure_langfuse_tracing,
configure_otel_metrics,
get_langfuse_tracing_status,
get_otel_metrics_status,
shutdown_headroom_tracing,
shutdown_otel_metrics,
)
from headroom.offline import apply_offline_env, is_offline
from headroom.pipeline import PipelineExtensionManager, PipelineStage
from headroom.providers.proxy_routes import register_provider_routes
from headroom.providers.registry import (
DEFAULT_ANTHROPIC_API_URL,
DEFAULT_CLOUDCODE_API_URL,
DEFAULT_GEMINI_API_URL,
DEFAULT_OPENAI_API_URL,
DEFAULT_VERTEX_API_URL,
build_proxy_provider_runtime,
create_proxy_backend,
format_backend_status,
resolve_api_targets,
)
from headroom.proxy import runtime_env
from headroom.proxy.audit import is_auditable_path, record_admin_action
from headroom.proxy.auth_mode import should_stamp_codex_client
from headroom.proxy.background_compression import BackgroundCompressor
from headroom.proxy.budget_basis_policy import resolve_estimated_basis_policy
from headroom.proxy.buffered_ccr_response import DEFAULT_BUFFERED_CCR_GRACE_SECONDS
# =============================================================================
# Extracted modules (re-exported for backward compatibility)
# =============================================================================
from headroom.proxy.cost import (
_CACHE_ECONOMICS, # noqa: F401
CostTracker, # noqa: F401
_summarize_transforms, # noqa: F401
build_prefix_cache_stats, # noqa: F401
build_session_summary, # noqa: F401
merge_cost_stats, # noqa: F401
)
from headroom.proxy.helpers import (
COMPRESSION_CACHE_MAX_ENTRIES,
COMPRESSION_CACHE_TTL_SECONDS,
COMPRESSION_TIMEOUT_SECONDS, # noqa: F401
EAGER_PRELOAD_TIMEOUT_SECONDS,
MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401
MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401
MAX_REQUEST_BODY_SIZE, # noqa: F401
MAX_SSE_BUFFER_SIZE, # noqa: F401
RETRYABLE_OVERLOAD_STATUSES,
_get_image_compressor, # noqa: F401
_read_request_json, # noqa: F401
_setup_file_logging, # noqa: F401
is_anthropic_auth, # noqa: F401
jitter_delay_ms,
resolve_display_provider,
retry_after_ms,
)
from headroom.proxy.loop_callback_failure_policy import is_known_websocket_callback_failure
from headroom.proxy.loopback_guard import is_loopback_host
from headroom.proxy.malloc_trim import trim_periodically
from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
# Data models (extracted to headroom/proxy/models.py for maintainability)
from headroom.proxy.model_router import ModelRouter, ModelRouterConfig
from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
from headroom.proxy.modes import (
PROXY_MODE_CACHE,
PROXY_MODE_TOKEN,
is_token_mode,
normalize_proxy_mode,
)
from headroom.proxy.probe_recorder import probe_recorder_from_env
from headroom.proxy.project_context import (
classify_project,
set_current_project,
strip_project_path_prefix,
)
from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
from headroom.proxy.request_logger import RequestLogger # noqa: F401
from headroom.proxy.savings_tracker import LITELLM_AVAILABLE
from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
from headroom.proxy.ssl_context import build_httpx_verify
from headroom.proxy.tool_schema_savings_policy import tool_schema_saved_from_tags
from headroom.proxy.warmup import WarmupRegistry
from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
from headroom.subscription.base import get_quota_registry, reset_quota_registry
from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state
from headroom.subscription.copilot_quota import get_copilot_quota_tracker
from headroom.subscription.tracker import (
configure_subscription_tracker,
get_subscription_tracker,
)
from headroom.telemetry import get_telemetry_collector
from headroom.telemetry.beacon import is_telemetry_enabled
from headroom.telemetry.toin import get_toin
from headroom.transforms import (
CacheAligner,
CodeAwareCompressor,
CodeCompressorConfig,
CompressionStrategy,
ContentRouter,
ContentRouterConfig,
TransformPipeline,
is_tree_sitter_available,
)
AnyLLMBackend: Any = None
LiteLLMBackend: Any = None
fcntl: Any = None
try:
import fcntl as _fcntl
fcntl = _fcntl
HAS_FCNTL = True
except ImportError:
HAS_FCNTL = False
_build_prefix_cache_stats = build_prefix_cache_stats
_build_session_summary = build_session_summary
_merge_cost_stats = merge_cost_stats
_AGENT_LABELS: dict[str, str] = {
"claude": "Claude",
"claude-code": "Claude",
"claude_cli": "Claude",
"claude-code-cli": "Claude",
"codex": "Codex",
"codex-cli": "Codex",
"cursor": "Cursor",
"grok_build": "Grok Build",
"copilot": "GitHub Copilot",
"github-copilot": "GitHub Copilot",
"aider": "Aider",
"zed": "Zed",
"opencode": "OpenCode",
"openclaw": "OpenClaw",
"gemini": "Gemini",
"google": "Gemini",
"vertex:google": "Gemini",
"anthropic": "Claude",
"openai": "OpenAI",
"unknown": "Unidentified",
}
_AGENT_SOURCE_PRIORITY: dict[str, int] = {
"unknown": 0,
"provider": 1,
"model": 2,
"stack": 3,
"client": 4,
}
def _normalize_agent_key(raw: Any) -> str | None:
if raw is None:
return None
value = str(raw).strip().lower()
if not value:
return None
value = value.replace(" ", "-").replace("_", "-")
if value.startswith("wrap-"):
value = value.removeprefix("wrap-")
if value in {"claude-cli", "claude-code", "claude-code-cli"}:
return "claude-code"
if value in {"codex-cli", "codex"}:
return "codex"
if value in {"github-copilot", "copilot"}:
return "copilot"
if value in {"google", "vertex-google", "vertex:google"}:
return "gemini"
return value
def _agent_label(agent_key: str) -> str:
if agent_key in _AGENT_LABELS:
return _AGENT_LABELS[agent_key]
return agent_key.replace("-", " ").replace("_", " ").title()
def _classify_agent_from_log(entry: dict[str, Any]) -> tuple[str, str, str]:
raw_tags = entry.get("tags")
tags = raw_tags if isinstance(raw_tags, dict) else {}
for source, candidate in (
("client", tags.get("client")),
("stack", tags.get("stack") or tags.get("headroom-stack")),
):
key = _normalize_agent_key(candidate)
if key:
return key, _agent_label(key), source
model = str(entry.get("model") or "").lower()
if "codex" in model:
return "codex", _agent_label("codex"), "model"
if "claude" in model:
return "claude-code", _agent_label("claude-code"), "model"
if "gemini" in model:
return "gemini", _agent_label("gemini"), "model"
key = _normalize_agent_key(entry.get("provider"))
if key:
return key, _agent_label(key), "provider"
return "unknown", _agent_label("unknown"), "unknown"
def _remap_provider_counts(counts: dict[str, int], config: ProxyConfig) -> dict[str, int]:
"""Relabel ``openai`` upstream counts with the configured display provider.
Display only — the stored metrics key stays ``openai`` (issue #1533).
Collisions (none today) are summed defensively.
"""
out: dict[str, int] = {}
for provider, count in counts.items():
display = resolve_display_provider(
provider,
openai_api_url=config.openai_api_url,
provider_name=config.provider_name,
)
out[display] = out.get(display, 0) + int(count)
return out
def _build_agent_usage_summary(
logs: list[dict[str, Any]],
*,
requests_by_provider: dict[str, int],
requests_by_model: dict[str, int],
global_before_tokens: int,
global_after_tokens: int,
global_tokens_saved: int,
global_output_tokens: int,
) -> dict[str, Any]:
agents: dict[str, dict[str, Any]] = {}
def _agent_row(agent_key: str, label: str, source: str) -> dict[str, Any]:
row = agents.setdefault(
agent_key,
{
"agent": agent_key,
"label": label,
"source": source,
"requests": 0,
"before_tokens": 0,
"after_tokens": 0,
"output_tokens": 0,
"tokens_saved": 0,
"models": {},
"providers": {},
"has_exact_tokens": False,
},
)
if _AGENT_SOURCE_PRIORITY.get(source, 0) > _AGENT_SOURCE_PRIORITY.get(
str(row.get("source") or "unknown"), 0
):
row["source"] = source
return row
for entry in logs:
agent_key, label, source = _classify_agent_from_log(entry)
row = _agent_row(agent_key, label, source)
before = max(0, int(entry.get("input_tokens_original") or 0))
after = max(0, int(entry.get("input_tokens_optimized") or 0))
saved = max(0, int(entry.get("tokens_saved") or 0))
output = max(0, int(entry.get("output_tokens") or 0))
provider = str(entry.get("provider") or "unknown")
model = str(entry.get("model") or "unknown")
row["requests"] += 1
row["before_tokens"] += before
row["after_tokens"] += after
row["output_tokens"] += output
row["tokens_saved"] += saved
row["providers"][provider] = int(row["providers"].get(provider, 0)) + 1
row["models"][model] = int(row["models"].get(model, 0)) + 1
if before > 0 or after > 0 or saved > 0:
row["has_exact_tokens"] = True
if not agents:
inferred_model_counts: dict[str, int] = {}
for model, count in requests_by_model.items():
model_lower = str(model).lower()
if "codex" in model_lower:
key = "codex"
elif "claude" in model_lower:
key = "claude-code"
elif "gemini" in model_lower:
key = "gemini"
else:
continue
inferred_model_counts[str(model)] = int(count)
provider_request_count = sum(max(0, int(count)) for count in requests_by_provider.values())
inferred_request_count = sum(max(0, count) for count in inferred_model_counts.values())
use_model_fallback = (
inferred_request_count > 0 and inferred_request_count == provider_request_count
)
if not use_model_fallback:
for provider, count in requests_by_provider.items():
key = _normalize_agent_key(provider) or "unknown"
row = _agent_row(key, _agent_label(key), "provider")
row["requests"] += int(count)
row["providers"][provider] = int(row["providers"].get(provider, 0)) + int(count)
for model, count in requests_by_model.items():
model_lower = str(model).lower()
if "codex" in model_lower:
key = "codex"
elif "claude" in model_lower:
key = "claude-code"
elif "gemini" in model_lower:
key = "gemini"
else:
continue
if not use_model_fallback:
continue
row = _agent_row(key, _agent_label(key), "model")
row["requests"] += int(count)
row["models"][str(model)] = int(row["models"].get(str(model), 0)) + int(count)
rows: list[dict[str, Any]] = []
for row in agents.values():
before = int(row["before_tokens"])
saved = int(row["tokens_saved"])
after = int(row["after_tokens"])
if before == 0 and (after > 0 or saved > 0):
before = after + saved
savings_percent = round((saved / before) * 100.0, 2) if before else 0.0
row["before_tokens"] = before
row["savings_percent"] = savings_percent
row["after_percent"] = round((after / before) * 100.0, 2) if before else 0.0
row["share_of_saved_percent"] = (
round((saved / global_tokens_saved) * 100.0, 2) if global_tokens_saved else 0.0
)
row["share_of_requests_percent"] = 0.0
rows.append(row)
total_requests = sum(int(row["requests"]) for row in rows)
for row in rows:
row["share_of_requests_percent"] = (
round((int(row["requests"]) / total_requests) * 100.0, 2) if total_requests else 0.0
)
rows.sort(
key=lambda row: (
int(row.get("tokens_saved", 0)),
int(row.get("before_tokens", 0)),
int(row.get("requests", 0)),
),
reverse=True,
)
return {
"agents": rows,
"totals": {
"requests": total_requests,
"before_tokens": global_before_tokens,
"after_tokens": global_after_tokens,
"output_tokens": global_output_tokens,
"tokens_saved": global_tokens_saved,
"savings_percent": (
round((global_tokens_saved / global_before_tokens) * 100.0, 2)
if global_before_tokens
else 0.0
),
},
"coverage": {
"logged_requests": len(logs),
"exact_token_rows": sum(1 for row in rows if row.get("has_exact_tokens")),
"mode": "request_logs" if logs else "aggregate_fallback",
},
}
# Suppress "[transformers] PyTorch was not found" warning emitted when
# transformers is imported for availability checks (e.g. kompress ONNX probe).
# PyTorch is optional in headroom; the warning is not actionable for operators.
os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger("headroom.proxy")
class _SuppressCancelledErrorFilter(logging.Filter):
"""Hide expected uvicorn CancelledError tracebacks during shutdown."""
def filter(self, record: logging.LogRecord) -> bool:
if record.levelno == logging.ERROR and record.exc_info:
exc_type = record.exc_info[0]
if exc_type is not None and issubclass(exc_type, asyncio.CancelledError):
return False
return True
LoopExceptionHandler = Callable[[asyncio.AbstractEventLoop, dict[str, Any]], object]
class LoopFailureDetails(TypedDict):
message: Any | None
exception: str | None
class LoopHealthState(TypedDict):
status: str
known_failures: int
last_known_failure: LoopFailureDetails | None
class CompressionQuarantinedError(asyncio.TimeoutError):
"""Compression was skipped while a timed-out worker was still running.
Inherit from ``asyncio.TimeoutError`` rather than the builtin directly so
Python 3.10 callers classify quarantine bypasses exactly like executor
timeouts. The two exception classes only became aliases in Python 3.11.
"""
_MULTI_WORKER_CONFIG_ENV = "HEADROOM_PROXY_CONFIG_JSON"
# Env var that opts out of the Rust core deployment smoke test (Hotfix-A0).
# Default behavior: hard-fail at startup if `headroom._core` is unimportable
# (Finding #2 in HEADROOM_PROXY_LOG_FINDINGS_2026_05_03.md — production
# deployment was silently running without the Rust extension and degrading
# every compressed request to a Python-only path or a no-op).
#
# Set to the literal string "false" to start the proxy in degraded
# Python-only mode. Any other value (including unset) keeps the
# fail-loud behavior.
_RUST_CORE_REQUIRED_ENV = "HEADROOM_REQUIRE_RUST_CORE"
# sysexits.h(3) — EX_CONFIG. Process supervisors (systemd, k8s, docker)
# treat this as a deliberate configuration failure rather than a crash, so
# they won't restart-loop on a broken deployment.
_EXIT_CONFIG = 78
def _check_rust_core() -> tuple[str, str | None]:
"""Verify the Rust extension `headroom._core` is loadable at startup.
Returns a `(status, error)` tuple:
- ``("loaded", None)`` — `headroom._core.hello()` returned the
expected sentinel.
- ``("disabled", reason)`` — opt-out env var was set; proxy starts
in Python-only degraded mode. `reason` carries the underlying
import error (or ``None`` if the import actually succeeded).
- ``("missing", reason)`` — never returned: this branch calls
``sys.exit(78)`` so the proxy refuses to start. The branch exists
only as a typed sentinel for callers that want to reason about
all three states (e.g. health endpoints).
Behavior is gated by the ``HEADROOM_REQUIRE_RUST_CORE`` env var:
any value other than ``"false"`` (case-insensitive) keeps the
fail-loud default.
"""
require = os.environ.get(_RUST_CORE_REQUIRED_ENV, "true").strip().lower() != "false"
try:
from headroom._core import hello as _rust_hello
marker = _rust_hello()
except Exception as exc: # ImportError, but also any init-time PyO3 failure
reason = f"{type(exc).__name__}: {exc}"
if not require:
logger.warning(
"event=rust_core_disabled reason=%r opt_out_env=%s=false mode=python_only_degraded",
reason,
_RUST_CORE_REQUIRED_ENV,
)
return ("disabled", reason)
# Fail loud. Print to stderr in addition to logging so operators
# see it even if the logging handler is mis-configured.
msg = (
f"FATAL: Rust extension `headroom._core` not loadable.\n"
f" error: {reason}\n"
f" fix: `make build-wheel && pip install --force-reinstall "
f"target/wheels/headroom_*.whl`\n"
f" opt-out: set {_RUST_CORE_REQUIRED_ENV}=false to start in "
f"degraded Python-only mode\n"
)
logger.error("event=rust_core_missing reason=%r action=exit_78", reason)
print(msg, file=sys.stderr, flush=True)
sys.exit(_EXIT_CONFIG)
# Import succeeded; sanity-check the marker so we catch a stale or
# mis-linked .so where the symbol name resolves but returns garbage.
if marker != "headroom-core":
reason = f"unexpected marker {marker!r}"
if not require:
logger.warning(
"event=rust_core_disabled reason=%r opt_out_env=%s=false",
reason,
_RUST_CORE_REQUIRED_ENV,
)
return ("disabled", reason)
msg = (
f"FATAL: Rust extension `headroom._core` is loaded but the "
f"marker function returned {marker!r}; expected 'headroom-core'.\n"
f" fix: rebuild: `make build-wheel && pip install "
f"--force-reinstall target/wheels/headroom_*.whl`\n"
)
logger.error("event=rust_core_marker_mismatch marker=%r action=exit_78", marker)
print(msg, file=sys.stderr, flush=True)
sys.exit(_EXIT_CONFIG)
logger.info("event=rust_core_loaded marker=%r", marker)
return ("loaded", None)
# Compression pipeline timeout in seconds
from headroom.proxy.handlers import ( # noqa: E402
AnthropicHandlerMixin,
BatchHandlerMixin,
BedrockHandlerMixin,
GeminiHandlerMixin,
OpenAIHandlerMixin,
StreamingMixin,
)
def _apply_stateless_persistence(config: ProxyConfig) -> None:
"""When the proxy runs stateless, force global persisters to in-memory so no
files are written to the workspace.
Covers TOIN (the always-on serving writer): it keeps learning patterns
in-memory but never reads or writes ``toin.json``. An empty ``storage_path``
makes the backend ``None``, which no-ops load/save/auto-save. The savings
subsystem is handled separately via ``PrometheusMetrics(stateless=...)``.
Note: setting ``HEADROOM_TOIN_BACKEND=none`` is NOT sufficient on its own —
``ToolIntelligenceNetwork`` falls back to ``config.storage_path`` when no
backend is passed, so we must clear the path explicitly here.
Concurrency: ``stateless`` is a per-process flag (set once at ``headroom
proxy`` launch), never a per-request/per-session value — every session a
process serves shares it, and two proxies with different settings run as
separate OS processes with independent TOIN singletons. In the rare case
where two HeadroomProxy instances with different ``stateless`` settings live
in ONE process (e.g. tests), this fails closed: the reset forces the
process-global TOIN in-memory, so a stateless proxy never persists (the safe
direction). A co-resident stateful proxy would then also stop persisting
TOIN — acceptable, since not-writing can never leak data.
"""
if not getattr(config, "stateless", False):
return
from headroom.telemetry.toin import TOINConfig, get_toin, reset_toin
# Reset first so this wins regardless of whether the singleton was already
# created with a filesystem backend earlier in the process.
reset_toin()
get_toin(TOINConfig(storage_path=""))
def _provider_httpx_client_options(
config: ProxyConfig,
verify: Any,
) -> tuple[bool, dict[str, Any]]:
client_kwargs: dict[str, Any] = {
"timeout": httpx.Timeout(
connect=config.connect_timeout_seconds,
read=config.request_timeout_seconds,
write=config.request_timeout_seconds,
pool=config.connect_timeout_seconds,
),
"limits": httpx.Limits(
max_connections=config.max_connections,
max_keepalive_connections=config.max_keepalive_connections,
keepalive_expiry=config.keepalive_expiry,
),
"verify": verify,
}
if config.http_proxy:
client_kwargs["proxy"] = config.http_proxy
return config.http2 and not config.http_proxy, client_kwargs
# Recognized built-in compressor names → the `ContentRouterConfig` `enable_*`
# flag that gates each one. This is the whole selection surface: a `--compressor`
# selection is mapped onto these existing flags with ZERO new dispatch logic, so
# the router's if/elif built-in dispatch stays byte-identical. Names outside this
# map (e.g. a third-party `headroom.compressor` entry point) are intentionally
# ignored here — they belong to the registry, not the built-in enable_* seam.
BUILTIN_COMPRESSOR_FLAGS: dict[str, str] = {
"smart_crusher": "enable_smart_crusher",
"kompress": "enable_kompress",
"code_aware": "enable_code_aware",
"search": "enable_search_compressor",
"log": "enable_log_compressor",
"tabular": "enable_tabular_compressor",
"config": "enable_config_compressor",
"html": "enable_html_extractor",
"image": "enable_image_optimizer",
}
def _apply_compressor_selection(
router_config: ContentRouterConfig,
compressors: set[str] | None,
) -> None:
"""Narrow the built-in compressor set on ``router_config`` in place.
``compressors is None`` (the default) is a no-op: every ``enable_*`` flag
keeps its dataclass default, so behavior is byte-identical to today. When a
selection is given, each recognized built-in in :data:`BUILTIN_COMPRESSOR_FLAGS`
is enabled iff it (or the wildcard ``"*"``) was selected, and disabled
otherwise. Unrecognized names select no flag (reserved for the compressor
registry) but are surfaced with a warning, because a typo'd selection is
otherwise indistinguishable from "compress nothing" (#2384). This maps a
selection onto the existing flags without adding any dispatch logic.
"""
if compressors is None:
return
selected = {name.strip() for name in compressors if name.strip()}
if not selected:
return
select_all = "*" in selected
unmatched = sorted(selected - set(BUILTIN_COMPRESSOR_FLAGS) - {"*"})
if unmatched:
if select_all or selected & set(BUILTIN_COMPRESSOR_FLAGS):
logger.warning(
"compressor selection: %s match no built-in compressor "
"(assumed registry names); built-ins: %s",
", ".join(unmatched),
", ".join(sorted(BUILTIN_COMPRESSOR_FLAGS)),
)
else:
logger.warning(
"compressor selection %s matches no built-in compressor — every "
"built-in compressor is now disabled. If this is a typo, valid "
"names are: %s (or '*' for all).",
", ".join(unmatched),
", ".join(sorted(BUILTIN_COMPRESSOR_FLAGS)),
)
for name, flag in BUILTIN_COMPRESSOR_FLAGS.items():
setattr(router_config, flag, select_all or name in selected)
def _external_compressor_selection(compressors: set[str] | None) -> list[str] | None:
"""Return the selected EXTERNAL (non-built-in) compressor names, or ``None``.
The built-in selection (:func:`_apply_compressor_selection`) consumes only
the names in :data:`BUILTIN_COMPRESSOR_FLAGS`; every OTHER selected name is a
third-party ``headroom.compressor`` entry point. This threads those to the
router (via ``ContentRouterConfig.active_external_compressors``) so it can
route matching blocks through them.
Returns ``None`` — the router's external-dispatch branch stays inert, so the
request path is byte-identical to today — when the selection is empty or
contains only recognized built-in names. ``"*"`` is preserved so the router
activates every discovered external compressor (mirroring the wildcard's
"select everything" meaning on the built-in side).
"""
if not compressors:
return None
selected = {name.strip() for name in compressors if name.strip()}
if not selected:
return None
if "*" in selected:
return ["*"]
external = sorted(selected - set(BUILTIN_COMPRESSOR_FLAGS))
return external or None
class HeadroomProxy(
StreamingMixin,
AnthropicHandlerMixin,
OpenAIHandlerMixin,
GeminiHandlerMixin,
BatchHandlerMixin,
BedrockHandlerMixin,
):
"""Production-ready Headroom optimization proxy."""
ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL
OPENAI_API_URL = DEFAULT_OPENAI_API_URL
GEMINI_API_URL = DEFAULT_GEMINI_API_URL
CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL
VERTEX_API_URL = DEFAULT_VERTEX_API_URL
def __init__(self, config: ProxyConfig):
self.config = config
self.config.mode = normalize_proxy_mode(self.config.mode)
# Record process-wide stateless mode so module-level persisters
# (output-savings recorder, etc.) can skip workspace writes.
from headroom import paths as _hr_paths
_hr_paths.set_process_stateless(config.stateless)
# Stateless: keep TOIN learning in-memory; never touch toin.json.
_apply_stateless_persistence(self.config)
pipeline_extensions = list(config.pipeline_extensions or [])
probe_recorder = probe_recorder_from_env()
if probe_recorder is not None:
pipeline_extensions.append(probe_recorder)
self.pipeline_extensions = PipelineExtensionManager(
hooks=config.hooks,
extensions=pipeline_extensions,
discover=config.discover_pipeline_extensions,
)
self.provider_runtime = build_proxy_provider_runtime(config)
api_targets = self.provider_runtime.api_targets
# Preserve the long-standing proxy compatibility surface while keeping
# provider_runtime as the source of truth for resolved upstream targets.
HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic
HeadroomProxy.OPENAI_API_URL = api_targets.openai
HeadroomProxy.GEMINI_API_URL = api_targets.gemini
HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode
HeadroomProxy.VERTEX_API_URL = api_targets.vertex
self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")
self.openai_provider = self.provider_runtime.pipeline_provider("openai")
# `metrics` is hoisted ahead of transform construction so the
# transforms can receive `self.metrics` as their compression
# observer at __init__ time. The forcing function for catching
# silent strategy regressions: per-strategy counters increment
# only when wired up here, so the wiring is mandatory, not
# something we patch in later. (See `RUST_DEV.md` audit notes.)
self.cost_tracker = (
CostTracker(
budget_limit_usd=config.budget_limit_usd,
budget_period=config.budget_period,
estimated_basis_policy=config.budget_estimated_basis,
)
if config.cost_tracking_enabled
else None
)
self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker, stateless=config.stateless)
# Cost-aware model routing (issue #1706). Disabled unless configured, so
# the default request path is unchanged.
self.model_router = ModelRouter(config.model_router)
# Initialize transforms based on routing mode.
#
# Phase B PR-B1 retired the IntelligentContextManager / RollingWindow
# message-dropping branch. Live-zone-only compression (PR-B2..B7) does
# not drop messages — it operates on content blocks within messages —
# so the proxy no longer needs a "context manager" transform stage.
# Reported via metrics as `_context_manager_status = "passthrough"`.
self._context_manager_status = "passthrough"
# ContentRouter is the single proxy routing surface. Provider handlers
# normalize their request shapes into messages or CompressionUnits, and
# the router chooses SmartCrusher, log/search/diff/code, or Kompress.
profile_kwargs = proxy_pipeline_kwargs(config)
router_config = ContentRouterConfig(
enable_code_aware=config.code_aware_enabled,
prefer_code_aware_for_code=_get_env_bool("HEADROOM_PREFER_CODE_AWARE_FOR_CODE", True),
tool_profiles=config.tool_profiles,
read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),
smart_crusher_max_items_after_crush=cast(
int | None,
profile_kwargs.get("max_items_after_crush"),
),
smart_crusher_with_compaction=cast(
bool,
profile_kwargs.get("smart_crusher_with_compaction", True),
),
ccr_inject_marker=config.ccr_inject_marker,
force_kompress_all=config.force_kompress_all,
lossless=config.lossless,
)
# Compressor selection (opt-in). None keeps every built-in enabled
# (default, byte-identical to today); a selection maps the recognized
# built-in names onto the `enable_*` flags just constructed above.
# Runs BEFORE the disable_kompress override below so that flag stays
# authoritative for turning Kompress off.
_apply_compressor_selection(router_config, config.compressors)
# External (non-built-in) `headroom.compressor` selections are ignored by
# `_apply_compressor_selection` (they have no enable_* flag). Thread them
# to the router here so it can route matching blocks through them; None
# (no external selected) keeps the external-dispatch branch inert.
router_config.active_external_compressors = _external_compressor_selection(
config.compressors
)
# No-CCR lossless mode: compress tool outputs with format-native
# lossless compaction and marker-free SmartCrusher, and suppress every
# retrieval marker + the retrieve-tool injection so no MCP round-trip is
# needed. Mirrors the force_kompress_all wiring precedent.
if config.lossless:
router_config.lossless = True
router_config.smart_crusher_lossless_only = True
router_config.ccr_inject_marker = False
if hasattr(config, "ccr_inject_tool"):
config.ccr_inject_tool = False
if config.disable_kompress:
router_config.enable_kompress = False
# Opt-in restore of the legacy behaviour: send fall-through content
# to PASSTHROUGH instead of the default KOMPRESS fallback strategy.
if config.disable_kompress_fallback:
router_config.fallback_strategy = CompressionStrategy.PASSTHROUGH
# `HEADROOM_LOSSLESS_ONLY=1` routes SmartCrusher through strict
# marker-free mode: lossless tabular compaction still applies, but
# any path that would emit a `<<ccr:…>>` marker (row-drop or
# opaque-blob offload) leaves the content uncompacted instead — so
# the session needs no CCR retrieval round-trips to stay recoverable.
if "HEADROOM_LOSSLESS_ONLY" in os.environ:
router_config.smart_crusher_lossless_only = _get_env_bool(
"HEADROOM_LOSSLESS_ONLY", False
)
# A non-None exclude_tools replaces DEFAULT_EXCLUDE_TOOLS in
# ContentRouter, so merge rather than assign.
if config.exclude_tools:
router_config.exclude_tools = set(DEFAULT_EXCLUDE_TOOLS) | config.exclude_tools
# protect_tool_results: force-merge named tools into the exclude set
# so their results are never lossy-compressed, regardless of mode.
if config.protect_tool_results:
base = (
router_config.exclude_tools
if router_config.exclude_tools is not None
else set(DEFAULT_EXCLUDE_TOOLS)
)
router_config.exclude_tools = base | config.protect_tool_results
# Token mode: allow compression of older excluded-tool results,
# and emit search results grouped by file (path once per file
# instead of repeated on every match line).
if is_token_mode(config.mode):
router_config.protect_recent_reads_fraction = 0.3
router_config.search_group_by_file = True
# Note: protect_tool_results runs AFTER token mode (ordering matters).
# It resets protect_recent_reads_fraction from 0.3→0.0, restoring
# full protection for ALL excluded-tool results regardless of age.
# This means naming any tool with --protect-tool-results also
# protects Read/Glob/Grep/Write/Edit results indefinitely.
if config.protect_tool_results:
router_config.protect_recent_reads_fraction = 0.0
# `--compress-user-messages` flips the router's default skip rule.
# Off by default for prefix-cache safety; enabled for workloads where
# user-message content dominates input (OpenAI/Azure chat with pasted
# code/RAG context — see issue #454).
if profile_kwargs.get("compress_user_messages"):
router_config.skip_user_messages = False
# Kompress (lossy ML text compression) is resolved per provider. The
# global `disable_kompress` above is the baseline for both; a per-
# provider override (disable_kompress_{anthropic,openai}) wins when set.
# Only `enable_kompress` differs between providers — routing, tool
# exclusion, and read-protection are identical — so when both resolve
# the same we reuse ONE ContentRouter instance and the Kompress model
# still loads once (startup warmup dedupes transforms by id()).
base_kompress_disabled = not router_config.enable_kompress
anthropic_kompress_disabled = (
base_kompress_disabled
if config.disable_kompress_anthropic is None
else config.disable_kompress_anthropic
)
openai_kompress_disabled = (
base_kompress_disabled
if config.disable_kompress_openai is None
else config.disable_kompress_openai
)
def _router_config_for(kompress_disabled: bool) -> ContentRouterConfig:
if kompress_disabled == base_kompress_disabled:
return router_config
return replace(router_config, enable_kompress=not kompress_disabled)
cache_aligner = CacheAligner(CacheAlignerConfig(enabled=False))
anthropic_router = ContentRouter(
_router_config_for(anthropic_kompress_disabled), observer=self.metrics
)
openai_router = (
anthropic_router
if openai_kompress_disabled == anthropic_kompress_disabled
else ContentRouter(_router_config_for(openai_kompress_disabled), observer=self.metrics)
)
self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"
_intercept_prefix: list = []
assert config.rollout is not None
if config.rollout.is_enabled("tool_result_interceptors"):
from headroom.proxy.interceptors import ToolResultInterceptorTransform
_intercept_prefix = [ToolResultInterceptorTransform()]
self.anthropic_pipeline = TransformPipeline(
transforms=[*_intercept_prefix, cache_aligner, anthropic_router],
provider=self.anthropic_provider,
)
self.openai_pipeline = TransformPipeline(
transforms=[*_intercept_prefix, cache_aligner, openai_router],
provider=self.openai_provider,
)
# Build the DEFAULT /v1/compress pipeline now, not on first request.
# It is a ContentRouter derived from `openai_router` (marker-free), so
# a lazy build would land inside the bounded compression executor on a
# cold pod's very first gateway request — paying router construction
# and, when the ML model runs in-process, model load against the 30 s
# compression budget. Building it here also lets startup warmup see it
# (`_eager_preload_transforms`). Never fatal: on failure the handler
# falls back to the original lazy path.
try:
self._no_ccr_pipeline()
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"Eager /v1/compress pipeline build failed (%s); deferring to first request",
exc,
)
# Initialize components
self.cache = (
SemanticCache(
max_entries=config.cache_max_entries,
ttl_seconds=config.cache_ttl_seconds,
)