Repository navigation
Expand file tree
/
Copy pathkompress_compressor.py
More file actions
2585 lines (2221 loc) · 107 KB
/
Copy pathkompress_compressor.py
File metadata and controls
2585 lines (2221 loc) · 107 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Kompress: ModernBERT token compressor for structured tool outputs.
Auto-downloads the model from HuggingFace (chopratejas/kompress-v2-base)
on first use.
Requires the [ml] extra: pip install headroom-ai[ml]
Usage:
>>> from headroom.transforms.kompress_compressor import KompressCompressor
>>> compressor = KompressCompressor()
>>> result = compressor.compress(long_tool_output)
>>> print(result.compressed)
"""
from __future__ import annotations
import contextlib
import gc
import hashlib
import itertools
import logging
import os
import re
import threading
import time
from dataclasses import dataclass
from functools import lru_cache
from typing import Any, Literal
from ..config import TransformResult
from ..offline import OFFLINE_ENV, OfflineEgressBlocked
from ..onnx_runtime import (
ONNX_CPU_ARENA_ENV,
_resolve_revision,
create_cpu_session_options,
hf_entry_known_absent,
hf_from_pretrained_local_first,
hf_hub_download_local_first,
trim_process_heap,
)
from ..tokenizer import Tokenizer
from .base import Transform
logger = logging.getLogger(__name__)
# Default HuggingFace model ID
HF_MODEL_ID = "chopratejas/kompress-v2-base"
# Tokens matching this pattern are always kept regardless of model score.
# Numbers, ALLCAPS identifiers, dotted paths, unix paths, file extensions,
# CLI flags, and CamelCase names carry semantic meaning that agents cannot
# reconstruct from context — dropping them degrades reasoning correctness.
# Disable with HEADROOM_KOMPRESS_MUST_KEEP=0.
_KOMPRESS_MUST_KEEP_RE = re.compile(
r"\b0x[0-9A-Fa-f]+\b" # hex addresses/IDs: 0x7fff2038
r"|(?<![\w.])\d+(?:\.\d+)?(?![\w.])" # standalone numbers: 42, 3.14
r"|[A-Z_]{2,}" # ALLCAPS: SIGILL, HTTP, EOF, ERROR
r"|[a-z_][a-z0-9_]*\.[a-z0-9_]+" # dotted.paths: libsystem_kernel.dylib
r"|/[a-z0-9/._-]{2,}" # unix paths: /usr/lib/python3.so
r"|\.[a-z]{2,4}\b" # extensions: .py .so .json
r"|--?[a-z][\w-]*" # flags: --verbose, -n
r"|\b[A-Z][a-z]+[A-Z]\w*" # CamelCase: EXC_BAD_INSTRUCTION, IndexError
# Directive words. Every other class above protects a token the model could
# not RECONSTRUCT; these protect tokens whose loss INVERTS the surrounding
# sentence. Compressed prompts carry instructions as often as they carry
# tool output, and "do not guess at model names" without its "not" is not a
# degraded instruction, it is the opposite instruction. Measured over 40
# real skill bodies with THIS pattern: negation retention rose
# 73.5% -> 91.9% and modal retention 65.5% -> 96.8%, for 0.2 percentage
# points of compression (ratio 0.716 -> 0.718) -- these are short, common
# words, so the model was already keeping most of them and pinning the rest
# costs almost nothing.
r"|(?i:\b(?:not|never|none|cannot|can't|don't|doesn't|didn't|won't|shouldn't"
r"|mustn't|isn't|aren't|avoid|refuse|prohibited|forbidden|disallow|unless"
r"|except|without|must|should|shall|required|always|only|mandatory)\b)"
# Boolean connectives, for the same reason as the directive words above:
# they decide WHICH predicates have to hold, so dropping one does not
# weaken the statement, it changes the condition. Negation was pinned but
# conjunction was not, so one line could lose its `or` and keep its `not`
# (issue #3545: 12 of 40 `or` lost from a repeated Python return line,
# while the same word survived 48/48 in prose). The surviving text is the
# dangerous part -- `a == b not c.startswith(d)` still reads as code, and
# nothing marks it as altered. Uppercase `AND`/`OR` were already held by
# the ALLCAPS class; only the lowercase forms leaked.
r"|(?i:\b(?:and|or|nor|xor)\b)"
)
_KOMPRESS_MUST_KEEP_ENV = "HEADROOM_KOMPRESS_MUST_KEEP"
KOMPRESS_BACKEND_ENV = "HEADROOM_KOMPRESS_BACKEND"
KOMPRESS_ONNX_FILENAME_ENV = "HEADROOM_KOMPRESS_ONNX_FILENAME"
def _add_kompress_must_keep_words(
kept_ids: set[int],
chunk_words: list[str],
chunk_start: int,
) -> None:
"""Add semantically fragile words that should never be model-dropped."""
if os.environ.get(_KOMPRESS_MUST_KEEP_ENV, "1") == "0":
return
for word_idx, word in enumerate(chunk_words):
if _KOMPRESS_MUST_KEEP_RE.search(word):
kept_ids.add(word_idx + chunk_start)
_TABULAR_ROW_SPLIT_RE = re.compile(r"\s{2,}")
def _words_by_line(content: str) -> tuple[list[str], list[int]]:
"""Split into words like content.split(), plus each word's source line index.
Reconstruction must never join words that came from different source lines
onto one line -- doing so can glue unrelated facts together (#3099).
"""
words: list[str] = []
line_ids: list[int] = []
for line_idx, line in enumerate(content.split("\n")):
for word in line.split():
words.append(word)
line_ids.append(line_idx)
return words, line_ids
def _join_kept_words_by_line(words: list[str], line_ids: list[int], kept_ids: set[int]) -> str:
"""Rejoin kept words, starting a new line wherever they crossed a line boundary."""
parts: list[str] = []
prev_line: int | None = None
for i in sorted(kept_ids):
if prev_line is not None:
parts.append("\n" if line_ids[i] != prev_line else " ")
parts.append(words[i])
prev_line = line_ids[i]
return "".join(parts)
def _tabular_protected_line_ids(content: str) -> set[int]:
"""Flag lines in a run of >=3 consecutive lines with a consistent fixed-width
column count (ls -l, ps, etc.), so compression keeps each such row whole
instead of dropping individual fields inconsistently row to row (#3099).
Shares the must-keep toggle: HEADROOM_KOMPRESS_MUST_KEEP=0 disables this too.
"""
if os.environ.get(_KOMPRESS_MUST_KEEP_ENV, "1") == "0":
return set()
cols_per_line = [
len(_TABULAR_ROW_SPLIT_RE.split(line.strip())) if line.strip() else 0
for line in content.split("\n")
]
protected: set[int] = set()
for cols, group in itertools.groupby(enumerate(cols_per_line), key=lambda p: p[1]):
indices = [idx for idx, _ in group]
if cols >= 3 and len(indices) >= 3:
protected.update(indices)
return protected
# ONNX artifacts are resolved against the model repo in this order, falling
# through on download miss OR session-load failure:
#
# - kompress-int8-wo.onnx: weight-only int8 (MatMulNBits), 261MB. Evaluated on
# the labeled dataset_v2 test split (n=500): f1=0.9130 vs fp32's 0.9128,
# must_keep_recall 0.9765 vs 0.9770, keep_rate 0.8097 vs 0.8100, 99.6%
# keep-decision agreement — fp32-equivalent at 2.2x less memory. Uses the
# com.microsoft MatMulNBits contrib op; older onnxruntime builds without the
# 8-bit kernel fail at session load and fall through to fp32.
# - kompress-fp32.onnx: lossless reference, 601MB.
# - kompress-int8.onnx: v1-era dynamic int8 (kept for custom domain repos).
#
# An operator can pin an exact file via HEADROOM_KOMPRESS_ONNX_FILENAME.
_DEFAULT_ONNX_FILENAMES = (
"onnx/kompress-int8-wo.onnx",
"onnx/kompress-fp32.onnx",
"onnx/kompress-int8.onnx",
)
KOMPRESS_ONNX_INTRA_THREADS_ENV = "HEADROOM_KOMPRESS_ONNX_INTRA_THREADS"
KOMPRESS_ONNX_INTER_THREADS_ENV = "HEADROOM_KOMPRESS_ONNX_INTER_THREADS"
KOMPRESS_COREML_CACHE_DIR_ENV = "HEADROOM_KOMPRESS_COREML_CACHE_DIR"
KOMPRESS_MAX_CONCURRENT_ENV = "HEADROOM_KOMPRESS_MAX_CONCURRENT"
# Consecutive inference failures before Kompress latches to passthrough for the
# rest of the process. 3 rides out a transient error while still catching a model
# that is broken for this install on the first few requests rather than the 200th.
# ponytail: fixed count, not a rate window — a broken artifact fails every call,
# so there is nothing a window would tell us that three strikes doesn't.
_INFERENCE_FAILURE_LATCH = 3
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV = "HEADROOM_KOMPRESS_EXECUTION_TIMEOUT_MS"
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT = 3000
KOMPRESS_BATCH_SIZE_ENV = "HEADROOM_KOMPRESS_BATCH_SIZE"
KOMPRESS_ACQUIRE_TIMEOUT_ENV = "HEADROOM_KOMPRESS_ACQUIRE_TIMEOUT_SECONDS"
KOMPRESS_TIME_BUDGET_ENV = "HEADROOM_KOMPRESS_TIME_BUDGET_SECONDS"
KOMPRESS_CANARY_THRESHOLD_ENV = "HEADROOM_KOMPRESS_CANARY_SECONDS"
KOMPRESS_REQUEST_DEADLINE_ENV = "HEADROOM_COMPRESSION_DEADLINE_MS"
# Both defaults sit well under the proxy's 30s compression-stage timeout so a
# slow model gives up (passthrough) before the request is abandoned. A thread
# abandoned by asyncio.wait_for cannot be killed; before these bounds existed,
# one pathologically slow inference would hold the execution semaphore forever
# and wedge every subsequent compression in the process.
_DEFAULT_ACQUIRE_TIMEOUT_SECONDS = 5.0
_DEFAULT_TIME_BUDGET_SECONDS = 20.0
_DEFAULT_CANARY_THRESHOLD_SECONDS = 5.0
KompressBackend = Literal["auto", "onnx", "onnx_cpu", "onnx_coreml", "pytorch", "pytorch_mps"]
# HuggingFace local-lookup errors that mean "asset not in cache" rather than a
# genuine failure. Caught when loading cache-only so startup can defer instead.
try:
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
_NOT_CACHED_ERRORS: tuple[type[BaseException], ...] = (
LocalEntryNotFoundError,
EntryNotFoundError,
OSError,
)
except Exception: # pragma: no cover - huggingface_hub always present with [ml]
_NOT_CACHED_ERRORS = (OSError,)
class KompressModelNotCached(RuntimeError):
"""Raised when a cache-only load is requested but the model is not cached.
Used by startup eager-preload (``allow_download=False``) so the caller can
defer the download to first use instead of blocking the proxy startup path
on a network fetch.
"""
def _hf_artifact(model_id: str, filename: str, *, allow_network: bool) -> str:
"""``hf_hub_download_local_first`` with the air-gap refusal translated.
``OfflineEgressBlocked`` is a ``BaseException`` so that no ``except
Exception:`` can silently degrade an air-gap refusal into "that feature
stopped working". At THIS boundary that degradation is the correct
behaviour and is written down rather than inherited: a model download is
not data leaving the box, every caller below already handles
"the model is not available locally", and failing a user's request because
an optional compressor could not fetch public weights would punish the
request for a decision the operator made about the host.
So the refusal is logged with the switch named and re-raised as
:class:`KompressModelNotCached`, which is exactly what it means: under
``HEADROOM_OFFLINE`` the only artifacts that will ever be available are the
cached ones. The original refusal is chained, so it is still in the
traceback.
"""
try:
return hf_hub_download_local_first(model_id, filename, allow_network=allow_network)
except OfflineEgressBlocked as blocked:
logger.warning(
"Kompress: %r for %s is not cached and %s forbids fetching it (%s). "
"Compression falls back to the non-ML path.",
filename,
model_id,
OFFLINE_ENV,
blocked,
)
raise KompressModelNotCached(model_id) from blocked
def _hf_pretrained(
loader: Any, name: str, *, purpose: str, allow_download: bool, **kwargs: Any
) -> Any:
"""``hf_from_pretrained_local_first`` with Kompress's two translations.
The ``from_pretrained`` counterpart of :func:`_hf_artifact`: cache first,
``guard_egress`` before any remote attempt, and both "not cached while
downloads are disallowed" and "not cached while ``HEADROOM_OFFLINE`` is set"
surface as :class:`KompressModelNotCached` — the error every Kompress caller
already treats as "use the non-ML path". A genuine remote failure while
downloads are allowed propagates unchanged.
"""
try:
return hf_from_pretrained_local_first(
loader, name, purpose=purpose, allow_network=allow_download, **kwargs
)
except OfflineEgressBlocked as blocked:
logger.warning(
"Kompress: the %s (%s) is not cached and %s forbids fetching it (%s). "
"Compression falls back to the non-ML path.",
purpose,
name,
OFFLINE_ENV,
blocked,
)
raise KompressModelNotCached(name) from blocked
except _NOT_CACHED_ERRORS as exc:
if allow_download:
raise
raise KompressModelNotCached(name) from exc
# Model cache: model_id -> (model, tokenizer, backend)
# Supports multiple models loaded simultaneously.
_kompress_cache: dict[str, tuple[Any, Any, str]] = {}
_kompress_lock = threading.Lock()
_execution_semaphores: dict[str, threading.BoundedSemaphore] = {}
_execution_semaphores_lock = threading.Lock()
_execution_metrics_lock = threading.Lock()
_execution_skip_counters: dict[str, int] = {
"timeout": 0,
}
_execution_wait_seconds_total: dict[str, float] = {
"timeout": 0.0,
}
def _execution_wait_budget_seconds() -> float:
raw = os.environ.get(KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV)
if raw is None:
return KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT / 1000.0
try:
parsed = int(raw)
except ValueError:
logger.warning(
"Invalid %s=%r; using %dms",
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV,
raw,
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT,
)
return KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT / 1000.0
if parsed < 0:
logger.warning(
"Negative %s=%r; disabling timeout and using fail-open.",
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV,
raw,
)
return 0.0
return parsed / 1000.0
def _request_deadline_seconds() -> float:
try:
return max(0.0, float(os.environ.get(KOMPRESS_REQUEST_DEADLINE_ENV, "20000")) / 1000.0)
except ValueError:
return 20.0
def _acquire_execution_slot(
backend: str,
device_type: str,
*,
timeout_seconds: float | None,
) -> tuple[threading.BoundedSemaphore | None, float]:
semaphore = _execution_semaphore(backend, device_type)
start = time.perf_counter()
if timeout_seconds is None:
semaphore.acquire()
wait_ms = (time.perf_counter() - start) * 1000.0
return semaphore, wait_ms
acquired = semaphore.acquire(blocking=False)
if not acquired and timeout_seconds > 0:
acquired = semaphore.acquire(timeout=timeout_seconds)
elif not acquired and timeout_seconds == 0:
acquired = False
wait_ms = (time.perf_counter() - start) * 1000.0
if not acquired:
with _execution_metrics_lock:
_execution_skip_counters["timeout"] += 1
_execution_wait_seconds_total["timeout"] += wait_ms / 1000.0
return None, wait_ms
return semaphore, wait_ms
def get_kompress_execution_stats() -> dict[str, int | float]:
"""Return execution-acquire observability counters."""
with _execution_metrics_lock:
return {
"execution_acquire_timeout_ms": int(_execution_wait_budget_seconds() * 1000),
"execution_timeout_skips_total": _execution_skip_counters["timeout"],
"execution_wait_seconds_total": _execution_wait_seconds_total["timeout"],
}
def _selected_backend() -> KompressBackend:
raw = os.environ.get(KOMPRESS_BACKEND_ENV, "auto").strip().lower().replace("-", "_")
aliases = {
"": "auto",
"cpu": "onnx_cpu",
"coreml": "onnx_coreml",
"mps": "pytorch_mps",
"torch": "pytorch",
"torch_mps": "pytorch_mps",
"onnx": "onnx",
"onnx_cpu": "onnx_cpu",
"onnx_coreml": "onnx_coreml",
"pytorch": "pytorch",
"pytorch_mps": "pytorch_mps",
"auto": "auto",
}
backend = aliases.get(raw)
if backend is None:
logger.warning(
"%s has unrecognized value %r; falling back to 'auto'. Valid values: %s",
KOMPRESS_BACKEND_ENV,
os.environ.get(KOMPRESS_BACKEND_ENV, ""),
", ".join(sorted(set(aliases.values()))),
)
return "auto"
return backend # type: ignore[return-value]
def _env_int(name: str) -> int | None:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return None
try:
value = int(raw)
except ValueError:
logger.warning("%s must be an integer, got %r; ignoring", name, raw)
return None
if value <= 0:
logger.warning("%s must be positive, got %r; ignoring", name, raw)
return None
return value
def _env_float(name: str) -> float | None:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return None
try:
return float(raw)
except ValueError:
logger.warning("%s must be a number, got %r; ignoring", name, raw)
return None
def _acquire_timeout_seconds() -> float | None:
"""Max seconds to wait for the execution semaphore. <=0 disables the bound."""
raw = _env_float(KOMPRESS_ACQUIRE_TIMEOUT_ENV)
if raw is None:
return _DEFAULT_ACQUIRE_TIMEOUT_SECONDS
return raw if raw > 0 else None
def _time_budget_seconds() -> float | None:
"""Wall-clock budget for one compress/compress_batch call. <=0 disables."""
raw = _env_float(KOMPRESS_TIME_BUDGET_ENV)
if raw is None:
return _DEFAULT_TIME_BUDGET_SECONDS
return raw if raw > 0 else None
def _capped_time_budget(cap: float | None) -> float | None:
"""The per-call budget, lowered to *cap* when the caller has less to give.
The router passes what is left of its request-scoped ML budget, so a call
-- including the first one of a request, which has no measured cost to be
refused on -- stops at a chunk boundary when that budget runs out instead
of running to its own, larger, per-call budget.
"""
budget = _time_budget_seconds()
if cap is None:
return budget
cap = max(0.0, cap)
return cap if budget is None else min(budget, cap)
def _canary_threshold_seconds() -> float | None:
"""Startup canary threshold; inference slower than this disables Kompress.
<=0 disables the canary entirely.
"""
raw = _env_float(KOMPRESS_CANARY_THRESHOLD_ENV)
if raw is None:
return _DEFAULT_CANARY_THRESHOLD_SECONDS
return raw if raw > 0 else None
_CANARY_SENTENCE = (
"Headroom probes model latency at startup so a degraded runtime is "
"detected before live traffic depends on it."
)
def _canary_words(target_words: int = 120) -> list[str]:
"""Deterministic probe input sized like a small real compression chunk."""
words = _CANARY_SENTENCE.split()
reps = (target_words // len(words)) + 1
return (words * reps)[:target_words]
# First give-up is a WARNING with remediation hints; repeats drop to DEBUG so a
# persistently slow machine doesn't emit one warning per compressible message.
_giveup_warned = False
def _log_giveup(reason: str, *, backend: str, device_type: str, n_words: int) -> None:
global _giveup_warned
level = logging.DEBUG if _giveup_warned else logging.WARNING
_giveup_warned = True
logger.log(
level,
"Kompress giving up (%s) backend=%s device=%s words=%d — content passes "
"through uncompressed. If this persists, ML inference on this machine is "
"too slow for inline compression; tune %s / %s or disable Kompress.",
reason,
backend,
device_type,
n_words,
KOMPRESS_TIME_BUDGET_ENV,
KOMPRESS_ACQUIRE_TIMEOUT_ENV,
)
def _onnx_session_options(ort: Any) -> Any:
return create_cpu_session_options(
ort,
intra_op_num_threads=_env_int(KOMPRESS_ONNX_INTRA_THREADS_ENV),
inter_op_num_threads=_env_int(KOMPRESS_ONNX_INTER_THREADS_ENV),
)
def _model_device_type(model: Any, backend: str) -> str:
if backend.startswith("onnx"):
return backend
if hasattr(model, "parameters"):
try:
return str(next(model.parameters()).device.type)
except Exception:
return "unknown"
return "unknown"
def _default_max_concurrent(backend: str, device_type: str) -> int:
# MPS/CUDA execution is usually serialized under the hood; letting many
# Codex unit workers call the same model concurrently mostly adds queueing,
# memory pressure, and timeout leaks. CPU defaults to 1 as well because ONNX
# already owns its intra/inter-op threads.
if backend.startswith("onnx"):
return 1
if backend == "pytorch" and device_type in {"cuda", "mps", "cpu"}:
return 1
return 1
def _execution_limit(backend: str, device_type: str) -> int:
return _env_int(KOMPRESS_MAX_CONCURRENT_ENV) or _default_max_concurrent(backend, device_type)
def _execution_semaphore(backend: str, device_type: str) -> threading.BoundedSemaphore:
limit = _execution_limit(backend, device_type)
key = f"{backend}:{device_type}:{limit}"
with _execution_semaphores_lock:
semaphore = _execution_semaphores.get(key)
if semaphore is None:
semaphore = threading.BoundedSemaphore(limit)
_execution_semaphores[key] = semaphore
return semaphore
def _batch_size() -> int:
return _env_int(KOMPRESS_BATCH_SIZE_ENV) or 32
def _bucket_count(value: int) -> str:
"""Return a coarse, privacy-preserving size bucket."""
if value <= 0:
return "0"
lower = 1 << (value.bit_length() - 1)
upper = lower << 1
return f"{lower}-{upper}"
def _kompress_content_signature(content: str) -> Any:
"""Create a first-class TOIN signature for Kompress/plain-text content.
This intentionally keys on shape, not values. Retrieval pressure should
teach TOIN about this class of compressed content without storing the
content or treating it as an anonymous fallback.
"""
from ..telemetry.models import ToolSignature
words = content.split()
line_count = content.count("\n") + 1 if content else 0
nonempty_lines = [line for line in content.splitlines() if line.strip()]
avg_line_chars = (
sum(len(line) for line in nonempty_lines) // len(nonempty_lines) if nonempty_lines else 0
)
has_paths = "/" in content or "\\" in content
has_assignment_like_tokens = any("=" in word for word in words[:200])
has_brackets = any(ch in content for ch in "{}[]()")
has_error_terms = any(
term in content.lower() for term in ("error", "exception", "traceback", "failed", "fatal")
)
shape = "|".join(
(
"kompress-text",
f"chars:{_bucket_count(len(content))}",
f"words:{_bucket_count(len(words))}",
f"lines:{_bucket_count(line_count)}",
f"avg_line:{_bucket_count(avg_line_chars)}",
f"paths:{int(has_paths)}",
f"assign:{int(has_assignment_like_tokens)}",
f"brackets:{int(has_brackets)}",
f"errors:{int(has_error_terms)}",
)
)
structure_hash = hashlib.sha256(shape.encode()).hexdigest()[:24]
return ToolSignature(
structure_hash=structure_hash,
field_count=0,
has_nested_objects=False,
has_arrays=False,
max_depth=0,
string_field_count=1,
has_error_like_field=has_error_terms,
has_message_like_field=True,
)
@lru_cache(maxsize=1)
def _is_onnx_available() -> bool:
"""Check if ONNX Runtime is available (lightweight, no torch needed)."""
try:
import onnxruntime # noqa: F401
import transformers # noqa: F401
return True
except (ImportError, OSError):
# OSError: the package is installed but its native library will not
# load. See _is_pytorch_available for the full case.
return False
@lru_cache(maxsize=1)
def _is_pytorch_available() -> bool:
"""Check if full PyTorch stack is available (requires [ml] extra)."""
try:
import safetensors # noqa: F401
import torch # noqa: F401
import transformers # noqa: F401
return True
except (ImportError, OSError):
# An installed-but-unloadable native dependency raises OSError, not
# ImportError: on Windows without the MSVC redistributable, importing
# torch fails with WinError 126 loading c10.dll. Catching only
# ImportError let that escape a probe whose contract is to return a
# bool, and callers guard the call with `except ImportError` too, so it
# propagated into the request path instead of degrading to "no
# kompress".
return False
def is_kompress_available() -> bool:
"""Check if Kompress can run — ONNX (lightweight) or PyTorch (full)."""
return _is_onnx_available() or _is_pytorch_available()
# ── Model Architecture (must match training) ──────────────────────────
# torch/transformers are imported lazily — only when actually needed.
# This allows `from kompress_compressor import is_kompress_available`
# to work without torch installed.
def _get_model_class() -> type:
"""Return the HeadroomCompressorModel class, importing torch on demand."""
import torch
import torch.nn as nn
from transformers import AutoModel
class HeadroomCompressorModel(nn.Module):
"""Dual-head ModernBERT: token classification + span importance CNN."""
def __init__(
self,
model_name: str = "answerdotai/ModernBERT-base",
*,
revision: str | None = None,
allow_download: bool = True,
):
super().__init__()
# Same pin as the tokenizer: the fine-tuned heads were trained on
# this encoder snapshot. Unknown repos resolve to None (floating).
# Cache first, air-gap guard before any download (see _hf_pretrained).
self.encoder = _hf_pretrained(
AutoModel.from_pretrained,
model_name,
purpose="Kompress PyTorch encoder",
allow_download=allow_download,
attn_implementation="eager",
revision=_resolve_revision(model_name, revision),
)
hidden_size = self.encoder.config.hidden_size # 768
# Head 1: Token keep/discard
self.token_dropout = nn.Dropout(0.1)
self.token_head = nn.Linear(hidden_size, 2)
# Head 2: Span importance (1D CNN)
self.span_conv = nn.Sequential(
nn.Conv1d(hidden_size, 256, kernel_size=5, padding=2),
nn.GELU(),
nn.Conv1d(256, 1, kernel_size=3, padding=1),
nn.Sigmoid(),
)
def get_keep_mask(
self, input_ids: torch.Tensor, attention_mask: torch.Tensor
) -> torch.Tensor:
"""Get per-token keep/discard decision. True = keep."""
with torch.no_grad():
hidden = self.encoder(input_ids, attention_mask=attention_mask).last_hidden_state
# Token head: binary classifier — argmax decides keep/discard
token_logits = self.token_head(hidden) # [B, L, 2]
token_keep = (
token_logits[:, :, 1] > token_logits[:, :, 0]
) # True if class 1 > class 0
# Span head: boost tokens in important spans
# If a token is borderline but its span is important, keep it
span_scores = self.span_conv(hidden.transpose(1, 2)).squeeze(1)
span_boost = span_scores > 0.5 # span says this region matters
# Keep if: token head says keep, OR token is borderline and span says keep
token_probs = torch.softmax(token_logits, dim=-1)[:, :, 1]
borderline = (token_probs > 0.3) & (token_probs <= 0.5)
keep = token_keep | (borderline & span_boost)
return keep # type: ignore[no-any-return]
def get_scores(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
"""Get per-token importance scores (for ranking when target_ratio is set)."""
with torch.no_grad():
hidden = self.encoder(input_ids, attention_mask=attention_mask).last_hidden_state
token_probs = torch.softmax(self.token_head(hidden), dim=-1)[:, :, 1]
span_scores = self.span_conv(hidden.transpose(1, 2)).squeeze(1)
return token_probs * (0.5 + 0.5 * span_scores) # type: ignore[no-any-return]
return HeadroomCompressorModel
# ── Model Loading ─────────────────────────────────────────────────────
class _OnnxModel:
"""Thin wrapper so ONNX session has the same interface as PyTorch model."""
def __init__(self, session: Any):
self._session = session
def get_scores(self, input_ids: Any, attention_mask: Any) -> Any:
"""Return [batch, seq] scores via ONNX Runtime."""
import numpy as np
scores = self._session.run(
["final_scores"],
{
"input_ids": np.asarray(input_ids, dtype=np.int64),
"attention_mask": np.asarray(attention_mask, dtype=np.int64),
},
)
return scores[0] # [batch, seq] numpy array
def get_keep_mask(self, input_ids: Any, attention_mask: Any) -> Any:
"""Return [batch, seq] boolean mask (score > 0.5)."""
import numpy as np
scores = self.get_scores(input_ids, attention_mask)
return (np.array(scores) > 0.5).tolist()
def _onnx_filename_candidates() -> tuple[str, ...]:
"""ONNX repo paths to try, honoring an optional exact-file override."""
override = os.environ.get(KOMPRESS_ONNX_FILENAME_ENV, "").strip()
if override:
# Put the override first but keep the defaults as a safety net.
return (override, *(f for f in _DEFAULT_ONNX_FILENAMES if f != override))
return _DEFAULT_ONNX_FILENAMES
def _smoke_run(session: Any) -> None:
"""Run one tiny forward pass so a broken artifact fails HERE, not per request.
Some ONNX Runtime builds accept a session and then reject it at execution.
The int8 weight-only artifact carries ``MatMulNBits`` with ``bits=8``; ORT's
CPU kernel only handles 8-bit through the prepacked MLAS path, so a build or
ISA without an 8-bit ``SQNBitGemm`` kernel falls into ``ComputeBUnpacked``,
which hard-asserts ``nbits_ == 4``. That raises on ``session.run()`` — after
construction succeeded — so a load-only check never sees it and the fp32
fallback below is unreachable. Observed in the wild as 207 consecutive
per-request failures over three days with ML compression silently dead.
Two tokens through the real graph, so it costs milliseconds rather than the
seconds the timed canary takes (kernel dispatch is what fails, not compute).
"""
import numpy as np
session.run(
["final_scores"],
{
"input_ids": np.zeros((1, 2), dtype=np.int64),
"attention_mask": np.ones((1, 2), dtype=np.int64),
},
)
def _create_onnx_session(
model_id: str, providers: list[Any], *, allow_download: bool = True
) -> Any:
"""Resolve and load the model's ONNX artifact, trying candidates in order.
A candidate is skipped on download miss (file not in the repo), on
session-load failure, or on smoke-run failure (e.g. the weight-only int8
artifact uses the MatMulNBits contrib op, which some onnxruntime builds
accept at load and then reject at execution — those installs fall through to
the fp32 artifact instead of losing Kompress). See :func:`_smoke_run`.
When ``allow_download`` is ``False`` candidates are resolved from the local
cache only; if none is cached, :class:`KompressModelNotCached` is raised
instead of hitting the network. ``onnxruntime`` is imported only after a
candidate resolves, so a cache-only miss never requires it.
"""
last_err: Exception | None = None
cache_miss = False
refused: KompressModelNotCached | None = None
ort: Any = None
for filename in _onnx_filename_candidates():
try:
onnx_path = _hf_artifact(
model_id, filename, allow_network=allow_download and refused is None
)
except KompressModelNotCached as exc:
# Only _hf_artifact's air-gap translation raises this here. It says
# THIS candidate is not cached; a later one may be. Keep looking,
# cache-only, so the refusal is logged once and never retried.
refused = exc
continue
except Exception as exc:
last_err = exc
cache_miss = cache_miss or isinstance(exc, _NOT_CACHED_ERRORS)
logger.debug("ONNX artifact %r unavailable for %s: %s", filename, model_id, exc)
continue
if ort is None:
import onnxruntime
ort = onnxruntime
try:
session = ort.InferenceSession(
onnx_path,
_onnx_session_options(ort),
providers=providers,
)
_smoke_run(session)
return session
except Exception as exc:
last_err = exc
logger.warning(
"ONNX artifact %r from %s is unusable (%s); trying next candidate",
filename,
model_id,
exc,
)
if refused is not None:
raise refused
if not allow_download and cache_miss:
raise KompressModelNotCached(model_id) from last_err
raise FileNotFoundError(
f"No loadable ONNX artifact in {model_id}; tried {_onnx_filename_candidates()}"
) from last_err
def _load_kompress_onnx(
model_id: str,
*,
use_coreml: bool = False,
allow_download: bool = True,
) -> tuple[Any, Any, str]:
"""Download ONNX INT8 model from HuggingFace and load with onnxruntime.
When ``allow_download`` is ``False`` the model and tokenizer are loaded from
the local cache only; a cache miss raises :class:`KompressModelNotCached`
instead of hitting the network.
"""
with _kompress_lock:
if model_id in _kompress_cache:
return _kompress_cache[model_id]
logger.info("Downloading Kompress ONNX model from %s ...", model_id)
backend = "onnx_coreml" if use_coreml else "onnx"
providers: list[Any]
if use_coreml:
from headroom import paths as _paths
coreml_cache_dir = os.environ.get(KOMPRESS_COREML_CACHE_DIR_ENV, "").strip()
cache_dir = (
coreml_cache_dir
if coreml_cache_dir
else str(_paths.workspace_dir() / "cache" / "coreml")
)
os.makedirs(cache_dir, exist_ok=True)
providers = [
(
"CoreMLExecutionProvider",
{
"ModelFormat": "NeuralNetwork",
"MLComputeUnits": "ALL",
"RequireStaticInputShapes": "1",
"ModelCacheDirectory": cache_dir,
},
),
"CPUExecutionProvider",
]
else:
providers = ["CPUExecutionProvider"]
session = _create_onnx_session(model_id, providers, allow_download=allow_download)
model = _OnnxModel(session)
from transformers import AutoTokenizer
tokenizer = _load_modernbert_tokenizer(AutoTokenizer, allow_download=allow_download)
_kompress_cache[model_id] = (model, tokenizer, backend)
logger.info("Kompress ONNX loaded: %s backend=%s", model_id, backend)
return model, tokenizer, backend
_MODERNBERT_TOKENIZER_REPO = "answerdotai/ModernBERT-base"
def _load_modernbert_tokenizer(auto_tokenizer: Any, *, allow_download: bool) -> Any:
"""Load the ModernBERT tokenizer, cache-only when ``allow_download`` is False.
Always tries the local cache FIRST, even when downloading is allowed. With
``local_files_only=False`` transformers re-validates against the Hub on every
load — a tree listing plus a HEAD per tokenizer file — even when the repo is
fully cached. MEASURED ~900ms warm-cache versus ~150ms local-only, i.e. ~750ms
of pure network round-trip on every process start, and it is also what makes
a cold start slow on a bad network rather than merely offline.
Same files, same tokenizer, so the loaded object is identical; this only
changes whether the Hub is consulted to confirm what is already on disk.
Goes through ``onnx_runtime.hf_from_pretrained_local_first`` (via
:func:`_hf_pretrained`), the ``from_pretrained`` twin of the
``hf_hub_download_local_first`` the ONNX half of this loader uses, so a cache
miss under ``HEADROOM_OFFLINE`` is refused before any remote attempt even
when ``HF_HUB_OFFLINE=0`` is set explicitly.
The load is pinned to the same immutable revision as the model artifacts
(``onnx_runtime._PINNED_REVISIONS``): the ONNX export was produced against
that tokenizer snapshot, so a floating ``main`` could shift token ids under
the shipped weights without any code change. ``HEADROOM_HF_PIN=off`` floats
it, exactly as for the weights.
"""
return _hf_pretrained(
auto_tokenizer.from_pretrained,
_MODERNBERT_TOKENIZER_REPO,
purpose="ModernBERT tokenizer",
allow_download=allow_download,
revision=_resolve_revision(_MODERNBERT_TOKENIZER_REPO, None),
)
# Sub-state-dict keys inside a merged v2-style checkpoint (see
# scripts/export_kompress_v2_onnx.py, which this mirrors).
_MERGED_CHECKPOINT_KEYS = ("encoder_state_dict", "token_head_state_dict", "span_conv_state_dict")
def _load_merged_state_dict(model: Any, ckpt_path: str, model_id: str) -> None:
"""Load a merged v2-style checkpoint (LoRA already folded into the encoder).
The checkpoint is a dict of per-submodule state-dicts
(``encoder_state_dict`` / ``token_head_state_dict`` / ``span_conv_state_dict``)
rather than a single flat state-dict, so each piece is loaded into its
matching submodule directly instead of via a single ``load_state_dict``
call on the whole model.
"""
import torch
ckpt = torch.load(ckpt_path, map_location="cpu")
missing_sections = [k for k in _MERGED_CHECKPOINT_KEYS if k not in ckpt]
if missing_sections:
raise RuntimeError(
f"merged.pt for {model_id} is missing {missing_sections}; found keys: "
f"{sorted(ckpt)}. This checkpoint format is not what the loader expects."
)
for section, submodule in (
("encoder_state_dict", model.encoder),
("token_head_state_dict", model.token_head),
("span_conv_state_dict", model.span_conv),
):
missing, unexpected = submodule.load_state_dict(ckpt[section], strict=False)
if missing or unexpected:
raise RuntimeError(
f"{model_id} {section}: state_dict mismatch against {type(submodule).__name__} "
f"(missing={list(missing)[:5]}, unexpected={list(unexpected)[:5]}). "
"The checkpoint no longer matches HeadroomCompressorModel's architecture."
)
def _load_plain_state_dict(model: Any, weights_path: str, model_id: str) -> None:
"""Load a plain, already-merged full state-dict (the pre-v2 / non-PEFT format)."""
from safetensors.torch import load_file
state_dict = load_file(weights_path)