Skip to content

Commit 1b0b40d

Browse files
feat(sparse): allow MiniCOIL sequence length limit
1 parent 7d36728 commit 1b0b40d

3 files changed

Lines changed: 141 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,15 @@ embeddings = list(model.embed(documents))
9898
# ]
9999
```
100100

101+
For MiniCOIL, use `max_sequence_length` to cap the tokens sent to the model when
102+
embedding long texts. The limit includes special tokens and must leave room for at least one
103+
text token. Values above the model's own tokenizer limit are capped at that limit.
104+
105+
```python
106+
model = SparseTextEmbedding(model_name="Qdrant/minicoil-v1", max_sequence_length=512)
107+
embeddings = list(model.embed(documents))
108+
```
109+
101110
<!--
102111
* BM42 - ([link](ToDo))
103112

‎fastembed/sparse/minicoil.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ def __init__(
8484
lazy_load: bool = False,
8585
device_id: int | None = None,
8686
specific_model_path: str | None = None,
87+
max_sequence_length: int | None = None,
8788
**kwargs: Any,
8889
):
8990
"""
@@ -108,12 +109,23 @@ def __init__(
108109
Should be set to True when using multiple-gpu and parallel encoding. Defaults to False.
109110
device_id (Optional[int], optional): The device id to use for loading the model in the worker process.
110111
specific_model_path (Optional[str], optional): The specific path to the onnx model dir if it should be imported from somewhere else
112+
max_sequence_length (Optional[int], optional): Maximum number of input tokens, including
113+
special tokens. Defaults to the model's tokenizer limit. Values above the model's limit
114+
are capped at that limit. Applies to document and query embeddings, including workers.
111115
112116
Raises:
113117
ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-base-en.
114118
"""
115119

120+
if max_sequence_length is not None and (
121+
isinstance(max_sequence_length, bool)
122+
or not isinstance(max_sequence_length, int)
123+
or max_sequence_length <= 0
124+
):
125+
raise ValueError("max_sequence_length must be a positive integer or None")
126+
116127
super().__init__(model_name, cache_dir, threads, **kwargs)
128+
self.max_sequence_length = max_sequence_length
117129
self.providers = providers
118130
self.lazy_load = lazy_load
119131
self.device_ids = device_ids
@@ -164,10 +176,26 @@ def load_onnx_model(self) -> None:
164176
def _load_tokenizer(self, model_dir: Path) -> None:
165177
super()._load_tokenizer(model_dir=model_dir)
166178
assert self.tokenizer is not None
179+
if self.max_sequence_length is not None:
180+
assert self.tokenizer.truncation is not None
181+
minimum_length = self.tokenizer.num_special_tokens_to_add(is_pair=False) + 1
182+
if self.max_sequence_length < minimum_length:
183+
raise ValueError(
184+
f"max_sequence_length must be at least {minimum_length} to include a text token"
185+
)
186+
model_max_length = self.tokenizer.truncation["max_length"]
187+
self.tokenizer.enable_truncation(
188+
max_length=min(self.max_sequence_length, model_max_length)
189+
)
167190
self.invert_vocab = {idx: token for token, idx in self.tokenizer.get_vocab().items()}
168191
self.special_tokens = set(self.special_token_to_id.keys())
169192
self.special_tokens_ids = set(self.special_token_to_id.values())
170193

194+
def _get_worker_init_kwargs(self) -> dict[str, Any]:
195+
if self.max_sequence_length is None:
196+
return {}
197+
return {"max_sequence_length": self.max_sequence_length}
198+
171199
def _load_post_processing_state(self) -> None:
172200
"""Load the vocab resolver, the encoder and the sparse vector converter.
173201
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
from types import SimpleNamespace
2+
3+
import numpy as np
4+
import pytest
5+
from tokenizers import Tokenizer, models, pre_tokenizers, processors
6+
7+
from fastembed.sparse.minicoil import MiniCOIL
8+
from fastembed.sparse.sparse_text_embedding import SparseTextEmbedding
9+
10+
11+
@pytest.fixture
12+
def local_minicoil(monkeypatch, tmp_path):
13+
"""Use a real tokenizer but avoid downloading model weights for sequence-length tests."""
14+
15+
def load_tokenizer(model_dir):
16+
vocab = {"[UNK]": 0, "[PAD]": 1}
17+
vocab.update({f"word{i}": i + 2 for i in range(12)})
18+
tokenizer = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]"))
19+
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
20+
tokenizer.enable_truncation(max_length=8)
21+
tokenizer.enable_padding(pad_id=1, pad_token="[PAD]")
22+
return tokenizer, {}
23+
24+
monkeypatch.setattr("fastembed.text.onnx_text_model.load_tokenizer", load_tokenizer)
25+
monkeypatch.setattr(MiniCOIL, "download_model", lambda *args, **kwargs: tmp_path)
26+
27+
def make_model(max_sequence_length=None):
28+
kwargs = (
29+
{} if max_sequence_length is None else {"max_sequence_length": max_sequence_length}
30+
)
31+
return SparseTextEmbedding("Qdrant/minicoil-v1", lazy_load=True, **kwargs).model
32+
33+
return make_model
34+
35+
36+
@pytest.mark.parametrize(
37+
("max_sequence_length", "expected_length"),
38+
[(None, 8), (4, 4), (20, 8)],
39+
)
40+
def test_minicoil_limit_reaches_onnx_input(
41+
local_minicoil, monkeypatch, max_sequence_length, expected_length
42+
) -> None:
43+
model = local_minicoil(max_sequence_length)
44+
model._ensure_tokenizer()
45+
model.model = SimpleNamespace(get_inputs=lambda: [SimpleNamespace(name="input_ids")])
46+
monkeypatch.setattr(
47+
model,
48+
"_run_model",
49+
lambda onnx_input, onnx_output_names=None: np.zeros(
50+
(*onnx_input["input_ids"].shape, 1), dtype=np.float32
51+
),
52+
)
53+
54+
text = " ".join(f"word{i}" for i in range(12))
55+
output = model.onnx_embed([text])
56+
57+
assert output.input_ids.shape == (1, expected_length)
58+
assert model.token_count(text) == expected_length
59+
60+
61+
@pytest.mark.parametrize("invalid_length", [0, -1, True, 2.5])
62+
def test_minicoil_rejects_invalid_sequence_limit(local_minicoil, invalid_length) -> None:
63+
with pytest.raises(ValueError, match="max_sequence_length must be a positive integer"):
64+
local_minicoil(invalid_length)
65+
66+
67+
@pytest.mark.parametrize("max_sequence_length", [1, 2])
68+
def test_minicoil_limit_must_leave_room_for_text_and_special_tokens(
69+
local_minicoil, monkeypatch, max_sequence_length
70+
) -> None:
71+
tokenizer = Tokenizer(
72+
models.WordLevel({"[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "word": 3}, unk_token="[UNK]")
73+
)
74+
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
75+
tokenizer.post_processor = processors.TemplateProcessing(
76+
single="[CLS] $A [SEP]", special_tokens=[("[CLS]", 1), ("[SEP]", 2)]
77+
)
78+
tokenizer.enable_truncation(max_length=8)
79+
monkeypatch.setattr(
80+
"fastembed.text.onnx_text_model.load_tokenizer",
81+
lambda model_dir: (tokenizer, {"[CLS]": 1, "[SEP]": 2}),
82+
)
83+
model = local_minicoil(max_sequence_length)
84+
85+
with pytest.raises(ValueError, match="max_sequence_length must be at least 3"):
86+
model._ensure_tokenizer()
87+
88+
89+
def test_minicoil_parallel_workers_receive_sequence_limit(local_minicoil, monkeypatch) -> None:
90+
model = local_minicoil(4)
91+
captured = {}
92+
93+
class RecordingPool:
94+
def __init__(self, **kwargs):
95+
pass
96+
97+
def ordered_map(self, batches, **kwargs):
98+
captured.update(kwargs)
99+
return iter(())
100+
101+
monkeypatch.setattr("fastembed.text.onnx_text_model.ParallelWorkerPool", RecordingPool)
102+
103+
assert list(model.embed(["word0", "word1"], batch_size=1, parallel=2)) == []
104+
assert captured["max_sequence_length"] == 4

0 commit comments

Comments
 (0)