Skip to content

Commit 3793741

Browse files
fix(sparse): reject invalid MiniCOIL limits on retry
1 parent 1b0b40d commit 3793741

2 files changed

Lines changed: 15 additions & 3 deletions

File tree

‎fastembed/sparse/minicoil.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ def __init__(
162162
self.load_onnx_model()
163163

164164
def load_onnx_model(self) -> None:
165+
self._ensure_tokenizer()
165166
self._load_onnx_model(
166167
model_dir=self._model_dir,
167168
model_file=self.model_description.model_file,
@@ -180,6 +181,8 @@ def _load_tokenizer(self, model_dir: Path) -> None:
180181
assert self.tokenizer.truncation is not None
181182
minimum_length = self.tokenizer.num_special_tokens_to_add(is_pair=False) + 1
182183
if self.max_sequence_length < minimum_length:
184+
self.tokenizer = None
185+
self.special_token_to_id = {}
183186
raise ValueError(
184187
f"max_sequence_length must be at least {minimum_length} to include a text token"
185188
)

‎tests/test_minicoil_sequence_length.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,9 @@ def test_minicoil_rejects_invalid_sequence_limit(local_minicoil, invalid_length)
6565

6666

6767
@pytest.mark.parametrize("max_sequence_length", [1, 2])
68+
@pytest.mark.parametrize("operation", ["token_count", "embed", "query_embed"])
6869
def test_minicoil_limit_must_leave_room_for_text_and_special_tokens(
69-
local_minicoil, monkeypatch, max_sequence_length
70+
local_minicoil, monkeypatch, max_sequence_length, operation
7071
) -> None:
7172
tokenizer = Tokenizer(
7273
models.WordLevel({"[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "word": 3}, unk_token="[UNK]")
@@ -81,9 +82,17 @@ def test_minicoil_limit_must_leave_room_for_text_and_special_tokens(
8182
lambda model_dir: (tokenizer, {"[CLS]": 1, "[SEP]": 2}),
8283
)
8384
model = local_minicoil(max_sequence_length)
85+
monkeypatch.setattr(
86+
model,
87+
"_load_onnx_model",
88+
lambda **kwargs: pytest.fail("invalid sequence limit must fail before loading ONNX"),
89+
)
8490

85-
with pytest.raises(ValueError, match="max_sequence_length must be at least 3"):
86-
model._ensure_tokenizer()
91+
for _ in range(2):
92+
with pytest.raises(ValueError, match="max_sequence_length must be at least 3"):
93+
result = getattr(model, operation)("word word")
94+
if operation != "token_count":
95+
list(result)
8796

8897

8998
def test_minicoil_parallel_workers_receive_sequence_limit(local_minicoil, monkeypatch) -> None:

0 commit comments

Comments
 (0)