|
| 1 | +from types import SimpleNamespace |
| 2 | + |
| 3 | +import numpy as np |
| 4 | +import pytest |
| 5 | +from tokenizers import Tokenizer, models, pre_tokenizers, processors |
| 6 | + |
| 7 | +from fastembed.sparse.minicoil import MiniCOIL |
| 8 | +from fastembed.sparse.sparse_text_embedding import SparseTextEmbedding |
| 9 | + |
| 10 | + |
| 11 | +@pytest.fixture |
| 12 | +def local_minicoil(monkeypatch, tmp_path): |
| 13 | + """Use a real tokenizer but avoid downloading model weights for sequence-length tests.""" |
| 14 | + |
| 15 | + def load_tokenizer(model_dir): |
| 16 | + vocab = {"[UNK]": 0, "[PAD]": 1} |
| 17 | + vocab.update({f"word{i}": i + 2 for i in range(12)}) |
| 18 | + tokenizer = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]")) |
| 19 | + tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() |
| 20 | + tokenizer.enable_truncation(max_length=8) |
| 21 | + tokenizer.enable_padding(pad_id=1, pad_token="[PAD]") |
| 22 | + return tokenizer, {} |
| 23 | + |
| 24 | + monkeypatch.setattr("fastembed.text.onnx_text_model.load_tokenizer", load_tokenizer) |
| 25 | + monkeypatch.setattr(MiniCOIL, "download_model", lambda *args, **kwargs: tmp_path) |
| 26 | + |
| 27 | + def make_model(max_sequence_length=None): |
| 28 | + kwargs = ( |
| 29 | + {} if max_sequence_length is None else {"max_sequence_length": max_sequence_length} |
| 30 | + ) |
| 31 | + return SparseTextEmbedding("Qdrant/minicoil-v1", lazy_load=True, **kwargs).model |
| 32 | + |
| 33 | + return make_model |
| 34 | + |
| 35 | + |
| 36 | +@pytest.mark.parametrize( |
| 37 | + ("max_sequence_length", "expected_length"), |
| 38 | + [(None, 8), (4, 4), (20, 8)], |
| 39 | +) |
| 40 | +def test_minicoil_limit_reaches_onnx_input( |
| 41 | + local_minicoil, monkeypatch, max_sequence_length, expected_length |
| 42 | +) -> None: |
| 43 | + model = local_minicoil(max_sequence_length) |
| 44 | + model._ensure_tokenizer() |
| 45 | + model.model = SimpleNamespace(get_inputs=lambda: [SimpleNamespace(name="input_ids")]) |
| 46 | + monkeypatch.setattr( |
| 47 | + model, |
| 48 | + "_run_model", |
| 49 | + lambda onnx_input, onnx_output_names=None: np.zeros( |
| 50 | + (*onnx_input["input_ids"].shape, 1), dtype=np.float32 |
| 51 | + ), |
| 52 | + ) |
| 53 | + |
| 54 | + text = " ".join(f"word{i}" for i in range(12)) |
| 55 | + output = model.onnx_embed([text]) |
| 56 | + |
| 57 | + assert output.input_ids.shape == (1, expected_length) |
| 58 | + assert model.token_count(text) == expected_length |
| 59 | + |
| 60 | + |
| 61 | +@pytest.mark.parametrize("invalid_length", [0, -1, True, 2.5]) |
| 62 | +def test_minicoil_rejects_invalid_sequence_limit(local_minicoil, invalid_length) -> None: |
| 63 | + with pytest.raises(ValueError, match="max_sequence_length must be a positive integer"): |
| 64 | + local_minicoil(invalid_length) |
| 65 | + |
| 66 | + |
| 67 | +@pytest.mark.parametrize("max_sequence_length", [1, 2]) |
| 68 | +def test_minicoil_limit_must_leave_room_for_text_and_special_tokens( |
| 69 | + local_minicoil, monkeypatch, max_sequence_length |
| 70 | +) -> None: |
| 71 | + tokenizer = Tokenizer( |
| 72 | + models.WordLevel({"[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "word": 3}, unk_token="[UNK]") |
| 73 | + ) |
| 74 | + tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() |
| 75 | + tokenizer.post_processor = processors.TemplateProcessing( |
| 76 | + single="[CLS] $A [SEP]", special_tokens=[("[CLS]", 1), ("[SEP]", 2)] |
| 77 | + ) |
| 78 | + tokenizer.enable_truncation(max_length=8) |
| 79 | + monkeypatch.setattr( |
| 80 | + "fastembed.text.onnx_text_model.load_tokenizer", |
| 81 | + lambda model_dir: (tokenizer, {"[CLS]": 1, "[SEP]": 2}), |
| 82 | + ) |
| 83 | + model = local_minicoil(max_sequence_length) |
| 84 | + |
| 85 | + with pytest.raises(ValueError, match="max_sequence_length must be at least 3"): |
| 86 | + model._ensure_tokenizer() |
| 87 | + |
| 88 | + |
| 89 | +def test_minicoil_parallel_workers_receive_sequence_limit(local_minicoil, monkeypatch) -> None: |
| 90 | + model = local_minicoil(4) |
| 91 | + captured = {} |
| 92 | + |
| 93 | + class RecordingPool: |
| 94 | + def __init__(self, **kwargs): |
| 95 | + pass |
| 96 | + |
| 97 | + def ordered_map(self, batches, **kwargs): |
| 98 | + captured.update(kwargs) |
| 99 | + return iter(()) |
| 100 | + |
| 101 | + monkeypatch.setattr("fastembed.text.onnx_text_model.ParallelWorkerPool", RecordingPool) |
| 102 | + |
| 103 | + assert list(model.embed(["word0", "word1"], batch_size=1, parallel=2)) == [] |
| 104 | + assert captured["max_sequence_length"] == 4 |
0 commit comments