Skip to content

Commit a90ad58

Browse files
Merge branch 'main' into feat/544-minicoil-sequence-limit
2 parents 3793741 + d076f08 commit a90ad58

33 files changed

Lines changed: 1647 additions & 919 deletions

‎.github/workflows/python-publish.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,18 @@ jobs:
3232
poetry install
3333
- name: Build package
3434
run: poetry build
35+
- name: Smoke-test built package
36+
# run outside the checkout so the source tree's fastembed/ can't shadow the installed wheel
37+
working-directory: ${{ runner.temp }}
38+
run: |
39+
python -m venv smoke-venv
40+
smoke-venv/bin/pip install --quiet "$GITHUB_WORKSPACE"/dist/*.whl
41+
smoke-venv/bin/python -c "
42+
from fastembed import TextEmbedding
43+
embedding = next(iter(TextEmbedding('sentence-transformers/all-MiniLM-L6-v2').embed(['hello world'])))
44+
assert embedding.shape == (384,), embedding.shape
45+
print('Smoke test OK')
46+
"
3547
- name: Publish package
3648
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
3749
with:

‎.github/workflows/weekly-tests.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Weekly tests
2+
3+
# PR runs test one representative model per family, scheduled and manual runs test every model,
4+
# see is_manual_run in tests/utils.py
5+
on:
6+
schedule:
7+
- cron: '5 9 * * 1' # Mondays, 09:05 UTC
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
test:
15+
16+
# linux runners have 16 GB of RAM, enough for every model tested in ci (colpali is skipped in ci,
17+
# gpu tests are skipped everywhere)
18+
runs-on: ubuntu-latest
19+
timeout-minutes: 180
20+
21+
name: Python 3.14 weekly test
22+
23+
outputs:
24+
report: ${{ steps.report.outputs.report }}
25+
26+
steps:
27+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28+
with:
29+
persist-credentials: false
30+
- name: Set up Python
31+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
32+
with:
33+
python-version: '3.14.x'
34+
- name: Install dependencies
35+
run: |
36+
python -m pip install poetry
37+
poetry config virtualenvs.create false
38+
poetry install --no-interaction --no-ansi --without dev,docs
39+
40+
- name: Run pytest
41+
# explicit bash runs with pipefail, so tee keeps pytest's exit code
42+
shell: bash
43+
env:
44+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
45+
PYTHONUNBUFFERED: '1'
46+
run: |
47+
poetry run pytest --durations=20 2>&1 | tee pytest.log
48+
49+
- name: Collect failure report
50+
id: report
51+
if: failure() || cancelled()
52+
shell: bash
53+
run: |
54+
{
55+
if grep -q 'short test summary info' pytest.log 2>/dev/null; then
56+
sed -n '/short test summary info/,$p' pytest.log
57+
elif [ -s pytest.log ]; then
58+
echo "No pytest test summary, last lines of its output:"
59+
tail -n 50 pytest.log
60+
fi
61+
} > summary.txt
62+
head -c 60000 summary.txt > report.txt # issue bodies are limited to 65536 characters
63+
if [ -s report.txt ]; then
64+
delimiter="EOF_$(openssl rand -hex 16)"
65+
{
66+
echo "report<<$delimiter"
67+
sed '$a\' report.txt # appends a newline only if the cut left the last line without one
68+
echo "$delimiter"
69+
} >> "$GITHUB_OUTPUT"
70+
fi
71+
72+
report:
73+
# a timed out job is cancelled rather than failed, so check the result instead of failure()
74+
needs: test
75+
if: always() && github.event_name == 'schedule' && needs.test.result != 'success'
76+
runs-on: ubuntu-latest
77+
permissions:
78+
issues: write # open the failure issue or comment on it
79+
80+
name: Report weekly test failure
81+
82+
steps:
83+
- name: Open an issue or comment on the open one
84+
env:
85+
GH_TOKEN: ${{ github.token }}
86+
GH_REPO: ${{ github.repository }}
87+
RESULT: ${{ needs.test.result }}
88+
REPORT: ${{ needs.test.outputs.report }}
89+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
90+
SHA: ${{ github.sha }}
91+
run: |
92+
title="Weekly tests failed"
93+
{
94+
echo "Run: $RUN_URL"
95+
echo "Result: $RESULT"
96+
echo "Commit: $SHA"
97+
echo
98+
if [ -n "$REPORT" ]; then
99+
echo '```text'
100+
printf '%s\n' "$REPORT"
101+
echo '```'
102+
else
103+
echo "No pytest summary: tests failed to start or the job timed out, see the run log."
104+
fi
105+
} > body.md
106+
issue=$(gh issue list --state open --search "in:title \"$title\"" --json number,title \
107+
--jq "map(select(.title == \"$title\")) | first | .number // empty")
108+
if [ -n "$issue" ]; then
109+
gh issue comment "$issue" --body-file body.md
110+
else
111+
gh issue create --title "$title" --body-file body.md --assignee joein
112+
fi

‎docs/examples/Supported_Models.ipynb‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
"source": [
3636
"import pandas as pd\n",
3737
"\n",
38+
"pd.set_option(\"display.max_colwidth\", None)\n",
39+
"\n",
3840
"from fastembed import (\n",
3941
" SparseTextEmbedding,\n",
4042
" TextEmbedding,\n",
@@ -84,7 +86,7 @@
8486
"6 snowflake/snowflake-arctic-embed-s 384 \n",
8587
"7 nomic-ai/nomic-embed-text-v1.5-Q 768 \n",
8688
"8 BAAI/bge-base-en-v1.5 768 \n",
87-
"9 sentence-transformers/paraphrase-multilingual-... 384 \n",
89+
"9 sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 384 \n",
8890
"10 Qdrant/clip-ViT-B-32-text 512 \n",
8991
"11 jinaai/jina-embeddings-v2-base-de 768 \n",
9092
"12 BAAI/bge-base-en 768 \n",
@@ -95,7 +97,7 @@
9597
"17 snowflake/snowflake-arctic-embed-m-long 768 \n",
9698
"18 mixedbread-ai/mxbai-embed-large-v1 1024 \n",
9799
"19 jinaai/jina-embeddings-v2-base-code 768 \n",
98-
"20 sentence-transformers/paraphrase-multilingual-... 768 \n",
100+
"20 sentence-transformers/paraphrase-multilingual-mpnet-base-v2 768 \n",
99101
"21 snowflake/snowflake-arctic-embed-l 1024 \n",
100102
"22 thenlper/gte-large 1024 \n",
101103
"23 BAAI/bge-large-en-v1.5 1024 \n",
@@ -229,7 +231,7 @@
229231
" </tr>\n",
230232
" <tr>\n",
231233
" <th>9</th>\n",
232-
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
234+
" <td>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</td>\n",
233235
" <td>384</td>\n",
234236
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
235237
" <td>apache-2.0</td>\n",
@@ -317,7 +319,7 @@
317319
" </tr>\n",
318320
" <tr>\n",
319321
" <th>20</th>\n",
320-
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
322+
" <td>sentence-transformers/paraphrase-multilingual-mpnet-base-v2</td>\n",
321323
" <td>768</td>\n",
322324
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
323325
" <td>apache-2.0</td>\n",

‎fastembed/common/model_management.py‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -264,11 +264,14 @@ def _verify_files_from_metadata(
264264
return False
265265

266266
def _collect_file_metadata(
267-
model_dir: Path, repo_files: list[RepoFile]
267+
model_dir: Path, revision: str, repo_files: list[RepoFile]
268268
) -> dict[str, dict[str, int | str]]:
269269
meta: dict[str, dict[str, int | str]] = {}
270270
file_info_map = {f.path: f for f in repo_files}
271-
for file_path in model_dir.rglob("*"):
271+
# Only look at the downloaded revision's folder. The cache may also hold folders of
272+
# older revisions, whose files differ from the ones on the hub now, so checking
273+
# them would report a good download as corrupted.
274+
for file_path in (model_dir / "snapshots" / revision).rglob("*"):
272275
if file_path.is_file() and file_path.name != cls.METADATA_FILE:
273276
relative_path = file_path.relative_to(model_dir)
274277
repo_file = file_info_map.get(_repo_relative_path(relative_path))
@@ -399,9 +402,18 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]])
399402

400403
verified_metadata = False
401404

402-
if snapshot_dir.exists() and metadata_file.exists():
405+
# a forced download re-fetches every file, so show progress and re-collect the metadata
406+
if not kwargs.get("force_download") and snapshot_dir.exists() and metadata_file.exists():
403407
metadata = json.loads(metadata_file.read_text())
404-
verified_metadata = _verify_files_from_metadata(snapshot_dir, metadata, repo_files)
408+
# metadata lacking a requested file, e.g. a new model_file, can't vouch for it
409+
requested_files = {f.path for f in repo_files if f.path in allow_patterns}
410+
stored_files = {_repo_relative_path(Path(rel_path)) for rel_path in metadata}
411+
# empty metadata lists no files, so it can't vouch for the cached ones
412+
verified_metadata = (
413+
bool(metadata)
414+
and requested_files.issubset(stored_files)
415+
and _verify_files_from_metadata(snapshot_dir, metadata, repo_files)
416+
)
405417

406418
if verified_metadata:
407419
disable_progress_bars()
@@ -418,7 +430,10 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]])
418430
not verified_metadata
419431
): # metadata is not up-to-date, update it and check whether the files have been
420432
# downloaded correctly
421-
metadata = _collect_file_metadata(snapshot_dir, repo_files)
433+
# result is <cache_dir>/.../snapshots/<revision>. Only the revision is taken from
434+
# it: the hub resolves cache_dir, so the rest of the path may not match ours.
435+
downloaded_revision = Path(result).name
436+
metadata = _collect_file_metadata(snapshot_dir, downloaded_revision, repo_files)
422437

423438
download_successful = _verify_files_from_metadata(
424439
snapshot_dir, metadata, repo_files=[]
@@ -428,7 +443,10 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]])
428443
"Files have been corrupted during downloading process. "
429444
"Please check your internet connection and try again."
430445
)
431-
_save_file_metadata(snapshot_dir, metadata)
446+
# Empty means no file was checked, e.g. the files went to a local_dir instead of
447+
# the cache. Saving it would make later downloads skip this check.
448+
if metadata:
449+
_save_file_metadata(snapshot_dir, metadata)
432450

433451
return result
434452

‎fastembed/common/preprocessor_utils.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,13 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]:
9595
tokens_map = load_special_tokens(model_dir)
9696

9797
tokenizer = Tokenizer.from_file(str(tokenizer_path))
98-
tokenizer.enable_truncation(max_length=max_context)
98+
# enable_truncation resets the direction to right unless it is passed. The direction is
99+
# resolved as in transformers: tokenizer_config.json, then tokenizer.json, then right.
100+
truncation = tokenizer.truncation or {}
101+
tokenizer.enable_truncation(
102+
max_length=max_context,
103+
direction=tokenizer_config.get("truncation_side") or truncation.get("direction", "right"),
104+
)
99105

100106
# Registered before the padding is resolved: the map may name a pad token that
101107
# tokenizer.json does not carry, and it only gets an id once it is added.
@@ -107,8 +113,10 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]:
107113

108114
# Padding is always normalized to batch-longest. A serialized fixed length shorter than the
109115
# truncation limit leaves longer encodings untouched, which produces ragged batches, and a
110-
# fixed length equal to it pads every batch to the maximum. Direction and pad token metadata
111-
# are taken from the serialized settings, since some models pad on the left.
116+
# fixed length equal to it pads every batch to the maximum. Pad token metadata is taken from
117+
# the serialized settings. The direction follows transformers: `padding_side` from
118+
# tokenizer_config.json wins over the serialized direction, since some models pad on the left
119+
# and set it only in the config.
112120
padding = tokenizer.padding or {}
113121
pad_token = padding.get("pad_token") or tokenizer_config.get("pad_token")
114122
if pad_token is None:
@@ -123,7 +131,7 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]:
123131
raise ValueError(f"Could not resolve an id for the pad token {pad_token!r} in {model_dir}")
124132

125133
tokenizer.enable_padding(
126-
direction=padding.get("direction", "right"),
134+
direction=tokenizer_config.get("padding_side") or padding.get("direction", "right"),
127135
pad_id=pad_id,
128136
pad_type_id=padding.get("pad_type_id", 0),
129137
pad_token=pad_token,

‎fastembed/image/transform/functional.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ def resize_longest_edge(
186186
if new_width % 2 != 0:
187187
new_width += 1
188188

189-
return image.resize((new_width, new_height), resample)
189+
# A very narrow image can round down to zero along its shorter edge.
190+
return image.resize((max(1, new_width), max(1, new_height)), resample)
190191

191192

192193
def crop_ndarray(

‎fastembed/image/transform/operators.py‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,15 @@ def __init__(
191191
self.resample = resample
192192

193193
def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type: ignore[override]
194+
patches, _ = self.split_with_grid(images)
195+
return patches
196+
197+
def split_with_grid(
198+
self, images: list[NumpyArray]
199+
) -> tuple[list[list[NumpyArray]], list[tuple[int, int]]]:
200+
"""Return patches and their (rows, cols), or (0, 0) for unsplit images."""
194201
result = []
202+
image_grid = []
195203

196204
for image in images:
197205
# Assume (C, H, W) format
@@ -204,6 +212,7 @@ def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type:
204212
# Calculate the number of splits needed
205213
num_splits_h = math.ceil(height / max_height)
206214
num_splits_w = math.ceil(width / max_width)
215+
image_grid.append((num_splits_h, num_splits_w))
207216

208217
# Calculate optimal patch dimensions
209218
optimal_height = math.ceil(height / num_splits_h)
@@ -234,12 +243,13 @@ def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type:
234243
frames.append(global_view)
235244
else:
236245
# Image is small enough, no splitting needed
246+
image_grid.append((0, 0))
237247
frames.append(image)
238248

239249
# Append (not extend) to preserve per-image grouping
240250
result.append(frames)
241251

242-
return result
252+
return result, image_grid
243253

244254

245255
class SquareResize(Transform):
@@ -272,10 +282,23 @@ def __init__(self, transforms: list[Transform]):
272282
self.transforms = transforms
273283

274284
def __call__(
275-
self, images: list[Image.Image] | list[NumpyArray]
285+
self,
286+
images: list[Image.Image] | list[NumpyArray],
287+
*,
288+
metadata: dict[str, Any] | None = None,
276289
) -> list[NumpyArray] | list[Image.Image]:
290+
"""Apply transforms without changing their usual return type.
291+
292+
If supplied, metadata["image_grid"] records (rows, cols) for each image,
293+
or (0, 0) for images that are not split.
294+
"""
295+
if metadata is not None:
296+
metadata["image_grid"] = [(0, 0) for _ in images]
277297
for transform in self.transforms:
278-
images = transform(images)
298+
if metadata is not None and isinstance(transform, ImageSplitter):
299+
images, metadata["image_grid"] = transform.split_with_grid(images) # type: ignore[assignment,arg-type]
300+
else:
301+
images = transform(images)
279302
return images
280303

281304
@classmethod
@@ -344,6 +367,9 @@ def _get_resize(cls, transforms: list[Transform], config: dict[str, Any]) -> Non
344367
)
345368
)
346369
elif mode == "ConvNextFeatureExtractor":
370+
# HF defaults do_resize to True for ConvNeXT; it also gates the coupled crop
371+
if not config.get("do_resize", True):
372+
return
347373
if "size" in config and "shortest_edge" not in config["size"]:
348374
raise ValueError(
349375
f"Size dictionary must contain 'shortest_edge' key. Got {config['size'].keys()}"

‎fastembed/late_interaction/colbert.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@
4141
class Colbert(LateInteractionTextEmbeddingBase, OnnxTextModel[NumpyArray]):
4242
QUERY_MARKER_TOKEN_ID = 1
4343
DOCUMENT_MARKER_TOKEN_ID = 2
44-
MIN_QUERY_LENGTH = 31 # it's 32, we add one additional special token in the beginning
44+
# tokens `_preprocess_onnx_input` inserts that the tokenizer metadata does not count
45+
RESERVED_MARKER_TOKENS = 1
46+
# it's 32, we add one additional special token in the beginning. `None` turns the query
47+
# expansion off: the query is then padded to the longest one in its batch, as a document is
48+
MIN_QUERY_LENGTH: int | None = 31
4549
MASK_TOKEN = "[MASK]"
4650

4751
def _post_process_onnx_output(
@@ -120,7 +124,7 @@ def token_count(
120124
token_num += sum(tokens.attention_mask)
121125
else:
122126
attend_count = sum(tokens.attention_mask)
123-
if include_extension:
127+
if include_extension and self.MIN_QUERY_LENGTH is not None:
124128
token_num += max(attend_count, self.MIN_QUERY_LENGTH)
125129

126130
else:
@@ -242,7 +246,7 @@ def _load_tokenizer(self, model_dir: Path) -> None:
242246
for symbol in string.punctuation
243247
}
244248
# ensure not to overflow after adding document-marker
245-
max_length = tokenizer.truncation["max_length"] - 1
249+
max_length = tokenizer.truncation["max_length"] - self.RESERVED_MARKER_TOKENS
246250
tokenizer.enable_truncation(max_length=max_length)
247251
query_tokenizer.enable_truncation(max_length=max_length)
248252
query_tokenizer.enable_padding(

0 commit comments

Comments
 (0)