Skip to content

Commit e5e26b8

Browse files
authored
fix: do not accept empty vectors and multivectors in local mode (#1405)
* fix: do not accept empty vectors and multivectors in local mode * fix: do not modify points if vector does not pass validation
1 parent df683c7 commit e5e26b8

3 files changed

Lines changed: 256 additions & 10 deletions

File tree

‎qdrant_client/local/local_collection.py‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,25 @@ def to_jsonable_python(x: Any) -> Any:
100100
return json.loads(json.dumps(x, allow_nan=True, default=_to_jsonable_python))
101101

102102

103+
def validate_dense_vector(vector: Any, vector_name: str) -> None:
104+
"""Reject empty dense vectors, as the server does at write time."""
105+
if len(vector) == 0:
106+
raise ValueError(f"Wrong input: Dense vector must not be empty for vector '{vector_name}'")
107+
108+
109+
def validate_multivector(vector: Any, vector_name: str) -> None:
110+
"""Reject empty multivectors and multivectors holding empty vectors, as the server does."""
111+
if len(vector) == 0:
112+
raise ValueError(f"Wrong input: Multivector must not be empty for vector '{vector_name}'")
113+
114+
for sub_vector in vector:
115+
if hasattr(sub_vector, "__len__") and len(sub_vector) == 0:
116+
raise ValueError(
117+
"Wrong input: All vectors of a multivector must be non-empty "
118+
f"for vector '{vector_name}'"
119+
)
120+
121+
103122
class LocalCollection:
104123
"""
105124
LocalCollection is a class that represents a collection of vectors in the local storage.
@@ -393,6 +412,20 @@ def get_vector_params(self, name: str) -> models.VectorParams:
393412

394413
raise ValueError(f"Malformed config.vectors: {self.config.vectors}")
395414

415+
def _validate_dense_or_multivector(self, vector: Any, vector_name: str) -> None:
416+
"""Reject empty vectors on the write path, the way the server does.
417+
418+
Sparse vectors are validated by `validate_sparse_vector`; an empty sparse vector is
419+
legitimate, so it is not routed here.
420+
"""
421+
if vector is None:
422+
return
423+
424+
if vector_name in self.multivectors:
425+
validate_multivector(vector, vector_name)
426+
elif vector_name in self.vectors:
427+
validate_dense_vector(vector, vector_name)
428+
396429
@classmethod
397430
def _check_include_pattern(cls, pattern: str, key: str) -> bool:
398431
"""
@@ -2613,6 +2646,8 @@ def _upsert_point(
26132646
validate_sparse_vector(vector)
26142647
# sort sparse vector by indices before persistence
26152648
updated_sparse_vectors[vector_name] = sort_sparse_vector(vector)
2649+
else:
2650+
self._validate_dense_or_multivector(vector, vector_name)
26162651
# update point.vector with the modified values after iteration
26172652
point.vector.update(updated_sparse_vectors)
26182653
else:
@@ -2627,6 +2662,7 @@ def _upsert_point(
26272662
)
26282663
if not self.vectors and not self.multivectors:
26292664
raise ValueError("Wrong input: Not existing vector name error")
2665+
self._validate_dense_or_multivector(point.vector, DEFAULT_VECTOR_NAME)
26302666

26312667
if isinstance(point.id, uuid.UUID):
26322668
point.id = str(point.id)
@@ -2703,23 +2739,31 @@ def upsert(
27032739
def _update_named_vectors(
27042740
self, idx: int, vectors: dict[str, list[float] | SparseVector | list[list[float]]]
27052741
) -> None:
2742+
validated: list[tuple[str, Any]] = []
27062743
for vector_name, vector in vectors.items():
27072744
if vector_name not in self._all_vectors_keys:
27082745
raise ValueError(f"Wrong input: Not existing vector name error: {vector_name}")
27092746

2710-
self.deleted_per_vector[vector_name][idx] = 0
2711-
27122747
if isinstance(vector, SparseVector):
27132748
validate_sparse_vector(vector)
2714-
old_vector = self.sparse_vectors[vector_name][idx]
2715-
self._update_idf_remove(old_vector, vector_name)
2716-
new_vector = sort_sparse_vector(vector)
2717-
self.sparse_vectors[vector_name][idx] = new_vector
2718-
self._update_idf_append(new_vector, vector_name)
2749+
validated.append((vector_name, sort_sparse_vector(vector)))
27192750
continue
27202751

2752+
self._validate_dense_or_multivector(vector, vector_name)
27212753
vector_np = np.array(vector, dtype=np.float32)
27222754
assert not np.isnan(vector_np).any(), "Vector contains NaN values"
2755+
validated.append((vector_name, vector_np))
2756+
2757+
for vector_name, vector_np in validated:
2758+
self.deleted_per_vector[vector_name][idx] = 0
2759+
2760+
if isinstance(vector_np, SparseVector):
2761+
old_vector = self.sparse_vectors[vector_name][idx]
2762+
self._update_idf_remove(old_vector, vector_name)
2763+
self.sparse_vectors[vector_name][idx] = vector_np
2764+
self._update_idf_append(vector_np, vector_name)
2765+
continue
2766+
27232767
params = self.get_vector_params(vector_name)
27242768
if vector_name in self.vectors:
27252769
if params.distance == models.Distance.COSINE:

‎tests/congruence_tests/test_multivector_updates.py‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import pytest
66

77
from qdrant_client.http import models
8+
from qdrant_client.http.exceptions import UnexpectedResponse
89
from tests.congruence_tests.test_common import (
910
COLLECTION_NAME,
1011
compare_collections,
@@ -57,9 +58,7 @@ def test_upsert():
5758
COLLECTION_NAME,
5859
scroll_filter=id_filter,
5960
limit=1,
60-
)[
61-
0
62-
][0]
61+
)[0][0]
6362
remote_old_point = remote_client.scroll(COLLECTION_NAME, scroll_filter=id_filter, limit=1)[0][
6463
0
6564
]
@@ -203,3 +202,96 @@ def test_upload_uuid_in_batches():
203202
UPLOAD_NUM_VECTORS,
204203
attrs=("points_count",),
205204
)
205+
206+
207+
def test_upsert_empty_multivector():
208+
"""Both clients must reject an empty multivector on every write path."""
209+
points = generate_multivector_fixtures(UPLOAD_NUM_VECTORS)
210+
211+
local_client = init_local()
212+
init_client(local_client, points, vectors_config=multi_vector_config)
213+
214+
remote_client = init_remote()
215+
init_client(remote_client, points, vectors_config=multi_vector_config)
216+
217+
existing_id = points[0].id
218+
new_point_id = UPLOAD_NUM_VECTORS + 1
219+
220+
# a multivector with no vectors at all, and one holding an empty vector
221+
cases = (
222+
([], "Multivector must not be empty"),
223+
([[]], "vectors of a multivector must be non-empty"),
224+
)
225+
for empty_multivector, local_error in cases:
226+
upsert_structs = (
227+
[models.PointStruct(id=new_point_id, vector={"multi-text": empty_multivector})],
228+
[models.PointStruct(id=existing_id, vector={"multi-text": empty_multivector})],
229+
models.Batch(ids=[new_point_id], vectors={"multi-text": [empty_multivector]}),
230+
)
231+
for upsert_struct in upsert_structs:
232+
with pytest.raises(ValueError, match=local_error):
233+
local_client.upsert(COLLECTION_NAME, upsert_struct)
234+
235+
with pytest.raises(UnexpectedResponse):
236+
remote_client.upsert(COLLECTION_NAME, upsert_struct)
237+
238+
point_vectors = [
239+
models.PointVectors(id=existing_id, vector={"multi-text": empty_multivector})
240+
]
241+
with pytest.raises(ValueError, match=local_error):
242+
local_client.update_vectors(COLLECTION_NAME, points=point_vectors)
243+
244+
with pytest.raises(UnexpectedResponse):
245+
remote_client.update_vectors(COLLECTION_NAME, points=point_vectors)
246+
247+
compare_collections(
248+
local_client,
249+
remote_client,
250+
UPLOAD_NUM_VECTORS,
251+
attrs=("points_count",),
252+
)
253+
254+
255+
def test_rejected_empty_multivector_update_leaves_point_untouched():
256+
"""A rejected update_vectors request must not make an absent multivector visible.
257+
258+
Regression: validation used to run after `deleted_per_vector` was cleared, so rejecting
259+
an empty multivector still exposed the placeholder to retrieval and search.
260+
"""
261+
local_client = init_local()
262+
remote_client = init_remote()
263+
264+
mvc = models.MultiVectorConfig(comparator=models.MultiVectorComparator.MAX_SIM)
265+
vectors_config = {
266+
"a": models.VectorParams(size=4, distance=models.Distance.COSINE, multivector_config=mvc),
267+
"b": models.VectorParams(size=4, distance=models.Distance.COSINE, multivector_config=mvc),
268+
}
269+
local_client.create_collection(COLLECTION_NAME, vectors_config=vectors_config)
270+
if remote_client.collection_exists(collection_name=COLLECTION_NAME):
271+
remote_client.delete_collection(collection_name=COLLECTION_NAME)
272+
remote_client.create_collection(COLLECTION_NAME, vectors_config=vectors_config)
273+
274+
query = [[1.0, 0.0, 0.0, 0.0]]
275+
276+
# point 1 has no "a" at all, so "a" is backed by a placeholder marked deleted
277+
absent_a = [models.PointStruct(id=1, vector={"b": query})]
278+
local_client.upsert(COLLECTION_NAME, absent_a)
279+
remote_client.upsert(COLLECTION_NAME, absent_a, wait=True)
280+
281+
for empty_multivector, local_error in (
282+
([], "Multivector must not be empty"),
283+
([[]], "vectors of a multivector must be non-empty"),
284+
):
285+
point_vectors = [models.PointVectors(id=1, vector={"a": empty_multivector})]
286+
with pytest.raises(ValueError, match=local_error):
287+
local_client.update_vectors(COLLECTION_NAME, points=point_vectors)
288+
with pytest.raises(UnexpectedResponse):
289+
remote_client.update_vectors(COLLECTION_NAME, points=point_vectors, wait=True)
290+
291+
for client in (local_client, remote_client):
292+
record = client.retrieve(COLLECTION_NAME, ids=[1], with_vectors=True)[0]
293+
assert "a" not in record.vector
294+
hits = client.query_points(COLLECTION_NAME, query=query, using="a", limit=5).points
295+
assert hits == []
296+
297+
compare_collections(local_client, remote_client, 1, attrs=("points_count",))

‎tests/congruence_tests/test_updates.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,116 @@ def test_upload_wrong_vectors():
343343
)
344344

345345

346+
def test_upsert_empty_dense_vector():
347+
"""Both clients must reject an empty dense vector on every write path.
348+
349+
An empty *sparse* vector, on the other hand, is legitimate and must stay accepted.
350+
"""
351+
local_client = init_local()
352+
remote_client = init_remote()
353+
354+
vectors_config = {"text": models.VectorParams(size=2, distance=models.Distance.COSINE)}
355+
sparse_vectors_config = {"text-sparse": models.SparseVectorParams()}
356+
357+
local_client.create_collection(
358+
collection_name=COLLECTION_NAME,
359+
vectors_config=vectors_config,
360+
sparse_vectors_config=sparse_vectors_config,
361+
)
362+
if remote_client.collection_exists(collection_name=COLLECTION_NAME):
363+
remote_client.delete_collection(collection_name=COLLECTION_NAME)
364+
remote_client.create_collection(
365+
collection_name=COLLECTION_NAME,
366+
vectors_config=vectors_config,
367+
sparse_vectors_config=sparse_vectors_config,
368+
)
369+
370+
# a valid point to overwrite later
371+
valid_points = [models.PointStruct(id=1, vector={"text": [0.1, 0.3]})]
372+
local_client.upsert(COLLECTION_NAME, valid_points)
373+
remote_client.upsert(COLLECTION_NAME, valid_points)
374+
375+
empty_points = [models.PointStruct(id=2, vector={"text": []})]
376+
overwrite_points = [models.PointStruct(id=1, vector={"text": []})]
377+
empty_batch = models.Batch(ids=[3], vectors={"text": [[]]})
378+
379+
for points in (empty_points, overwrite_points, empty_batch):
380+
with pytest.raises(ValueError, match="Dense vector must not be empty"):
381+
local_client.upsert(COLLECTION_NAME, points)
382+
383+
with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
384+
remote_client.upsert(COLLECTION_NAME, points)
385+
386+
empty_point_vectors = [models.PointVectors(id=1, vector={"text": []})]
387+
with pytest.raises(ValueError, match="Dense vector must not be empty"):
388+
local_client.update_vectors(COLLECTION_NAME, points=empty_point_vectors)
389+
390+
with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
391+
remote_client.update_vectors(COLLECTION_NAME, points=empty_point_vectors)
392+
393+
# an empty sparse vector is valid input for both clients
394+
empty_sparse_points = [
395+
models.PointStruct(
396+
id=4, vector={"text-sparse": models.SparseVector(indices=[], values=[])}
397+
)
398+
]
399+
local_client.upsert(COLLECTION_NAME, empty_sparse_points)
400+
remote_client.upsert(COLLECTION_NAME, empty_sparse_points, wait=True)
401+
402+
compare_collections(local_client, remote_client, UPLOAD_NUM_VECTORS)
403+
404+
405+
def test_rejected_empty_vector_update_leaves_point_untouched():
406+
"""A rejected update_vectors request must not change the point in any way.
407+
408+
Regression: validation used to run after `deleted_per_vector` was cleared, so rejecting
409+
an empty vector still made an absent vector's placeholder visible to search, and a valid
410+
vector supplied alongside the rejected one was written anyway.
411+
"""
412+
local_client = init_local()
413+
remote_client = init_remote()
414+
415+
vectors_config = {
416+
"a": models.VectorParams(size=4, distance=models.Distance.COSINE),
417+
"b": models.VectorParams(size=4, distance=models.Distance.COSINE),
418+
}
419+
local_client.create_collection(COLLECTION_NAME, vectors_config=vectors_config)
420+
if remote_client.collection_exists(collection_name=COLLECTION_NAME):
421+
remote_client.delete_collection(collection_name=COLLECTION_NAME)
422+
remote_client.create_collection(COLLECTION_NAME, vectors_config=vectors_config)
423+
424+
orig = [1.0, 0.0, 0.0, 0.0]
425+
other = [0.0, 1.0, 0.0, 0.0]
426+
427+
# point 1 has no "a" at all, so "a" is backed by a placeholder marked deleted
428+
absent_a = [models.PointStruct(id=1, vector={"b": orig})]
429+
local_client.upsert(COLLECTION_NAME, absent_a)
430+
remote_client.upsert(COLLECTION_NAME, absent_a, wait=True)
431+
432+
empty_a = [models.PointVectors(id=1, vector={"a": []})]
433+
with pytest.raises(ValueError, match="Dense vector must not be empty"):
434+
local_client.update_vectors(COLLECTION_NAME, points=empty_a)
435+
with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
436+
remote_client.update_vectors(COLLECTION_NAME, points=empty_a, wait=True)
437+
438+
# "a" must still be absent for both clients, and must not be searchable
439+
for client in (local_client, remote_client):
440+
assert "a" not in client.retrieve(COLLECTION_NAME, ids=[1], with_vectors=True)[0].vector
441+
assert client.query_points(COLLECTION_NAME, query=orig, using="a", limit=5).points == []
442+
443+
# a valid vector supplied next to the rejected one must not be written either
444+
mixed = [models.PointVectors(id=1, vector={"b": other, "a": []})]
445+
with pytest.raises(ValueError, match="Dense vector must not be empty"):
446+
local_client.update_vectors(COLLECTION_NAME, points=mixed)
447+
with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
448+
remote_client.update_vectors(COLLECTION_NAME, points=mixed, wait=True)
449+
450+
for client in (local_client, remote_client):
451+
assert client.retrieve(COLLECTION_NAME, ids=[1], with_vectors=True)[0].vector["b"] == orig
452+
453+
compare_collections(local_client, remote_client, UPLOAD_NUM_VECTORS)
454+
455+
346456
def test_upsert_without_vector_name():
347457
local_client = init_local()
348458
remote_client = init_remote()

0 commit comments

Comments
 (0)