Skip to content

Commit 541c48c

Browse files
committed
Fix score-orientation handling for recommend/discovery/context/feedback queries in local mode
Recommend, discovery, context and relevance-feedback queries score points from the internal core distance, which is oriented so that a higher score is always better regardless of the collection's distance metric. The raw distance order therefore does not describe how their scores should be sorted or thresholded. Two places got this wrong on Euclid/Manhattan collections in local mode: - Result ordering already special-cased the recommend/discovery/context family, but omitted NaiveFeedbackQuery, so relevance-feedback queries returned the farthest points first instead of the nearest. - score_threshold filtering keyed off the raw distance order for every query type, so for the whole higher-is-better family it compared in the wrong direction and dropped all results (or applied no filtering) instead of removing the low-scoring points. Derive a single higher_score_is_better flag from the query type and use it for both the ordering and the threshold, and add NaiveFeedbackQuery to the family. Adds regression tests covering both the ordering and the threshold on Euclid and Manhattan.
1 parent a50a16a commit 541c48c

2 files changed

Lines changed: 105 additions & 3 deletions

File tree

‎qdrant_client/local/local_collection.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,12 @@ def search(
697697

698698
required_order = distance_to_order(distance)
699699

700-
if required_order == DistanceOrder.BIGGER_IS_BETTER or isinstance(
700+
# The recommend/discovery/context/feedback queries score points from the
701+
# internal core distance, which is oriented so that a higher score is
702+
# always better, regardless of the collection's distance metric. The raw
703+
# distance order therefore does not describe how their scores should be
704+
# sorted or thresholded, so treat those queries as higher-is-better here.
705+
higher_score_is_better = required_order == DistanceOrder.BIGGER_IS_BETTER or isinstance(
701706
query_vector,
702707
(
703708
DiscoveryQuery,
@@ -706,8 +711,11 @@ def search(
706711
MultiDiscoveryQuery,
707712
MultiContextQuery,
708713
MultiRecoQuery,
714+
NaiveFeedbackQuery,
709715
), # sparse structures are not required, sparse always uses DOT
710-
):
716+
)
717+
718+
if higher_score_is_better:
711719
order = np.argsort(scores)[::-1]
712720
else:
713721
order = np.argsort(scores)
@@ -727,7 +735,7 @@ def search(
727735
point_id = self.ids_inv[idx]
728736

729737
if score_threshold is not None:
730-
if required_order == DistanceOrder.BIGGER_IS_BETTER:
738+
if higher_score_is_better:
731739
if score < score_threshold:
732740
break
733741
else:

‎tests/test_in_memory.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,3 +294,97 @@ def test_fusion_dbsf_score_threshold(qdrant: QdrantClient):
294294
f"Expected 3 points after filtering (threshold 1.0), got {len(result_with_threshold.points)}. "
295295
f"Scores: {[p.score for p in result_no_threshold.points]}"
296296
)
297+
298+
299+
@pytest.mark.parametrize("distance", [models.Distance.EUCLID, models.Distance.MANHATTAN])
300+
def test_relevance_feedback_orders_nearest_first_on_smaller_is_better(
301+
qdrant: QdrantClient, distance: models.Distance
302+
):
303+
# NaiveFeedbackQuery scores come from the internal core distance, which is
304+
# oriented so that bigger is always better (like the recommend/discovery/
305+
# context queries). On EUCLID/MANHATTAN the results must therefore be sorted
306+
# descending by score; otherwise the farthest points are returned first.
307+
qdrant.create_collection(
308+
collection_name="test_collection",
309+
vectors_config=models.VectorParams(size=2, distance=distance),
310+
)
311+
qdrant.upsert(
312+
collection_name="test_collection",
313+
points=[
314+
models.PointStruct(id=1, vector=[0.1, 0.0]), # nearest to target [0, 0]
315+
models.PointStruct(id=5, vector=[5.0, 0.0]),
316+
models.PointStruct(id=9, vector=[9.0, 0.0]), # farthest
317+
],
318+
)
319+
320+
result = qdrant.query_points(
321+
collection_name="test_collection",
322+
query=models.RelevanceFeedbackQuery(
323+
relevance_feedback=models.RelevanceFeedbackInput(
324+
target=[0.0, 0.0],
325+
# equal scores -> zero confidence -> no context pairs, so the
326+
# score reduces to a * core_distance(target), isolating ordering.
327+
feedback=[
328+
models.FeedbackItem(example=[3.0, 0.0], score=0.5),
329+
models.FeedbackItem(example=[4.0, 0.0], score=0.5),
330+
],
331+
strategy=models.NaiveFeedbackStrategy(
332+
naive=models.NaiveFeedbackStrategyParams(a=1.0, b=1.0, c=1.0)
333+
),
334+
)
335+
),
336+
limit=3,
337+
).points
338+
339+
assert [point.id for point in result] == [1, 5, 9]
340+
# scores must be non-increasing (best match first)
341+
scores = [point.score for point in result]
342+
assert scores == sorted(scores, reverse=True)
343+
344+
345+
@pytest.mark.parametrize("distance", [models.Distance.EUCLID, models.Distance.MANHATTAN])
346+
def test_score_threshold_on_recommend_is_applied_correctly_on_smaller_is_better(
347+
qdrant: QdrantClient, distance: models.Distance
348+
):
349+
# Recommend (and discovery/context/feedback) scores are higher-is-better
350+
# regardless of the metric, so score_threshold must keep points whose score
351+
# is >= the threshold. The threshold direction used to follow the raw
352+
# distance order, which inverted it on Euclid/Manhattan and dropped every
353+
# point instead of filtering the low-scoring ones.
354+
qdrant.create_collection(
355+
collection_name="test_collection",
356+
vectors_config=models.VectorParams(size=2, distance=distance),
357+
)
358+
qdrant.upsert(
359+
collection_name="test_collection",
360+
points=[
361+
models.PointStruct(id=1, vector=[0.0, 0.0]),
362+
models.PointStruct(id=2, vector=[1.0, 0.0]),
363+
models.PointStruct(id=3, vector=[5.0, 0.0]),
364+
models.PointStruct(id=4, vector=[9.0, 0.0]),
365+
],
366+
)
367+
368+
def recommend(threshold=None):
369+
return qdrant.query_points(
370+
collection_name="test_collection",
371+
query=models.RecommendQuery(
372+
recommend=models.RecommendInput(
373+
positive=[[0.0, 0.0]],
374+
negative=[],
375+
strategy=models.RecommendStrategy.BEST_SCORE,
376+
)
377+
),
378+
limit=10,
379+
score_threshold=threshold,
380+
).points
381+
382+
ranked = recommend()
383+
assert [point.id for point in ranked] == [1, 2, 3, 4] # higher score first
384+
scores = [point.score for point in ranked]
385+
386+
# A threshold between the 2nd and 3rd score must keep exactly the top two.
387+
threshold = (scores[1] + scores[2]) / 2
388+
filtered = recommend(threshold)
389+
assert [point.id for point in filtered] == [1, 2]
390+
assert all(point.score >= threshold for point in filtered)

0 commit comments

Comments
 (0)