Skip to content

Commit 7999eb8

Browse files
authored
[Backport 3.0] fix: preserve search iterator external filtering (#3729) (#3730)
Backport of #3729 to `3.0`. Signed-off-by: pymilvus-bot <[email protected]> Signed-off-by: pymilvus-bot <[email protected]>
1 parent 602496b commit 7999eb8

2 files changed

Lines changed: 64 additions & 2 deletions

File tree

‎pymilvus/milvus_client/milvus_client.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import copy
22
import logging
33
import time
4-
from typing import Dict, List, Optional, Union
4+
from typing import Callable, Dict, List, Optional, Union
55

66
from pymilvus.client import type_info
77
from pymilvus.client.abstract import AnnSearchRequest, BaseRanker
@@ -10,6 +10,7 @@
1010
from pymilvus.client.embedding_list import EmbeddingList
1111
from pymilvus.client.iterator import QueryIterator, SearchIterator, SearchIteratorV2
1212
from pymilvus.client.search_aggregation import SearchAggregation
13+
from pymilvus.client.search_result import Hit, Hits
1314
from pymilvus.client.types import (
1415
CompactionPlans,
1516
ExceptionsMessage,
@@ -605,6 +606,8 @@ def search_iterator(
605606
partition_names: Optional[List[str]] = None,
606607
anns_field: Optional[str] = None,
607608
round_decimal: int = -1,
609+
*,
610+
external_filter_func: Optional[Callable[[Hits], Union[Hits, List[Hit]]]] = None,
608611
**kwargs,
609612
) -> Union[SearchIteratorV2, SearchIterator]:
610613
"""Creates an iterator for searching vectors in batches.
@@ -630,6 +633,9 @@ def search_iterator(
630633
there is only one vector field in the collection.
631634
round_decimal (int, optional): Number of decimal places for distance values.
632635
Defaults to -1 (no rounding).
636+
external_filter_func (Callable, optional): A client-side callback that filters each
637+
page of search hits. The callback receives Hits and returns Hits or a list of Hit
638+
objects. This argument is keyword-only and is supported by Search Iterator V2.
633639
**kwargs: Additional arguments to pass to the search operation.
634640
635641
Returns:
@@ -669,6 +675,7 @@ def search_iterator(
669675
partition_names=partition_names,
670676
anns_field=anns_field or "",
671677
round_decimal=round_decimal,
678+
external_filter_func=external_filter_func,
672679
rpc_options=kwargs,
673680
)
674681
except ServerVersionIncompatibleException:

‎tests/unit/test_iterator_ownership.py‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import ast
22
import importlib
3+
import inspect
34
import warnings
45
from pathlib import Path
56
from types import SimpleNamespace
6-
from unittest.mock import patch
7+
from unittest.mock import Mock, patch
78

89
import pytest
910
from pymilvus import PyMilvusDeprecationWarning
@@ -546,6 +547,60 @@ def test_milvus_client_uses_shared_search_v2_owner_and_one_context():
546547
assert all(call["cluster_id"] == "cluster" for call in handler.search_calls)
547548

548549

550+
def test_milvus_client_search_iterator_external_filter_is_explicit_keyword_only():
551+
parameters = inspect.signature(MilvusClient.search_iterator).parameters
552+
parameter = parameters["external_filter_func"]
553+
554+
assert parameter.kind is inspect.Parameter.KEYWORD_ONLY
555+
assert parameter.default is None
556+
assert all(
557+
parameters[name].kind is inspect.Parameter.POSITIONAL_OR_KEYWORD
558+
for name in (
559+
"batch_size",
560+
"filter",
561+
"limit",
562+
"output_fields",
563+
"search_params",
564+
"timeout",
565+
"partition_names",
566+
"anns_field",
567+
"round_decimal",
568+
)
569+
)
570+
571+
572+
def test_milvus_client_search_v2_preserves_external_filter_callback():
573+
handler = _SwitchingSearchV2Handler()
574+
handler.search = Mock(
575+
side_effect=[
576+
_SearchV2Result([_SearchV2Hit(0, "probe")]),
577+
_SearchV2Result([_SearchV2Hit(1, "rejected")]),
578+
_SearchV2Result([_SearchV2Hit(2, "accepted")]),
579+
]
580+
)
581+
context = CallContext(db_name="db", client_request_id="request")
582+
client = object.__new__(MilvusClient)
583+
client._handler = handler
584+
external_filter = Mock(side_effect=lambda hits: [hit for hit in hits if hit.id == 2])
585+
586+
with patch.object(client, "_generate_call_context", return_value=context) as generate_context:
587+
iterator = client.search_iterator(
588+
"collection",
589+
data=[[0.1, 0.2]],
590+
batch_size=1,
591+
external_filter_func=external_filter,
592+
)
593+
594+
page = iterator.next()
595+
596+
assert page.ids() == [2]
597+
assert external_filter.call_count == 2
598+
assert len(handler.search.call_args_list) == 3
599+
assert "external_filter_func" not in generate_context.call_args.kwargs
600+
assert "external_filter_func" not in handler.describe_calls[0][1]
601+
assert all("external_filter_func" not in call.kwargs for call in handler.search.call_args_list)
602+
603+
549604
def test_milvus_client_preserves_search_v2_public_preparation():
550605
handler = _SwitchingSearchV2Handler()
551606
context = CallContext(db_name="db", client_request_id="request")

0 commit comments

Comments
 (0)