7878T = TypeVar ("T" , bound = "LanceModel" )
7979AnalyzePlanDistributedMetrics = Literal ["aggregate" , "per_worker" , "full" ]
8080
81+ # Number of rows a hybrid query returns when no limit was set on it. This
82+ # mirrors the default the Rust query builder applies to its sub-queries.
83+ DEFAULT_HYBRID_LIMIT = 10
84+
8185
8286@runtime_checkable
8387class _LanceScanner (Protocol ):
@@ -3856,14 +3860,54 @@ def rerank(
38563860
38573861 return self
38583862
3863+ def _create_child_queries (
3864+ self ,
3865+ ) -> Tuple ["AsyncFTSQuery" , "AsyncVectorQuery" , int , int ]:
3866+ """Build the sub-queries that make up this hybrid query.
3867+
3868+ Execution, `explain_plan` and `analyze_plan` all go through here so that
3869+ the plans that are reported are the plans that actually run.
3870+
3871+ Returns the two sub-queries along with the effective limit and offset of
3872+ the hybrid query itself.
3873+ """
3874+ fts_query = AsyncFTSQuery (self ._inner .to_fts_query (), self ._table )
3875+ vec_query = AsyncVectorQuery (self ._inner .to_vector_query (), self ._table )
3876+
3877+ fts_req = fts_query ._inner .to_query_request ()
3878+ vec_req = vec_query ._inner .to_query_request ()
3879+
3880+ # Only one of the two sub-queries carries the limit when it was never
3881+ # set explicitly: nearest_to()/nearest_to_text() build the sibling query
3882+ # from scratch, and that is where the default gets filled in. Which one
3883+ # that is depends on the order the hybrid query was built in, so look at
3884+ # both rather than at a single side.
3885+ limit = fts_req .limit if fts_req .limit is not None else vec_req .limit
3886+ if limit is None :
3887+ limit = DEFAULT_HYBRID_LIMIT
3888+ offset = fts_req .offset or vec_req .offset or 0
3889+
3890+ fts_query .with_row_id ()
3891+ vec_query .with_row_id ()
3892+
3893+ # offset() pushes the offset down into both sub-queries, which would make
3894+ # each of them skip its own first `offset` rows. The window has to be
3895+ # taken out of the combined, reranked results instead, so fetch the
3896+ # skipped prefix here too and slice it off afterwards.
3897+ fts_query .limit (limit + offset )
3898+ vec_query .limit (limit + offset )
3899+ fts_query .offset (0 )
3900+ vec_query .offset (0 )
3901+
3902+ return fts_query , vec_query , limit , offset
3903+
38593904 async def to_batches (
38603905 self ,
38613906 * ,
38623907 max_batch_length : Optional [int ] = None ,
38633908 timeout : Optional [timedelta ] = None ,
38643909 ) -> AsyncRecordBatchReader :
3865- fts_query = AsyncFTSQuery (self ._inner .to_fts_query (), self ._table )
3866- vec_query = AsyncVectorQuery (self ._inner .to_vector_query (), self ._table )
3910+ fts_query , vec_query , limit , offset = self ._create_child_queries ()
38673911
38683912 req = fts_query ._inner .to_query_request ()
38693913 blob_auto_row_id = False
@@ -3882,22 +3926,6 @@ async def to_batches(
38823926 self ._blob_auto_row_id = blob_auto_row_id
38833927 self ._blob_paths = blob_paths
38843928
3885- fts_query .with_row_id ()
3886- vec_query .with_row_id ()
3887-
3888- limit = self ._inner .get_limit ()
3889- offset = req .offset or 0
3890- if offset :
3891- # offset() pushes the offset down into both sub-queries, which would
3892- # make each of them skip its own first `offset` rows. The window has
3893- # to be taken out of the combined, reranked results instead, so
3894- # fetch the skipped prefix here too and slice it off afterwards.
3895- if limit :
3896- fts_query .limit (limit + offset )
3897- vec_query .limit (limit + offset )
3898- fts_query .offset (0 )
3899- vec_query .offset (0 )
3900-
39013929 fts_results , vector_results = await asyncio .gather (
39023930 fts_query .to_arrow (timeout = timeout ),
39033931 vec_query .to_arrow (timeout = timeout ),
@@ -3940,14 +3968,14 @@ async def explain_plan(self, verbose: Optional[bool] = False):
39403968 ... print(plan)
39413969 >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
39423970 RRFReranker(K=60)
3943- ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance]
3971+ ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance, _rowid@1 as _rowid ]
39443972 LanceRead: uri=..., projection=[text], source=stream(_rowid)
39453973 GlobalLimitExec: skip=0, fetch=10
39463974 FilterExec: _distance@2 IS NOT NULL
39473975 SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
39483976 KNNVectorDistance: metric=l2
39493977 LanceRead: uri=..., projection=[vector], ...
3950- ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score]
3978+ ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score, _rowid@0 as _rowid ]
39513979 LanceRead: uri=..., projection=[vector, text], source=stream(_rowid)
39523980 GlobalLimitExec: skip=0, fetch=10
39533981 MatchQuery: column=text, query=[hello]
@@ -3962,8 +3990,9 @@ async def explain_plan(self, verbose: Optional[bool] = False):
39623990 plan : str
39633991 """ # noqa: E501
39643992
3965- vector_plan = await self ._inner .to_vector_query ().explain_plan (verbose )
3966- fts_plan = await self ._inner .to_fts_query ().explain_plan (verbose )
3993+ fts_query , vec_query , _ , _ = self ._create_child_queries ()
3994+ vector_plan = await vec_query .explain_plan (verbose )
3995+ fts_plan = await fts_query .explain_plan (verbose )
39673996 # Indent sub-plans under the reranker
39683997 indented_vector = "\n " .join (" " + line for line in vector_plan .splitlines ())
39693998 indented_fts = "\n " .join (" " + line for line in fts_plan .splitlines ())
@@ -3990,14 +4019,12 @@ async def analyze_plan(
39904019 -------
39914020 plan : str
39924021 """
4022+ fts_query , vec_query , _ , _ = self ._create_child_queries ()
4023+
39934024 results = ["Vector Search Query:" ]
3994- results .append (
3995- await self ._inner .to_vector_query ().analyze_plan (distributed_metrics )
3996- )
4025+ results .append (await vec_query .analyze_plan (distributed_metrics ))
39974026 results .append ("FTS Search Query:" )
3998- results .append (
3999- await self ._inner .to_fts_query ().analyze_plan (distributed_metrics )
4000- )
4027+ results .append (await fts_query .analyze_plan (distributed_metrics ))
40014028
40024029 return "\n " .join (results )
40034030
0 commit comments