Skip to content

Commit 80a6aab

Browse files
committed
[fix](lance) retain search protocol version and correct regression assertions
1 parent d38fba4 commit 80a6aab

10 files changed

Lines changed: 57 additions & 107 deletions

File tree

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,7 @@ Status LanceTableReader::_validate_external_search_request() const {
506506
}
507507

508508
const auto& request = lance_scan_params.external_search_request;
509-
if (request.schema_version != 1 && request.schema_version != 2) {
509+
if (request.schema_version != 1) {
510510
return Status::NotSupported("unsupported external search schema version: {}",
511511
request.schema_version);
512512
}
@@ -538,10 +538,10 @@ Status LanceTableReader::_validate_external_search_request() const {
538538
static_cast<int>(query_vector.element_type));
539539
}
540540
const bool multi_vector = query_vector.__isset.num_vectors;
541-
if (multi_vector != (request.schema_version == 2) ||
542-
(multi_vector && query_vector.num_vectors <= 0)) {
541+
// The optional count distinguishes a query matrix, including a one-row matrix.
542+
if (multi_vector && query_vector.num_vectors <= 0) {
543543
return Status::InvalidArgument(
544-
"Lance multi-vector queries require schema version 2 and positive num_vectors");
544+
"Lance multi-vector queries require positive num_vectors");
545545
}
546546
if (multi_vector && (query_vector.element_type == TVectorElementType::UINT8 ||
547547
query_vector.element_type == TVectorElementType::INT8 ||
@@ -587,9 +587,6 @@ Status LanceTableReader::_validate_external_search_request() const {
587587
}
588588
} else {
589589
DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT);
590-
if (request.schema_version != 1) {
591-
return Status::InvalidArgument("Lance full-text search requires schema version 1");
592-
}
593590
const auto& full_text = request.search_query.full_text_search;
594591
if (!full_text.__isset.column || full_text.column.empty() ||
595592
full_text.column.find('\0') != std::string::npos) {

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ TEST(LanceTableReaderVectorSearchTest, ValidatesMultiVectorWireShapeBeforeDatase
494494
RuntimeState state(globals);
495495
auto valid = make_float32_vector_search_params({1.0F, 0.0F, 0.0F}, 4, 0);
496496
auto& request = valid.lance_scan_params.external_search_request;
497-
request.__set_schema_version(2);
497+
request.__set_schema_version(1);
498498
auto& query = request.search_query.vector_search.query_vector;
499499
query.__set_dimension(3);
500500
query.__set_num_vectors(1);
@@ -519,13 +519,14 @@ TEST(LanceTableReaderVectorSearchTest, ValidatesMultiVectorWireShapeBeforeDatase
519519
.__set_num_vectors(count);
520520
check(invalid, false);
521521
}
522-
auto old_version = valid;
523-
old_version.lance_scan_params.external_search_request.__set_schema_version(1);
524-
check(old_version, false);
522+
auto unsupported_version = valid;
523+
unsupported_version.lance_scan_params.external_search_request.__set_schema_version(2);
524+
check(unsupported_version, false);
525525
auto missing_count = valid;
526526
missing_count.lance_scan_params.external_search_request.search_query.vector_search.query_vector
527527
.__isset.num_vectors = false;
528-
check(missing_count, false);
528+
// Without num_vectors, the same payload is an ordinary single-vector request.
529+
check(missing_count, true);
529530
auto hamming = valid;
530531
hamming.lance_scan_params.external_search_request.search_query.vector_search.__set_metric(
531532
TVectorMetric::HAMMING);
@@ -884,7 +885,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexed
884885
RuntimeProfile profile("multi_vector_search");
885886
auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0);
886887
auto& request = params.lance_scan_params.external_search_request;
887-
request.__set_schema_version(2);
888+
request.__set_schema_version(1);
888889
request.vector_search_options.__set_use_index(indexed);
889890
request.vector_search_options.__set_nprobes(1);
890891
auto& search = request.search_query.vector_search;
@@ -1006,7 +1007,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorSegmentTopKMatchesIndependentG
10061007
// FE sends top_k + offset to each split, and applies offset only after merging.
10071008
auto params = make_float32_vector_search_params({0, 0, 0}, 20, 0);
10081009
auto& request = params.lance_scan_params.external_search_request;
1009-
request.__set_schema_version(2);
1010+
request.__set_schema_version(1);
10101011
request.vector_search_options.__set_use_index(split < 2);
10111012
request.vector_search_options.__set_nprobes(4);
10121013
request.vector_search_options.__set_refine_factor(64);
@@ -1075,7 +1076,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorTopOnePreservesPrecisionAndBat
10751076
" indexed=" + std::to_string(indexed));
10761077
auto params = make_float32_vector_search_params({0, 0, 0}, 1, 0);
10771078
auto& request = params.lance_scan_params.external_search_request;
1078-
request.__set_schema_version(2);
1079+
request.__set_schema_version(1);
10791080
request.vector_search_options.__set_use_index(indexed);
10801081
request.vector_search_options.__set_nprobes(1);
10811082
auto& search = request.search_query.vector_search;
@@ -1118,7 +1119,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) {
11181119
for (const bool zero_query : {false, true}) {
11191120
auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0);
11201121
auto& request = params.lance_scan_params.external_search_request;
1121-
request.__set_schema_version(2);
1122+
request.__set_schema_version(1);
11221123
request.vector_search_options.__set_use_index(indexed);
11231124
request.vector_search_options.__set_nprobes(1);
11241125
auto& search = request.search_query.vector_search;
@@ -1163,7 +1164,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcros
11631164
for (const auto fragment : fixture.fragment_ids) {
11641165
auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0);
11651166
auto& request = params.lance_scan_params.external_search_request;
1166-
request.__set_schema_version(2);
1167+
request.__set_schema_version(1);
11671168
request.vector_search_options.__set_use_index(true);
11681169
auto& search = request.search_query.vector_search;
11691170
search.__set_column("vectors32");
@@ -1204,7 +1205,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorRejectsActualNullAndNonFiniteE
12041205
SCOPED_TRACE(column);
12051206
auto params = make_float32_vector_search_params({0, 0, 0}, 1, 0);
12061207
auto& request = params.lance_scan_params.external_search_request;
1207-
request.__set_schema_version(2);
1208+
request.__set_schema_version(1);
12081209
auto& search = request.search_query.vector_search;
12091210
search.__set_column(column);
12101211
search.query_vector.__set_dimension(3);

‎docs/lance-multivector-search.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@ subvectors. The score is a sum, not an average, so duplicating a query subvector
5555
changes the score. `_distance` uses Doris FLOAT, including for Float64 inputs.
5656

5757
`top_k` and `offset` count table rows, not subvectors. The TVF `filter` is applied
58-
before candidate search; an outer SQL WHERE filters the search's results.
58+
before candidate search. An outer SQL WHERE filters each split's returned candidates
59+
before Doris merges them into the global TopK; it does not rerun Lance search to
60+
replace rejected candidates within a split.
5961
Distributed fragments return row candidates for the global TopK. Ties at the
6062
TopK boundary have no guaranteed row order.
6163

@@ -92,9 +94,6 @@ are searched together with indexed rows.
9294
at most 100,000, with a default refinement factor of 1. These limits bound
9395
per-query ANN plan expansion and candidate allocation, independently of wire size.
9496
- Extension and dictionary vector encodings are not supported.
95-
- Multi-vector requests use a new protocol version. Old BEs reject these requests;
96-
finish the BE upgrade before enabling multi-vector searches. Ordinary vector
97-
requests retain their previous protocol.
9897

9998
For example, a compatible PyArrow schema is:
10099

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,7 @@ protected void doInitialize() throws UserException {
156156
}
157157
super.doInitialize();
158158
checkAdditionalTypeBackendCompatibility(
159-
projectsCurrentReaderType() || (externalSearchRequest != null
160-
&& externalSearchRequest.getSchemaVersion() >= 2), backendPolicy.getBackends());
159+
projectsCurrentReaderType(), backendPolicy.getBackends());
161160
ExternalUtil.initSchemaInfo(params, -1L, sourceColumns);
162161

163162
if (searchKind != SearchKind.NORMAL) {
@@ -191,7 +190,7 @@ private boolean projectsCurrentReaderType() {
191190
return false;
192191
}
193192

194-
/** Rejects old smooth-upgrade source BEs for Lance features requiring the current reader. */
193+
/** Rejects old smooth-upgrade source BEs for additional Lance type projections. */
195194
@VisibleForTesting
196195
public static void checkAdditionalTypeBackendCompatibility(
197196
boolean requiresCurrentReader, Iterable<Backend> backends) throws UserException {
@@ -201,7 +200,7 @@ public static void checkAdditionalTypeBackendCompatibility(
201200
for (Backend backend : backends) {
202201
if (backend.isSmoothUpgradeSrc()) {
203202
throw new UserException(
204-
"Additional Lance types or multi-vector search are unavailable while backend "
203+
"Additional Lance types are unavailable while backend "
205204
+ backend.getId() + " is a smooth upgrade source");
206205
}
207206
}

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ private static PreparedSearch prepare(Map<String, String> properties)
8686
throw new AnalysisException("Lance multi-vector search supports l2, cosine, and dot metrics");
8787
}
8888
TExternalSearchRequest searchRequest = new TExternalSearchRequest()
89-
.setSchemaVersion(queryVector.isSetNumVectors() ? 2 : 1)
89+
.setSchemaVersion(1)
9090
.setSearchQuery(TExternalSearchQuery.vector_search(vectorParams));
9191
TVectorSearchOptions vectorSearchOptions = buildVectorSearchOptions(params, useIndex);
9292
if (vectorSearchOptions != null) {

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,14 +484,14 @@ public void testMultiVectorSplitRetainsMatrixAndProtocolVersion() throws Excepti
484484
org.apache.doris.thrift.TSearchVector matrix = new org.apache.doris.thrift.TSearchVector()
485485
.setElementType(org.apache.doris.thrift.TVectorElementType.FLOAT32)
486486
.setDimension(2).setNumVectors(2).setValues(new byte[16]);
487-
logical.setSchemaVersion(2);
487+
logical.setSchemaVersion(1);
488488
logical.getSearchQuery().getVectorSearch().setQueryVector(matrix);
489489
logical.getSearchQuery().getVectorSearch().unsetMetric();
490490
LanceScanNode node = LanceScanNode.forExternalSearch(
491491
new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), null,
492492
null, -1, logical, new SessionVariable());
493493
TExternalSearchRequest split = node.createSplitSearchRequest();
494-
Assert.assertEquals(2, split.getSchemaVersion());
494+
Assert.assertEquals(1, split.getSchemaVersion());
495495
Assert.assertEquals(TVectorMetric.L2, split.getSearchQuery().getVectorSearch().getMetric());
496496
Assert.assertEquals(matrix, split.getSearchQuery().getVectorSearch().getQueryVector());
497497
Assert.assertEquals(7, split.getSearchQuery().getVectorSearch().getTopK());

‎fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceLazyMaterializationUpgradeTest.java‎

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -75,25 +75,6 @@ public void testFullTextSearchLazyNullUpgradeFence() throws Exception {
7575
assertUpgradeFence(false);
7676
}
7777

78-
@Test
79-
public void testMultiVectorSearchFencesOldBackendWithoutVectorProjection() throws Exception {
80-
ConnectContext previous = ConnectContext.get();
81-
ConnectContext context = new ConnectContext();
82-
context.setThreadLocalInfo();
83-
try {
84-
RuntimeException error = Assertions.assertThrows(RuntimeException.class,
85-
() -> translate(true, false, true, true));
86-
Assertions.assertTrue(error.toString().contains("smooth upgrade source"));
87-
translate(true, false, false, true);
88-
} finally {
89-
if (previous == null) {
90-
ConnectContext.remove();
91-
} else {
92-
previous.setThreadLocalInfo();
93-
}
94-
}
95-
}
96-
9778
private void assertUpgradeFence(boolean vector) throws Exception {
9879
ConnectContext previous = ConnectContext.get();
9980
ConnectContext context = new ConnectContext();
@@ -115,10 +96,6 @@ private void assertUpgradeFence(boolean vector) throws Exception {
11596
}
11697

11798
private void translate(boolean vector, boolean lazyNull, boolean mixedVersion) throws Exception {
118-
translate(vector, lazyNull, mixedVersion, false);
119-
}
120-
121-
private void translate(boolean vector, boolean lazyNull, boolean mixedVersion, boolean multiVector) throws Exception {
12299
String functionName = vector ? "vector_search" : "full_text_search";
123100
String scoreName = vector ? "_distance" : "_score";
124101
Field nestedNull = new Field("nested_null", FieldType.nullable(ArrowType.List.INSTANCE),
@@ -167,8 +144,7 @@ private void translate(boolean vector, boolean lazyNull, boolean mixedVersion, b
167144
: TExternalSearchQuery.full_text_search(new TFullTextSearchParams().setColumn("text"));
168145
LanceScanNode node = LanceScanNode.forExternalSearch(call.getArgument(0, PlanNodeId.class),
169146
call.getArgument(1, TupleDescriptor.class), source, metadata, 0,
170-
new TExternalSearchRequest().setSchemaVersion(multiVector ? 2 : 1).setSearchQuery(query),
171-
call.getArgument(2, SessionVariable.class));
147+
new TExternalSearchRequest().setSearchQuery(query), call.getArgument(2, SessionVariable.class));
172148
java.lang.reflect.Field backendPolicy = ExternalScanNode.class.getDeclaredField("backendPolicy");
173149
backendPolicy.setAccessible(true);
174150
backendPolicy.set(node, policy);

‎regression-test/data/external_table_p0/lance/test_lance_vector_search_index_types.out‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,3 @@ row_id bigint No false \N
5151
4 item-0004 144.0
5252
5 item-0005 256.0
5353

54-
-- !ivf_hnsw_sq_ef_5 --
55-
518 item-0518 0.0
56-
517 item-0517 66.278755
57-
516 item-0516 132.55751
58-
515 item-0515 198.83627
59-
514 item-0514 248.54533
60-
61-
-- !ivf_hnsw_sq_ef_50 --
62-
518 item-0518 0.0
63-
519 item-0519 49.70907
64-
517 item-0517 66.278755
65-
520 item-0520 115.98782
66-
516 item-0516 132.55751
67-

‎regression-test/suites/external_table_p0/lance/test_lance_multivector_coverage.groovy‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ suite("test_lance_multivector_coverage", "p0,external") {
2525
def saved = [:]
2626
for (String variable : ["batch_size", "topn_lazy_materialization_threshold", "enable_file_scanner_v2",
2727
"enable_sql_cache", "enable_query_cache"]) {
28-
saved[variable] = (sql "SHOW VARIABLES LIKE '${variable}'")[0][1]
28+
// Experimental variables are omitted by SHOW VARIABLES under their unprefixed names.
29+
saved[variable] = (sql "SELECT @@session.${variable}")[0][0]
2930
}
3031
def parseVectors = { value -> new JsonSlurper().parseText(value.toString()) }
3132
def assertVectors = { expected, actual ->
@@ -176,12 +177,18 @@ suite("test_lance_multivector_coverage", "p0,external") {
176177
assertEquals([], sql("SELECT row_id FROM ${source(table, 'vectors', query, 'cosine', indexed, 3, 0, 'row_id < 0')}"))
177178
}
178179
}
179-
// A TVF prefilter can find the next eligible row; an outer WHERE cannot refill TopK.
180-
int nearest = ranked(false)[0][0]
180+
// Outer WHERE filters each split's candidates before the global merge. Remove
181+
// every split winner so another split cannot refill TopK and mask postfiltering.
182+
def splitWinners = (0..2).collect { fragment ->
183+
ranked(false).find { (it[0] - 1) % 3 == fragment }[0]
184+
}
185+
assertEquals(3, splitWinners.unique(false).size())
186+
String predicate = "row_id NOT IN (${splitWinners.join(',')})"
181187
String full = source("multivector_ivf_flat", "vectors", query, "cosine", false, 1, 0, null)
182-
assertEquals([], sql("SELECT row_id FROM ${full} WHERE row_id != ${nearest}"))
183-
String prefiltered = source("multivector_ivf_flat", "vectors", query, "cosine", false, 1, 0, "row_id != ${nearest}")
184-
assertEquals(ranked(false)[1][0], (sql "SELECT row_id FROM ${prefiltered}")[0][0] as int)
188+
assertEquals([], sql("SELECT row_id FROM ${full} WHERE ${predicate}"))
189+
String prefiltered = source("multivector_ivf_flat", "vectors", query, "cosine", false, 1, 0, predicate)
190+
int nextEligible = ranked(false).find { !splitWinners.contains(it[0]) }[0]
191+
assertEquals(nextEligible, (sql "SELECT row_id FROM ${prefiltered}")[0][0] as int)
185192
for (String column : ["integer_vectors", "nullable_vectors"]) {
186193
test {
187194
sql "SELECT _distance FROM ${source('multivector_dimensions', column, [[1, 2, 3]], 'l2', false, 1, 0, null)}"

‎regression-test/suites/external_table_p0/lance/test_lance_vector_search_index_types.groovy‎

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -141,39 +141,24 @@ suite("test_lance_vector_search_index_types", "p0,external") {
141141
ORDER BY _distance, row_id
142142
"""
143143

144-
// The graph counterpart of the nprobes discriminator: with ef=5 the traversal has to
145-
// settle for a worse fifth neighbour than with ef=50, so ef demonstrably reached the
146-
// index instead of being dropped on the way. Both queries probe all four partitions and
147-
// neither reranks, because refine_factor with exact distances is precisely what would
148-
// hide the effect. IVF_HNSW_SQ is the table the fixture generator pins this on: on 1024
149-
// collinear vectors the FLAT and PQ graphs still return the exact rows at ef=5, so only
150-
// this one can carry the assertion.
151-
def narrowEf = sql """
152-
SELECT row_id, _distance
153-
FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="5"')}
154-
ORDER BY _distance, row_id
155-
"""
156-
def wideEf = sql """
157-
SELECT row_id, _distance
158-
FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="50"')}
159-
ORDER BY _distance, row_id
160-
"""
161-
assertEquals(5, narrowEf.size())
162-
assertEquals(5, wideEf.size())
163-
assertFalse(narrowEf.collect { it[1] }.equals(wideEf.collect { it[1] }),
164-
"IVF_HNSW_SQ returned the same distances for ef=5 and ef=50, so ef never reached "
165-
+ "the graph search. ef=5=" + narrowEf + " ef=50=" + wideEf)
166-
167-
qt_ivf_hnsw_sq_ef_5 """
168-
SELECT row_id, label, _distance
169-
FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="5"')}
170-
ORDER BY _distance, row_id
171-
"""
172-
qt_ivf_hnsw_sq_ef_50 """
173-
SELECT row_id, label, _distance
174-
FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="50"')}
175-
ORDER BY _distance, row_id
176-
"""
144+
// ANN results may coincide at different ef values, and quantized scores can vary
145+
// across Lance versions. Validate the result contract; the rejection below proves
146+
// that ef reaches Lance's graph-search candidate-width validation.
147+
for (int ef : [5, 50]) {
148+
def rows = sql """
149+
SELECT row_id, _distance
150+
FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="' + ef + '"')}
151+
ORDER BY _distance, row_id
152+
"""
153+
assertEquals(5, rows.size())
154+
assertEquals(5, rows.collect { it[0] }.unique().size())
155+
assertEquals(518L, rows[0][0] as long)
156+
assertEquals(0d, rows[0][1] as double)
157+
rows.each { row ->
158+
assertTrue((row[0] as long) >= 1 && (row[0] as long) <= 1024)
159+
assertTrue(Double.isFinite(row[1] as double) && (row[1] as double) >= 0)
160+
}
161+
}
177162

178163
// Lance reranks a refined query over top_k * refine_factor candidates, so ef has to be
179164
// at least that wide. Too small an ef is a diagnosable error rather than a quietly

0 commit comments

Comments
 (0)