Skip to content

Commit ebe7d6f

Browse files
committed
[fix](lance) Search vector splits without index segments flat
The FE plans a vector-search split without index segments as a flat search, and EXPLAIN and the profile count it as one. The BE forwarded use_index only when the user set it, so Lance searched such a split with the first index on the column in manifest order, which need not be the index the FE selected, for example while an index is rebuilt under a new name, or an index the FE could not plan with at all. The BE now sets use_index to whether the split carries index segments. A split that carries index segments while the request says use_index=false is rejected. The FE plans segments only when use_index allows them, so such a split contradicts itself, and the BE would otherwise have to override either the plan or the request.
1 parent da030be commit ebe7d6f

4 files changed

Lines changed: 155 additions & 10 deletions

File tree

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,6 +1043,9 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
10431043
DORIS_CHECK(scanner != nullptr);
10441044
DORIS_CHECK(_scan_params != nullptr);
10451045
DORIS_CHECK(_scan_params->__isset.lance_scan_params);
1046+
const auto& lance_scan_params = _scan_params->lance_scan_params;
1047+
DORIS_CHECK(lance_scan_params.__isset.external_search_request);
1048+
const auto& request = lance_scan_params.external_search_request;
10461049
std::vector<uint64_t> fragment_ids;
10471050
RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids));
10481051
RETURN_IF_ERROR(_check_fragment_ids(fragment_ids));
@@ -1053,6 +1056,15 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
10531056
std::vector<uint8_t> segment_uuids;
10541057
size_t segment_count = 0;
10551058
RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count));
1059+
// FE plans index segments only when use_index allows them. A split carrying both contradicts
1060+
// itself, and the BE would have to override either FE's plan or the request.
1061+
if (segment_count > 0 && request.__isset.vector_search_options &&
1062+
request.vector_search_options.__isset.use_index &&
1063+
!request.vector_search_options.use_index) {
1064+
return Status::InvalidArgument(
1065+
"Lance vector search split carries {} index segments although use_index is false",
1066+
segment_count);
1067+
}
10561068
if (segment_count > 0 &&
10571069
lance_scanner_set_index_segments(scanner, segment_uuids.data(), segment_count) != 0) {
10581070
return lance_error("set Lance vector scanner index segments");
@@ -1062,9 +1074,6 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
10621074
if (lance_scanner_set_prefilter(scanner, true) != 0) {
10631075
return lance_error("enable Lance vector prefilter");
10641076
}
1065-
const auto& lance_scan_params = _scan_params->lance_scan_params;
1066-
DORIS_CHECK(lance_scan_params.__isset.external_search_request);
1067-
const auto& request = lance_scan_params.external_search_request;
10681077
const auto& vector = request.search_query.vector_search;
10691078
const auto& query = vector.query_vector;
10701079
const auto dimension = static_cast<size_t>(query.dimension);
@@ -1179,10 +1188,13 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
11791188
lance_scanner_set_ef(scanner, static_cast<uint32_t>(options.ef)) != 0) {
11801189
return lance_error("set Lance vector ef");
11811190
}
1182-
if (options.__isset.use_index &&
1183-
lance_scanner_set_use_index(scanner, options.use_index) != 0) {
1184-
return lance_error("set Lance vector use_index");
1185-
}
1191+
}
1192+
// FE plans a split without index segments as a flat search, and EXPLAIN and the profile count
1193+
// it as one. Lance would otherwise search it with the first index on the column in manifest
1194+
// order, which need not be the index FE selected. A split with segments searches exactly
1195+
// those; one that also carries use_index=false was rejected above.
1196+
if (lance_scanner_set_use_index(scanner, segment_count > 0) != 0) {
1197+
return lance_error("set Lance vector use_index");
11861198
}
11871199
if (lance_scanner_set_offset(scanner, vector.offset) != 0) {
11881200
return lance_error("set Lance vector offset");

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1079,6 +1079,130 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorSegmentTopKMatchesIndependentG
10791079
}
10801080
}
10811081

1082+
TEST(LanceTableReaderVectorSearchTest, SplitWithoutIndexSegmentsSearchesFlat) {
1083+
// FE plans a split without index segments as a flat search. Lance must not search it with
1084+
// an index of its own choosing, even one covering the split and with use_index=true.
1085+
const std::filesystem::path uri =
1086+
"./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance";
1087+
LanceFixtureInfo fixture;
1088+
ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
1089+
ASSERT_EQ(3, fixture.fragment_ids.size());
1090+
std::vector<std::string> segments;
1091+
ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok());
1092+
ASSERT_EQ(2, segments.size());
1093+
const auto query = representative_multi_vector(7, 0);
1094+
const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
1095+
projected_column("_distance", TYPE_FLOAT, true)};
1096+
TQueryGlobals globals;
1097+
RuntimeState state(globals);
1098+
std::map<std::string, std::vector<std::pair<int64_t, float>>> results;
1099+
for (const auto* mode : {"segment", "no_segment_default", "no_segment_use_index"}) {
1100+
SCOPED_TRACE(mode);
1101+
RuntimeProfile profile("lance_segmentless_split");
1102+
auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0);
1103+
auto& request = params.lance_scan_params.external_search_request;
1104+
request.vector_search_options.__isset.use_index = false;
1105+
if (std::string(mode) == "no_segment_use_index") {
1106+
request.vector_search_options.__set_use_index(true);
1107+
}
1108+
request.vector_search_options.__set_nprobes(64);
1109+
auto& search = request.search_query.vector_search;
1110+
search.__set_column("vectors");
1111+
search.__set_metric(TVectorMetric::COSINE);
1112+
search.query_vector.__set_dimension(128);
1113+
search.query_vector.__set_num_vectors(1);
1114+
std::string bytes(query.size() * sizeof(float), '\0');
1115+
for (size_t j = 0; j < query.size(); ++j) {
1116+
LittleEndian::Store32(bytes.data() + j * sizeof(float),
1117+
std::bit_cast<uint32_t>(query[j]));
1118+
}
1119+
search.query_vector.__set_values(bytes);
1120+
auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]});
1121+
if (std::string(mode) == "segment") {
1122+
range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]});
1123+
}
1124+
LanceTableReader reader;
1125+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &params).ok());
1126+
ASSERT_TRUE(prepare_range(&reader, range).ok());
1127+
Block block;
1128+
add_output_columns(&block, columns);
1129+
results[mode] = read_vector_search_rows(&reader, &block);
1130+
ASSERT_EQ(5, results[mode].size());
1131+
EXPECT_TRUE(reader.close().ok());
1132+
auto* partitions = profile.get_counter("LanceIVFPartitionsSearched");
1133+
ASSERT_NE(nullptr, partitions);
1134+
auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount");
1135+
ASSERT_NE(nullptr, flat);
1136+
if (std::string(mode) == "segment") {
1137+
EXPECT_GT(partitions->value(), 0);
1138+
EXPECT_EQ(0, flat->value());
1139+
} else {
1140+
EXPECT_EQ(0, partitions->value());
1141+
EXPECT_EQ(1, flat->value());
1142+
}
1143+
}
1144+
EXPECT_EQ(results["no_segment_default"], results["no_segment_use_index"]);
1145+
}
1146+
1147+
TEST(LanceTableReaderVectorSearchTest, RejectsIndexSegmentsWhenUseIndexIsFalse) {
1148+
// FE plans index segments only when use_index allows them. A split carrying segments under
1149+
// use_index=false is rejected instead of being searched with or without the index.
1150+
const std::filesystem::path uri =
1151+
"./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance";
1152+
LanceFixtureInfo fixture;
1153+
ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
1154+
std::vector<std::string> segments;
1155+
ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok());
1156+
ASSERT_FALSE(segments.empty());
1157+
const auto query = representative_multi_vector(7, 0);
1158+
const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
1159+
projected_column("_distance", TYPE_FLOAT, true)};
1160+
TQueryGlobals globals;
1161+
RuntimeState state(globals);
1162+
for (const bool with_segment : {true, false}) {
1163+
SCOPED_TRACE(with_segment ? "with segment" : "without segment");
1164+
RuntimeProfile profile("lance_segment_with_use_index_false");
1165+
auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0);
1166+
auto& request = params.lance_scan_params.external_search_request;
1167+
ASSERT_TRUE(request.vector_search_options.__isset.use_index);
1168+
ASSERT_FALSE(request.vector_search_options.use_index);
1169+
auto& search = request.search_query.vector_search;
1170+
search.__set_column("vectors");
1171+
search.__set_metric(TVectorMetric::COSINE);
1172+
search.query_vector.__set_dimension(128);
1173+
search.query_vector.__set_num_vectors(1);
1174+
std::string bytes(query.size() * sizeof(float), '\0');
1175+
for (size_t j = 0; j < query.size(); ++j) {
1176+
LittleEndian::Store32(bytes.data() + j * sizeof(float),
1177+
std::bit_cast<uint32_t>(query[j]));
1178+
}
1179+
search.query_vector.__set_values(bytes);
1180+
auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]});
1181+
if (with_segment) {
1182+
range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]});
1183+
}
1184+
LanceTableReader reader;
1185+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &params).ok());
1186+
const auto status = prepare_range(&reader, range);
1187+
if (with_segment) {
1188+
EXPECT_FALSE(status.ok());
1189+
EXPECT_NE(
1190+
status.to_string().find("carries 1 index segments although use_index is false"),
1191+
std::string::npos)
1192+
<< status.to_string();
1193+
} else {
1194+
ASSERT_TRUE(status.ok()) << status.to_string();
1195+
Block block;
1196+
add_output_columns(&block, columns);
1197+
EXPECT_EQ(5, read_vector_search_rows(&reader, &block).size());
1198+
auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount");
1199+
ASSERT_NE(nullptr, flat);
1200+
EXPECT_EQ(1, flat->value());
1201+
}
1202+
EXPECT_TRUE(reader.close().ok());
1203+
}
1204+
}
1205+
10821206
TEST(LanceTableReaderSnapshotTest, RejectsFragmentIdsMissingFromTheSnapshot) {
10831207
// lance-c silently skips a fragment id its snapshot lacks, which would drop rows.
10841208
const std::filesystem::path dataset_uri =
@@ -1349,6 +1473,9 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) {
13491473
}
13501474
}
13511475

1476+
// FE plans an omitted metric as L2, and this column's only index is cosine, so FE plans every split
1477+
// flat. BE must then search flat too, with the same L2 metric in every split, although the request
1478+
// allows indexes.
13521479
TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcrossFragments) {
13531480
const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance";
13541481
LanceFixtureInfo fixture;
@@ -1381,6 +1508,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcros
13811508
auto rows = read_vector_search_rows(&reader, &block);
13821509
all_rows.insert(all_rows.end(), rows.begin(), rows.end());
13831510
EXPECT_TRUE(reader.close().ok());
1511+
EXPECT_EQ(0, profile.get_counter("LanceIVFPartitionsSearched")->value());
13841512
}
13851513
std::sort(all_rows.begin(), all_rows.end());
13861514
EXPECT_EQ((std::vector<std::pair<int64_t, float>> {{1, 0}, {2, 1}, {3, 4}, {6, 1}}), all_rows);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/index/LanceDatasetIndexDiscovery.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,9 @@ private LanceDatasetIndexDiscovery() {
4747
* Describes only user-created indexes. The Lance JNI bulk describe path also tries to
4848
* materialize details for internal indexes, whose details are not supported by the SDK.
4949
* Legacy indexes that cannot be described because all their segments lack details are
50-
* omitted. This disables their FE segment planning, not the scanner's automatic index use.
50+
* omitted, so the FE plans no segments for them. Vector and full-text searches use only the
51+
* segments the FE plans and therefore never use such an index; an ordinary scan may still use
52+
* a legacy scalar index through Lance's own index selection.
5153
*/
5254
public static List<IndexDescription> describeUserIndexes(Dataset dataset) {
5355
List<String> listedNames = dataset.listIndexes();
@@ -98,7 +100,7 @@ public static List<IndexDescription> describeUserIndexes(Dataset dataset) {
98100
if (indexesWithoutDetails.contains(name)) {
99101
LOG.warn("Skipping FE metadata and segment planning for legacy Lance index '{}' "
100102
+ "at dataset version {}: all physical segments lack index details; "
101-
+ "scanner index selection remains unchanged", name, dataset.version());
103+
+ "vector and full-text searches will not use it", name, dataset.version());
102104
continue;
103105
}
104106
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,10 @@ static SearchKind fromSearchRequest(TExternalSearchRequest request) {
5252
}
5353
}
5454

55-
/** Explains FE segment selection, not whether the BE may automatically use a Lance index. */
55+
/**
56+
* Explains FE segment selection. The BE searches an index only through the segments the FE
57+
* plans, so a vector search under any status other than USED is a flat search.
58+
*/
5659
enum VectorIndexStatus {
5760
/** No vector planning result exists yet (also used by ordinary and FTS scans). */
5861
NOT_PLANNED,

0 commit comments

Comments
 (0)