@@ -1079,6 +1079,130 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorSegmentTopKMatchesIndependentG
10791079 }
10801080}
10811081
1082+ TEST (LanceTableReaderVectorSearchTest, SplitWithoutIndexSegmentsSearchesFlat) {
1083+ // FE plans a split without index segments as a flat search. Lance must not search it with
1084+ // an index of its own choosing, even one covering the split and with use_index=true.
1085+ const std::filesystem::path uri =
1086+ " ./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance" ;
1087+ LanceFixtureInfo fixture;
1088+ ASSERT_TRUE (get_fixture_info (uri, &fixture).ok ());
1089+ ASSERT_EQ (3 , fixture.fragment_ids .size ());
1090+ std::vector<std::string> segments;
1091+ ASSERT_TRUE (get_index_segment_uuids (uri, " vectors_idx" , &segments).ok ());
1092+ ASSERT_EQ (2 , segments.size ());
1093+ const auto query = representative_multi_vector (7 , 0 );
1094+ const Columns columns {projected_column (" row_id" , TYPE_BIGINT , false ),
1095+ projected_column (" _distance" , TYPE_FLOAT , true )};
1096+ TQueryGlobals globals;
1097+ RuntimeState state (globals);
1098+ std::map<std::string, std::vector<std::pair<int64_t , float >>> results;
1099+ for (const auto * mode : {" segment" , " no_segment_default" , " no_segment_use_index" }) {
1100+ SCOPED_TRACE (mode);
1101+ RuntimeProfile profile (" lance_segmentless_split" );
1102+ auto params = make_float32_vector_search_params ({0 , 0 , 0 }, 5 , 0 );
1103+ auto & request = params.lance_scan_params .external_search_request ;
1104+ request.vector_search_options .__isset .use_index = false ;
1105+ if (std::string (mode) == " no_segment_use_index" ) {
1106+ request.vector_search_options .__set_use_index (true );
1107+ }
1108+ request.vector_search_options .__set_nprobes (64 );
1109+ auto & search = request.search_query .vector_search ;
1110+ search.__set_column (" vectors" );
1111+ search.__set_metric (TVectorMetric::COSINE );
1112+ search.query_vector .__set_dimension (128 );
1113+ search.query_vector .__set_num_vectors (1 );
1114+ std::string bytes (query.size () * sizeof (float ), ' \0 ' );
1115+ for (size_t j = 0 ; j < query.size (); ++j) {
1116+ LittleEndian::Store32 (bytes.data () + j * sizeof (float ),
1117+ std::bit_cast<uint32_t >(query[j]));
1118+ }
1119+ search.query_vector .__set_values (bytes);
1120+ auto range = make_lance_range (uri, fixture.version , {fixture.fragment_ids [0 ]});
1121+ if (std::string (mode) == " segment" ) {
1122+ range.table_format_params .lance_params .__set_index_segment_uuids ({segments[0 ]});
1123+ }
1124+ LanceTableReader reader;
1125+ ASSERT_TRUE (init_reader (&reader, columns, &state, &profile, ¶ms).ok ());
1126+ ASSERT_TRUE (prepare_range (&reader, range).ok ());
1127+ Block block;
1128+ add_output_columns (&block, columns);
1129+ results[mode] = read_vector_search_rows (&reader, &block);
1130+ ASSERT_EQ (5 , results[mode].size ());
1131+ EXPECT_TRUE (reader.close ().ok ());
1132+ auto * partitions = profile.get_counter (" LanceIVFPartitionsSearched" );
1133+ ASSERT_NE (nullptr , partitions);
1134+ auto * flat = profile.get_counter (" LancePlannedFlatSearchFragmentCount" );
1135+ ASSERT_NE (nullptr , flat);
1136+ if (std::string (mode) == " segment" ) {
1137+ EXPECT_GT (partitions->value (), 0 );
1138+ EXPECT_EQ (0 , flat->value ());
1139+ } else {
1140+ EXPECT_EQ (0 , partitions->value ());
1141+ EXPECT_EQ (1 , flat->value ());
1142+ }
1143+ }
1144+ EXPECT_EQ (results[" no_segment_default" ], results[" no_segment_use_index" ]);
1145+ }
1146+
1147+ TEST (LanceTableReaderVectorSearchTest, RejectsIndexSegmentsWhenUseIndexIsFalse) {
1148+ // FE plans index segments only when use_index allows them. A split carrying segments under
1149+ // use_index=false is rejected instead of being searched with or without the index.
1150+ const std::filesystem::path uri =
1151+ " ./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance" ;
1152+ LanceFixtureInfo fixture;
1153+ ASSERT_TRUE (get_fixture_info (uri, &fixture).ok ());
1154+ std::vector<std::string> segments;
1155+ ASSERT_TRUE (get_index_segment_uuids (uri, " vectors_idx" , &segments).ok ());
1156+ ASSERT_FALSE (segments.empty ());
1157+ const auto query = representative_multi_vector (7 , 0 );
1158+ const Columns columns {projected_column (" row_id" , TYPE_BIGINT , false ),
1159+ projected_column (" _distance" , TYPE_FLOAT , true )};
1160+ TQueryGlobals globals;
1161+ RuntimeState state (globals);
1162+ for (const bool with_segment : {true , false }) {
1163+ SCOPED_TRACE (with_segment ? " with segment" : " without segment" );
1164+ RuntimeProfile profile (" lance_segment_with_use_index_false" );
1165+ auto params = make_float32_vector_search_params ({0 , 0 , 0 }, 5 , 0 );
1166+ auto & request = params.lance_scan_params .external_search_request ;
1167+ ASSERT_TRUE (request.vector_search_options .__isset .use_index );
1168+ ASSERT_FALSE (request.vector_search_options .use_index );
1169+ auto & search = request.search_query .vector_search ;
1170+ search.__set_column (" vectors" );
1171+ search.__set_metric (TVectorMetric::COSINE );
1172+ search.query_vector .__set_dimension (128 );
1173+ search.query_vector .__set_num_vectors (1 );
1174+ std::string bytes (query.size () * sizeof (float ), ' \0 ' );
1175+ for (size_t j = 0 ; j < query.size (); ++j) {
1176+ LittleEndian::Store32 (bytes.data () + j * sizeof (float ),
1177+ std::bit_cast<uint32_t >(query[j]));
1178+ }
1179+ search.query_vector .__set_values (bytes);
1180+ auto range = make_lance_range (uri, fixture.version , {fixture.fragment_ids [0 ]});
1181+ if (with_segment) {
1182+ range.table_format_params .lance_params .__set_index_segment_uuids ({segments[0 ]});
1183+ }
1184+ LanceTableReader reader;
1185+ ASSERT_TRUE (init_reader (&reader, columns, &state, &profile, ¶ms).ok ());
1186+ const auto status = prepare_range (&reader, range);
1187+ if (with_segment) {
1188+ EXPECT_FALSE (status.ok ());
1189+ EXPECT_NE (
1190+ status.to_string ().find (" carries 1 index segments although use_index is false" ),
1191+ std::string::npos)
1192+ << status.to_string ();
1193+ } else {
1194+ ASSERT_TRUE (status.ok ()) << status.to_string ();
1195+ Block block;
1196+ add_output_columns (&block, columns);
1197+ EXPECT_EQ (5 , read_vector_search_rows (&reader, &block).size ());
1198+ auto * flat = profile.get_counter (" LancePlannedFlatSearchFragmentCount" );
1199+ ASSERT_NE (nullptr , flat);
1200+ EXPECT_EQ (1 , flat->value ());
1201+ }
1202+ EXPECT_TRUE (reader.close ().ok ());
1203+ }
1204+ }
1205+
10821206TEST (LanceTableReaderSnapshotTest, RejectsFragmentIdsMissingFromTheSnapshot) {
10831207 // lance-c silently skips a fragment id its snapshot lacks, which would drop rows.
10841208 const std::filesystem::path dataset_uri =
@@ -1349,6 +1473,9 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) {
13491473 }
13501474}
13511475
1476+ // FE plans an omitted metric as L2, and this column's only index is cosine, so FE plans every split
1477+ // flat. BE must then search flat too, with the same L2 metric in every split, although the request
1478+ // allows indexes.
13521479TEST (LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcrossFragments) {
13531480 const std::filesystem::path uri = " ./be/test/format_v2/table/lance/data/multivector.lance" ;
13541481 LanceFixtureInfo fixture;
@@ -1381,6 +1508,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcros
13811508 auto rows = read_vector_search_rows (&reader, &block);
13821509 all_rows.insert (all_rows.end (), rows.begin (), rows.end ());
13831510 EXPECT_TRUE (reader.close ().ok ());
1511+ EXPECT_EQ (0 , profile.get_counter (" LanceIVFPartitionsSearched" )->value ());
13841512 }
13851513 std::sort (all_rows.begin (), all_rows.end ());
13861514 EXPECT_EQ ((std::vector<std::pair<int64_t , float >> {{1 , 0 }, {2 , 1 }, {3 , 4 }, {6 , 1 }}), all_rows);
0 commit comments