@@ -1220,6 +1220,131 @@ TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter)
12201220 EXPECT_TRUE (combined_reader.close ().ok ());
12211221}
12221222
1223+ TEST (LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithoutLosingRows) {
1224+ const auto unique_suffix = std::chrono::steady_clock::now ().time_since_epoch ().count ();
1225+ for (const auto index_type : {LANCE_SCALAR_BTREE , LANCE_SCALAR_BITMAP }) {
1226+ const auto dataset_uri = std::filesystem::temp_directory_path () /
1227+ (" doris_lance_scalar_segment_" + std::to_string (unique_suffix) +
1228+ " _" + std::to_string (index_type) + " .lance" );
1229+ Defer cleanup {[&] {
1230+ std::error_code error;
1231+ std::filesystem::remove_all (dataset_uri, error);
1232+ }};
1233+ const auto schema = arrow::schema ({arrow::field (" row_id" , arrow::int64 (), false )});
1234+ for (int64_t first_id : {1 , 3 }) {
1235+ arrow::Int64Builder values;
1236+ ASSERT_TRUE (values.AppendValues ({first_id, first_id + 1 }).ok ());
1237+ auto array = values.Finish ();
1238+ ASSERT_TRUE (array.ok ());
1239+ auto batch = arrow::RecordBatch::Make (schema, 2 , {std::move (array).ValueUnsafe ()});
1240+ auto batches = arrow::RecordBatchReader::Make ({batch}, schema);
1241+ ASSERT_TRUE (batches.ok ());
1242+ ArrowArrayStream stream {};
1243+ ASSERT_TRUE (
1244+ arrow::ExportRecordBatchReader (std::move (batches).ValueUnsafe (), &stream).ok ());
1245+ auto dataset = ::lance::Dataset::write (
1246+ dataset_uri.string (), &stream,
1247+ first_id == 1 ? ::lance::WriteMode::Create : ::lance::WriteMode::Append);
1248+ if (first_id == 1 ) {
1249+ // The second append remains uncovered by this index segment.
1250+ dataset.create_scalar_index (" row_id" , index_type, " row_id_idx" );
1251+ }
1252+ }
1253+ LanceFixtureInfo fixture;
1254+ ASSERT_TRUE (get_fixture_info (dataset_uri, &fixture).ok ());
1255+ ASSERT_EQ (2 , fixture.fragment_ids .size ());
1256+ std::vector<std::string> segments;
1257+ ASSERT_TRUE (get_index_segment_uuids (dataset_uri, " row_id_idx" , &segments).ok ());
1258+ ASSERT_EQ (1 , segments.size ());
1259+
1260+ const auto read_domain = [&](const std::vector<int64_t >& fragments, bool use_segment,
1261+ const std::vector<int64_t >& expected, int64_t searched,
1262+ int64_t fallbacks) {
1263+ TQueryGlobals globals;
1264+ RuntimeState state (globals);
1265+ RuntimeProfile profile (" lance_scalar_segment" );
1266+ TFileScanRangeParams scan_params;
1267+ const Columns columns {projected_column (" row_id" , TYPE_BIGINT , false )};
1268+ LanceTableReader reader;
1269+ auto filter = create_int64_runtime_in_conjunct (" row_id" , {2 , 4 }, 41 );
1270+ ASSERT_TRUE (
1271+ init_reader (&reader, columns, &state, &profile, &scan_params, {filter}).ok ());
1272+ auto range = make_lance_range (dataset_uri, fixture.version , fragments);
1273+ if (use_segment) {
1274+ range.table_format_params .lance_params .__set_index_segment_uuids (segments);
1275+ } else {
1276+ range.table_format_params .lance_params .__set_use_scalar_index (false );
1277+ }
1278+ ASSERT_TRUE (prepare_range (&reader, std::move (range)).ok ());
1279+ Block block;
1280+ add_output_columns (&block, columns);
1281+ std::vector<int64_t > actual;
1282+ bool eos = false ;
1283+ while (!eos) {
1284+ auto status = reader.get_block (&block, &eos);
1285+ ASSERT_TRUE (status.ok ()) << status.to_string ();
1286+ if (!eos) {
1287+ const auto & ids =
1288+ assert_cast<const ColumnInt64&>(*block.get_by_position (0 ).column );
1289+ actual.insert (actual.end (), ids.get_data ().begin (), ids.get_data ().end ());
1290+ }
1291+ }
1292+ std::ranges::sort (actual);
1293+ EXPECT_EQ (expected, actual);
1294+ EXPECT_EQ (use_segment ? 1 : 0 ,
1295+ profile.get_counter (" LanceScalarIndexSegmentsRequested" )->value ());
1296+ EXPECT_EQ (searched, profile.get_counter (" LanceScalarIndexSegmentsSearched" )->value ());
1297+ EXPECT_EQ (fallbacks, profile.get_counter (" LanceScalarIndexSegmentFallbacks" )->value ());
1298+ if (!use_segment) {
1299+ EXPECT_EQ (0 , profile.get_counter (" LanceIndexComparisons" )->value ());
1300+ }
1301+ EXPECT_TRUE (reader.close ().ok ());
1302+ };
1303+ read_domain ({fixture.fragment_ids [0 ]}, true , {2 }, 1 , 0 );
1304+ read_domain ({fixture.fragment_ids [1 ]}, false , {4 }, 0 , 0 );
1305+ // A segment which cannot cover the whole task must filter the entire explicit domain.
1306+ read_domain (fixture.fragment_ids , true , {2 , 4 }, 0 , 1 );
1307+ }
1308+ }
1309+
1310+ TEST (LanceTableReaderScalarSegmentTest, RejectsInvalidSegmentAssignments) {
1311+ const std::filesystem::path dataset_uri =
1312+ " ./be/test/format_v2/table/lance/data/all_types.lance" ;
1313+ LanceFixtureInfo fixture;
1314+ ASSERT_TRUE (get_fixture_info (dataset_uri, &fixture).ok ());
1315+ const auto check_invalid = [&](TLanceFileDesc params, const std::string& message) {
1316+ TQueryGlobals globals;
1317+ RuntimeState state (globals);
1318+ RuntimeProfile profile (" lance_invalid_scalar_segment" );
1319+ TFileScanRangeParams scan_params;
1320+ LanceTableReader reader;
1321+ ASSERT_TRUE (init_reader (&reader, {projected_column (" row_id" , TYPE_BIGINT , false )}, &state,
1322+ &profile, &scan_params)
1323+ .ok ());
1324+ auto range = make_lance_range (dataset_uri, fixture.version , fixture.fragment_ids );
1325+ range.table_format_params .__set_lance_params (std::move (params));
1326+ auto status = prepare_range (&reader, std::move (range));
1327+ EXPECT_FALSE (status.ok ());
1328+ EXPECT_NE (status.to_string ().find (message), std::string::npos) << status.to_string ();
1329+ EXPECT_TRUE (reader.close ().ok ());
1330+ };
1331+ auto params = make_lance_range (dataset_uri, fixture.version , fixture.fragment_ids )
1332+ .table_format_params .lance_params ;
1333+ params.__set_index_segment_uuids ({" too-short" });
1334+ check_invalid (params, " 16 bytes" );
1335+ params.__set_index_segment_uuids ({std::string (16 , ' a' ), std::string (16 , ' b' )});
1336+ check_invalid (params, " only one scalar index segment" );
1337+ params.__set_index_segment_uuids ({std::string (16 , ' a' )});
1338+ params.__set_fragment_ids ({});
1339+ check_invalid (params, " nonempty fragment ids" );
1340+ params.__set_fragment_ids (fixture.fragment_ids );
1341+ params.__set_version (0 );
1342+ check_invalid (params, " fixed version" );
1343+ params.__set_version (fixture.version );
1344+ params.__set_use_scalar_index (false );
1345+ check_invalid (params, " use_scalar_index=false" );
1346+ }
1347+
12231348TEST (LanceTableReaderFilterTest, PushesRuntimeInFilterIntoLanceScanner) {
12241349 const std::filesystem::path dataset_uri =
12251350 " ./be/test/format_v2/table/lance/data/all_types.lance" ;
0 commit comments