@@ -1064,7 +1064,9 @@ class TableReaderCastTestHelper final : public TableReader {
10641064 using TableReader::_materialize_mapping_column;
10651065 using TableReader::_materialize_present_child_mapping_column;
10661066 using TableReader::_materialize_struct_mapping_column;
1067- using TableReader::_project_collection_parent_null_map;
1067+ using TableReader::_project_collection_parent_null_map_for_hidden_entries;
1068+ using TableReader::_requires_collection_parent_null_map;
1069+ using TableReader::_requires_parent_null_map_for_alignment;
10681070};
10691071
10701072TEST (TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) {
@@ -3149,6 +3151,76 @@ TEST(TableReaderTest, ScalarProjectionRejectsNullInRequiredTableColumn) {
31493151 std::string::npos);
31503152}
31513153
3154+ TEST (TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenNearerMaskCoversVisibleNull) {
3155+ constexpr size_t visible_entries = 500000 ;
3156+ const size_t entries = visible_entries + 1 ;
3157+ const auto int_type = std::make_shared<DataTypeInt32>();
3158+ const auto struct_type =
3159+ std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {" value" });
3160+ const auto nullable_struct_type = make_nullable (struct_type);
3161+
3162+ auto values = ColumnInt32::create (entries, 0 );
3163+ auto value_null_map = ColumnUInt8::create (entries, 0 );
3164+ value_null_map->get_data ().back () = 1 ;
3165+ MutableColumns struct_children;
3166+ struct_children.push_back (ColumnNullable::create (std::move (values), std::move (value_null_map)));
3167+ auto element_null_map = ColumnUInt8::create (entries, 0 );
3168+ element_null_map->get_data ().back () = 1 ;
3169+ ColumnPtr nullable_elements = ColumnNullable::create (
3170+ ColumnStruct::create (std::move (struct_children)), std::move (element_null_map));
3171+ NullMap parent_null_map {1 , 0 };
3172+ ColumnArray::Offsets64 offsets {1 , entries};
3173+
3174+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
3175+ &parent_null_map, nullable_elements, nullable_struct_type, 2 , offsets));
3176+ }
3177+
3178+ TEST (TableReaderTest, CollectionParentMaskSkipsLargeMapWhenNearerMaskCoversVisibleNull) {
3179+ constexpr size_t visible_entries = 500000 ;
3180+ const size_t entries = visible_entries + 1 ;
3181+ const auto int_type = std::make_shared<DataTypeInt32>();
3182+ const auto struct_type =
3183+ std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {" value" });
3184+ const auto nullable_struct_type = make_nullable (struct_type);
3185+
3186+ auto values = ColumnInt32::create (entries, 0 );
3187+ auto value_null_map = ColumnUInt8::create (entries, 0 );
3188+ value_null_map->get_data ().back () = 1 ;
3189+ MutableColumns struct_children;
3190+ struct_children.push_back (ColumnNullable::create (std::move (values), std::move (value_null_map)));
3191+ auto value_struct_null_map = ColumnUInt8::create (entries, 0 );
3192+ value_struct_null_map->get_data ().back () = 1 ;
3193+ ColumnPtr nullable_value_structs = ColumnNullable::create (
3194+ ColumnStruct::create (std::move (struct_children)), std::move (value_struct_null_map));
3195+ NullMap parent_null_map {1 , 0 };
3196+ ColumnArray::Offsets64 offsets {1 , entries};
3197+
3198+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
3199+ &parent_null_map, nullable_value_structs, nullable_struct_type, 2 , offsets));
3200+ }
3201+
3202+ TEST (TableReaderTest, CollectionParentMaskKeepsRequiredNullInHiddenEntry) {
3203+ const auto int_type = std::make_shared<DataTypeInt32>();
3204+ const auto struct_type =
3205+ std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {" value" });
3206+ const auto nullable_struct_type = make_nullable (struct_type);
3207+
3208+ auto values = ColumnInt32::create (2 , 0 );
3209+ auto value_null_map = ColumnUInt8::create ();
3210+ value_null_map->get_data ().assign ({1 , 1 });
3211+ MutableColumns struct_children;
3212+ struct_children.push_back (ColumnNullable::create (std::move (values), std::move (value_null_map)));
3213+ auto element_null_map = ColumnUInt8::create ();
3214+ element_null_map->get_data ().assign ({0 , 1 });
3215+ ColumnPtr nullable_elements = ColumnNullable::create (
3216+ ColumnStruct::create (std::move (struct_children)), std::move (element_null_map));
3217+ NullMap parent_null_map {1 , 0 };
3218+ ColumnArray::Offsets64 offsets {1 , 2 };
3219+
3220+ EXPECT_TRUE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
3221+ &parent_null_map, nullable_elements, nullable_struct_type, 2 , offsets));
3222+ }
3223+
31523224TEST (TableReaderTest, ReopenSplitAfterClose) {
31533225 const auto test_dir = std::filesystem::temp_directory_path () / " doris_table_reader_test" ;
31543226 std::filesystem::remove_all (test_dir);
@@ -4924,6 +4996,112 @@ TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) {
49244996 }
49254997}
49264998
4999+ TEST (TableReaderTest, TrivialArrayChildProjectsNullableStructParentMask) {
5000+ const auto int_type = std::make_shared<DataTypeInt32>();
5001+ const auto element_struct_type =
5002+ std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings {" value" });
5003+ const auto nullable_element_struct_type = make_nullable (element_struct_type);
5004+ const auto array_type = std::make_shared<DataTypeArray>(nullable_element_struct_type);
5005+ const auto table_struct_type = make_nullable (std::make_shared<DataTypeStruct>(
5006+ DataTypes {array_type, int_type}, Strings {" items" , " added" }));
5007+ const auto file_struct_type = make_nullable (
5008+ std::make_shared<DataTypeStruct>(DataTypes {array_type}, Strings {" items" }));
5009+
5010+ auto table_items = make_table_column (0 , " items" , array_type);
5011+ table_items.type = array_type;
5012+ auto table_element = make_table_column (0 , " element" , element_struct_type);
5013+ table_element.type = element_struct_type;
5014+ auto table_value = make_table_column (0 , " value" , int_type);
5015+ table_value.type = int_type;
5016+ table_element.children = {table_value};
5017+ table_items.children = {table_element};
5018+ auto table_added = make_table_column (1 , " added" , int_type);
5019+ table_added.type = int_type;
5020+ auto table_struct = make_table_column (0 , " payload" , table_struct_type);
5021+ table_struct.type = table_struct_type;
5022+ table_struct.children = {table_items, table_added};
5023+
5024+ auto file_items = make_file_column (0 , " items" , array_type);
5025+ file_items.type = array_type;
5026+ auto file_element = make_file_column (0 , " element" , element_struct_type);
5027+ file_element.type = element_struct_type;
5028+ auto file_value = make_file_column (0 , " value" , int_type);
5029+ file_value.type = int_type;
5030+ file_element.children = {file_value};
5031+ file_items.children = {file_element};
5032+ auto file_struct = make_file_column (0 , " payload" , file_struct_type);
5033+ file_struct.type = file_struct_type;
5034+ file_struct.children = {file_items};
5035+
5036+ TableColumnMapper mapper ({.mode = TableColumnMappingMode::BY_NAME });
5037+ ASSERT_TRUE (mapper.create_mapping ({table_struct}, {}, {file_struct}).ok ());
5038+ ASSERT_EQ (mapper.mappings ().size (), 1 );
5039+ ASSERT_FALSE (mapper.mappings ()[0 ].is_trivial );
5040+ ASSERT_TRUE (mapper.mappings ()[0 ].child_mappings [0 ].is_trivial );
5041+
5042+ auto values = ColumnInt32::create ();
5043+ values->get_data ().assign ({0 , 7 });
5044+ auto value_null_map = ColumnUInt8::create ();
5045+ value_null_map->get_data ().assign ({1 , 0 });
5046+ MutableColumns element_children;
5047+ element_children.push_back (
5048+ ColumnNullable::create (std::move (values), std::move (value_null_map)));
5049+ auto element_null_map = ColumnUInt8::create (2 , 0 );
5050+ auto array_values = ColumnNullable::create (ColumnStruct::create (std::move (element_children)),
5051+ std::move (element_null_map));
5052+ auto offsets = ColumnArray::ColumnOffsets::create ();
5053+ offsets->get_data ().assign ({1 , 2 });
5054+ MutableColumns struct_children;
5055+ struct_children.push_back (ColumnArray::create (std::move (array_values), std::move (offsets)));
5056+ auto parent_null_map = ColumnUInt8::create ();
5057+ parent_null_map->get_data ().assign ({1 , 0 });
5058+ ColumnPtr file_data = ColumnNullable::create (ColumnStruct::create (std::move (struct_children)),
5059+ std::move (parent_null_map));
5060+
5061+ TableReaderCastTestHelper reader;
5062+ ColumnPtr result;
5063+ const auto status =
5064+ reader._materialize_struct_mapping_column (mapper.mappings ()[0 ], file_data, 2 , &result);
5065+ ASSERT_TRUE (status.ok ()) << status.to_string ();
5066+ const auto & result_struct = assert_cast<const ColumnStruct&>(
5067+ assert_cast<const ColumnNullable&>(*result).get_nested_column ());
5068+ const auto & result_array = assert_cast<const ColumnArray&>(result_struct.get_column (0 ));
5069+ const auto & result_elements = assert_cast<const ColumnNullable&>(result_array.get_data ());
5070+ const auto & result_element_struct =
5071+ assert_cast<const ColumnStruct&>(result_elements.get_nested_column ());
5072+ ASSERT_FALSE (result_element_struct.get_column (0 ).is_nullable ());
5073+ EXPECT_EQ (assert_cast<const ColumnInt32&>(result_element_struct.get_column (0 )).get_element (1 ),
5074+ 7 );
5075+ }
5076+
5077+ TEST (TableReaderTest, ParentMaskProjectionOnlyWhenRequiredDescendantCanConsumeIt) {
5078+ const auto int_type = std::make_shared<DataTypeInt32>();
5079+ auto mutable_values = ColumnInt32::create ();
5080+ mutable_values->get_data ().assign ({1 , 2 });
5081+ ColumnPtr values = std::move (mutable_values);
5082+ EXPECT_FALSE (
5083+ TableReaderCastTestHelper::_requires_parent_null_map_for_alignment (values, int_type));
5084+
5085+ ColumnPtr nullable_values = ColumnNullable::create (values->clone (), ColumnUInt8::create (2 , 0 ));
5086+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_parent_null_map_for_alignment (nullable_values,
5087+ int_type));
5088+ auto null_map = ColumnUInt8::create (2 , 0 );
5089+ null_map->get_data ()[0 ] = 1 ;
5090+ ColumnPtr nullable_values_with_null =
5091+ ColumnNullable::create (values->clone (), std::move (null_map));
5092+ EXPECT_TRUE (TableReaderCastTestHelper::_requires_parent_null_map_for_alignment (
5093+ nullable_values_with_null, int_type));
5094+
5095+ NullMap all_clear_parent_mask (2 , 0 );
5096+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
5097+ &all_clear_parent_mask, nullable_values_with_null, int_type));
5098+ NullMap hidden_parent_mask {1 , 0 };
5099+ EXPECT_TRUE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
5100+ &hidden_parent_mask, nullable_values_with_null, int_type));
5101+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
5102+ nullptr , nullable_values_with_null, int_type));
5103+ }
5104+
49275105TEST (TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) {
49285106 const auto int_type = std::make_shared<DataTypeInt32>();
49295107 const std::vector<ColumnDefinition> projected_columns = {
@@ -6452,9 +6630,52 @@ TEST(TableReaderTest, CollectionParentMaskFastPathSkipsEntryScratchForClearMasks
64526630 ColumnArray::Offsets64 offsets {500000 , 1000000 };
64536631 NullMap projected_null_map;
64546632
6455- const auto * result = TableReaderCastTestHelper::_project_collection_parent_null_map (
6456- &container_null_map, &ancestor_null_map, 2 , offsets, offsets.back (),
6457- &projected_null_map);
6633+ const auto * result =
6634+ TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries (
6635+ &container_null_map, &ancestor_null_map, 2 , offsets, offsets.back (),
6636+ &projected_null_map);
6637+
6638+ EXPECT_EQ (nullptr , result);
6639+ EXPECT_TRUE (projected_null_map.empty ());
6640+ }
6641+
6642+ TEST (TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenOnlyEmptyRowIsHidden) {
6643+ constexpr size_t entries = 500000 ;
6644+ auto values = ColumnInt32::create (entries, 0 );
6645+ auto value_null_map = ColumnUInt8::create (entries, 0 );
6646+ value_null_map->get_data ().back () = 1 ;
6647+ ColumnPtr nullable_values =
6648+ ColumnNullable::create (std::move (values), std::move (value_null_map));
6649+ NullMap parent_null_map {1 , 0 };
6650+ ColumnArray::Offsets64 offsets {0 , entries};
6651+ NullMap projected_null_map;
6652+
6653+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
6654+ &parent_null_map, nullable_values, std::make_shared<DataTypeInt32>(), 2 , offsets));
6655+ const auto * result =
6656+ TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries (
6657+ nullptr , &parent_null_map, 2 , offsets, entries, &projected_null_map);
6658+
6659+ EXPECT_EQ (nullptr , result);
6660+ EXPECT_TRUE (projected_null_map.empty ());
6661+ }
6662+
6663+ TEST (TableReaderTest, CollectionParentMaskSkipsLargeMapWhenOnlyEmptyRowIsHidden) {
6664+ constexpr size_t entries = 500000 ;
6665+ auto values = ColumnInt32::create (entries, 0 );
6666+ auto value_null_map = ColumnUInt8::create (entries, 0 );
6667+ value_null_map->get_data ().back () = 1 ;
6668+ ColumnPtr nullable_values =
6669+ ColumnNullable::create (std::move (values), std::move (value_null_map));
6670+ NullMap parent_null_map {1 , 0 };
6671+ ColumnArray::Offsets64 offsets {0 , entries};
6672+ NullMap projected_null_map;
6673+
6674+ EXPECT_FALSE (TableReaderCastTestHelper::_requires_collection_parent_null_map (
6675+ &parent_null_map, nullable_values, std::make_shared<DataTypeInt32>(), 2 , offsets));
6676+ const auto * result =
6677+ TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries (
6678+ nullptr , &parent_null_map, 2 , offsets, entries, &projected_null_map);
64586679
64596680 EXPECT_EQ (nullptr , result);
64606681 EXPECT_TRUE (projected_null_map.empty ());
0 commit comments