|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +#include <gtest/gtest.h> |
| 19 | + |
| 20 | +#include <cstdlib> |
| 21 | + |
| 22 | +#include "core/column/column_array.h" |
| 23 | +#include "core/column/column_struct.h" |
| 24 | +#include "core/data_type/data_type_array.h" |
| 25 | +#include "core/data_type/data_type_struct.h" |
| 26 | +#include "core/data_type/data_type_varbinary.h" |
| 27 | +#include "format/parquet/vparquet_reader.h" |
| 28 | +#include "io/fs/file_meta_cache.h" |
| 29 | +#include "io/fs/local_file_system.h" |
| 30 | +#include "util/timezone_utils.h" |
| 31 | + |
| 32 | +namespace doris { |
| 33 | + |
| 34 | +TEST(ParquetUuidBinaryTest, LegacyReaderPreservesNestedBytesAcrossBatches) { |
| 35 | + // Timestamp-semantics negotiation can force V2 even with its session flag disabled. |
| 36 | + // Exercise the legacy reader directly to keep its nested BINARY mapping covered. |
| 37 | + const char* source_root = std::getenv("ROOT"); |
| 38 | + ASSERT_NE(source_root, nullptr); |
| 39 | + const std::vector<std::string> bytes = {std::string(16, '\0'), |
| 40 | + std::string(16, '\0'), |
| 41 | + std::string(15, '\0') + std::string(1, '\1'), |
| 42 | + std::string(1, '\x7f') + std::string(15, '\xff'), |
| 43 | + std::string(1, '\x80') + std::string(15, '\0'), |
| 44 | + std::string(16, '\xff')}; |
| 45 | + auto check_value = [&](const IColumn& column, size_t row, size_t id) { |
| 46 | + EXPECT_EQ(column.is_null_at(row), id % 6 == 0); |
| 47 | + if (id % 6 != 0) { |
| 48 | + EXPECT_EQ(assert_cast<const ColumnNullable&>(column) |
| 49 | + .get_nested_column() |
| 50 | + .get_data_at(row) |
| 51 | + .to_string(), |
| 52 | + bytes[id % 6]); |
| 53 | + } |
| 54 | + }; |
| 55 | + const auto binary = make_nullable(std::make_shared<DataTypeVarbinary>()); |
| 56 | + const DataTypes types = { |
| 57 | + make_nullable(std::make_shared<DataTypeInt32>()), binary, |
| 58 | + make_nullable(std::make_shared<DataTypeArray>(binary)), |
| 59 | + make_nullable(std::make_shared<DataTypeStruct>(DataTypes {binary}, Strings {"k"}))}; |
| 60 | + const std::vector<std::string> names = {"id", "u", "a", "s"}; |
| 61 | + std::unordered_map<std::string, uint32_t> indexes = {{"id", 0}, {"u", 1}, {"a", 2}, {"s", 3}}; |
| 62 | + cctz::time_zone zone; |
| 63 | + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("UTC", zone)); |
| 64 | + FileMetaCache cache(1024); |
| 65 | + for (const std::string encoding : {"plain", "dictionary"}) { |
| 66 | + io::FileReaderSPtr file; |
| 67 | + ASSERT_TRUE( |
| 68 | + io::global_local_filesystem() |
| 69 | + ->open_file( |
| 70 | + std::string(source_root) + |
| 71 | + "/regression-test/data/datatype_p0/uuid/test_data/uuid_" + |
| 72 | + encoding + ".parquet", |
| 73 | + &file) |
| 74 | + .ok()); |
| 75 | + TFileScanRangeParams params; |
| 76 | + params.__set_enable_mapping_varbinary(true); |
| 77 | + params.__set_file_type(TFileType::FILE_LOCAL); |
| 78 | + TFileRangeDesc range; |
| 79 | + range.__set_start_offset(0); |
| 80 | + range.__set_size(file->size()); |
| 81 | + range.__set_file_size(file->size()); |
| 82 | + range.__set_path(file->path().native()); |
| 83 | + ParquetReader reader(nullptr, params, range, 37, &zone, nullptr, nullptr, &cache); |
| 84 | + phmap::flat_hash_map<int, std::vector<std::shared_ptr<ColumnPredicate>>> predicates; |
| 85 | + auto status = reader.init_reader(names, &indexes, {}, predicates, nullptr, nullptr, nullptr, |
| 86 | + nullptr, nullptr); |
| 87 | + ASSERT_TRUE(status.ok()) << status; |
| 88 | + std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>> partitions; |
| 89 | + std::unordered_map<std::string, VExprContextSPtr> missing; |
| 90 | + ASSERT_TRUE(reader.set_fill_columns(partitions, missing).ok()); |
| 91 | + bool eof = false; |
| 92 | + size_t total = 0; |
| 93 | + while (!eof) { |
| 94 | + Block block; |
| 95 | + for (size_t i = 0; i < names.size(); ++i) { |
| 96 | + block.insert({types[i]->create_column(), types[i], names[i]}); |
| 97 | + } |
| 98 | + size_t rows = 0; |
| 99 | + status = reader.get_next_block(&block, &rows, &eof); |
| 100 | + ASSERT_TRUE(status.ok()) << status; |
| 101 | + for (size_t row = 0; row < rows; ++row) { |
| 102 | + const size_t id = total + row; |
| 103 | + check_value(*block.get_by_position(1).column, row, id); |
| 104 | + const auto& array_nullable = |
| 105 | + assert_cast<const ColumnNullable&>(*block.get_by_position(2).column); |
| 106 | + EXPECT_EQ(array_nullable.is_null_at(row), id % 7 == 0); |
| 107 | + const auto& array = |
| 108 | + assert_cast<const ColumnArray&>(array_nullable.get_nested_column()); |
| 109 | + const auto begin = array.get_offsets()[static_cast<ssize_t>(row) - 1]; |
| 110 | + const auto end = array.get_offsets()[row]; |
| 111 | + if (id % 7 > 1) { |
| 112 | + ASSERT_EQ(end - begin, 2); |
| 113 | + check_value(array.get_data(), begin, id); |
| 114 | + EXPECT_TRUE(array.get_data().is_null_at(begin + 1)); |
| 115 | + } else { |
| 116 | + EXPECT_EQ(end, begin); |
| 117 | + } |
| 118 | + const auto& structure = |
| 119 | + assert_cast<const ColumnNullable&>(*block.get_by_position(3).column); |
| 120 | + EXPECT_EQ(structure.is_null_at(row), id % 5 == 0); |
| 121 | + if (id % 5 != 0) { |
| 122 | + const auto& fields = |
| 123 | + assert_cast<const ColumnStruct&>(structure.get_nested_column()); |
| 124 | + check_value(fields.get_column(0), row, id); |
| 125 | + } |
| 126 | + } |
| 127 | + total += rows; |
| 128 | + } |
| 129 | + EXPECT_EQ(total, 4097); |
| 130 | + } |
| 131 | +} |
| 132 | + |
| 133 | +} // namespace doris |
0 commit comments