|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +#include <gtest/gtest.h> |
| 19 | +#include <unistd.h> |
| 20 | + |
| 21 | +#include <filesystem> |
| 22 | +#include <orc/OrcFile.hh> |
| 23 | + |
| 24 | +#include "core/block/block.h" |
| 25 | +#include "core/column/column_nullable.h" |
| 26 | +#include "core/column/column_struct.h" |
| 27 | +#include "core/column/column_vector.h" |
| 28 | +#include "core/data_type/data_type_nullable.h" |
| 29 | +#include "core/data_type/data_type_struct.h" |
| 30 | +#include "format_v2/orc/orc_reader.h" |
| 31 | +#include "io/io_common.h" |
| 32 | +#include "runtime/runtime_profile.h" |
| 33 | +#include "runtime/runtime_state.h" |
| 34 | +#include "util/timezone_utils.h" |
| 35 | + |
| 36 | +namespace doris { |
| 37 | + |
| 38 | +class OrcTimestampProjectionTest : public testing::Test { |
| 39 | +protected: |
| 40 | + void SetUp() override { |
| 41 | + TimezoneUtils::load_timezones_to_cache(); |
| 42 | + path = (std::filesystem::temp_directory_path() / |
| 43 | + ("orc_timestamp_projection_" + std::to_string(getpid()) + ".orc")) |
| 44 | + .string(); |
| 45 | + } |
| 46 | + void TearDown() override { std::filesystem::remove(path); } |
| 47 | + std::string path; |
| 48 | +}; |
| 49 | + |
| 50 | +TEST_F(OrcTimestampProjectionTest, OnlyProjectedTimestampLeavesReadWriterTimezones) { |
| 51 | + auto type = ::orc::Type::buildTypeFromString( |
| 52 | + "struct<ns:timestamp,s:struct<i:int,unused:timestamp>>"); |
| 53 | + const_cast<::orc::Type*>(type->getSubtype(0))->setAttribute("iceberg.timestamp-unit", "NANOS"); |
| 54 | + auto output = ::orc::writeLocalFile(path); |
| 55 | + ::orc::WriterOptions options; |
| 56 | + options.setTimezoneName("UTC"); |
| 57 | + options.setStripeSize(0); |
| 58 | + auto writer = ::orc::createWriter(*type, output.get(), options); |
| 59 | + constexpr size_t stripes = 32; |
| 60 | + for (size_t i = 0; i < stripes; ++i) { |
| 61 | + auto batch = writer->createRowBatch(1); |
| 62 | + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); |
| 63 | + auto& ns = dynamic_cast<::orc::TimestampVectorBatch&>(*root.fields[0]); |
| 64 | + auto& record = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[1]); |
| 65 | + auto& number = dynamic_cast<::orc::LongVectorBatch&>(*record.fields[0]); |
| 66 | + auto& timestamp = dynamic_cast<::orc::TimestampVectorBatch&>(*record.fields[1]); |
| 67 | + root.numElements = ns.numElements = record.numElements = number.numElements = |
| 68 | + timestamp.numElements = 1; |
| 69 | + number.data[0] = i; |
| 70 | + ns.data[0] = timestamp.data[0] = 1704153600; |
| 71 | + ns.nanoseconds[0] = timestamp.nanoseconds[0] = 123456000; |
| 72 | + writer->add(*batch); |
| 73 | + } |
| 74 | + writer->close(); |
| 75 | + output->close(); |
| 76 | + |
| 77 | + for (bool project_timestamp : {false, true}) { |
| 78 | + for (bool predicate_column : {false, true}) { |
| 79 | + SCOPED_TRACE(testing::Message() << project_timestamp << ", " << predicate_column); |
| 80 | + RuntimeProfile profile("timestamp_projection"); |
| 81 | + auto properties = std::make_shared<io::FileSystemProperties>(); |
| 82 | + properties->system_type = TFileType::FILE_LOCAL; |
| 83 | + auto description = std::make_unique<io::FileDescription>(); |
| 84 | + description->path = path; |
| 85 | + description->file_size = std::filesystem::file_size(path); |
| 86 | + format::orc::OrcReader reader(properties, description, nullptr, &profile); |
| 87 | + RuntimeState state {TQueryOptions(), TQueryGlobals()}; |
| 88 | + ASSERT_TRUE(reader.init(&state).ok()); |
| 89 | + std::vector<format::ColumnDefinition> schema; |
| 90 | + ASSERT_TRUE(reader.get_schema(&schema).ok()); |
| 91 | + auto request = std::make_shared<format::FileScanRequest>(); |
| 92 | + auto projection = format::LocalColumnIndex::partial_local(1); |
| 93 | + projection.children = {format::LocalColumnIndex::local(0)}; |
| 94 | + if (project_timestamp) { |
| 95 | + projection.children.push_back(format::LocalColumnIndex::local(1)); |
| 96 | + } |
| 97 | + (predicate_column ? request->predicate_columns : request->non_predicate_columns) |
| 98 | + .push_back(projection); |
| 99 | + ASSERT_TRUE(reader.open(request).ok()); |
| 100 | + DataTypePtr projected_type = schema[1].type; |
| 101 | + if (!project_timestamp) { |
| 102 | + const auto& structure = |
| 103 | + assert_cast<const DataTypeStruct&>(*remove_nullable(projected_type)); |
| 104 | + projected_type = make_nullable(std::make_shared<DataTypeStruct>( |
| 105 | + DataTypes {structure.get_elements()[0]}, Strings {"i"})); |
| 106 | + } |
| 107 | + bool eof = false; |
| 108 | + size_t total = 0; |
| 109 | + while (!eof) { |
| 110 | + Block block; |
| 111 | + block.insert({projected_type->create_column(), projected_type, "s"}); |
| 112 | + size_t rows = 0; |
| 113 | + auto status = reader.get_block(&block, &rows, &eof); |
| 114 | + ASSERT_TRUE(status.ok()) << status; |
| 115 | + const auto& record = assert_cast<const ColumnStruct&>( |
| 116 | + assert_cast<const ColumnNullable&>(*block.get_by_position(0).column) |
| 117 | + .get_nested_column()); |
| 118 | + const auto& values = assert_cast<const ColumnInt32&>( |
| 119 | + assert_cast<const ColumnNullable&>(record.get_column(0)) |
| 120 | + .get_nested_column()); |
| 121 | + for (size_t row = 0; row < rows; ++row) { |
| 122 | + EXPECT_EQ(values.get_element(row), total + row); |
| 123 | + if (project_timestamp) { |
| 124 | + const auto& struct_type = assert_cast<const DataTypeStruct&>( |
| 125 | + *remove_nullable(schema[1].type)); |
| 126 | + EXPECT_EQ( |
| 127 | + struct_type.get_elements()[1]->to_string( |
| 128 | + record.get_column(1), row, DataTypeSerDe::FormatOptions {}), |
| 129 | + "2024-01-02 00:00:00.123456"); |
| 130 | + } |
| 131 | + } |
| 132 | + total += rows; |
| 133 | + } |
| 134 | + EXPECT_EQ(total, stripes); |
| 135 | + // An unread plain timestamp must not turn a nested integer projection into |
| 136 | + // per-stripe timezone discovery just because another file column is NANOS. |
| 137 | + EXPECT_EQ(profile.get_counter("OrcTimestampStripeFootersRead")->value(), |
| 138 | + project_timestamp ? stripes : 0); |
| 139 | + if (!project_timestamp) { |
| 140 | + EXPECT_EQ(profile.get_counter("OrcRowReadersCreated")->value(), 1); |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | +} |
| 145 | + |
| 146 | +} // namespace doris |
0 commit comments