Skip to content

Commit 19ef150

Browse files
committed
[fix](datalake) Preserve nanos schemas and safe Iceberg filters
### What problem does this PR solve? Related PR: #68934 Problem Summary: TVF schema discovery rounded nanosecond files before V2 decoding. Generated Iceberg overwrite/conflict predicates bypassed retained DATE-bound safeguards, and partial predicates under NOT could prune matching files. Historical ORC producer semantics, metadata MIN/MAX, dropped partition sources, per-stripe reader setup, and nullable Parquet nanos dispatch also needed correction. Route ORC/Parquet schema discovery through V2, require exact negated predicates, reject unsafe static overwrite, and preserve conservative conflict filtering. Keep Java/C++ historical carriers distinct, fall back from unsafe metadata aggregates, coalesce equal-zone stripes, and restore fused nullable decoding. SDK sources and legacy LOAD/V1 decoding are unchanged. ### Release note Preserve nanosecond precision in ORC/Parquet TVF schema discovery and correct affected Iceberg filtering and ORC timestamp reads. ### Check List (For Author) - Test: 425 FE unit tests; 949 BE unit tests; Iceberg nanosecond regression with real S3 ORC/Parquet TVF reads; BE/FE builds; FE Checkstyle; clang-format 16; clang-tidy; build hygiene; GCC 15 syntax check. - Behavior changed: Yes, preserve TVF nanos precision, reject unsafe static overwrites, and avoid incorrect pruning or historical timestamp results. - Does this need documentation: No, review corrections to the existing feature.
1 parent 3f7020a commit 19ef150

24 files changed

Lines changed: 607 additions & 47 deletions

‎be/src/core/data_type_serde/data_type_datetimev2_serde.cpp‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ static const int64_t micro_to_nano_second = 1000;
5353

5454
namespace {
5555

56-
Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view,
57-
const cctz::time_zone& timezone) {
56+
Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view) {
5857
const auto* orc_batch = dynamic_cast<const ::orc::TimestampVectorBatch*>(orc_view.batch);
5958
if (orc_batch == nullptr) {
6059
return Status::InternalError("Unexpected ORC timestamp batch type {}",
@@ -87,7 +86,8 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum
8786
// so rounding cannot jump across a daylight-saving gap or fold.
8887
status = orc_serde_utils::orc_timestamp_to_datetime(
8988
is_timestamp_instant ? timestamp.seconds : seconds, timestamp.microseconds,
90-
timezone, !is_timestamp_instant && timestamp.carry, &value);
89+
orc_view.timestamp_context->timezone_for(*orc_view.file_type, seconds),
90+
!is_timestamp_instant && timestamp.carry, &value);
9191
if (!status.ok()) {
9292
data.resize(old_data_size);
9393
return status;
@@ -1051,8 +1051,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column,
10511051
if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) {
10521052
return Status::OK();
10531053
}
1054-
return decode_timestamp_orc_values(column, view,
1055-
view.timestamp_context->timezone_for(*view.file_type));
1054+
return decode_timestamp_orc_values(column, view);
10561055
}
10571056

10581057
} // namespace doris

‎be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,6 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column,
480480
const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows);
481481
DORIS_CHECK(view.file_type != nullptr);
482482
DORIS_CHECK(view.timestamp_context != nullptr);
483-
const auto& timezone = view.timestamp_context->timezone_for(*view.file_type);
484483
for (size_t row = 0; row < rows; ++row) {
485484
const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows);
486485
if (orc_serde_utils::orc_row_is_null(batch, source)) {
@@ -494,6 +493,7 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column,
494493
}
495494
DateV2Value<DateTimeV2ValueType> civil;
496495
const int64_t seconds = batch.data[source];
496+
const auto timezone = view.timestamp_context->timezone_for(*view.file_type, seconds);
497497
auto status = orc_serde_utils::orc_timestamp_to_datetime(seconds, nanos / 1000, timezone,
498498
false, &civil);
499499
TimeStampNsValue value;

‎be/src/core/data_type_serde/orc_timestamp_decode_context.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,15 @@
2323

2424
namespace doris {
2525

26-
const cctz::time_zone& OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type) const {
26+
cctz::time_zone OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type,
27+
int64_t seconds) const {
2728
const auto kind = file_type.getKind();
2829
DORIS_CHECK(kind == orc::TIMESTAMP || kind == orc::TIMESTAMP_INSTANT);
2930
// Plain timestamps reconstruct the SDK reader's civil value. Instant-to-DATETIME conversion
3031
// uses the session zone, which must not follow a plain sibling's stripe writer timezone.
32+
if (kind == orc::TIMESTAMP && writer_timezone) {
33+
return writer_timezone(seconds);
34+
}
3135
return kind == orc::TIMESTAMP ? reader_timezone : session_timezone;
3236
}
3337

‎be/src/core/data_type_serde/orc_timestamp_decode_context.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
#include <cctz/time_zone.h>
2121

22+
#include <functional>
23+
2224
namespace orc {
2325
class Type;
2426
}
@@ -30,7 +32,10 @@ struct OrcTimestampDecodeContext {
3032
explicit OrcTimestampDecodeContext(const cctz::time_zone& timezone)
3133
: reader_timezone(timezone), session_timezone(timezone) {}
3234

33-
const cctz::time_zone& timezone_for(const orc::Type& file_type) const;
35+
cctz::time_zone timezone_for(const orc::Type& file_type, int64_t seconds = 0) const;
36+
37+
// Java carriers follow the SDK writer-zone rules rather than cctz historical LMT.
38+
std::function<cctz::time_zone(int64_t)> writer_timezone;
3439

3540
cctz::time_zone reader_timezone;
3641
cctz::time_zone session_timezone;

‎be/src/format/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES})
2828

2929
add_library(Format STATIC ${SRC_FILES})
3030

31+
# Read the pinned SDK timezone rules without changing SDK decoding or public interfaces.
32+
target_include_directories(Format PRIVATE
33+
${CMAKE_CURRENT_SOURCE_DIR}/../../../contrib/apache-orc/c++/src
34+
${BUILD_DIR}/apache-orc/c++/src)
35+
3136
pch_reuse(Format)
3237

3338
# Unity build scoped to format_v2/: the v2 readers are the maintained line and

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 64 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <cctz/time_zone.h>
2121
#include <gen_cpp/Types_types.h>
2222

23+
#include <Timezone.hh>
2324
#include <algorithm>
2425
#include <atomic>
2526
#include <cctype>
@@ -766,7 +767,7 @@ class OrcTimestampReadPlan {
766767
return requires_stripe_reader() && has_timestamp(column_type, true);
767768
}
768769

769-
Status configure_stripe(const ::orc::StripeInformation& stripe,
770+
Status configure_stripe(const ::orc::StripeInformation& stripe, ::orc::WriterId writer,
770771
::orc::RowReaderOptions* options) {
771772
DORIS_CHECK(requires_stripe_reader());
772773
const auto& name = stripe.getWriterTimezone();
@@ -778,6 +779,22 @@ class OrcTimestampReadPlan {
778779
// Matching zones preserves the SDK's UTC carrier until its negative-fraction correction
779780
// is complete. SerDes then apply historical offsets to that original second with cctz.
780781
options->setTimezoneName(writer_timezone);
782+
if (writer == ::orc::ORC_JAVA_WRITER) {
783+
// Java and C++ ancient carriers differ. Preserve the SDK Java writer rules
784+
// instead of applying cctz LMT to a Java standard-offset carrier.
785+
const auto* timezone = &::orc::getTimezoneByName(writer_timezone);
786+
_context.writer_timezone = [timezone,
787+
zones = std::unordered_map<int64_t, cctz::time_zone> {}](
788+
int64_t seconds) mutable {
789+
const auto offset = timezone->getVariant(seconds).gmtOffset;
790+
auto it = zones.find(offset);
791+
if (it == zones.end()) {
792+
// Cache the few SDK offsets; fixed_time_zone takes a global cache lock.
793+
it = zones.emplace(offset, cctz::fixed_time_zone(cctz::seconds(offset))).first;
794+
}
795+
return it->second;
796+
};
797+
}
781798
return Status::OK();
782799
}
783800

@@ -888,6 +905,8 @@ void OrcReader::_init_profile() {
888905
file_scan_profile::ensure_hierarchy(_profile);
889906
_orc_profile.total_time =
890907
ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1);
908+
_orc_profile.row_readers_created = ADD_CHILD_COUNTER_WITH_LEVEL(
909+
_profile, "OrcRowReadersCreated", TUnit::UNIT, orc_profile, 1);
891910
_orc_profile.reader_call =
892911
ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1);
893912
_orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL(
@@ -1786,20 +1805,33 @@ Status OrcReader::_prepare_timestamp_stripe_ranges() {
17861805
try {
17871806
uint64_t first_row = 0;
17881807
size_t selected = 0;
1808+
std::string previous_timezone;
17891809
for (uint64_t index = 0; index <= stripes.back(); ++index) {
17901810
const auto stripe = _state->reader->getStripe(index);
17911811
const auto end_row = first_row + stripe->getNumberOfRows();
17921812
if (index == stripes[selected]) {
1793-
// A RowReader has one timezone, whereas writer timezones belong to stripes.
1794-
// Read footer timezone metadata only when entering the selected stripe.
1795-
_state->selected_stripe_ranges.push_back({
1796-
.first_stripe = index,
1797-
.last_stripe = index + 1,
1798-
.offset = stripe->getOffset(),
1799-
.length = stripe->getLength(),
1800-
.first_row = first_row,
1801-
.end_row = end_row,
1802-
});
1813+
const auto& name = stripe->getWriterTimezone();
1814+
const std::string timezone = name.empty() ? "GMT" : name;
1815+
auto& ranges = _state->selected_stripe_ranges;
1816+
// Each SDK RowReader scans the entire stripe directory. Reuse it across
1817+
// adjacent stripes with the same timezone to avoid quadratic setup work.
1818+
if (!ranges.empty() && ranges.back().last_stripe == index &&
1819+
previous_timezone == timezone) {
1820+
auto& range = ranges.back();
1821+
range.last_stripe = index + 1;
1822+
range.length = stripe->getOffset() + stripe->getLength() - range.offset;
1823+
range.end_row = end_row;
1824+
} else {
1825+
ranges.push_back({
1826+
.first_stripe = index,
1827+
.last_stripe = index + 1,
1828+
.offset = stripe->getOffset(),
1829+
.length = stripe->getLength(),
1830+
.first_row = first_row,
1831+
.end_row = end_row,
1832+
});
1833+
}
1834+
previous_timezone = timezone;
18031835
++selected;
18041836
}
18051837
first_row = end_row;
@@ -1820,8 +1852,8 @@ Status OrcReader::_apply_current_stripe_range() {
18201852
if (_state->timestamp_plan.requires_stripe_reader()) {
18211853
try {
18221854
const auto stripe = _state->reader->getStripe(stripe_range.first_stripe);
1823-
RETURN_IF_ERROR(
1824-
_state->timestamp_plan.configure_stripe(*stripe, &_state->row_reader_options));
1855+
RETURN_IF_ERROR(_state->timestamp_plan.configure_stripe(
1856+
*stripe, _state->reader->getWriterId(), &_state->row_reader_options));
18251857
} catch (const std::exception& e) {
18261858
if (is_orc_stop(_io_ctx.get(), e)) {
18271859
return Status::EndOfFile("stop");
@@ -1854,6 +1886,9 @@ Status OrcReader::_create_row_reader() {
18541886
_state->row_reader = _state->reader->createRowReader(
18551887
_state->row_reader_options,
18561888
_state->orc_lazy_read_enabled ? _orc_filter.get() : nullptr);
1889+
if (_profile != nullptr) {
1890+
COUNTER_UPDATE(_orc_profile.row_readers_created, 1);
1891+
}
18571892
_state->selected_type = &_state->row_reader->getSelectedType();
18581893
DORIS_CHECK(_state->selected_type->getKind() == ::orc::TypeKind::STRUCT);
18591894
// Row-id fetch seeks before every read; a one-row batch preserves exact selection instead
@@ -2426,6 +2461,22 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque
24262461
}
24272462
std::unique_ptr<::orc::StripeStatistics> stripe_statistics;
24282463
try {
2464+
if (leaf_type->getKind() == ::orc::TIMESTAMP &&
2465+
orc_serde_utils::has_nanosecond_timestamp(*_state->root_type) &&
2466+
_state->reader->getWriterId() == ::orc::ORC_CPP_WRITER) {
2467+
// Named-zone C++ statistics use SDK historical offsets, whereas rows use cctz.
2468+
// Preserve exact metadata aggregates for fixed-zone writers; otherwise scan rows.
2469+
const auto stripe = _state->reader->getStripe(stripe_index);
2470+
const auto& name = stripe->getWriterTimezone();
2471+
cctz::time_zone writer_zone;
2472+
int32_t offset = 0;
2473+
if (!TimezoneUtils::find_cctz_time_zone(name.empty() ? "GMT" : name,
2474+
writer_zone) ||
2475+
!TimezoneUtils::try_get_fixed_offset_seconds(writer_zone, &offset)) {
2476+
return Status::NotSupported(
2477+
"ORC C++ named-zone mixed nanos timestamps require row min/max");
2478+
}
2479+
}
24292480
stripe_statistics = _state->reader->getStripeStatistics(stripe_index);
24302481
} catch (const std::exception& e) {
24312482
return Status::InternalError("Failed to read ORC stripe statistics {}: {}",

‎be/src/format_v2/orc/orc_reader.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ class OrcReader final : public format::FileReader {
7373

7474
private:
7575
struct OrcProfile {
76+
RuntimeProfile::Counter* row_readers_created = nullptr;
7677
RuntimeProfile::Counter* total_time = nullptr;
7778
RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall
7879
RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs

‎be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -592,6 +592,8 @@ bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) {
592592
VISIT_COLUMN(ColumnUUID)
593593
VISIT_COLUMN(ColumnTimeV2)
594594
VISIT_COLUMN(ColumnTimeStampTz)
595+
// Nanos columns also support in-place NULL expansion; do not decode once per null run.
596+
VISIT_COLUMN(ColumnTimeStampNs)
595597
VISIT_COLUMN(ColumnOffset32)
596598
VISIT_COLUMN(ColumnOffset64)
597599
VISIT_COLUMN(ColumnDecimal32)

‎be/src/format_v2/schema_reader.cpp‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "format_v2/schema_reader.h"
19+
20+
#include "format_v2/orc/orc_reader.h"
21+
#include "format_v2/parquet/parquet_reader.h"
22+
#include "io/file_factory.h"
23+
#include "runtime/runtime_state.h"
24+
25+
namespace doris::format {
26+
27+
Status fetch_file_schema(const TFileScanRangeParams& params, const TFileRangeDesc& range,
28+
const std::shared_ptr<io::IOContext>& io_ctx, RuntimeProfile* profile,
29+
std::vector<ColumnDefinition>* schema) {
30+
auto properties = std::make_shared<io::FileSystemProperties>();
31+
// Schema RPCs carry the filesystem kind on the range, not necessarily in scan params.
32+
properties->system_type = range.__isset.file_type ? range.file_type : params.file_type;
33+
properties->properties = params.properties;
34+
properties->hdfs_params = params.hdfs_params;
35+
properties->broker_addresses = params.broker_addresses;
36+
auto file = std::make_unique<io::FileDescription>();
37+
file->path = range.path;
38+
file->file_size = range.__isset.file_size ? range.file_size : -1;
39+
file->mtime = range.__isset.modification_time ? range.modification_time : 0;
40+
file->fs_name = range.fs_name;
41+
if (range.__isset.file_cache_admission) {
42+
file->file_cache_admission = range.file_cache_admission;
43+
}
44+
const bool timestamp_tz =
45+
params.__isset.enable_mapping_timestamp_tz && params.enable_mapping_timestamp_tz;
46+
const bool varbinary =
47+
params.__isset.enable_mapping_varbinary && params.enable_mapping_varbinary;
48+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
49+
std::unique_ptr<FileReader> reader;
50+
// TVF targets must retain file precision before column mapping; a microsecond schema
51+
// irreversibly rounds nanos even when the subsequent row scan uses the V2 reader.
52+
if (params.format_type == TFileFormatType::FORMAT_PARQUET) {
53+
reader = std::make_unique<parquet::ParquetReader>(properties, file, io_ctx, profile,
54+
std::nullopt, timestamp_tz, varbinary);
55+
} else if (params.format_type == TFileFormatType::FORMAT_ORC) {
56+
reader = std::make_unique<orc::OrcReader>(properties, file, io_ctx, profile, std::nullopt,
57+
timestamp_tz, varbinary);
58+
} else {
59+
return Status::NotSupported("V2 schema inference requires Parquet or ORC");
60+
}
61+
RETURN_IF_ERROR(reader->init(&state));
62+
RETURN_IF_ERROR(reader->get_schema(schema));
63+
return reader->close();
64+
}
65+
66+
} // namespace doris::format

‎be/src/format_v2/schema_reader.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <memory>
21+
#include <vector>
22+
23+
#include "common/status.h"
24+
25+
namespace doris {
26+
class RuntimeProfile;
27+
class TFileScanRangeParams;
28+
class TFileRangeDesc;
29+
namespace io {
30+
struct IOContext;
31+
}
32+
namespace format {
33+
struct ColumnDefinition;
34+
35+
Status fetch_file_schema(const TFileScanRangeParams& params, const TFileRangeDesc& range,
36+
const std::shared_ptr<io::IOContext>& io_ctx, RuntimeProfile* profile,
37+
std::vector<ColumnDefinition>* schema);
38+
} // namespace format
39+
} // namespace doris

0 commit comments

Comments
 (0)