Skip to content

Commit 1a0cfb1

Browse files
committed
[fix](external) Address timestamp planning review findings
1 parent ad56ece commit 1a0cfb1

7 files changed

Lines changed: 312 additions & 43 deletions

File tree

‎be/src/format_v2/orc/orc_reader.cpp‎

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -755,29 +755,49 @@ Status find_projected_minmax_leaf(const ::orc::Type& root_type,
755755
// resulting decode context and never need to infer which reader strategy produced a batch.
756756
class OrcTimestampReadPlan {
757757
public:
758-
void configure(const ::orc::Type& file_schema,
759-
const std::vector<format::LocalColumnId>& read_columns,
760-
const cctz::time_zone& session_timezone) {
758+
Status configure(const ::orc::Type& file_schema,
759+
const std::vector<format::LocalColumnId>& read_columns,
760+
const format::FileScanRequest& request,
761+
const cctz::time_zone& session_timezone) {
761762
_context = OrcTimestampDecodeContext(session_timezone);
762-
// An unprojected NANOS sibling can affect the SDK's ordinary timestamp decoding.
763-
const bool needs_writer_timezone =
764-
orc_serde_utils::has_nanosecond_timestamp(file_schema) &&
765-
std::ranges::any_of(read_columns, [&](const auto column_id) {
766-
return !is_virtual_column(column_id) &&
767-
has_timestamp(*file_schema.getSubtype(column_id.value()), false);
768-
});
763+
_timezone_source = TimezoneSource::SESSION;
764+
_timestamp_columns.clear();
765+
if (!orc_serde_utils::has_nanosecond_timestamp(file_schema)) {
766+
return Status::OK();
767+
}
768+
bool needs_writer_timezone = false;
769+
for (const auto column_id : read_columns) {
770+
if (is_virtual_column(column_id)) {
771+
continue;
772+
}
773+
const auto& type = *file_schema.getSubtype(column_id.value());
774+
const auto* projection = find_request_projection(request, column_id);
775+
DORIS_CHECK(projection != nullptr);
776+
std::set<uint64_t> projected_ids;
777+
const auto* selected_ids = projection->project_all_children ? nullptr : &projected_ids;
778+
if (selected_ids != nullptr) {
779+
RETURN_IF_ERROR(collect_projected_type_ids(type, *projection, &projected_ids));
780+
}
781+
// A NANOS sibling affects the SDK only when a plain timestamp is decoded. Match
782+
// includeTypes so unread nested leaves cannot trigger footer reads or dense decoding.
783+
needs_writer_timezone |= has_timestamp(type, false, selected_ids);
784+
if (has_timestamp(type, true, selected_ids)) {
785+
_timestamp_columns.insert(column_id);
786+
}
787+
}
769788
_timezone_source =
770789
needs_writer_timezone ? TimezoneSource::STRIPE_WRITER : TimezoneSource::SESSION;
790+
return Status::OK();
771791
}
772792

773793
bool requires_stripe_reader() const {
774794
return _timezone_source == TimezoneSource::STRIPE_WRITER;
775795
}
776796

777-
bool requires_dense_decoding(const ::orc::Type& column_type) const {
797+
bool requires_dense_decoding(format::LocalColumnId column_id) const {
778798
// The pinned SDK's selected timestamp decoder checks a fraction at the wrong row index.
779799
// Decode timestamp leaders densely while retaining lazy reads for other followers.
780-
return requires_stripe_reader() && has_timestamp(column_type, true);
800+
return requires_stripe_reader() && _timestamp_columns.contains(column_id);
781801
}
782802

783803
Status configure_stripe(const std::string& writer_timezone, ::orc::WriterId writer,
@@ -814,19 +834,24 @@ class OrcTimestampReadPlan {
814834
private:
815835
enum class TimezoneSource { SESSION, STRIPE_WRITER };
816836

817-
static bool has_timestamp(const ::orc::Type& type, bool include_instant) {
837+
static bool has_timestamp(const ::orc::Type& type, bool include_instant,
838+
const std::set<uint64_t>* selected_ids) {
839+
if (selected_ids != nullptr && !selected_ids->contains(type.getColumnId())) {
840+
return false;
841+
}
818842
if (type.getKind() == ::orc::TIMESTAMP ||
819843
(include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) {
820844
return true;
821845
}
822846
for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) {
823-
if (has_timestamp(*type.getSubtype(child), include_instant)) {
847+
if (has_timestamp(*type.getSubtype(child), include_instant, selected_ids)) {
824848
return true;
825849
}
826850
}
827851
return false;
828852
}
829853

854+
std::set<format::LocalColumnId> _timestamp_columns;
830855
TimezoneSource _timezone_source = TimezoneSource::SESSION;
831856
OrcTimestampDecodeContext _context {cctz::utc_time_zone()};
832857
};
@@ -1593,14 +1618,13 @@ Status OrcReader::_configure_row_reader_projection() {
15931618
DORIS_CHECK(projection != nullptr);
15941619
has_complex_projection = has_complex_projection || has_pruned_projection(*projection);
15951620
}
1596-
_state->timestamp_plan.configure(*_state->root_type, _state->read_columns,
1597-
_state->timezone_obj);
1621+
RETURN_IF_ERROR(_state->timestamp_plan.configure(*_state->root_type, _state->read_columns,
1622+
*_request, _state->timezone_obj));
15981623
auto filter_projections = _request->predicate_columns;
15991624
if (_state->orc_lazy_read_enabled) {
16001625
for (const auto column_id : _state->read_columns) {
16011626
if (!is_virtual_column(column_id) &&
1602-
_state->timestamp_plan.requires_dense_decoding(
1603-
*_state->root_type->getSubtype(column_id.value())) &&
1627+
_state->timestamp_plan.requires_dense_decoding(column_id) &&
16041628
std::ranges::none_of(filter_projections, [column_id](const auto& projection) {
16051629
return projection.column_id() == column_id;
16061630
})) {
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <gtest/gtest.h>
19+
#include <unistd.h>
20+
21+
#include <filesystem>
22+
#include <orc/OrcFile.hh>
23+
24+
#include "core/block/block.h"
25+
#include "core/column/column_nullable.h"
26+
#include "core/column/column_struct.h"
27+
#include "core/column/column_vector.h"
28+
#include "core/data_type/data_type_nullable.h"
29+
#include "core/data_type/data_type_struct.h"
30+
#include "format_v2/orc/orc_reader.h"
31+
#include "io/io_common.h"
32+
#include "runtime/runtime_profile.h"
33+
#include "runtime/runtime_state.h"
34+
#include "util/timezone_utils.h"
35+
36+
namespace doris {
37+
38+
class OrcTimestampProjectionTest : public testing::Test {
39+
protected:
40+
void SetUp() override {
41+
TimezoneUtils::load_timezones_to_cache();
42+
path = (std::filesystem::temp_directory_path() /
43+
("orc_timestamp_projection_" + std::to_string(getpid()) + ".orc"))
44+
.string();
45+
}
46+
void TearDown() override { std::filesystem::remove(path); }
47+
std::string path;
48+
};
49+
50+
TEST_F(OrcTimestampProjectionTest, OnlyProjectedTimestampLeavesReadWriterTimezones) {
51+
auto type = ::orc::Type::buildTypeFromString(
52+
"struct<ns:timestamp,s:struct<i:int,unused:timestamp>>");
53+
const_cast<::orc::Type*>(type->getSubtype(0))->setAttribute("iceberg.timestamp-unit", "NANOS");
54+
auto output = ::orc::writeLocalFile(path);
55+
::orc::WriterOptions options;
56+
options.setTimezoneName("UTC");
57+
options.setStripeSize(0);
58+
auto writer = ::orc::createWriter(*type, output.get(), options);
59+
constexpr size_t stripes = 32;
60+
for (size_t i = 0; i < stripes; ++i) {
61+
auto batch = writer->createRowBatch(1);
62+
auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch);
63+
auto& ns = dynamic_cast<::orc::TimestampVectorBatch&>(*root.fields[0]);
64+
auto& record = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[1]);
65+
auto& number = dynamic_cast<::orc::LongVectorBatch&>(*record.fields[0]);
66+
auto& timestamp = dynamic_cast<::orc::TimestampVectorBatch&>(*record.fields[1]);
67+
root.numElements = ns.numElements = record.numElements = number.numElements =
68+
timestamp.numElements = 1;
69+
number.data[0] = i;
70+
ns.data[0] = timestamp.data[0] = 1704153600;
71+
ns.nanoseconds[0] = timestamp.nanoseconds[0] = 123456000;
72+
writer->add(*batch);
73+
}
74+
writer->close();
75+
output->close();
76+
77+
for (bool project_timestamp : {false, true}) {
78+
for (bool predicate_column : {false, true}) {
79+
SCOPED_TRACE(testing::Message() << project_timestamp << ", " << predicate_column);
80+
RuntimeProfile profile("timestamp_projection");
81+
auto properties = std::make_shared<io::FileSystemProperties>();
82+
properties->system_type = TFileType::FILE_LOCAL;
83+
auto description = std::make_unique<io::FileDescription>();
84+
description->path = path;
85+
description->file_size = std::filesystem::file_size(path);
86+
format::orc::OrcReader reader(properties, description, nullptr, &profile);
87+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
88+
ASSERT_TRUE(reader.init(&state).ok());
89+
std::vector<format::ColumnDefinition> schema;
90+
ASSERT_TRUE(reader.get_schema(&schema).ok());
91+
auto request = std::make_shared<format::FileScanRequest>();
92+
auto projection = format::LocalColumnIndex::partial_local(1);
93+
projection.children = {format::LocalColumnIndex::local(0)};
94+
if (project_timestamp) {
95+
projection.children.push_back(format::LocalColumnIndex::local(1));
96+
}
97+
(predicate_column ? request->predicate_columns : request->non_predicate_columns)
98+
.push_back(projection);
99+
ASSERT_TRUE(reader.open(request).ok());
100+
DataTypePtr projected_type = schema[1].type;
101+
if (!project_timestamp) {
102+
const auto& structure =
103+
assert_cast<const DataTypeStruct&>(*remove_nullable(projected_type));
104+
projected_type = make_nullable(std::make_shared<DataTypeStruct>(
105+
DataTypes {structure.get_elements()[0]}, Strings {"i"}));
106+
}
107+
bool eof = false;
108+
size_t total = 0;
109+
while (!eof) {
110+
Block block;
111+
block.insert({projected_type->create_column(), projected_type, "s"});
112+
size_t rows = 0;
113+
auto status = reader.get_block(&block, &rows, &eof);
114+
ASSERT_TRUE(status.ok()) << status;
115+
const auto& record = assert_cast<const ColumnStruct&>(
116+
assert_cast<const ColumnNullable&>(*block.get_by_position(0).column)
117+
.get_nested_column());
118+
const auto& values = assert_cast<const ColumnInt32&>(
119+
assert_cast<const ColumnNullable&>(record.get_column(0))
120+
.get_nested_column());
121+
for (size_t row = 0; row < rows; ++row) {
122+
EXPECT_EQ(values.get_element(row), total + row);
123+
if (project_timestamp) {
124+
const auto& struct_type = assert_cast<const DataTypeStruct&>(
125+
*remove_nullable(schema[1].type));
126+
EXPECT_EQ(
127+
struct_type.get_elements()[1]->to_string(
128+
record.get_column(1), row, DataTypeSerDe::FormatOptions {}),
129+
"2024-01-02 00:00:00.123456");
130+
}
131+
}
132+
total += rows;
133+
}
134+
EXPECT_EQ(total, stripes);
135+
// An unread plain timestamp must not turn a nested integer projection into
136+
// per-stripe timezone discovery just because another file column is NANOS.
137+
EXPECT_EQ(profile.get_counter("OrcTimestampStripeFootersRead")->value(),
138+
project_timestamp ? stripes : 0);
139+
if (!project_timestamp) {
140+
EXPECT_EQ(profile.get_counter("OrcRowReadersCreated")->value(), 1);
141+
}
142+
}
143+
}
144+
}
145+
146+
} // namespace doris

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTimestampUtils.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.analysis.Expr;
2121
import org.apache.doris.analysis.SlotRef;
22+
import org.apache.doris.nereids.analyzer.UnboundSlot;
2223
import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
2324

2425
import org.apache.iceberg.Schema;
@@ -83,6 +84,11 @@ public static boolean referencesUnsafe(Expr expr, Schema schema, Set<Integer> un
8384

8485
public static boolean referencesUnsafe(org.apache.doris.nereids.trees.expressions.Expression expr,
8586
Schema schema, Set<Integer> unsafe) {
86-
return expr.getInputSlots().stream().anyMatch(slot -> unsafeName(slot.getName(), schema, unsafe));
87+
return expr.getInputSlots().stream().anyMatch(slot -> {
88+
// Match IcebergNereidsUtils: SQL-rendered names add quoting to literal dots/backticks.
89+
String name = slot instanceof UnboundSlot && ((UnboundSlot) slot).getNameParts().size() == 1
90+
? ((UnboundSlot) slot).getNameParts().get(0) : slot.getName();
91+
return unsafeName(name, schema, unsafe);
92+
});
8793
}
8894
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2458,16 +2458,8 @@ protected List<FileScanTask> loadFileScanTasksWithManifestCache(
24582458
deleteFiles.addAll(value.getDeleteFiles());
24592459
}
24602460

2461-
// DATE bounds in historical manifests are days, not nanos. The SDK's equality-delete
2462-
// overlap check cannot compare those units; retain candidate deletes without metrics pruning.
2463-
boolean unsafeDeleteMetrics = deleteFiles.stream().anyMatch(file ->
2464-
file.content() == FileContent.EQUALITY_DELETES
2465-
&& file.equalityFieldIds().stream().anyMatch(unsafeNanosFields::contains));
2466-
List<DeleteFile> indexedDeleteFiles = unsafeDeleteMetrics
2467-
? deleteFiles.stream().map(DeleteFile::copyWithoutStats).collect(Collectors.toList()) : deleteFiles;
2468-
24692461
// Build delete file index for efficient lookup of deletes applicable to each data file
2470-
DeleteFileIndex.Builder deleteIndexBuilder = DeleteFileIndex.builderFor(indexedDeleteFiles)
2462+
DeleteFileIndex.Builder deleteIndexBuilder = DeleteFileIndex.builderFor(deleteFiles)
24712463
.specsById(specsById)
24722464
.caseSensitive(caseSensitive);
24732465
// Equality deletes may reference fields dropped from the current schema, so resolve their
@@ -2514,8 +2506,7 @@ protected List<FileScanTask> loadFileScanTasksWithManifestCache(
25142506
}
25152507
// Find all delete files that apply to this data file based on sequence number
25162508
List<DeleteFile> deletes = Arrays.asList(
2517-
deleteIndex.forDataFile(dataFile.dataSequenceNumber(),
2518-
unsafeDeleteMetrics ? dataFile.copyWithoutStats() : dataFile));
2509+
deleteIndex.forDataFile(dataFile.dataSequenceNumber(), dataFile));
25192510

25202511
// Create a FileScanTask containing the data file, associated deletes, and metadata
25212512
tasks.add(new BaseFileScanTask(

‎fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,9 @@ private static <T> boolean rangesOverlap(
290290
T deleteLower,
291291
T deleteUpper) {
292292
Type.PrimitiveType type = field.type().asPrimitiveType();
293+
if (isHistoricalDateBound(type, dataLowerBuf) || isHistoricalDateBound(type, dataUpperBuf)) {
294+
return true;
295+
}
293296
Comparator<T> comparator = Comparators.forType(type);
294297

295298
T dataLower = Conversions.fromByteBuffer(type, dataLowerBuf);
@@ -305,6 +308,13 @@ private static <T> boolean rangesOverlap(
305308
return true;
306309
}
307310

311+
private static boolean isHistoricalDateBound(Type type, ByteBuffer bound) {
312+
// DATE-to-nanos promotion preserves four-byte day bounds in both data and delete manifests.
313+
// Widening them to long does not convert their units, even after the key leaves the scan schema.
314+
return type.typeId() == Type.TypeID.TIMESTAMP_NANO
315+
&& bound != null && bound.remaining() == Integer.BYTES;
316+
}
317+
308318
private static boolean allNonNull(Map<Integer, Long> nullValueCounts, Types.NestedField field) {
309319
if (field.isRequired()) {
310320
return true;
@@ -929,7 +939,7 @@ private Map<Integer, Object> convertBounds(Map<Integer, ByteBuffer> bounds) {
929939
Type type = field.type();
930940
if (type.isPrimitiveType()) {
931941
ByteBuffer bound = bounds.get(id);
932-
if (bound != null) {
942+
if (bound != null && !isHistoricalDateBound(type, bound)) {
933943
converted.put(id, Conversions.fromByteBuffer(type, bound));
934944
}
935945
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFilePlannerTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,32 @@ public void setUp() {
9999
Mockito.when(mockTableScan.ignoreResiduals()).thenReturn(mockTableScan);
100100
}
101101

102+
@Test
103+
public void testQuotedPromotedTimestampRejectsUnsafeRewritePruning() throws Exception {
104+
Mockito.when(mockTable.newScan()).thenReturn(mockTableScan);
105+
Mockito.when(mockTableScan.filter(Mockito.any())).thenReturn(mockTableScan);
106+
Mockito.when(mockTableScan.planFiles()).thenReturn(
107+
CloseableIterable.withNoopClose(Collections.emptyList()));
108+
for (String name : Arrays.asList("ts.day", "has space", "tick`name")) {
109+
Schema date = new Schema(0, Types.NestedField.optional(1, name, Types.DateType.get()));
110+
Schema nanos = new Schema(1,
111+
Types.NestedField.optional(1, name, Types.TimestampNanoType.withoutZone()));
112+
Mockito.when(mockTable.schema()).thenReturn(nanos);
113+
Mockito.when(mockTable.schemas()).thenReturn(
114+
com.google.common.collect.ImmutableMap.of(0, date, 1, nanos));
115+
org.apache.doris.nereids.trees.expressions.Expression predicate =
116+
new org.apache.doris.nereids.parser.NereidsParser().parseExpression(
117+
"`" + name.replace("`", "``") + "` = '2024-01-02'");
118+
RewriteDataFilePlanner.Parameters parameters = new RewriteDataFilePlanner.Parameters(
119+
128L, 64L, 256L, 2, false, 512L, 3, 0.1, 0L, Optional.of(predicate));
120+
UserException error = Assertions.assertThrows(UserException.class,
121+
() -> new RewriteDataFilePlanner(parameters).planAndOrganizeTasks(mockTable));
122+
Assertions.assertTrue(error.getMessage().contains("cannot prune historical DATE bounds"),
123+
error.getMessage());
124+
}
125+
Mockito.verify(mockTableScan, Mockito.never()).filter(Mockito.any());
126+
}
127+
102128
@Test
103129
public void testParametersGetters() {
104130
Assertions.assertEquals(128 * 1024 * 1024L, defaultParameters.getTargetFileSizeBytes());

0 commit comments

Comments
 (0)