Skip to content

Commit b9d008e

Browse files
committed
fix(external): complete connector mappings and catalog migration
1 parent 619c016 commit b9d008e

168 files changed

Lines changed: 4910 additions & 2258 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "core/data_type/data_type_number.h"
3434
#include "core/data_type/data_type_string.h"
3535
#include "core/data_type/data_type_struct.h"
36+
#include "core/data_type/data_type_varbinary.h"
3637
#include "exprs/vexpr.h"
3738
#include "format/table/parquet_utils.h"
3839
#include "format_v2/table/iceberg_position_delete_sys_table_reader.h"
@@ -573,6 +574,41 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) {
573574
EXPECT_EQ(3, file_reader_stats.read_rows);
574575
}
575576

577+
TEST(IcebergPositionDeleteSysTableV2ReaderTest, PreservesUuidPartitionBytes) {
578+
ObjectPool pool;
579+
const auto partition_type = make_nullable(std::make_shared<DataTypeStruct>(
580+
DataTypes {make_nullable(std::make_shared<DataTypeVarbinary>())}, Strings {"p"}));
581+
auto* slot = make_slot(&pool, 0, "partition", partition_type);
582+
TIcebergFileDesc descriptor;
583+
descriptor.__set_partition_data_json(R"({"p":"0x123E4567E89B12D3A456426614174000"})");
584+
auto verify = [&](auto& reader) {
585+
reader._iceberg_file_desc = &descriptor;
586+
auto column = partition_type->create_column();
587+
ASSERT_TRUE(reader._append_partition_column(column, *slot).ok());
588+
ASSERT_TRUE(reader._append_partition_column(column, *slot).ok());
589+
const auto& structure = assert_cast<const ColumnStruct&>(
590+
assert_cast<const ColumnNullable&>(*column).get_nested_column());
591+
const auto& field = assert_cast<const ColumnNullable&>(structure.get_column(0));
592+
for (size_t row = 0; row < 2; ++row) {
593+
ASSERT_FALSE(field.is_null_at(row));
594+
EXPECT_EQ(
595+
std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00",
596+
16),
597+
field.get_nested_column().get_data_at(row).to_string());
598+
}
599+
};
600+
format::iceberg::IcebergPositionDeleteSysTableV2Reader reader;
601+
verify(reader);
602+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
603+
RuntimeProfile profile("test_profile");
604+
TFileRangeDesc range;
605+
TFileScanRangeParams params;
606+
std::vector<SlotDescriptor*> slots {slot};
607+
IcebergPositionDeleteSysTableReader legacy_reader(slots, &state, &profile, range, &params,
608+
std::make_shared<io::IOContext>(), nullptr);
609+
verify(legacy_reader);
610+
}
611+
576612
TEST(IcebergPositionDeleteSysTableV2ReaderTest, CachesAndClearsPartitionValue) {
577613
ObjectPool pool;
578614
const auto nullable_int32 = make_nullable(std::make_shared<DataTypeInt32>());

‎fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,21 @@ public byte getByte() {
113113

114114
@Override
115115
public LocalDateTime getTimeStampTz() {
116-
return ((Timestamp) fieldData).toLocalDateTime();
116+
// Different Hudi readers expose the same instant through different physical carriers.
117+
Instant instant;
118+
if (fieldData instanceof Timestamp) {
119+
instant = ((Timestamp) fieldData).toInstant();
120+
} else if (fieldData instanceof TimestampWritableV2) {
121+
org.apache.hadoop.hive.common.type.Timestamp timestamp =
122+
((TimestampObjectInspector) fieldInspector).getPrimitiveJavaObject(fieldData);
123+
instant = Instant.ofEpochSecond(timestamp.toEpochSecond(), timestamp.getNanos());
124+
} else {
125+
long value = ((LongWritable) fieldData).get();
126+
long units = dorisType.getPrecision() == 3 ? 1000L : 1_000_000L;
127+
instant = Instant.ofEpochSecond(Math.floorDiv(value, units),
128+
Math.floorMod(value, units) * (1_000_000_000L / units));
129+
}
130+
return LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC);
117131
}
118132

119133
@Override
@@ -131,25 +145,9 @@ public LocalDate getDate() {
131145
public LocalDateTime getDateTime() {
132146
if (fieldData instanceof Timestamp) {
133147
return ((Timestamp) fieldData).toLocalDateTime();
134-
} else if (fieldData instanceof TimestampWritableV2) {
135-
return LocalDateTime.ofInstant(Instant.ofEpochSecond((((TimestampObjectInspector) fieldInspector)
136-
.getPrimitiveJavaObject(fieldData)).toEpochSecond()), zoneId);
137-
} else {
138-
long datetime = ((LongWritable) fieldData).get();
139-
long seconds;
140-
long nanoseconds;
141-
if (dorisType.getPrecision() == 3) {
142-
seconds = datetime / 1000;
143-
nanoseconds = (datetime % 1000) * 1000000;
144-
} else if (dorisType.getPrecision() == 6) {
145-
seconds = datetime / 1000000;
146-
nanoseconds = (datetime % 1000000) * 1000;
147-
} else {
148-
throw new RuntimeException("Hoodie timestamp only support milliseconds and microseconds, "
149-
+ "wrong precision = " + dorisType.getPrecision());
150-
}
151-
return LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds, nanoseconds), zoneId);
152148
}
149+
// DATETIMEV2 now denotes local-timestamp annotations: decode their fields without a zone shift.
150+
return getTimeStampTz();
153151
}
154152

155153
@Override
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.hudi;
19+
20+
import org.apache.doris.common.jni.vec.ColumnType;
21+
22+
import org.apache.hadoop.hive.common.type.Timestamp;
23+
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
24+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
25+
import org.apache.hadoop.io.LongWritable;
26+
import org.junit.Assert;
27+
import org.junit.Test;
28+
29+
import java.time.LocalDateTime;
30+
import java.time.ZoneId;
31+
32+
public class HadoopHudiColumnValueTest {
33+
@Test
34+
public void testInstantCarriersAcrossTimezonesAndPrecisions() {
35+
java.util.TimeZone original = java.util.TimeZone.getDefault();
36+
try {
37+
for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) {
38+
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone(zone));
39+
for (int precision : new int[] {3, 6}) {
40+
for (long epoch : new long[] {-1, 0, 1636263000123L, 1636266600123L}) {
41+
if (precision == 6 && epoch > 0) {
42+
epoch = epoch * 1000 + 456;
43+
}
44+
long units = precision == 3 ? 1000L : 1_000_000L;
45+
java.time.Instant instant = java.time.Instant.ofEpochSecond(Math.floorDiv(epoch, units),
46+
Math.floorMod(epoch, units) * (1_000_000_000L / units));
47+
LocalDateTime expected = LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC);
48+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneId.of(zone));
49+
value.setField(ColumnType.parseType("ts", "timestamptz(" + precision + ")"),
50+
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
51+
value.setRow(new LongWritable(epoch));
52+
Assert.assertEquals(expected, value.getTimeStampTz());
53+
value.setRow(java.sql.Timestamp.from(instant));
54+
Assert.assertEquals(expected, value.getTimeStampTz());
55+
value.setRow(new TimestampWritableV2(Timestamp.ofEpochSecond(
56+
instant.getEpochSecond(), instant.getNano())));
57+
Assert.assertEquals(expected, value.getTimeStampTz());
58+
value.setRow(null);
59+
Assert.assertTrue(value.isNull());
60+
}
61+
}
62+
}
63+
} finally {
64+
java.util.TimeZone.setDefault(original);
65+
}
66+
}
67+
68+
@Test
69+
public void testInstantTimestampUsesUtcComponents() {
70+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneId.of("Asia/Shanghai"));
71+
value.setField(ColumnType.parseType("ts", "timestamptz(6)"), null);
72+
value.setRow(new LongWritable(-1));
73+
Assert.assertEquals(LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999000), value.getTimeStampTz());
74+
value.setField(ColumnType.parseType("ts", "timestamptz(6)"),
75+
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
76+
value.setRow(new TimestampWritableV2(Timestamp.ofEpochSecond(1, 111333000)));
77+
Assert.assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 1, 111333000), value.getTimeStampTz());
78+
}
79+
80+
@Test
81+
public void testInt64LocalTimestampIgnoresSessionTimezone() {
82+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneId.of("America/Los_Angeles"));
83+
value.setField(ColumnType.parseType("ts", "datetimev2(6)"), null);
84+
value.setRow(new LongWritable(0));
85+
86+
Assert.assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0), value.getDateTime());
87+
}
88+
89+
@Test
90+
public void testInt96LocalTimestampIgnoresSessionTimezone() {
91+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneId.of("America/Los_Angeles"));
92+
value.setField(ColumnType.parseType("ts", "datetimev2(6)"),
93+
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
94+
value.setRow(new TimestampWritableV2(Timestamp.ofEpochSecond(0)));
95+
96+
Assert.assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0), value.getDateTime());
97+
}
98+
}

‎fe/be-java-extensions/jdbc-scanner/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ under the License.
3535
</properties>
3636

3737
<dependencies>
38+
<dependency>
39+
<groupId>org.mockito</groupId>
40+
<artifactId>mockito-core</artifactId>
41+
<scope>test</scope>
42+
</dependency>
3843
<dependency>
3944
<groupId>org.apache.doris</groupId>
4045
<artifactId>java-common</artifactId>

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,11 @@ private int insert(VectorTable data) throws SQLException {
677677
return data.getNumRows();
678678
}
679679

680+
protected void setTimestampTz(int parameterIndex, java.time.LocalDateTime value) throws SQLException {
681+
// JNI supplies UTC components; valueOf would reinterpret them in the JVM timezone.
682+
preparedStatement.setObject(parameterIndex, Timestamp.from(value.toInstant(java.time.ZoneOffset.UTC)));
683+
}
684+
680685
private void insertColumn(int rowIdx, int colIdx, VectorColumn column) throws SQLException {
681686
int parameterIndex = colIdx + 1;
682687
ColumnType.Type dorisType = column.getColumnPrimitiveType();
@@ -723,8 +728,7 @@ private void insertColumn(int rowIdx, int colIdx, VectorColumn column) throws SQ
723728
parameterIndex, Timestamp.valueOf(column.getDateTime(rowIdx)));
724729
break;
725730
case TIMESTAMPTZ:
726-
preparedStatement.setObject(
727-
parameterIndex, Timestamp.valueOf(column.getTimeStampTz(rowIdx)));
731+
setTimestampTz(parameterIndex, column.getTimeStampTz(rowIdx));
728732
break;
729733
case CHAR:
730734
case VARCHAR:

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/ClickHouseJdbcExecutor.java‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.sql.SQLException;
3232
import java.time.LocalDate;
3333
import java.time.LocalDateTime;
34+
import java.time.ZoneOffset;
3435
import java.util.ArrayList;
3536
import java.util.List;
3637

@@ -55,6 +56,12 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int
5556
@Override
5657
protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
5758
switch (type.getType()) {
59+
case TIMESTAMPTZ: {
60+
// The projection preserves epoch microseconds before JDBC v1 can lose a DST-fold offset.
61+
long micros = resultSet.getLong(columnIndex + 1);
62+
return resultSet.wasNull() ? null : LocalDateTime.ofEpochSecond(Math.floorDiv(micros, 1_000_000),
63+
(int) Math.floorMod(micros, 1_000_000) * 1000, ZoneOffset.UTC);
64+
}
5865
case BOOLEAN:
5966
return resultSet.getObject(columnIndex + 1, Boolean.class);
6067
case TINYINT:
@@ -86,8 +93,10 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
8693
case VARCHAR:
8794
case STRING:
8895
return resultSet.getObject(columnIndex + 1, String.class);
89-
case ARRAY:
90-
return convertArrayToList(resultSet.getArray(columnIndex + 1).getArray());
96+
case ARRAY: {
97+
java.sql.Array value = resultSet.getArray(columnIndex + 1);
98+
return value == null ? null : convertArrayToList(value.getArray());
99+
}
91100
default:
92101
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
93102
}
@@ -125,6 +134,20 @@ private List<?> convertArray(List<?> array, ColumnType type) {
125134
return null;
126135
}
127136
switch (type.getType()) {
137+
case TIMESTAMPTZ: {
138+
List<LocalDateTime> result = Lists.newArrayList();
139+
// The scan projection sends epoch microseconds, avoiding zone-less JDBC array values.
140+
for (Object element : array) {
141+
if (element == null) {
142+
result.add(null);
143+
} else {
144+
long micros = ((Number) element).longValue();
145+
result.add(LocalDateTime.ofEpochSecond(Math.floorDiv(micros, 1_000_000),
146+
(int) Math.floorMod(micros, 1_000_000) * 1000, ZoneOffset.UTC));
147+
}
148+
}
149+
return result;
150+
}
128151
case SMALLINT: {
129152
List<Short> result = Lists.newArrayList();
130153
for (Object element : array) {
@@ -212,8 +235,10 @@ private List<?> convertArray(List<?> array, ColumnType type) {
212235
if (element == null) {
213236
resultArray.add(null);
214237
} else {
215-
resultArray.add(
216-
Lists.newArrayList(convertArray((List<?>) element, type.getChildTypes().get(0))));
238+
// Drivers may return nested Java arrays rather than Lists.
239+
List<?> elements = element instanceof List
240+
? (List<?>) element : convertArrayToList(element);
241+
resultArray.add(Lists.newArrayList(convertArray(elements, type.getChildTypes().get(0))));
217242
}
218243
}
219244
return resultArray;

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@
3535
import java.sql.Connection;
3636
import java.sql.ResultSet;
3737
import java.sql.SQLException;
38+
import java.sql.Statement;
39+
import java.sql.Timestamp;
3840
import java.sql.Types;
3941
import java.time.LocalDate;
4042
import java.time.LocalDateTime;
43+
import java.time.ZoneOffset;
4144
import java.time.format.DateTimeFormatterBuilder;
4245
import java.time.temporal.ChronoField;
4346
import java.util.List;
@@ -70,6 +73,14 @@ protected void abortReadConnection(Connection connection, ResultSet resultSet)
7073

7174
@Override
7275
protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, String sql) throws SQLException {
76+
if (usesMySqlTimestampProtocol(config)) {
77+
// MySQL sends TIMESTAMP as session-local fields. Read and bind those fields in UTC
78+
// without the driver's legacy Calendar conversion, which may apply the JVM offset.
79+
// Set this on every checkout because pooled sessions may have been modified by a query.
80+
try (Statement timezoneStatement = conn.createStatement()) {
81+
timezoneStatement.execute("SET SESSION time_zone = '+00:00'");
82+
}
83+
}
7384
if (config.getOp() == TJdbcOperation.READ) {
7485
conn.setAutoCommit(false);
7586
Preconditions.checkArgument(sql != null, "SQL statement cannot be null for READ operation.");
@@ -168,13 +179,33 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
168179
return data;
169180
}
170181
case TIMESTAMPTZ: {
171-
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
182+
if (usesMySqlTimestampProtocol(config)) {
183+
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
184+
}
185+
Timestamp value = resultSet.getTimestamp(columnIndex + 1);
186+
return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), ZoneOffset.UTC);
172187
}
173188
default:
174189
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
175190
}
176191
}
177192

193+
private static boolean usesMySqlTimestampProtocol(JdbcDataSourceConfig config) {
194+
// OceanBase's MySQL mode shares both the TIMESTAMP mapping and session-local wire format.
195+
return config.getTableType() == TOdbcTableType.MYSQL || config.getTableType() == TOdbcTableType.OCEANBASE;
196+
}
197+
198+
@Override
199+
protected void setTimestampTz(int parameterIndex, LocalDateTime value) throws SQLException {
200+
if (usesMySqlTimestampProtocol(config)) {
201+
// Connector/J 5.x can ignore Calendar during timestamp conversion. Bind the UTC
202+
// session's civil fields directly so neither driver nor JVM applies another offset.
203+
preparedStatement.setString(parameterIndex, value.toString().replace('T', ' '));
204+
} else {
205+
super.setTimestampTz(parameterIndex, value);
206+
}
207+
}
208+
178209
@Override
179210
protected ColumnValueConverter getOutputConverter(ColumnType columnType, String replaceString) {
180211
switch (columnType.getType()) {

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int
6767

6868
@Override
6969
protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
70+
if (type.getType() == Type.TIMESTAMPTZ) {
71+
// Both driver paths must preserve the instant instead of passing a local wall clock to JNI.
72+
Timestamp value = resultSet.getTimestamp(columnIndex + 1);
73+
return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), java.time.ZoneOffset.UTC);
74+
}
7075
if (isNewJdbcVersion) {
7176
return newGetColumnValue(columnIndex, type, replaceStringList);
7277
} else {
@@ -103,9 +108,6 @@ private Object newGetColumnValue(int columnIndex, ColumnType type, String[] repl
103108
return resultSet.getObject(columnIndex + 1);
104109
case VARBINARY:
105110
return resultSet.getObject(columnIndex + 1, byte[].class);
106-
case TIMESTAMPTZ:
107-
Timestamp ts = resultSet.getObject(columnIndex + 1, Timestamp.class);
108-
return ts == null ? null : LocalDateTime.ofInstant(ts.toInstant(), java.time.ZoneOffset.UTC);
109111
default:
110112
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
111113
}

0 commit comments

Comments
 (0)