Skip to content

Commit 294d1f7

Browse files
Gabriel39Gabriel39
committed
[fix](external) Backport missing type mapping behavior to branch-4.2
1 parent e2a836e commit 294d1f7

23 files changed

Lines changed: 752 additions & 112 deletions

‎fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import org.apache.doris.common.jni.vec.ColumnValue;
2121

2222
import org.apache.iceberg.StructLike;
23+
import org.apache.iceberg.types.Type;
24+
import org.apache.iceberg.types.Types;
2325

2426
import java.math.BigDecimal;
2527
import java.math.BigInteger;
@@ -38,14 +40,20 @@ public class IcebergSysTableColumnValue implements ColumnValue {
3840

3941
private final Object fieldData;
4042
private final String timezone;
43+
private final Type icebergType;
4144

4245
public IcebergSysTableColumnValue(Object fieldData) {
4346
this(fieldData, DEFAULT_TIME_ZONE);
4447
}
4548

4649
public IcebergSysTableColumnValue(Object fieldData, String timezone) {
50+
this(fieldData, timezone, null);
51+
}
52+
53+
public IcebergSysTableColumnValue(Object fieldData, String timezone, Type icebergType) {
4754
this.fieldData = fieldData;
4855
this.timezone = timezone;
56+
this.icebergType = icebergType;
4957
}
5058

5159
@Override
@@ -136,14 +144,22 @@ public LocalDate getDate() {
136144

137145
@Override
138146
public LocalDateTime getDateTime() {
139-
Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000));
140-
return LocalDateTime.ofInstant(instant, ZoneId.of(timezone));
147+
// NTZ is a wall-clock value; only timestamptz mapped to DATETIME uses the session zone.
148+
ZoneId zone = icebergType instanceof Types.TimestampType
149+
&& !((Types.TimestampType) icebergType).shouldAdjustToUTC()
150+
? ZoneOffset.UTC : ZoneId.of(timezone);
151+
return LocalDateTime.ofInstant(timestampInstant(), zone);
141152
}
142153

143154
@Override
144155
public LocalDateTime getTimeStampTz() {
145-
Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000));
146-
return LocalDateTime.ofInstant(instant, ZoneId.of("UTC"));
156+
return LocalDateTime.ofInstant(timestampInstant(), ZoneOffset.UTC);
157+
}
158+
159+
private Instant timestampInstant() {
160+
long micros = (long) fieldData;
161+
return Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000L),
162+
Math.floorMod(micros, 1_000_000L) * 1_000L);
147163
}
148164

149165
@Override
@@ -162,16 +178,19 @@ public byte[] getBytes() {
162178
public void unpackArray(List<ColumnValue> values) {
163179
List<?> items = (List<?>) fieldData;
164180
for (Object item : items) {
165-
values.add(new IcebergSysTableColumnValue(item, timezone));
181+
values.add(new IcebergSysTableColumnValue(item, timezone,
182+
icebergType == null ? null : icebergType.asListType().elementType()));
166183
}
167184
}
168185

169186
@Override
170187
public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
171188
Map<?, ?> data = (Map<?, ?>) fieldData;
172189
data.forEach((key, value) -> {
173-
keys.add(new IcebergSysTableColumnValue(key, timezone));
174-
values.add(new IcebergSysTableColumnValue(value, timezone));
190+
keys.add(new IcebergSysTableColumnValue(key, timezone,
191+
icebergType == null ? null : icebergType.asMapType().keyType()));
192+
values.add(new IcebergSysTableColumnValue(value, timezone,
193+
icebergType == null ? null : icebergType.asMapType().valueType()));
175194
});
176195
}
177196

@@ -180,7 +199,8 @@ public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue> value
180199
StructLike record = (StructLike) fieldData;
181200
for (Integer fieldIndex : structFieldIndex) {
182201
Object rawValue = record.get(fieldIndex, Object.class);
183-
values.add(new IcebergSysTableColumnValue(rawValue, timezone));
202+
values.add(new IcebergSysTableColumnValue(rawValue, timezone,
203+
icebergType == null ? null : icebergType.asStructType().fields().get(fieldIndex).type()));
184204
}
185205
}
186206
}

‎fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ protected int getNext() throws IOException {
109109
for (int i = 0; i < fields.size(); i++) {
110110
SelectedField field = fields.get(i);
111111
Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass());
112-
ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone);
112+
ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone, field.field.type());
113113
appendData(i, columnValue);
114114
}
115115
rows++;

‎fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.iceberg;
19+
20+
import org.apache.doris.common.jni.vec.ColumnValue;
21+
22+
import org.apache.iceberg.PartitionData;
23+
import org.apache.iceberg.types.Types;
24+
import org.junit.jupiter.api.Assertions;
25+
import org.junit.jupiter.api.Test;
26+
27+
import java.time.LocalDateTime;
28+
import java.util.ArrayList;
29+
import java.util.Arrays;
30+
import java.util.Collections;
31+
import java.util.List;
32+
33+
public class IcebergSysTableColumnValueTest {
34+
@Test
35+
public void preservesMicrosAndTimestampSemantics() {
36+
long[] micros = {-1000001L, -1L, 1L, 999L, 1000L, 1234567L};
37+
String[] expected = {"1969-12-31T23:59:58.999999", "1969-12-31T23:59:59.999999",
38+
"1970-01-01T00:00:00.000001", "1970-01-01T00:00:00.000999",
39+
"1970-01-01T00:00:00.001", "1970-01-01T00:00:01.234567"};
40+
for (int i = 0; i < micros.length; i++) {
41+
LocalDateTime utc = LocalDateTime.parse(expected[i]);
42+
for (String zone : Arrays.asList("UTC", "Asia/Shanghai")) {
43+
ColumnValue ntz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withoutZone());
44+
ColumnValue tz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withZone());
45+
Assertions.assertEquals(utc, ntz.getDateTime());
46+
Assertions.assertEquals(zone.equals("UTC") ? utc : utc.plusHours(8), tz.getDateTime());
47+
Assertions.assertEquals(utc, tz.getTimeStampTz());
48+
}
49+
}
50+
}
51+
52+
@Test
53+
public void propagatesTypesThroughNestedContainers() {
54+
Types.StructType struct = Types.StructType.of(
55+
Types.NestedField.optional(1, "ts", Types.TimestampType.withoutZone()),
56+
Types.NestedField.optional(2, "tz", Types.TimestampType.withZone()));
57+
PartitionData record = new PartitionData(struct);
58+
record.set(0, -1L);
59+
record.set(1, -1L);
60+
List<ColumnValue> fields = new ArrayList<>();
61+
new IcebergSysTableColumnValue(record, "Asia/Shanghai", struct)
62+
.unpackStruct(Arrays.asList(1, 0), fields);
63+
LocalDateTime utc = LocalDateTime.parse("1969-12-31T23:59:59.999999");
64+
Assertions.assertEquals(utc.plusHours(8), fields.get(0).getDateTime());
65+
Assertions.assertEquals(utc, fields.get(1).getDateTime());
66+
67+
List<ColumnValue> elements = new ArrayList<>();
68+
new IcebergSysTableColumnValue(Collections.singletonList(-1L), "Asia/Shanghai",
69+
Types.ListType.ofOptional(3, Types.TimestampType.withoutZone())).unpackArray(elements);
70+
Assertions.assertEquals(utc, elements.get(0).getDateTime());
71+
72+
List<ColumnValue> keys = new ArrayList<>();
73+
List<ColumnValue> values = new ArrayList<>();
74+
new IcebergSysTableColumnValue(Collections.singletonMap(-1L, -1L), "Asia/Shanghai",
75+
Types.MapType.ofOptional(4, 5, Types.TimestampType.withoutZone(), Types.TimestampType.withZone()))
76+
.unpackMap(keys, values);
77+
Assertions.assertEquals(utc, keys.get(0).getDateTime());
78+
Assertions.assertEquals(utc.plusHours(8), values.get(0).getDateTime());
79+
}
80+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,10 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
208208
precision = scale;
209209
}
210210
return createDecimalOrStringType(precision, scale);
211+
// Oracle binary floating-point types retain IEEE precision instead of becoming unsupported.
212+
case "BINARY_FLOAT":
213+
return Type.FLOAT;
214+
case "BINARY_DOUBLE":
211215
case "FLOAT":
212216
return Type.DOUBLE;
213217
case "DATE":
@@ -227,8 +231,6 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
227231
return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize());
228232
case "NCLOB":
229233
case "BFILE":
230-
case "BINARY_FLOAT":
231-
case "BINARY_DOUBLE":
232234
default:
233235
return Type.UNSUPPORTED;
234236
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java‎

Lines changed: 120 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,19 @@
2121
import org.apache.doris.catalog.Type;
2222
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
2323

24+
import java.sql.Types;
25+
import java.util.Locale;
26+
import java.util.regex.Matcher;
27+
import java.util.regex.Pattern;
28+
2429
public class JdbcSQLServerClient extends JdbcClient {
2530

31+
// TYPE_NAME of an IDENTITY column decorates the base type: "int identity", "decimal() identity",
32+
// "numeric(18, 0) identity", "decimal(18,0) IDENTITY(1,1)". IDENTITY is only allowed on these base types.
33+
private static final Pattern IDENTITY_TYPE_NAME = Pattern.compile(
34+
"^(tinyint|smallint|int|bigint|decimal|numeric)\\s*(\\([^)]*\\))?\\s+identity(\\s*\\([^)]*\\))?$",
35+
Pattern.CASE_INSENSITIVE);
36+
2637
protected JdbcSQLServerClient(JdbcClientConfig jdbcClientConfig) {
2738
super(jdbcClientConfig);
2839
}
@@ -34,18 +45,48 @@ protected String getSchemaPatternForDatabaseNameList() {
3445
return "%";
3546
}
3647

48+
/**
49+
* The base type of an IDENTITY column's TYPE_NAME, or the name unchanged.
50+
* <p>
51+
* The decoration is trusted only when {@code DATA_TYPE} is the code of the named base type: an alias type
52+
* may legally be named like that ({@code CREATE TYPE dbo.[int identity] FROM varchar(10)}, or
53+
* {@code dbo.[int alias]}), and it then has to be resolved by its code, not by the words of its name.
54+
*/
55+
static String identityBaseType(String typeName, int dataType) {
56+
Matcher matcher = IDENTITY_TYPE_NAME.matcher(typeName);
57+
if (!matcher.matches()) {
58+
return typeName;
59+
}
60+
String baseType = matcher.group(1).toLowerCase(Locale.ROOT);
61+
boolean codeMatches;
62+
switch (baseType) {
63+
case "tinyint":
64+
codeMatches = dataType == Types.TINYINT;
65+
break;
66+
case "smallint":
67+
codeMatches = dataType == Types.SMALLINT;
68+
break;
69+
case "int":
70+
codeMatches = dataType == Types.INTEGER;
71+
break;
72+
case "bigint":
73+
codeMatches = dataType == Types.BIGINT;
74+
break;
75+
default:
76+
codeMatches = dataType == Types.DECIMAL || dataType == Types.NUMERIC;
77+
break;
78+
}
79+
return codeMatches ? baseType : typeName;
80+
}
81+
3782
@Override
3883
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
3984
String originSqlserverType = fieldSchema.getDataTypeName().orElse("unknown");
40-
// For sqlserver IDENTITY type, such as 'INT IDENTITY'
41-
// originSqlserverType is "int identity", so we only get "int".
42-
// For types with parameters like 'decimal(18,0) IDENTITY(1,1)', we need to extract the base type
43-
String sqlserverType = originSqlserverType.split(" ")[0];
44-
45-
// Handle types with parentheses like decimal(18,0), varchar(50), etc.
46-
if (sqlserverType.contains("(")) {
47-
sqlserverType = sqlserverType.substring(0, sqlserverType.indexOf("("));
48-
}
85+
// An IDENTITY column is reported as "int identity" or "decimal(18,0) identity": only the base type
86+
// is matched below. Any other name is matched as it is: system type names are single words, and a
87+
// user-defined alias type may be named with spaces or parentheses ("int alias") and must not be
88+
// mistaken for the system type its name starts with.
89+
String sqlserverType = identityBaseType(originSqlserverType, fieldSchema.getDataType());
4990

5091
switch (sqlserverType) {
5192
case "bit":
@@ -100,6 +141,76 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
100141
case "binary":
101142
case "varbinary":
102143
return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize());
144+
case "xml":
145+
case "sql_variant":
146+
case "geometry":
147+
case "geography":
148+
case "hierarchyid":
149+
case "json":
150+
case "vector":
151+
// SQL Server system types that Doris does not support. They are listed explicitly
152+
// so that they never reach the JDBC type code fallback below.
153+
return Type.UNSUPPORTED;
154+
default:
155+
return jdbcTypeCodeToDoris(fieldSchema);
156+
}
157+
}
158+
159+
/**
160+
* Fallback for type names that are not SQL Server system types.
161+
* <p>
162+
* User-defined alias types ({@code CREATE TYPE dbo.my_type FROM varchar(50)}) are reported by
163+
* {@code DatabaseMetaData.getColumns()} with {@code TYPE_NAME} set to the alias name, so they can not be
164+
* matched by name. {@code DATA_TYPE}, {@code COLUMN_SIZE} and {@code DECIMAL_DIGITS} still describe the
165+
* base type, so the standard {@link Types} code is used to resolve the Doris type. The mapping mirrors
166+
* the name based one above.
167+
* <p>
168+
* Binary codes are deliberately not mapped: mssql-jdbc also reports CLR user-defined types
169+
* (geometry, geography, hierarchyid, ...) as {@link Types#VARBINARY}, so they can not be told apart from
170+
* an alias over a binary type by the type code alone. Vendor specific codes stay unsupported as well.
171+
*/
172+
private Type jdbcTypeCodeToDoris(JdbcFieldSchema fieldSchema) {
173+
switch (fieldSchema.getDataType()) {
174+
case Types.BIT:
175+
case Types.BOOLEAN:
176+
return Type.BOOLEAN;
177+
// SQL Server tinyint is unsigned (0 to 255), so it needs SMALLINT
178+
case Types.TINYINT:
179+
case Types.SMALLINT:
180+
return Type.SMALLINT;
181+
case Types.INTEGER:
182+
return Type.INT;
183+
case Types.BIGINT:
184+
return Type.BIGINT;
185+
case Types.REAL:
186+
return Type.FLOAT;
187+
case Types.FLOAT:
188+
case Types.DOUBLE:
189+
return Type.DOUBLE;
190+
case Types.DECIMAL:
191+
case Types.NUMERIC: {
192+
// money and smallmoney are reported as DECIMAL(19,4) and DECIMAL(10,4)
193+
int precision = fieldSchema.getColumnSize().orElse(0);
194+
int scale = fieldSchema.getDecimalDigits().orElse(0);
195+
return createDecimalOrStringType(precision, scale);
196+
}
197+
case Types.DATE:
198+
return ScalarType.createDateV2Type();
199+
case Types.TIMESTAMP: {
200+
int scale = fieldSchema.getDecimalDigits().orElse(0);
201+
if (scale > 6) {
202+
scale = 6;
203+
}
204+
return ScalarType.createDatetimeV2Type(scale);
205+
}
206+
case Types.CHAR:
207+
case Types.VARCHAR:
208+
case Types.LONGVARCHAR:
209+
case Types.NCHAR:
210+
case Types.NVARCHAR:
211+
case Types.LONGNVARCHAR:
212+
case Types.TIME:
213+
return ScalarType.createStringType();
103214
default:
104215
return Type.UNSUPPORTED;
105216
}

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -253,12 +253,9 @@ && getTFileType() != TFileType.FILE_S3) {
253253
throw new AnalysisException("Lance format is supported only by local() and s3() TVFs");
254254
}
255255

256-
// The catalog property was removed, but this TVF-only option must remain explicit because
257-
// changing an ad-hoc TVF result schema also breaks CTAS type inference.
258-
String enableMappingVarbinaryStr = getOrDefaultAndRemove(copiedProps,
259-
FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "false");
260-
fileFormatProperties.enableMappingVarbinary = Boolean.parseBoolean(enableMappingVarbinaryStr);
261-
256+
// Binary file fields must preserve bytes even when a legacy opt-out is supplied.
257+
copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY);
258+
fileFormatProperties.enableMappingVarbinary = true;
262259
// Consume the legacy option, but let file logical types determine timezone semantics.
263260
copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_TIMESTAMP_TZ);
264261
fileFormatProperties.enableMappingTimestampTz = true;

0 commit comments

Comments
 (0)