Skip to content

Commit ad56ece

Browse files
committed
fix(external): address UUID review and FE regression failures
1 parent 7f5dc61 commit ad56ece

6 files changed

Lines changed: 186 additions & 14 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/UuidLiteral.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@ protected String toSqlImpl() {
7373
protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType,
7474
TableIf table) {
7575
if (needExternalSql && table instanceof org.apache.doris.catalog.JdbcTable
76-
&& ((org.apache.doris.catalog.JdbcTable) table).getJdbcTableType()
77-
== org.apache.doris.thrift.TOdbcTableType.TRINO) {
78-
// Trino does not implicitly coerce VARCHAR literals for UUID comparisons or IN.
76+
&& (((org.apache.doris.catalog.JdbcTable) table).getJdbcTableType()
77+
== org.apache.doris.thrift.TOdbcTableType.TRINO
78+
|| ((org.apache.doris.catalog.JdbcTable) table).getJdbcTableType()
79+
== org.apache.doris.thrift.TOdbcTableType.PRESTO)) {
80+
// Both dialects expose native UUIDs and reject implicit VARCHAR coercion in predicates.
7981
return "UUID '" + getStringValue() + "'";
8082
}
8183
return needExternalSql ? "'" + getStringValue() + "'" : toSqlImpl();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUuidWriteConverter.java‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.doris.nereids.types.ArrayType;
4040
import org.apache.doris.nereids.types.DataType;
4141
import org.apache.doris.nereids.types.MapType;
42+
import org.apache.doris.nereids.types.StringType;
4243
import org.apache.doris.nereids.types.StructField;
4344
import org.apache.doris.nereids.types.StructType;
4445
import org.apache.doris.nereids.types.UuidType;
@@ -74,6 +75,10 @@ private static Expression convertUuid(Expression input, DataType targetType) {
7475
if (!needsConversion(input.getDataType(), targetType)) {
7576
return input;
7677
}
78+
// Parse serialized containers with UUID leaves kept as text. Nullable complex SerDes
79+
// otherwise consume malformed UUIDs as NULL before the write validator can see them.
80+
input = TypeCoercionUtils.castIfNotSameType(input,
81+
uuidIntermediateType(input.getDataType(), targetType, true));
7782
if (targetType instanceof ArrayType) {
7883
DataType elementTarget = ((ArrayType) targetType).getItemType();
7984
ArrayItemReference item = new ArrayItemReference("item", input);
@@ -102,9 +107,9 @@ private static Expression convertUuid(Expression input, DataType targetType) {
102107
}
103108
return TypeCoercionUtils.processBoundFunction(new CreateMap(children.toArray(new Expression[0])));
104109
}
105-
// Cast the entire map once: separately extracting keys and values would evaluate
106-
// a volatile source twice and could pair keys with values from a different map.
107-
return new Cast(input, targetType, false, true);
110+
// Evaluate the map once, but change only UUID text leaves in the strict cast.
111+
// Narrowing non-UUID siblings here would override ordinary sink cast semantics.
112+
return new Cast(input, uuidIntermediateType(input.getDataType(), targetType, false), false, true);
108113
}
109114
if (targetType instanceof StructType) {
110115
if (!(input instanceof SlotReference) && !(input instanceof StructLiteral)) {
@@ -136,6 +141,9 @@ private static Expression convertUuid(Expression input, DataType targetType) {
136141
}
137142

138143
private static boolean needsConversion(DataType input, DataType semantic) {
144+
if (input.isStringLikeType() && containsUuid(semantic)) {
145+
return true;
146+
}
139147
if (semantic instanceof UuidType) {
140148
return input.isStringLikeType();
141149
}
@@ -162,4 +170,59 @@ private static boolean needsConversion(DataType input, DataType semantic) {
162170
return false;
163171
}
164172

173+
/** Preserve source siblings while parsing containers or validating UUID text leaves. */
174+
private static DataType uuidIntermediateType(DataType input, DataType target, boolean parseContainers) {
175+
if (input.isStringLikeType() && containsUuid(target)) {
176+
if (target.isUuidType()) {
177+
return parseContainers ? input : target;
178+
}
179+
return parseContainers ? uuidTextType(target) : input;
180+
}
181+
if (input instanceof ArrayType && target instanceof ArrayType) {
182+
return ArrayType.of(uuidIntermediateType(((ArrayType) input).getItemType(),
183+
((ArrayType) target).getItemType(), parseContainers));
184+
}
185+
if (input instanceof MapType && target instanceof MapType) {
186+
return MapType.of(uuidIntermediateType(((MapType) input).getKeyType(),
187+
((MapType) target).getKeyType(), parseContainers),
188+
uuidIntermediateType(((MapType) input).getValueType(),
189+
((MapType) target).getValueType(), parseContainers));
190+
}
191+
if (input instanceof StructType && target instanceof StructType) {
192+
List<StructField> sourceFields = ((StructType) input).getFields();
193+
List<StructField> targetFields = ((StructType) target).getFields();
194+
if (sourceFields.size() == targetFields.size()) {
195+
List<StructField> fields = new ArrayList<>();
196+
for (int i = 0; i < sourceFields.size(); i++) {
197+
StructField source = sourceFields.get(i);
198+
fields.add(source.withDataType(uuidIntermediateType(source.getDataType(),
199+
targetFields.get(i).getDataType(), parseContainers)));
200+
}
201+
return new StructType(fields);
202+
}
203+
}
204+
return input;
205+
}
206+
207+
private static DataType uuidTextType(DataType type) {
208+
if (type.isUuidType()) {
209+
return StringType.INSTANCE;
210+
}
211+
if (type instanceof ArrayType) {
212+
return ArrayType.of(uuidTextType(((ArrayType) type).getItemType()));
213+
}
214+
if (type instanceof MapType) {
215+
return MapType.of(uuidTextType(((MapType) type).getKeyType()),
216+
uuidTextType(((MapType) type).getValueType()));
217+
}
218+
if (type instanceof StructType) {
219+
List<StructField> fields = new ArrayList<>();
220+
for (StructField field : ((StructType) type).getFields()) {
221+
fields.add(field.withDataType(uuidTextType(field.getDataType())));
222+
}
223+
return new StructType(fields);
224+
}
225+
return type;
226+
}
227+
165228
}

‎fe/fe-core/src/test/java/org/apache/doris/analysis/UuidLiteralTest.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,26 @@ public void testClickHouseEqualityAndInPredicatesQuoteUuidValues() throws Except
5858
}
5959

6060
@Test
61-
public void testTrinoUsesTypedUuidLiterals() throws Exception {
61+
public void testTrinoAndPrestoUseTypedUuidLiterals() throws Exception {
6262
org.apache.doris.catalog.JdbcTable table = org.mockito.Mockito.mock(org.apache.doris.catalog.JdbcTable.class);
63-
org.mockito.Mockito.when(table.getJdbcTableType()).thenReturn(TOdbcTableType.TRINO);
6463
UuidLiteral value = new UuidLiteral("80000000000000000000000000000000");
65-
Assert.assertEquals("UUID '80000000-0000-0000-0000-000000000000'",
66-
value.toExternalSql(TableType.JDBC_EXTERNAL_TABLE, table));
64+
SlotRef slot = new SlotRef(null, "u");
65+
slot.setType(Type.UUID);
66+
for (TOdbcTableType dialect : Arrays.asList(TOdbcTableType.TRINO, TOdbcTableType.PRESTO)) {
67+
org.mockito.Mockito.when(table.getJdbcTableType()).thenReturn(dialect);
68+
org.mockito.Mockito.when(table.getProperRemoteColumnName(dialect, "u")).thenReturn("u");
69+
Assert.assertEquals("UUID '80000000-0000-0000-0000-000000000000'",
70+
value.toExternalSql(TableType.JDBC_EXTERNAL_TABLE, table));
71+
Assert.assertEquals("(u = UUID '80000000-0000-0000-0000-000000000000')",
72+
JdbcScanNode.conjunctExprToString(dialect,
73+
new BinaryPredicate(BinaryPredicate.Operator.EQ, slot, value), table));
74+
for (boolean notIn : Arrays.asList(false, true)) {
75+
Assert.assertEquals("u" + (notIn ? " NOT" : "")
76+
+ " IN (UUID '80000000-0000-0000-0000-000000000000')",
77+
JdbcScanNode.conjunctExprToString(dialect,
78+
new InPredicate(slot, Arrays.asList(value), notIn), table));
79+
}
80+
}
6781
}
6882

6983
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -705,7 +705,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th
705705
mockWritableTableLease(mockedIcebergUtils, dorisTable, icebergTable);
706706

707707
ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"),
708-
new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L);
708+
new Column("top_uuid", Type.UUID, true), null, 1L);
709709
ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.tz_value"),
710710
new Column("tz_value", ScalarType.createTimeStampTzType(6), true), null, 1L);
711711
}
@@ -1678,15 +1678,15 @@ private Schema mappedComplexSchema() {
16781678
}
16791679

16801680
private StructType mappedPayloadDorisType(Type metricType, int fixedLength, Type mapKeyType) {
1681+
// UUID annotations retain their logical type even when binary mapping is enabled.
16811682
return new StructType(
1682-
new StructField("uuid_value", ScalarType.createVarbinaryType(16)),
1683+
new StructField("uuid_value", Type.UUID),
16831684
new StructField("binary_value",
16841685
ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH)),
16851686
new StructField("fixed_value", ScalarType.createVarbinaryType(fixedLength)),
16861687
new StructField("tz_value", ScalarType.createTimeStampTzType(6)),
16871688
new StructField("metric", metricType),
1688-
new StructField("events", ArrayType.create(
1689-
ScalarType.createVarbinaryType(16), true)),
1689+
new StructField("events", ArrayType.create(Type.UUID, true)),
16901690
new StructField("attrs", new MapType(
16911691
mapKeyType, ScalarType.createTimeStampTzType(6))));
16921692
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUuidMappingTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
2525
import org.apache.doris.nereids.trees.expressions.literal.UuidLiteral;
2626
import org.apache.doris.nereids.types.ArrayType;
27+
import org.apache.doris.nereids.types.BigIntType;
28+
import org.apache.doris.nereids.types.DataType;
29+
import org.apache.doris.nereids.types.IntegerType;
2730
import org.apache.doris.nereids.types.MapType;
2831
import org.apache.doris.nereids.types.StringType;
2932
import org.apache.doris.nereids.types.StructField;
@@ -35,6 +38,7 @@
3538
import org.junit.Assert;
3639
import org.junit.Test;
3740

41+
import java.util.Arrays;
3842
import java.util.Collections;
3943
import java.util.UUID;
4044

@@ -82,6 +86,59 @@ public void uuidWriteCastsRejectInvalidTextAtEveryDepth() {
8286
Assert.assertSame(text, IcebergUuidWriteConverter.convert(text, StringType.INSTANCE));
8387
}
8488

89+
@Test
90+
public void mapUuidValidationPreservesOrdinarySiblingCasts() {
91+
SlotReference source = new SlotReference("m", MapType.of(StringType.INSTANCE, BigIntType.INSTANCE));
92+
MapType target = MapType.of(UuidType.INSTANCE, IntegerType.INSTANCE);
93+
Expression result = IcebergUuidWriteConverter.convert(source, target);
94+
Assert.assertEquals(target, result.getDataType());
95+
Assert.assertTrue(result instanceof Cast);
96+
Assert.assertFalse(((Cast) result).isStrict());
97+
Cast validation = (Cast) result.child(0);
98+
Assert.assertTrue(validation.isStrict());
99+
Assert.assertEquals(MapType.of(UuidType.INSTANCE, BigIntType.INSTANCE), validation.getDataType());
100+
Assert.assertSame(source, validation.child(0));
101+
}
102+
103+
@Test
104+
public void mapValidationPreservesNestedSiblingTypesAndNullability() {
105+
StructType sourceValue = new StructType(Arrays.asList(
106+
new StructField("u", StringType.INSTANCE, true, ""),
107+
new StructField("n", BigIntType.INSTANCE, true, "")));
108+
StructType targetValue = new StructType(Arrays.asList(
109+
new StructField("u", UuidType.INSTANCE, true, ""),
110+
new StructField("n", IntegerType.INSTANCE, true, "")));
111+
SlotReference source = new SlotReference("m", MapType.of(StringType.INSTANCE, ArrayType.of(sourceValue)));
112+
Expression result = IcebergUuidWriteConverter.convert(source,
113+
MapType.of(StringType.INSTANCE, ArrayType.of(targetValue)));
114+
Assert.assertFalse(((Cast) result).isStrict());
115+
Cast validation = (Cast) result.child(0);
116+
Assert.assertTrue(validation.isStrict());
117+
Assert.assertSame(source, validation.child(0));
118+
Assert.assertEquals(MapType.of(StringType.INSTANCE, ArrayType.of(new StructType(Arrays.asList(
119+
new StructField("u", UuidType.INSTANCE, true, ""),
120+
new StructField("n", BigIntType.INSTANCE, true, ""))))), validation.getDataType());
121+
}
122+
123+
@Test
124+
public void serializedContainersValidateUuidTextBeforeOrdinaryCoercion() {
125+
SlotReference source = new SlotReference("text", StringType.INSTANCE);
126+
for (DataType target : Arrays.asList(ArrayType.of(UuidType.INSTANCE),
127+
MapType.of(UuidType.INSTANCE, IntegerType.INSTANCE),
128+
new StructType(Arrays.asList(new StructField("u", UuidType.INSTANCE, true, ""),
129+
new StructField("n", IntegerType.INSTANCE, true, ""))),
130+
ArrayType.of(MapType.of(StringType.INSTANCE, UuidType.INSTANCE)))) {
131+
Expression result = IcebergUuidWriteConverter.convert(source, target);
132+
Assert.assertEquals(target, result.getDataType());
133+
Assert.assertTrue("Missing UUID validation for " + target, result.anyMatch(
134+
e -> e instanceof Cast && ((Cast) e).isStrict()
135+
&& IcebergUuidWriteConverter.containsUuid(((Cast) e).getDataType())));
136+
Assert.assertFalse("Serialized UUID text must survive initial parsing", result.anyMatch(
137+
e -> e instanceof Cast && e.child(0).equals(source)
138+
&& IcebergUuidWriteConverter.containsUuid(((Cast) e).getDataType())));
139+
}
140+
}
141+
85142
@Test
86143
public void incompatibleUuidBoundsDoNotPruneFiles() throws Exception {
87144
Assert.assertNull(IcebergUtils.extractDorisLiteral(Types.UUIDType.get(),

‎regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_uuid_inputs.groovy‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ suite("test_iceberg_write_uuid_inputs", "p0,external,iceberg,external_docker,ext
2929
String database = "uuid_input_roundtrip"
3030
String canonical = "00112233-4455-6677-8899-aabbccddeeff"
3131
String compact = canonical.replace("-", "")
32+
def originalStrictCast = sql("SELECT @@enable_strict_cast")[0][0]
3233
try {
3334
sql "DROP CATALOG IF EXISTS ${catalog}"
3435
sql """CREATE CATALOG ${catalog} PROPERTIES (
@@ -181,6 +182,40 @@ suite("test_iceberg_write_uuid_inputs", "p0,external,iceberg,external_docker,ext
181182
MAP('${otherUuid}', '${otherUuid}')) FROM numbers("number"="32")"""
182183
"qt_${maps}_volatile" """SELECT COUNT(*), SUM(MAP_KEYS(m)[1] = MAP_VALUES(m)[1])
183184
FROM ${maps} WHERE id >= 100"""
185+
sql "SET enable_strict_cast=false"
186+
String mixed = "uuid_mixed_${format}"
187+
sql "DROP TABLE IF EXISTS ${mixed}"
188+
sql """CREATE TABLE ${mixed} (id INT, m MAP<UUID,INT>)
189+
PROPERTIES("write.format.default"="${format}")"""
190+
// UUID validation must not make the unrelated BIGINT-to-INT overflow strict.
191+
sql """INSERT INTO ${mixed} SELECT number,
192+
IF(number = 0, map('${canonical}', CAST(2147483648 AS BIGINT)),
193+
map('${compact}', CAST(7 AS BIGINT))) FROM numbers("number"="2")"""
194+
assertEquals([[0, canonical, null], [1, canonical, 7]], sql("""SELECT id,
195+
CAST(map_keys(m)[1] AS STRING), map_values(m)[1] FROM ${mixed} ORDER BY id"""))
196+
sql """INSERT INTO ${mixed} SELECT 2,
197+
IF(number = 0, '{"${compact}":2147483648}', NULL) FROM numbers("number"="1")"""
198+
assertEquals([[canonical, null]], sql("""SELECT CAST(map_keys(m)[1] AS STRING),
199+
map_values(m)[1] FROM ${mixed} WHERE id = 2"""))
200+
// Serialized containers must retain UUID text until strict leaf validation runs.
201+
for (def input : [
202+
[table: nested, column: "items", value: '["invalid-uuid"]'],
203+
[table: nested, column: "record", value: '{"invalid-uuid", "text"}'],
204+
[table: maps, column: "m", value: '{"invalid-uuid":"' + canonical + '"}'],
205+
[table: maps, column: "m", value: '{"' + canonical + '":"invalid-uuid"}'],
206+
[table: maps, column: "a", value: '[{"k":"invalid-uuid"}]']]) {
207+
test {
208+
sql """INSERT INTO ${input.table}(id, ${input.column}) SELECT 200,
209+
IF(number = 0, '${input.value}', NULL) FROM numbers("number"="1")"""
210+
exception "parse uuid failed"
211+
}
212+
}
213+
sql """INSERT INTO ${nested}(id, items, record) SELECT number + 200,
214+
IF(number = 0, '["${compact}", null]', NULL),
215+
IF(number = 0, '{"${compact}", "text"}', NULL) FROM numbers("number"="2")"""
216+
assertEquals([[200, canonical, null, canonical, "text"], [201, null, null, null, null]],
217+
sql("""SELECT id, CAST(items[1] AS STRING), CAST(items[2] AS STRING),
218+
CAST(record.u AS STRING), record.text FROM ${nested} WHERE id >= 200 ORDER BY id"""))
184219
test {
185220
sql "INSERT INTO ${maps} VALUES (99, map('invalid-uuid', '${canonical}'), NULL, NULL)"
186221
exception "uuid"
@@ -191,6 +226,7 @@ suite("test_iceberg_write_uuid_inputs", "p0,external,iceberg,external_docker,ext
191226
}
192227
}
193228
} finally {
229+
sql "SET enable_strict_cast=${originalStrictCast}"
194230
sql "SWITCH internal"
195231
}
196232
}

0 commit comments

Comments
 (0)