Skip to content

Commit 374e8d3

Browse files
committed
[fix](fe) Preserve external table column name case
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Creating Iceberg or Paimon external tables with mixed-case partition columns could fail because Doris converted top-level external column names to lower case while building external schemas and partition specs. Reading external table schemas and partition metadata also normalized some Paimon and Iceberg column names to lower case, so SHOW CREATE and partition helpers could lose the original external column spelling. This change preserves the original top-level external field names when converting Doris columns to Iceberg/Paimon schemas, resolves partition and primary key names case-insensitively back to the external canonical names, and stops schema/partition parsing paths from lowercasing external column names. ### Release note Fix Iceberg and Paimon external table column name casing for mixed-case partition columns. ### Check List (For Author) - Test: Unit Test - Maven focused FE test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl fe-core -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=CreateIcebergTableTest,PaimonMetadataOpsTest,IcebergUtilsTest#testParseSchemaPreservesNonLowercaseColumnNames,PaimonUtilTest#testParseSchemaPreservesNonLowercaseColumnNames - git diff --check - A broader focused run including two existing Mockito-based IcebergUtilsTest methods compiled successfully but those two methods failed locally because Mockito inline Byte Buddy could not self-attach to the Homebrew JDK 17 VM. - Behavior changed: Yes. Iceberg and Paimon external schemas, partition specs, and partition metadata now preserve external column name casing. - Does this need documentation: No
1 parent 2eb8add commit 374e8d3

14 files changed

Lines changed: 161 additions & 41 deletions

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.iceberg.types.Type;
3030
import org.apache.iceberg.types.Types;
3131

32+
import java.util.Collections;
3233
import java.util.List;
3334

3435

@@ -37,14 +38,21 @@
3738
*/
3839
public class DorisTypeToIcebergType extends DorisTypeVisitor<Type> {
3940
private final StructType root;
41+
private final List<String> rootFieldNames;
4042
private int nextId = 0;
4143

4244
public DorisTypeToIcebergType() {
4345
this.root = null;
46+
this.rootFieldNames = Collections.emptyList();
4447
}
4548

4649
public DorisTypeToIcebergType(StructType root) {
50+
this(root, Collections.emptyList());
51+
}
52+
53+
public DorisTypeToIcebergType(StructType root, List<String> rootFieldNames) {
4754
this.root = root;
55+
this.rootFieldNames = rootFieldNames;
4856
// the root struct's fields use the first ids
4957
this.nextId = root.getFields().size();
5058
}
@@ -65,10 +73,11 @@ public Type struct(StructType struct, List<Type> types) {
6573
Type type = types.get(i);
6674

6775
int id = isRoot ? i : getNextId();
76+
String fieldName = isRoot && !rootFieldNames.isEmpty() ? rootFieldNames.get(i) : field.getName();
6877
if (field.getContainsNull()) {
69-
newFields.add(Types.NestedField.optional(id, field.getName(), type, field.getComment()));
78+
newFields.add(Types.NestedField.optional(id, fieldName, type, field.getComment()));
7079
} else {
71-
newFields.add(Types.NestedField.required(id, field.getName(), type, field.getComment()));
80+
newFields.add(Types.NestedField.required(id, fieldName, type, field.getComment()));
7281
}
7382
}
7483
return Types.StructType.of(newFields);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,8 +368,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx
368368
.map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull()))
369369
.collect(Collectors.toList());
370370
StructType structType = new StructType(new ArrayList<>(collect));
371-
Type visit =
372-
DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType));
371+
List<String> rootFieldNames = columns.stream().map(Column::getName).collect(Collectors.toList());
372+
Type visit = DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType, rootFieldNames));
373373
Schema schema = new Schema(visit.asNestedType().asStructType().fields());
374374
Map<String, String> properties = createTableInfo.getProperties();
375375
properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,6 @@
137137
import java.util.HashMap;
138138
import java.util.LinkedHashSet;
139139
import java.util.List;
140-
import java.util.Locale;
141140
import java.util.Map;
142141
import java.util.Optional;
143142
import java.util.Set;
@@ -585,37 +584,43 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes
585584
PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
586585
for (Expr expr : partitionExprs) {
587586
if (expr instanceof SlotRef) {
588-
builder.identity(((SlotRef) expr).getColumnName());
587+
builder.identity(getIcebergColumnName(schema, ((SlotRef) expr).getColumnName()));
589588
} else if (expr instanceof FunctionCallExpr) {
590589
String exprName = expr.accept(ExprToExprNameVisitor.INSTANCE, null);
591590
List<Expr> params = ((FunctionCallExpr) expr).getParams().exprs();
592591
switch (exprName.toLowerCase()) {
593592
case "bucket":
594593
builder.bucket(
595-
params.get(1).accept(ExprToExprNameVisitor.INSTANCE, null),
594+
getIcebergColumnName(schema,
595+
params.get(1).accept(ExprToExprNameVisitor.INSTANCE, null)),
596596
Integer.parseInt(params.get(0).getStringValue()));
597597
break;
598598
case "year":
599599
case "years":
600-
builder.year(params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null));
600+
builder.year(getIcebergColumnName(schema,
601+
params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null)));
601602
break;
602603
case "month":
603604
case "months":
604-
builder.month(params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null));
605+
builder.month(getIcebergColumnName(schema,
606+
params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null)));
605607
break;
606608
case "date":
607609
case "day":
608610
case "days":
609-
builder.day(params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null));
611+
builder.day(getIcebergColumnName(schema,
612+
params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null)));
610613
break;
611614
case "date_hour":
612615
case "hour":
613616
case "hours":
614-
builder.hour(params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null));
617+
builder.hour(getIcebergColumnName(schema,
618+
params.get(0).accept(ExprToExprNameVisitor.INSTANCE, null)));
615619
break;
616620
case "truncate":
617621
builder.truncate(
618-
params.get(1).accept(ExprToExprNameVisitor.INSTANCE, null),
622+
getIcebergColumnName(schema,
623+
params.get(1).accept(ExprToExprNameVisitor.INSTANCE, null)),
619624
Integer.parseInt(params.get(0).getStringValue()));
620625
break;
621626
default:
@@ -626,6 +631,11 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes
626631
return builder.build();
627632
}
628633

634+
private static String getIcebergColumnName(Schema schema, String columnName) {
635+
Types.NestedField field = schema.caseInsensitiveFindField(columnName);
636+
return field == null ? columnName : field.name();
637+
}
638+
629639
private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Type.PrimitiveType primitive,
630640
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
631641
switch (primitive.typeId()) {
@@ -771,7 +781,7 @@ public static List<String> getIdentityPartitionColumns(Table table) {
771781
}
772782
String columnName = table.schema().findColumnName(partitionField.sourceId());
773783
if (columnName != null) {
774-
partitionColumns.add(columnName.toLowerCase(Locale.ROOT));
784+
partitionColumns.add(columnName);
775785
}
776786
}
777787
}
@@ -803,8 +813,7 @@ public static Map<String, String> getIdentityPartitionInfoMap(PartitionData part
803813
}
804814
Object value = partitionData.get(i);
805815
try {
806-
partitionInfoMap.put(columnName.toLowerCase(Locale.ROOT),
807-
serializePartitionValue(field.type(), value, timeZone));
816+
partitionInfoMap.put(columnName, serializePartitionValue(field.type(), value, timeZone));
808817
} catch (UnsupportedOperationException e) {
809818
LOG.warn("Failed to serialize Iceberg table partition value for field {}: {}", field.name(),
810819
e.getMessage());
@@ -1168,7 +1177,7 @@ public static List<Column> parseSchema(Schema schema, boolean enableMappingVarbi
11681177
List<Types.NestedField> columns = schema.columns();
11691178
List<Column> resSchema = Lists.newArrayListWithCapacity(columns.size());
11701179
for (Types.NestedField field : columns) {
1171-
Column column = new Column(field.name().toLowerCase(Locale.ROOT),
1180+
Column column = new Column(field.name(),
11721181
IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz),
11731182
true, null,
11741183
true, field.doc(), true, -1);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ public Optional<SchemaCacheValue> initSchema(SchemaCacheKey key) {
346346
Set<String> partitionColumnNames = Sets.newHashSet(tableSchema.partitionKeys());
347347
List<Column> partitionColumns = Lists.newArrayList();
348348
for (DataField field : columns) {
349-
Column column = new Column(field.name().toLowerCase(),
349+
Column column = new Column(field.name(),
350350
PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(),
351351
getCatalog().getEnableMappingTimestampTz()),
352352
true,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import java.util.Collections;
5858
import java.util.HashMap;
5959
import java.util.List;
60+
import java.util.Locale;
6061
import java.util.Map;
6162
import java.util.Optional;
6263
import java.util.stream.Collectors;
@@ -218,7 +219,9 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx
218219
col.getComment(), col.isNullable()))
219220
.collect(Collectors.toList());
220221
StructType structType = new StructType(new ArrayList<>(collect));
221-
Schema schema = toPaimonSchema(structType, createTableInfo.getPartitionDesc(), createTableInfo.getProperties());
222+
List<String> rootFieldNames = columns.stream().map(ColumnDefinition::getName).collect(Collectors.toList());
223+
Schema schema = toPaimonSchema(structType, rootFieldNames, createTableInfo.getPartitionDesc(),
224+
createTableInfo.getProperties());
222225
try {
223226
catalog.createTable(new Identifier(createTableInfo.getDbName(), createTableInfo.getTableName()),
224227
schema, createTableInfo.isIfNotExists());
@@ -228,7 +231,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx
228231
return false;
229232
}
230233

231-
private Schema toPaimonSchema(StructType structType, PartitionDesc partitionDesc, Map<String, String> properties) {
234+
private Schema toPaimonSchema(StructType structType, List<String> rootFieldNames, PartitionDesc partitionDesc,
235+
Map<String, String> properties) {
232236
Map<String, String> normalizedProperties = new HashMap<>(properties);
233237
normalizedProperties.remove(PRIMARY_KEY_IDENTIFIER);
234238
normalizedProperties.remove(PROP_COMMENT);
@@ -242,19 +246,31 @@ private Schema toPaimonSchema(StructType structType, PartitionDesc partitionDesc
242246
.map(String::trim)
243247
.collect(Collectors.toList());
244248
List<String> partitionKeys = partitionDesc == null ? new ArrayList<>() : partitionDesc.getPartitionColNames();
249+
primaryKeys = getPaimonColumnNames(rootFieldNames, primaryKeys);
250+
partitionKeys = getPaimonColumnNames(rootFieldNames, partitionKeys);
245251
Schema.Builder schemaBuilder = Schema.newBuilder()
246252
.options(normalizedProperties)
247253
.primaryKey(primaryKeys)
248254
.partitionKeys(partitionKeys)
249255
.comment(properties.getOrDefault(PROP_COMMENT, null));
250-
for (StructField field : structType.getFields()) {
251-
schemaBuilder.column(field.getName(),
256+
List<StructField> fields = structType.getFields();
257+
for (int i = 0; i < fields.size(); i++) {
258+
StructField field = fields.get(i);
259+
schemaBuilder.column(rootFieldNames.get(i),
252260
toPaimontype(field.getType()).copy(field.getContainsNull()),
253261
field.getComment());
254262
}
255263
return schemaBuilder.build();
256264
}
257265

266+
private List<String> getPaimonColumnNames(List<String> paimonColumnNames, List<String> dorisColumnNames) {
267+
Map<String, String> paimonColumnNameMap = paimonColumnNames.stream()
268+
.collect(Collectors.toMap(name -> name.toLowerCase(Locale.ROOT), name -> name));
269+
return dorisColumnNames.stream()
270+
.map(name -> paimonColumnNameMap.getOrDefault(name.toLowerCase(Locale.ROOT), name))
271+
.collect(Collectors.toList());
272+
}
273+
258274
private DataType toPaimontype(Type type) {
259275
return DorisTypeVisitor.visit(type, new DorisToPaimonTypeVisitor());
260276
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ private List<Column> buildFullSchema() {
255255

256256
for (DataField field : fields) {
257257
Column column = new Column(
258-
field.name().toLowerCase(),
258+
field.name(),
259259
PaimonUtil.paimonTypeToDorisType(
260260
field.type(),
261261
getCatalog().getEnableMappingVarbinary(),

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@
9292
import java.util.Base64;
9393
import java.util.HashMap;
9494
import java.util.List;
95-
import java.util.Locale;
9695
import java.util.Map;
9796
import java.util.Optional;
9897
import java.util.regex.Pattern;
@@ -504,7 +503,7 @@ public static List<Column> parseSchema(RowType rowType, List<String> primaryKeys
504503
boolean enableTimestampTzMapping) {
505504
List<Column> resSchema = Lists.newArrayListWithCapacity(rowType.getFields().size());
506505
rowType.getFields().forEach(field -> {
507-
resSchema.add(new Column(field.name().toLowerCase(),
506+
resSchema.add(new Column(field.name(),
508507
PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping),
509508
primaryKeys.contains(field.name()),
510509
null,
@@ -553,7 +552,7 @@ public static Map<String, String> getPartitionInfoMap(Table table, BinaryRow par
553552
try {
554553
String partitionValue = serializePartitionValue(partitionType.getFields().get(i).type(),
555554
partitionValuesArray[i], timeZone);
556-
partitionInfoMap.put(partitionKeys.get(i).toLowerCase(Locale.ROOT), partitionValue);
555+
partitionInfoMap.put(partitionKeys.get(i), partitionValue);
557556
} catch (UnsupportedOperationException e) {
558557
LOG.warn("Failed to serialize table {} partition value for key {}: {}", table.name(),
559558
partitionKeys.get(i), e.getMessage());

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public class PaimonPredicateConverter {
4747

4848
public PaimonPredicateConverter(RowType rowType) {
4949
this.builder = new PredicateBuilder(rowType);
50-
this.fieldNames = rowType.getFields().stream().map(f -> f.name().toLowerCase()).collect(Collectors.toList());
50+
this.fieldNames = rowType.getFields().stream().map(DataField::name).collect(Collectors.toList());
5151
this.paimonFieldTypes = rowType.getFields().stream().map(DataField::type).collect(Collectors.toList());
5252
}
5353

@@ -100,7 +100,7 @@ private Predicate doInPredicate(InPredicate predicate) {
100100
return null;
101101
}
102102
String colName = slotRef.getColumnName();
103-
int idx = fieldNames.indexOf(colName);
103+
int idx = getFieldIndex(colName);
104104
DataType dataType = paimonFieldTypes.get(idx);
105105
List<Object> valueList = new ArrayList<>();
106106
for (int i = 1; i < predicate.getChildren().size(); i++) {
@@ -132,7 +132,7 @@ private Predicate binaryExprDesc(Expr dorisExpr) {
132132
return null;
133133
}
134134
String colName = slotRef.getColumnName();
135-
int idx = fieldNames.indexOf(colName);
135+
int idx = getFieldIndex(colName);
136136
DataType dataType = paimonFieldTypes.get(idx);
137137
Object value = dataType.accept(new PaimonValueConverter(literalExpr));
138138
if (value == null) {
@@ -174,6 +174,15 @@ private Predicate binaryExprDesc(Expr dorisExpr) {
174174
return null;
175175
}
176176

177+
private int getFieldIndex(String colName) {
178+
for (int i = 0; i < fieldNames.size(); i++) {
179+
if (fieldNames.get(i).equalsIgnoreCase(colName)) {
180+
return i;
181+
}
182+
}
183+
return fieldNames.indexOf(colName);
184+
}
185+
177186

178187
public static SlotRef convertDorisExprToSlotRef(Expr expr) {
179188
SlotRef slotRef = null;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
import java.util.Collections;
7272
import java.util.HashMap;
7373
import java.util.List;
74-
import java.util.Locale;
7574
import java.util.Map;
7675
import java.util.Optional;
7776
import java.util.concurrent.ConcurrentHashMap;
@@ -238,9 +237,7 @@ private List<String> getOrderedPathPartitionKeys() {
238237
&& !((PaimonSysExternalTable) externalTable).isDataTable()) {
239238
return Collections.emptyList();
240239
}
241-
return source.getPaimonTable().partitionKeys().stream()
242-
.map(key -> key.toLowerCase(Locale.ROOT))
243-
.collect(Collectors.toList());
240+
return source.getPaimonTable().partitionKeys();
244241
}
245242

246243
private void putHistorySchemaInfo(Long schemaId) {

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,28 @@ public void testPartition() throws UserException {
214214
Assert.assertEquals("b", table.properties().get("a"));
215215
}
216216

217+
@Test
218+
public void testPartitionPreservesNonLowercaseColumnNames() throws UserException {
219+
TableIdentifier tb = TableIdentifier.of(dbName, getTableName());
220+
String sql = "create table " + tb + " ("
221+
+ "data int, "
222+
+ "`PART` int, "
223+
+ "`mIxEd_COL` int"
224+
+ ") engine = iceberg "
225+
+ "partition by (`PART`, bucket(2, `mIxEd_COL`)) ()";
226+
createTable(sql);
227+
Table table = ops.getCatalog().loadTable(tb);
228+
Schema schema = table.schema();
229+
230+
Assert.assertEquals("PART", schema.columns().get(1).name());
231+
Assert.assertEquals("mIxEd_COL", schema.columns().get(2).name());
232+
PartitionSpec spec = PartitionSpec.builderFor(schema)
233+
.identity("PART")
234+
.bucket("mIxEd_COL", 2)
235+
.build();
236+
Assert.assertEquals(spec, table.spec());
237+
}
238+
217239
public void createTable(String sql) throws UserException {
218240
LogicalPlan plan = new NereidsParser().parseSingle(sql);
219241
Assertions.assertTrue(plan instanceof CreateTableCommand);

0 commit comments

Comments
 (0)