Skip to content

Commit ecc242b

Browse files
committed
[fix](fe) Resolve mixed-case external column references
Issue Number: None Related PR: None Problem Summary: Paimon scan projection still matched Doris slot names against Paimon field names with lower-case or exact string comparisons, so mixed-case columns could be dropped from the FE projection or rejected by the JNI scanner as missing. Iceberg table creation also passed sort-order column names to the Iceberg builder without resolving them through the case-insensitive schema lookup, so ORDER BY clauses that used normalized column spelling could fail to bind to mixed-case Iceberg fields. This change resolves Paimon field indexes case-insensitively in both FE and JNI scan paths and resolves Iceberg sort-order names to the canonical schema field name before building the sort order. Fix Paimon scan projection and Iceberg sort-order handling for mixed-case external column names. - Test: Unit Test - Remote FE UT: ssh [email protected], /mnt/disk3/gabriel/Workspace/dev3/doris, MAVEN_ARGS=-o ./run-fe-ut.sh --run org.apache.doris.datasource.paimon.source.PaimonScanNodeTest#testGetPathPartitionKeysReturnsTablePartitionKeys+testSetPaimonParamsUsesOrderedPartitionKeys+testGetFieldIndexMatchesMixedCaseColumns - Maven focused Paimon JNI test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=PaimonJniScannerTest#testGetFieldIndexMatchesMixedCaseColumns - Maven focused Iceberg FE test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl fe-core -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=CreateIcebergTableTest#testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively - git diff --check - Behavior changed: Yes. Paimon scan projection and Iceberg sort-order creation now resolve mixed-case external column names case-insensitively while preserving canonical schema names. - Does this need documentation: No
1 parent 374e8d3 commit ecc242b

6 files changed

Lines changed: 70 additions & 13 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,12 +254,21 @@ private static Object instantiateCustomIOManager(Class<?> implClass, String[] sp
254254

255255
private int[] getProjected() {
256256
return Arrays.stream(fields).mapToInt(fieldName -> {
257-
int index = paimonAllFieldNames.indexOf(fieldName);
257+
int index = getFieldIndex(paimonAllFieldNames, fieldName);
258258
Preconditions.checkArgument(index >= 0, "RequiredField %s not found in schema", fieldName);
259259
return index;
260260
}).toArray();
261261
}
262262

263+
static int getFieldIndex(List<String> fieldNames, String fieldName) {
264+
for (int i = 0; i < fieldNames.size(); i++) {
265+
if (fieldNames.get(i).equalsIgnoreCase(fieldName)) {
266+
return i;
267+
}
268+
}
269+
return -1;
270+
}
271+
263272
private List<Predicate> getPredicates() {
264273
List<Predicate> predicates = PaimonUtils.deserialize(paimonPredicate);
265274
if (LOG.isDebugEnabled()) {
@@ -281,7 +290,7 @@ private void resetDatetimeV2Precision() {
281290
if (types[i].isDateTimeV2()) {
282291
// paimon support precision > 6, but it has been reset as 6 in FE
283292
// try to get the right precision for datetimev2
284-
int index = paimonAllFieldNames.indexOf(fields[i]);
293+
int index = getFieldIndex(paimonAllFieldNames, fields[i]);
285294
if (index != -1) {
286295
DataType dataType = table.rowType().getTypeAt(index);
287296
if (dataType instanceof TimestampType) {

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.io.File;
3232
import java.lang.reflect.Field;
3333
import java.lang.reflect.Proxy;
34+
import java.util.Arrays;
3435
import java.util.Collections;
3536
import java.util.HashMap;
3637
import java.util.Map;
@@ -225,4 +226,14 @@ public BufferFileReader createBufferFileReader(FileIOChannel.ID channel) {
225226
public void close() {
226227
}
227228
}
229+
230+
@Test
231+
public void testGetFieldIndexMatchesMixedCaseColumns() {
232+
Assert.assertEquals(1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"),
233+
"mixed_col"));
234+
Assert.assertEquals(2, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"),
235+
"part"));
236+
Assert.assertEquals(-1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"),
237+
"missing_col"));
238+
}
228239
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1348,7 +1348,7 @@ private org.apache.iceberg.SortOrder buildSortOrder(List<SortFieldInfo> sortFiel
13481348

13491349
org.apache.iceberg.SortOrder.Builder builder = org.apache.iceberg.SortOrder.builderFor(schema);
13501350
for (SortFieldInfo sortField : sortFields) {
1351-
String columnName = sortField.getColumnName();
1351+
String columnName = getIcebergColumnName(schema, sortField.getColumnName());
13521352
if (sortField.isAscending()) {
13531353
if (sortField.isNullFirst()) {
13541354
builder.asc(columnName, org.apache.iceberg.NullOrder.NULLS_FIRST);
@@ -1365,4 +1365,9 @@ private org.apache.iceberg.SortOrder buildSortOrder(List<SortFieldInfo> sortFiel
13651365
}
13661366
return builder.build();
13671367
}
1368+
1369+
private static String getIcebergColumnName(Schema schema, String columnName) {
1370+
NestedField field = schema.caseInsensitiveFindField(columnName);
1371+
return field == null ? columnName : field.name();
1372+
}
13681373
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -604,13 +604,9 @@ public List<org.apache.paimon.table.source.Split> getPaimonSplitFromAPI() throws
604604
long startTime = System.currentTimeMillis();
605605
try {
606606
Table paimonTable = getProcessedTable();
607+
List<String> fieldNames = paimonTable.rowType().getFieldNames();
607608
int[] projected = desc.getSlots().stream().mapToInt(
608-
slot -> paimonTable.rowType()
609-
.getFieldNames()
610-
.stream()
611-
.map(String::toLowerCase)
612-
.collect(Collectors.toList())
613-
.indexOf(slot.getColumn().getName()))
609+
slot -> getFieldIndex(fieldNames, slot.getColumn().getName()))
614610
.filter(i -> i >= 0)
615611
.toArray();
616612
ReadBuilder readBuilder = paimonTable.newReadBuilder();
@@ -634,6 +630,16 @@ public List<org.apache.paimon.table.source.Split> getPaimonSplitFromAPI() throws
634630
}
635631
}
636632

633+
@VisibleForTesting
634+
static int getFieldIndex(List<String> fieldNames, String columnName) {
635+
for (int i = 0; i < fieldNames.size(); i++) {
636+
if (fieldNames.get(i).equalsIgnoreCase(columnName)) {
637+
return i;
638+
}
639+
}
640+
return -1;
641+
}
642+
637643
private String getFileFormat(String path) {
638644
return FileFormatUtils.getFileFormatBySuffix(path).orElse(source.getFileFormatFromTableProperties());
639645
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,23 @@ public void testPartitionPreservesNonLowercaseColumnNames() throws UserException
236236
Assert.assertEquals(spec, table.spec());
237237
}
238238

239+
@Test
240+
public void testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively() throws UserException {
241+
TableIdentifier tb = TableIdentifier.of(dbName, getTableName());
242+
String sql = "create table " + tb + " ("
243+
+ "data int, "
244+
+ "`mIxEd_COL` int"
245+
+ ") engine = iceberg "
246+
+ "order by (`mixed_col` asc)";
247+
createTable(sql);
248+
Table table = ops.getCatalog().loadTable(tb);
249+
Schema schema = table.schema();
250+
251+
Assert.assertEquals("mIxEd_COL", schema.columns().get(1).name());
252+
Assert.assertEquals(1, table.sortOrder().fields().size());
253+
Assert.assertEquals(schema.findField("mIxEd_COL").fieldId(), table.sortOrder().fields().get(0).sourceId());
254+
}
255+
239256
public void createTable(String sql) throws UserException {
240257
LogicalPlan plan = new NereidsParser().parseSingle(sql);
241258
Assertions.assertTrue(plan instanceof CreateTableCommand);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -578,7 +578,7 @@ public void testGetPathPartitionKeysReturnsTablePartitionKeys() throws Exception
578578
Mockito.when(sysTable.isDataTable()).thenReturn(true);
579579
node.setSource(source);
580580

581-
Assert.assertEquals(Arrays.asList("dt", "region"), node.getPathPartitionKeys());
581+
Assert.assertEquals(Arrays.asList("Dt", "Region"), node.getPathPartitionKeys());
582582
}
583583

584584
@Test
@@ -611,19 +611,28 @@ public void testSetPaimonParamsUsesOrderedPartitionKeys() throws Exception {
611611
rangeDesc.setColumnsFromPath(Collections.singletonList("old"));
612612
rangeDesc.setColumnsFromPathIsNull(Collections.singletonList(false));
613613
Map<String, String> partitionValues = new HashMap<>();
614-
partitionValues.put("dt", "2025-01-01");
615-
partitionValues.put("pt", "p1");
614+
partitionValues.put("Dt", "2025-01-01");
615+
partitionValues.put("Pt", "p1");
616616
PaimonSplit split = new PaimonSplit(createDataSplit("ordered.parquet"));
617617
split.setPaimonPartitionValues(partitionValues);
618618

619619
invokePrivateMethod(node, "setPaimonParams",
620620
new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split);
621621

622-
Assert.assertEquals(Arrays.asList("pt", "dt"), rangeDesc.getColumnsFromPathKeys());
622+
Assert.assertEquals(Arrays.asList("Pt", "Dt"), rangeDesc.getColumnsFromPathKeys());
623623
Assert.assertEquals(Arrays.asList("p1", "2025-01-01"), rangeDesc.getColumnsFromPath());
624624
Assert.assertEquals(Arrays.asList(false, false), rangeDesc.getColumnsFromPathIsNull());
625625
}
626626

627+
@Test
628+
public void testGetFieldIndexMatchesMixedCaseColumns() {
629+
List<String> fieldNames = Arrays.asList("data", "mIxEd_COL", "PART");
630+
631+
Assert.assertEquals(1, PaimonScanNode.getFieldIndex(fieldNames, "mixed_col"));
632+
Assert.assertEquals(2, PaimonScanNode.getFieldIndex(fieldNames, "part"));
633+
Assert.assertEquals(-1, PaimonScanNode.getFieldIndex(fieldNames, "missing_col"));
634+
}
635+
627636
private void mockJniReader(PaimonScanNode spyNode) {
628637
Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class));
629638
}

0 commit comments

Comments
 (0)