Skip to content

Commit 9f63be8

Browse files
authored
branch-4.2: [fix](iceberg) Align scan schema carrier names by field id (#69382) (#69709)
pick #69382 4.2 still uses the legacy IcebergScanNode/IcebergUtils path. The fix therefore resolves identity partition keys and values from the query-pinned scan schema by Iceberg field id, so path_partition_keys, schema info, and range partition metadata stay aligned. The master-only fe-connector dictionary/lowercasing changes are not applicable to this path.
1 parent ad87932 commit 9f63be8

6 files changed

Lines changed: 348 additions & 9 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,20 +1048,35 @@ public static List<String> getIdentityPartitionColumns(Table table) {
10481048
return getIdentityPartitionColumns(table, false, false);
10491049
}
10501050

1051+
public static List<String> getIdentityPartitionColumns(Table table, Schema scanSchema) {
1052+
return getIdentityPartitionColumns(table, scanSchema, false, false);
1053+
}
1054+
10511055
public static List<String> getIdentityPartitionColumns(Table table,
10521056
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
1057+
return getIdentityPartitionColumns(
1058+
table, table.schema(), enableMappingVarbinary, enableMappingTimestampTz);
1059+
}
1060+
1061+
/**
1062+
* Resolve identity partition names against the schema that owns this scan, rather than the
1063+
* table's latest schema. A pinned snapshot can expose a renamed field whose case-sensitive
1064+
* name must match the scan slots in {@code scanSchema}.
1065+
*/
1066+
public static List<String> getIdentityPartitionColumns(Table table, Schema scanSchema,
1067+
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
10531068
LinkedHashSet<String> partitionColumns = new LinkedHashSet<>();
10541069
for (PartitionSpec spec : table.specs().values()) {
10551070
for (PartitionField partitionField : spec.fields()) {
10561071
if (!partitionField.transform().isIdentity()) {
10571072
continue;
10581073
}
1059-
NestedField sourceField = table.schema().findField(partitionField.sourceId());
1074+
NestedField sourceField = scanSchema.findField(partitionField.sourceId());
10601075
if (sourceField == null || !isSupportedPartitionValueType(
10611076
sourceField.type(), enableMappingVarbinary, enableMappingTimestampTz)) {
10621077
continue;
10631078
}
1064-
String columnName = table.schema().findColumnName(partitionField.sourceId());
1079+
String columnName = topLevelSourceColumnName(scanSchema, partitionField.sourceId());
10651080
if (columnName != null) {
10661081
partitionColumns.add(columnName);
10671082
}
@@ -1079,11 +1094,21 @@ public static List<String> getCommonIdentityPartitionColumns(Table table) {
10791094
return getCommonIdentityPartitionColumns(table, false, false);
10801095
}
10811096

1097+
public static List<String> getCommonIdentityPartitionColumns(Table table, Schema scanSchema) {
1098+
return getCommonIdentityPartitionColumns(table, scanSchema, false, false);
1099+
}
1100+
10821101
public static List<String> getCommonIdentityPartitionColumns(Table table,
10831102
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
1103+
return getCommonIdentityPartitionColumns(
1104+
table, table.schema(), enableMappingVarbinary, enableMappingTimestampTz);
1105+
}
1106+
1107+
public static List<String> getCommonIdentityPartitionColumns(Table table, Schema scanSchema,
1108+
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
10841109
LinkedHashSet<Integer> commonSourceIds = new LinkedHashSet<>();
10851110
for (PartitionField field : table.spec().fields()) {
1086-
NestedField sourceField = table.schema().findField(field.sourceId());
1111+
NestedField sourceField = scanSchema.findField(field.sourceId());
10871112
if (field.transform().isIdentity() && sourceField != null
10881113
&& isSupportedPartitionValueType(
10891114
sourceField.type(), enableMappingVarbinary, enableMappingTimestampTz)) {
@@ -1098,19 +1123,35 @@ && isSupportedPartitionValueType(
10981123
commonSourceIds.retainAll(specIdentitySourceIds);
10991124
}
11001125
return commonSourceIds.stream()
1101-
.map(table.schema()::findColumnName)
1126+
.map(sourceId -> topLevelSourceColumnName(scanSchema, sourceId))
11021127
.filter(columnName -> columnName != null)
11031128
.collect(Collectors.toList());
11041129
}
11051130

11061131
public static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionData,
11071132
PartitionSpec partitionSpec, Table table, String timeZone) {
1108-
return getIdentityPartitionInfoMap(partitionData, partitionSpec, table, timeZone, false, false);
1133+
return getIdentityPartitionInfoMap(
1134+
partitionData, partitionSpec, table, timeZone, false, false);
1135+
}
1136+
1137+
public static Map<String, String> getIdentityPartitionInfoMap(
1138+
PartitionData partitionData, PartitionSpec partitionSpec, Schema scanSchema,
1139+
String timeZone) {
1140+
return getIdentityPartitionInfoMap(
1141+
partitionData, partitionSpec, scanSchema, timeZone, false, false);
11091142
}
11101143

11111144
public static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionData,
11121145
PartitionSpec partitionSpec, Table table, String timeZone,
11131146
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
1147+
return getIdentityPartitionInfoMap(
1148+
partitionData, partitionSpec, table.schema(), timeZone,
1149+
enableMappingVarbinary, enableMappingTimestampTz);
1150+
}
1151+
1152+
public static Map<String, String> getIdentityPartitionInfoMap(PartitionData partitionData,
1153+
PartitionSpec partitionSpec, Schema scanSchema, String timeZone,
1154+
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
11141155
Map<String, String> partitionInfoMap = Maps.newLinkedHashMap();
11151156
List<NestedField> fields = partitionData.getPartitionType().asNestedType().fields();
11161157
List<PartitionField> partitionFields = partitionSpec.fields();
@@ -1128,7 +1169,7 @@ public static Map<String, String> getIdentityPartitionInfoMap(PartitionData part
11281169
continue;
11291170
}
11301171

1131-
String columnName = table.schema().findColumnName(partitionField.sourceId());
1172+
String columnName = topLevelSourceColumnName(scanSchema, partitionField.sourceId());
11321173
if (columnName == null) {
11331174
continue;
11341175
}
@@ -1143,6 +1184,11 @@ public static Map<String, String> getIdentityPartitionInfoMap(PartitionData part
11431184
return partitionInfoMap;
11441185
}
11451186

1187+
private static String topLevelSourceColumnName(Schema schema, int sourceId) {
1188+
NestedField field = schema.asStruct().field(sourceId);
1189+
return field == null ? null : field.name();
1190+
}
1191+
11461192
private static boolean isSupportedPartitionValueType(org.apache.iceberg.types.Type type,
11471193
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) {
11481194
TypeID typeId = type.typeId();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ protected void doInitialize() throws UserException {
324324
icebergTable = source.getIcebergTable();
325325
icebergTable = useFrozenTableGeneration(icebergTable);
326326
partitionMapInfos = new HashMap<>();
327+
getQuerySchema();
327328
initializePartitionMetadata();
328329
isPartitionedTable = icebergTable.spec().isPartitioned();
329330
hasPartitionedSpec = IcebergUtils.hasPartitionedSpec(icebergTable);
@@ -517,6 +518,7 @@ private void initializePartitionMetadata() {
517518
orderedPartitionMetadataKeys = Collections.emptyList();
518519
return;
519520
}
521+
Schema scanSchema = getPartitionMetadataScanSchema();
520522
enableMappingVarbinaryForPartitionMetadata = getEnableMappingVarbinary();
521523
enableMappingTimestampTzForPartitionMetadata = getEnableMappingTimestampTz();
522524
if (sessionVariable.enableFileScannerV2) {
@@ -527,20 +529,31 @@ private void initializePartitionMetadata() {
527529
// for the column (a spec without it) are unaffected: the v2 column mapper falls back to the
528530
// physical field when the split has no partition value for a partition key.
529531
orderedPathPartitionKeys = Collections.unmodifiableList(
530-
IcebergUtils.getIdentityPartitionColumns(icebergTable,
532+
IcebergUtils.getIdentityPartitionColumns(icebergTable, scanSchema,
531533
enableMappingVarbinaryForPartitionMetadata,
532534
enableMappingTimestampTzForPartitionMetadata));
533535
} else {
534536
// v1 has no such per-file fallback: a partition key it cannot fill from the split is not read
535537
// from the file either. Keep the columns common to all specs so those files stay readable.
536538
orderedPathPartitionKeys = Collections.unmodifiableList(
537-
IcebergUtils.getCommonIdentityPartitionColumns(icebergTable,
539+
IcebergUtils.getCommonIdentityPartitionColumns(icebergTable, scanSchema,
538540
enableMappingVarbinaryForPartitionMetadata,
539541
enableMappingTimestampTzForPartitionMetadata));
540542
}
541543
orderedPartitionMetadataKeys = orderedPathPartitionKeys;
542544
}
543545

546+
private Schema getPartitionMetadataScanSchema() {
547+
if (plannedScanSchema != null) {
548+
return plannedScanSchema;
549+
}
550+
if (querySchema != null) {
551+
return querySchema;
552+
}
553+
return Preconditions.checkNotNull(
554+
icebergTable.schema(), "Schema for Iceberg scan is null");
555+
}
556+
544557
@VisibleForTesting
545558
void setPartitionValues(TFileRangeDesc rangeDesc, Map<String, String> partitionValues) {
546559
rangeDesc.unsetColumnsFromPathKeys();
@@ -2598,6 +2611,7 @@ private boolean requiresRowId() {
25982611

25992612
private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException {
26002613
DataFile dataFile = fileScanTask.file();
2614+
Schema scanSchema = getPartitionMetadataScanSchema();
26012615
String originalPath = dataFile.path().toString();
26022616
LocationPath locationPath = createLocationPathWithCache(originalPath);
26032617
IcebergSplit split = new IcebergSplit(
@@ -2659,7 +2673,7 @@ private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException
26592673
.anyMatch(field -> field.transform().isIdentity())) {
26602674
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
26612675
Pair.of(specId, partitionData), k -> IcebergUtils.getIdentityPartitionInfoMap(
2662-
partitionData, partitionSpec, icebergTable, sessionVariable.getTimeZone(),
2676+
partitionData, partitionSpec, scanSchema, sessionVariable.getTimeZone(),
26632677
enableMappingVarbinaryForPartitionMetadata,
26642678
enableMappingTimestampTzForPartitionMetadata));
26652679
// A spec may mix identity and transformed fields. Keep its identity values so a

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,6 +1091,31 @@ public void testGetIdentityPartitionColumnsIgnoresTransformPartitions() {
10911091
Assert.assertEquals(Arrays.asList("Dt", "id"), IcebergUtils.getIdentityPartitionColumns(table));
10921092
}
10931093

1094+
@Test
1095+
public void testIdentityPartitionColumnsUseAuthoritativeScanSchema() {
1096+
Schema pinnedSchema = new Schema(
1097+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
1098+
Types.NestedField.required(2, "City", Types.StringType.get()));
1099+
Schema currentSchema = new Schema(
1100+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
1101+
Types.NestedField.required(2, "city_name", Types.StringType.get()));
1102+
PartitionSpec spec = PartitionSpec.builderFor(pinnedSchema)
1103+
.identity("City")
1104+
.build();
1105+
1106+
Table table = Mockito.mock(Table.class);
1107+
Mockito.when(table.schema()).thenReturn(currentSchema);
1108+
Mockito.when(table.spec()).thenReturn(spec);
1109+
Mockito.when(table.specs()).thenReturn(Collections.singletonMap(spec.specId(), spec));
1110+
1111+
Assert.assertEquals(Collections.singletonList("City"),
1112+
IcebergUtils.getIdentityPartitionColumns(table, pinnedSchema));
1113+
Assert.assertEquals(Collections.singletonList("city_name"),
1114+
IcebergUtils.getIdentityPartitionColumns(table, currentSchema));
1115+
Assert.assertEquals(Collections.singletonList("City"),
1116+
IcebergUtils.getCommonIdentityPartitionColumns(table, pinnedSchema));
1117+
}
1118+
10941119
@Test
10951120
public void testGetCommonIdentityPartitionColumnsUsesSafeIntersection() {
10961121
Schema schema = new Schema(
@@ -1141,6 +1166,28 @@ public void testGetIdentityPartitionInfoMapReturnsIdentityColumnsOnly() {
11411166
Assert.assertEquals(Collections.singletonMap("Dt", "2025-01-01"), partitionInfoMap);
11421167
}
11431168

1169+
@Test
1170+
public void testIdentityPartitionInfoMapUsesAuthoritativeScanSchema() {
1171+
Schema pinnedSchema = new Schema(
1172+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
1173+
Types.NestedField.required(2, "City", Types.StringType.get()));
1174+
Schema currentSchema = new Schema(
1175+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
1176+
Types.NestedField.required(2, "city_name", Types.StringType.get()));
1177+
PartitionSpec partitionSpec = PartitionSpec.builderFor(pinnedSchema)
1178+
.identity("City")
1179+
.build();
1180+
PartitionData partitionData = new PartitionData(partitionSpec.partitionType());
1181+
partitionData.set(0, "Beijing");
1182+
1183+
Assert.assertEquals(Collections.singletonMap("City", "Beijing"),
1184+
IcebergUtils.getIdentityPartitionInfoMap(
1185+
partitionData, partitionSpec, pinnedSchema, "UTC"));
1186+
Assert.assertEquals(Collections.singletonMap("city_name", "Beijing"),
1187+
IcebergUtils.getIdentityPartitionInfoMap(
1188+
partitionData, partitionSpec, currentSchema, "UTC"));
1189+
}
1190+
11441191
@Test
11451192
public void testMappedTypesAreExcludedFromPartitionMetadata() {
11461193
Schema schema = new Schema(

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,36 @@ public void testDoesNotUseHiveParquetInt96TimeZone() {
175175
Assert.assertEquals("", node.getHiveParquetTimeZone());
176176
}
177177

178+
@Test
179+
public void testPartitionMetadataUsesQuerySchemaFieldId() throws Exception {
180+
Schema pinnedSchema = new Schema(
181+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
182+
Types.NestedField.required(2, "City", Types.StringType.get()));
183+
Schema currentSchema = new Schema(
184+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
185+
Types.NestedField.required(2, "city_name", Types.StringType.get()));
186+
PartitionSpec spec = PartitionSpec.builderFor(pinnedSchema)
187+
.identity("City")
188+
.build();
189+
Table table = Mockito.mock(Table.class);
190+
Mockito.when(table.schema()).thenReturn(currentSchema);
191+
Mockito.when(table.spec()).thenReturn(spec);
192+
Mockito.when(table.specs()).thenReturn(Collections.singletonMap(spec.specId(), spec));
193+
194+
SessionVariable sessionVariable = new SessionVariable();
195+
sessionVariable.enableFileScannerV2 = true;
196+
TestIcebergScanNode node = new TestIcebergScanNode(sessionVariable);
197+
setIcebergTable(node, table);
198+
setPrivateField(node, "querySchema", pinnedSchema);
199+
200+
Assert.assertEquals(ImmutableList.of("City"), orderedPathPartitionKeys(node));
201+
202+
setPrivateField(node, "querySchema", currentSchema);
203+
setPrivateField(node, "orderedPathPartitionKeys", null);
204+
setPrivateField(node, "orderedPartitionMetadataKeys", null);
205+
Assert.assertEquals(ImmutableList.of("city_name"), orderedPathPartitionKeys(node));
206+
}
207+
178208
@SuppressWarnings("unchecked")
179209
private static Optional<Map<Integer, List<String>>> extractNameMapping(
180210
IcebergScanNode node) throws Exception {
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !mixed_case_scan --
3+
1 one
4+
2 two
5+
3 three
6+
7+
-- !mixed_case_topn --
8+
one
9+
two
10+
11+
-- !mixed_case_nested --
12+
1 11 alpha
13+
2 22 beta
14+
3 33 gamma
15+
16+
-- !pinned_partition_schema --
17+
1 Beijing
18+
2 Beijing
19+
20+
-- !pinned_partition_filter --
21+
1 Beijing
22+
2 Beijing
23+
24+
-- !pinned_partition_topn --
25+
1 Beijing
26+
27+
-- !current_partition_schema --
28+
1 Beijing
29+
2 Beijing
30+
3 Shanghai
31+
32+
-- !pinned_count --
33+
2
34+
35+
-- !current_count --
36+
3

0 commit comments

Comments
 (0)