Skip to content

Commit c8c0ddc

Browse files
authored
branch-4.1: [fix](iceberg) keep identity partition values of old specs after evolving to unpartitioned(#68124) (#68259)
Cherry-picked from #68124.
1 parent 1512485 commit c8c0ddc

4 files changed

Lines changed: 151 additions & 11 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -982,6 +982,20 @@ public static Map<String, String> getPartitionInfoMap(PartitionData partitionDat
982982
return partitionInfoMap;
983983
}
984984

985+
/**
986+
* Whether <b>any</b> partition spec of the table is partitioned. Scan planning must use this instead of
987+
* the current default spec: after evolving to an unpartitioned spec, data files written under an older
988+
* partitioned spec still carry their partition metadata (identity values, spec id, partition data).
989+
*/
990+
public static boolean hasPartitionedSpec(Table table) {
991+
for (PartitionSpec spec : table.specs().values()) {
992+
if (spec.isPartitioned()) {
993+
return true;
994+
}
995+
}
996+
return false;
997+
}
998+
985999
public static List<String> getIdentityPartitionColumns(Table table) {
9861000
return getIdentityPartitionColumns(table, false, false);
9871001
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,15 @@ public class IcebergScanNode extends FileQueryScanNode {
199199
private List<String> orderedPartitionMetadataKeys;
200200
private boolean enableMappingVarbinaryForPartitionMetadata;
201201
private boolean enableMappingTimestampTzForPartitionMetadata;
202+
// Whether the table's CURRENT default spec is partitioned. Display/accounting only: it gates what
203+
// counts as a scanned partition (selectedPartitionNum, numApproximateSplits), preserving the legacy
204+
// numbers for a table that evolved to an unpartitioned spec.
202205
private boolean isPartitionedTable;
203206
private Boolean requiresRowId;
207+
// Whether ANY spec of the table is partitioned. This is the READ gate: data files written under an
208+
// older partitioned spec still carry their identity values, and must keep sending them even after
209+
// the default spec evolved to unpartitioned.
210+
private boolean hasPartitionedSpec;
204211
private int formatVersion;
205212
private ExecutionAuthenticator preExecutionAuthenticator;
206213
private IcebergRuntimeContext runtimeContext;
@@ -318,6 +325,7 @@ protected void doInitialize() throws UserException {
318325
partitionMapInfos = new HashMap<>();
319326
initializePartitionMetadata();
320327
isPartitionedTable = icebergTable.spec().isPartitioned();
328+
hasPartitionedSpec = IcebergUtils.hasPartitionedSpec(icebergTable);
321329
// Metadata tables (system tables) are not BaseTable instances, so we need to handle this case
322330
if (icebergTable instanceof BaseTable) {
323331
formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion();
@@ -510,18 +518,26 @@ private void initializePartitionMetadata() {
510518
}
511519
enableMappingVarbinaryForPartitionMetadata = getEnableMappingVarbinary();
512520
enableMappingTimestampTzForPartitionMetadata = getEnableMappingTimestampTz();
513-
orderedPathPartitionKeys = Collections.unmodifiableList(
514-
IcebergUtils.getCommonIdentityPartitionColumns(icebergTable,
515-
enableMappingVarbinaryForPartitionMetadata,
516-
enableMappingTimestampTzForPartitionMetadata));
517521
if (sessionVariable.enableFileScannerV2) {
518-
orderedPartitionMetadataKeys = Collections.unmodifiableList(
522+
// Classify every identity column of EVERY spec as a partition key, like master does. A file
523+
// written under an older identity spec carries its value in the manifest only, and BE uses a
524+
// split partition value just for a column FE marked as a partition key, so the intersection
525+
// below would leave that value unused and the column read as NULL. Files that carry no value
526+
// for the column (a spec without it) are unaffected: the v2 column mapper falls back to the
527+
// physical field when the split has no partition value for a partition key.
528+
orderedPathPartitionKeys = Collections.unmodifiableList(
519529
IcebergUtils.getIdentityPartitionColumns(icebergTable,
520530
enableMappingVarbinaryForPartitionMetadata,
521531
enableMappingTimestampTzForPartitionMetadata));
522532
} else {
523-
orderedPartitionMetadataKeys = orderedPathPartitionKeys;
533+
// v1 has no such per-file fallback: a partition key it cannot fill from the split is not read
534+
// from the file either. Keep the columns common to all specs so those files stay readable.
535+
orderedPathPartitionKeys = Collections.unmodifiableList(
536+
IcebergUtils.getCommonIdentityPartitionColumns(icebergTable,
537+
enableMappingVarbinaryForPartitionMetadata,
538+
enableMappingTimestampTzForPartitionMetadata));
524539
}
540+
orderedPartitionMetadataKeys = orderedPathPartitionKeys;
525541
}
526542

527543
@VisibleForTesting
@@ -2616,7 +2632,10 @@ private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException
26162632
split.setPartitionSpecId(specId);
26172633
PartitionData partitionData = (PartitionData) dataFile.partition();
26182634
boolean includePartitionData = requiresRowId();
2619-
if (partitionData != null && (includePartitionData || isPartitionedTable)) {
2635+
// Gate identity values on the table's spec HISTORY, not on the current default spec: a file written
2636+
// under an older identity spec keeps its partition values in the manifest, and they are the only
2637+
// source for an identity partition column that the physical file does not store.
2638+
if (partitionData != null && (includePartitionData || hasPartitionedSpec)) {
26202639
PartitionSpec partitionSpec = icebergTable.specs().get(specId);
26212640
Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
26222641
specId, icebergTable.name());
@@ -2631,7 +2650,10 @@ private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException
26312650
+ specId + ". Rewrite historical data files before DELETE or UPDATE.", e);
26322651
}
26332652
}
2634-
if (isPartitionedTable) {
2653+
// The cache doubles as the scanned-partition count only for a partitioned current spec. Otherwise
2654+
// cache only specs with identity fields, or transformed-only old partitions would grow it uselessly.
2655+
if (isPartitionedTable || partitionSpec.fields().stream()
2656+
.anyMatch(field -> field.transform().isIdentity())) {
26352657
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
26362658
Pair.of(specId, partitionData), k -> IcebergUtils.getIdentityPartitionInfoMap(
26372659
partitionData, partitionSpec, icebergTable, sessionVariable.getTimeZone(),
@@ -2813,7 +2835,7 @@ private List<Split> doGetSplits(int numBackends) throws UserException {
28132835
for (FileScanTask task : customFileScanTasks) {
28142836
splits.add(createIcebergSplit(task));
28152837
}
2816-
selectedPartitionNum = partitionMapInfos.size();
2838+
selectedPartitionNum = scannedPartitionNum();
28172839
recordManifestCacheProfile();
28182840
return splits;
28192841
}
@@ -2850,7 +2872,7 @@ private List<Split> doGetSplits(int numBackends) throws UserException {
28502872
}
28512873
}
28522874

2853-
selectedPartitionNum = partitionMapInfos.size();
2875+
selectedPartitionNum = scannedPartitionNum();
28542876
recordManifestCacheProfile();
28552877
return splits;
28562878
}
@@ -3233,9 +3255,21 @@ private void assignCountToSplits(List<Split> splits, long totalCount) {
32333255
((IcebergSplit) splits.get(size - 1)).setTableLevelRowCount(countPerSplit + totalCount % size);
32343256
}
32353257

3258+
/**
3259+
* The number of scanned partitions reported to EXPLAIN ({@code partition=N/M}), the
3260+
* {@code sql_block_rule} {@code partition_num} guard and the batch-mode split estimate. A table whose
3261+
* CURRENT spec is unpartitioned reports no partitions at all, so the partitions that files of an older
3262+
* spec still carry must not start being counted — {@code partitionMapInfos} now collects them because
3263+
* it doubles as the per-(spec, partition) identity-value cache for the read path.
3264+
*/
3265+
private int scannedPartitionNum() {
3266+
return isPartitionedTable ? partitionMapInfos.size() : 0;
3267+
}
3268+
32363269
@Override
32373270
public int numApproximateSplits() {
3238-
return NUM_SPLITS_PER_PARTITION * partitionMapInfos.size() > 0 ? partitionMapInfos.size() : 1;
3271+
int partitions = scannedPartitionNum();
3272+
return NUM_SPLITS_PER_PARTITION * partitions > 0 ? partitions : 1;
32393273
}
32403274

32413275
private Optional<NotSupportedException> checkNotSupportedException(Exception e) {

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,25 @@ public void testMalformedNameMappingFailsInsteadOfFallingBackToCurrentSchemaName
427427
Assert.assertTrue(exception.getMessage().contains("db.tbl"));
428428
}
429429

430+
@Test
431+
public void testHasPartitionedSpecLooksAtEverySpecNotOnlyTheCurrentOne() {
432+
Schema schema = new Schema(
433+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
434+
Types.NestedField.required(2, "p", Types.IntegerType.get()));
435+
PartitionSpec identity = PartitionSpec.builderFor(schema).identity("p").withSpecId(0).build();
436+
PartitionSpec unpartitioned = PartitionSpec.unpartitioned();
437+
438+
// A table that evolved identity(p) -> unpartitioned still has files carrying p in their
439+
// partition metadata, so scan planning must keep sending those values.
440+
Table evolved = Mockito.mock(Table.class);
441+
Mockito.when(evolved.specs()).thenReturn(ImmutableMap.of(0, identity, 1, unpartitioned));
442+
Assert.assertTrue(IcebergUtils.hasPartitionedSpec(evolved));
443+
444+
Table neverPartitioned = Mockito.mock(Table.class);
445+
Mockito.when(neverPartitioned.specs()).thenReturn(ImmutableMap.of(0, unpartitioned));
446+
Assert.assertFalse(IcebergUtils.hasPartitionedSpec(neverPartitioned));
447+
}
448+
430449
@Test
431450
public void testEmptyNameMappingStillParsesAsAuthoritativeMapping() throws Exception {
432451
Table table = Mockito.mock(Table.class);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1824,6 +1824,69 @@ public void testSchemaCarrierKeepsDroppedEqualityFieldDefault() throws Exception
18241824
IcebergUtils.getSerializedInitialDefaults(fields, false).get(7));
18251825
}
18261826

1827+
@Test
1828+
public void testSplitKeepsOldSpecIdentityValuesAfterEvolvingToUnpartitioned() throws Exception {
1829+
// A file written under identity(p) keeps p=7 only in its manifest partition metadata. Gating on
1830+
// the CURRENT spec drops that value once the default spec evolves to unpartitioned, and keying
1831+
// the partition-key classification off the columns common to all specs leaves BE reading p from
1832+
// a file that does not store it, so the column comes back NULL in both cases.
1833+
Schema schema = new Schema(
1834+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
1835+
Types.NestedField.required(2, "p", Types.IntegerType.get()));
1836+
PartitionSpec spec = PartitionSpec.builderFor(schema).identity("p").build();
1837+
String location = temporaryFolder.newFolder("identity_evolved_table").toURI().toString();
1838+
Table table = new HadoopTables(new Configuration()).create(schema, spec, location);
1839+
table.newFastAppend()
1840+
.appendFile(DataFiles.builder(table.spec())
1841+
.withPath(location + "data/p=7/old.parquet")
1842+
.withFileSizeInBytes(1024L)
1843+
.withRecordCount(2L)
1844+
.withFormat(FileFormat.PARQUET)
1845+
.withPartitionPath("p=7")
1846+
.build())
1847+
.commit();
1848+
table.updateSpec().removeField("p").commit();
1849+
Assert.assertTrue(table.spec().isUnpartitioned());
1850+
1851+
SessionVariable sessionVariable = new SessionVariable();
1852+
sessionVariable.enableFileScannerV2 = true;
1853+
TestIcebergScanNode node = new TestIcebergScanNode(sessionVariable);
1854+
setIcebergTable(node, table);
1855+
setPrivateField(node, "formatVersion", 2);
1856+
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
1857+
setPrivateField(node, "partitionMapInfos", new HashMap<>());
1858+
// Exactly what doInitialize computes for this table.
1859+
setPrivateField(node, "isPartitionedTable", table.spec().isPartitioned());
1860+
setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
1861+
1862+
// p must stay a partition key even though the current spec no longer mentions it; otherwise BE
1863+
// ignores the split value and reads the (absent) physical column.
1864+
Assert.assertEquals(ImmutableList.of("p"), orderedPathPartitionKeys(node));
1865+
1866+
FileScanTask task;
1867+
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
1868+
Iterator<FileScanTask> iterator = tasks.iterator();
1869+
Assert.assertTrue(iterator.hasNext());
1870+
task = iterator.next();
1871+
}
1872+
1873+
IcebergSplit split = createIcebergSplit(node, task);
1874+
Assert.assertEquals(ImmutableMap.of("p", "7"), split.getIcebergPartitionValues());
1875+
Assert.assertEquals(Integer.valueOf(0), split.getPartitionSpecId());
1876+
// A plain read does not project the row ID, so no row-ID partition JSON is generated.
1877+
Assert.assertNull(split.getPartitionDataJson());
1878+
1879+
// What BE actually receives: p=7 as a columns-from-path constant for this range.
1880+
TFileRangeDesc rangeDesc = new TFileRangeDesc();
1881+
setIcebergParams(node, rangeDesc, split);
1882+
Assert.assertEquals(ImmutableList.of("p"), rangeDesc.getColumnsFromPathKeys());
1883+
Assert.assertEquals(ImmutableList.of("7"), rangeDesc.getColumnsFromPath());
1884+
1885+
// Display parity: the CURRENT spec is unpartitioned, so the table still reports no scanned
1886+
// partitions and the batch-mode estimate stays at the unpartitioned default.
1887+
Assert.assertEquals(1, node.numApproximateSplits());
1888+
}
1889+
18271890
@Test
18281891
public void testBatchSplitCarriesDroppedEqualitySchemaThroughThrift() throws Exception {
18291892
Types.NestedField id = Types.NestedField.required(1, "id", Types.LongType.get());
@@ -1958,6 +2021,7 @@ public void testReadSplitsSkipPartitionJson() throws Exception {
19582021
node.addSlot(0, new Column("record_key", Type.INT));
19592022
setIcebergTable(node, table);
19602023
setPrivateField(node, "isPartitionedTable", keepPartitioned);
2024+
setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
19612025
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
19622026
setPrivateField(node, "formatVersion", 2);
19632027
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -2043,6 +2107,7 @@ private void assertSplitAfterDroppingPartitionSourceColumn(boolean keepPartition
20432107
TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
20442108
setIcebergTable(node, table);
20452109
setPrivateField(node, "isPartitionedTable", keepPartitioned);
2110+
setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
20462111
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
20472112
setPrivateField(node, "formatVersion", 2);
20482113
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -2181,6 +2246,7 @@ private void assertDeleteSplitPartitionMetadata(Table table, SessionVariable ses
21812246
node.addSlot(0, IcebergRowId.createHiddenColumn());
21822247
setIcebergTable(node, table);
21832248
setPrivateField(node, "isPartitionedTable", table.spec().isPartitioned());
2249+
setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
21842250
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
21852251
setPrivateField(node, "formatVersion", 2);
21862252
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -4097,6 +4163,13 @@ private static FileScanTask fileScanTask(String path, DeleteFile deleteFile) {
40974163
return task;
40984164
}
40994165

4166+
@SuppressWarnings("unchecked")
4167+
private static List<String> orderedPathPartitionKeys(IcebergScanNode node) throws Exception {
4168+
Method method = IcebergScanNode.class.getDeclaredMethod("getOrderedPathPartitionKeys");
4169+
method.setAccessible(true);
4170+
return (List<String>) method.invoke(node);
4171+
}
4172+
41004173
private static IcebergSplit createIcebergSplit(IcebergScanNode node, FileScanTask task)
41014174
throws Exception {
41024175
Method method = IcebergScanNode.class.getDeclaredMethod("createIcebergSplit", FileScanTask.class);

0 commit comments

Comments
 (0)