Skip to content

Commit 3c5652d

Browse files
committed
[fix](iceberg) Preserve reads with unknown historical partition types
1 parent 956eb7c commit 3c5652d

3 files changed

Lines changed: 130 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2598,8 +2598,19 @@ private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException
25982598
PartitionSpec partitionSpec = icebergTable.specs().get(specId);
25992599
Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
26002600
specId, icebergTable.name());
2601-
split.setPartitionDataJson(IcebergUtils.getPartitionDataJson(
2602-
partitionData, partitionSpec, sessionVariable.getTimeZone()));
2601+
try {
2602+
split.setPartitionDataJson(IcebergUtils.getPartitionDataJson(
2603+
partitionData, partitionSpec, sessionVariable.getTimeZone()));
2604+
} catch (UnsupportedOperationException e) {
2605+
// Dropped source columns leave UNKNOWN types in historical specs. Reads do not need
2606+
// this row-ID payload, but DML must never replace an unrepresentable partition with NULL.
2607+
boolean requiresRowId = desc.getSlots().stream().anyMatch(slot -> slot.getColumn() != null
2608+
&& Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName()));
2609+
if (requiresRowId) {
2610+
throw new UserException("Cannot produce Iceberg row IDs with unsupported partition types in spec "
2611+
+ specId + ". Rewrite historical data files before DELETE or UPDATE.", e);
2612+
}
2613+
}
26032614
if (isPartitionedTable) {
26042615
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
26052616
Pair.of(specId, partitionData), k -> IcebergUtils.getIdentityPartitionInfoMap(

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
5050
import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot;
5151
import org.apache.doris.datasource.iceberg.IcebergPartitionInfo;
52+
import org.apache.doris.datasource.iceberg.IcebergRowId;
5253
import org.apache.doris.datasource.iceberg.IcebergRuntimeContext;
5354
import org.apache.doris.datasource.iceberg.IcebergSnapshot;
5455
import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue;
@@ -1923,6 +1924,91 @@ public void testDeleteSplitKeepsOldAndNewSpecsAfterDroppingPartitionField() thro
19231924
assertDeleteSplitPartitionMetadata(table);
19241925
}
19251926

1927+
@Test
1928+
public void testReadSplitAfterDroppingPartitionSourceColumn() throws Exception {
1929+
for (boolean keepPartitioned : Arrays.asList(false, true)) {
1930+
for (Integer value : Arrays.asList(7, null)) {
1931+
assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, false);
1932+
}
1933+
}
1934+
}
1935+
1936+
@Test
1937+
public void testRowIdScanRejectsDroppedPartitionSourceColumn() throws Exception {
1938+
for (boolean keepPartitioned : Arrays.asList(false, true)) {
1939+
for (Integer value : Arrays.asList(7, null)) {
1940+
assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, true);
1941+
}
1942+
}
1943+
}
1944+
1945+
private void assertSplitAfterDroppingPartitionSourceColumn(
1946+
boolean keepPartitioned, Integer value, boolean requireRowId) throws Exception {
1947+
Schema schema = new Schema(Types.NestedField.optional(1, "record_key", Types.IntegerType.get()),
1948+
Types.NestedField.optional(2, "partition_key", Types.IntegerType.get()),
1949+
Types.NestedField.optional(3, "retained_key", Types.IntegerType.get()));
1950+
PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema).identity("partition_key");
1951+
if (keepPartitioned) {
1952+
specBuilder.identity("retained_key");
1953+
}
1954+
HadoopTables tables = new HadoopTables(new Configuration());
1955+
String location = temporaryFolder.newFolder().toURI().toString();
1956+
Table table = tables.create(schema, specBuilder.build(),
1957+
Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"), location);
1958+
PartitionData partition = new PartitionData(table.spec().partitionType());
1959+
partition.set(0, value);
1960+
if (keepPartitioned) {
1961+
partition.set(1, 3);
1962+
}
1963+
table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
1964+
.withPartition(partition).withRecordCount(1).withFileSizeInBytes(128).build()).commit();
1965+
table.updateSpec().removeField("partition_key").commit();
1966+
table.updateSchema().deleteColumn("partition_key").commit();
1967+
// Reload metadata so historical specs bind to the schema without the dropped source column.
1968+
table = tables.load(location);
1969+
Assert.assertEquals(keepPartitioned, table.spec().isPartitioned());
1970+
TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
1971+
setIcebergTable(node, table);
1972+
setPrivateField(node, "isPartitionedTable", keepPartitioned);
1973+
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
1974+
setPrivateField(node, "formatVersion", 2);
1975+
setPrivateField(node, "partitionMapInfos", new HashMap<>());
1976+
setPrivateField(node, "orderedPathPartitionKeys", Collections.emptyList());
1977+
setPrivateField(node, "orderedPartitionMetadataKeys", Collections.emptyList());
1978+
node.addSlot(0, new Column("record_key", Type.INT));
1979+
if (requireRowId) {
1980+
node.addSlot(1, IcebergRowId.createHiddenColumn());
1981+
}
1982+
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
1983+
int fileCount = 0;
1984+
for (FileScanTask task : tasks) {
1985+
PartitionData actualPartition = (PartitionData) task.file().partition();
1986+
Assert.assertEquals(Types.UnknownType.get(), actualPartition.getPartitionType().fields().get(0).type());
1987+
Assert.assertEquals(value, actualPartition.get(0));
1988+
if (requireRowId) {
1989+
InvocationTargetException thrown = Assert.assertThrows(InvocationTargetException.class,
1990+
() -> createIcebergSplit(node, task));
1991+
Assert.assertTrue(thrown.getCause() instanceof UserException);
1992+
Assert.assertTrue(thrown.getCause().getMessage().contains(
1993+
"Cannot produce Iceberg row IDs with unsupported partition types"));
1994+
} else {
1995+
IcebergSplit split = createIcebergSplit(node, task);
1996+
TFileRangeDesc range = new TFileRangeDesc();
1997+
setIcebergParams(node, range, split);
1998+
TIcebergFileDesc file = range.getTableFormatParams().getIcebergParams();
1999+
Assert.assertEquals(task.file().specId(), file.getPartitionSpecId());
2000+
Assert.assertFalse(file.isSetPartitionDataJson());
2001+
if (keepPartitioned) {
2002+
Assert.assertEquals(Collections.singletonMap("retained_key", "3"),
2003+
split.getIcebergPartitionValues());
2004+
}
2005+
}
2006+
fileCount++;
2007+
}
2008+
Assert.assertEquals(1, fileCount);
2009+
}
2010+
}
2011+
19262012
@Test
19272013
public void testDeleteSplitKeepsBinaryPartitionAfterDroppingField() throws Exception {
19282014
assertDeleteAfterDroppingPartitionField(Types.BinaryType.get(),

‎regression-test/suites/external_table_p0/iceberg/write/test_iceberg_delete_unpartitioned_evolution.groovy‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,37 @@ suite("test_iceberg_delete_unpartitioned_evolution",
118118
sql """update mixed_partition_specs set metric = metric + 100"""
119119
checkRows("mixed_partition_specs", expectedAfterUpdate)
120120

121+
// Dropping a source column makes its historical partition type unknown, even for non-null values.
122+
for (String fileFormat : ["parquet", "orc"]) {
123+
String table = "dropped_partition_source_${fileFormat}"
124+
spark_iceberg """drop table if exists demo.${dbName}.${table}"""
125+
spark_iceberg """
126+
create table demo.${dbName}.${table} (record_key int, partition_key int, metric int)
127+
using iceberg partitioned by (partition_key)
128+
tblproperties ('format-version' = '2', 'write.format.default' = '${fileFormat}',
129+
'write.delete.mode' = 'merge-on-read', 'write.update.mode' = 'merge-on-read')
130+
"""
131+
spark_iceberg """insert into demo.${dbName}.${table} values (1, 7, 10), (2, 7, 20), (3, null, 30)"""
132+
spark_iceberg """alter table demo.${dbName}.${table} drop partition field partition_key"""
133+
spark_iceberg """alter table demo.${dbName}.${table} drop column partition_key"""
134+
spark_iceberg """insert into demo.${dbName}.${table} values (4, 40)"""
135+
sql """refresh database ${dbName}"""
136+
def expected = spark_iceberg """
137+
select record_key, metric from demo.${dbName}.${table} order by record_key
138+
"""
139+
checkRows(table, expected)
140+
test {
141+
sql """delete from ${table} where record_key = 1"""
142+
exception "Cannot produce Iceberg row IDs with unsupported partition types"
143+
}
144+
test {
145+
sql """update ${table} set metric = metric + 100 where record_key = 2"""
146+
exception "Cannot produce Iceberg row IDs with unsupported partition types"
147+
}
148+
checkRows(table, expected)
149+
assertEquals([[0L]], sql("""select count(*) from ${table}\$delete_files"""))
150+
}
151+
121152
// Historical binary partitions must retain their bytes, not become NULL delete partitions.
122153
spark_iceberg """drop table if exists demo.${dbName}.historical_binary"""
123154
spark_iceberg """

0 commit comments

Comments
 (0)