|
49 | 49 | import org.apache.doris.datasource.iceberg.IcebergExternalTable; |
50 | 50 | import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; |
51 | 51 | import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; |
| 52 | +import org.apache.doris.datasource.iceberg.IcebergRowId; |
52 | 53 | import org.apache.doris.datasource.iceberg.IcebergRuntimeContext; |
53 | 54 | import org.apache.doris.datasource.iceberg.IcebergSnapshot; |
54 | 55 | import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; |
@@ -1923,6 +1924,91 @@ public void testDeleteSplitKeepsOldAndNewSpecsAfterDroppingPartitionField() thro |
1923 | 1924 | assertDeleteSplitPartitionMetadata(table); |
1924 | 1925 | } |
1925 | 1926 |
|
| 1927 | + @Test |
| 1928 | + public void testReadSplitAfterDroppingPartitionSourceColumn() throws Exception { |
| 1929 | + for (boolean keepPartitioned : Arrays.asList(false, true)) { |
| 1930 | + for (Integer value : Arrays.asList(7, null)) { |
| 1931 | + assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, false); |
| 1932 | + } |
| 1933 | + } |
| 1934 | + } |
| 1935 | + |
| 1936 | + @Test |
| 1937 | + public void testRowIdScanRejectsDroppedPartitionSourceColumn() throws Exception { |
| 1938 | + for (boolean keepPartitioned : Arrays.asList(false, true)) { |
| 1939 | + for (Integer value : Arrays.asList(7, null)) { |
| 1940 | + assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, true); |
| 1941 | + } |
| 1942 | + } |
| 1943 | + } |
| 1944 | + |
| 1945 | + private void assertSplitAfterDroppingPartitionSourceColumn( |
| 1946 | + boolean keepPartitioned, Integer value, boolean requireRowId) throws Exception { |
| 1947 | + Schema schema = new Schema(Types.NestedField.optional(1, "record_key", Types.IntegerType.get()), |
| 1948 | + Types.NestedField.optional(2, "partition_key", Types.IntegerType.get()), |
| 1949 | + Types.NestedField.optional(3, "retained_key", Types.IntegerType.get())); |
| 1950 | + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema).identity("partition_key"); |
| 1951 | + if (keepPartitioned) { |
| 1952 | + specBuilder.identity("retained_key"); |
| 1953 | + } |
| 1954 | + HadoopTables tables = new HadoopTables(new Configuration()); |
| 1955 | + String location = temporaryFolder.newFolder().toURI().toString(); |
| 1956 | + Table table = tables.create(schema, specBuilder.build(), |
| 1957 | + Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"), location); |
| 1958 | + PartitionData partition = new PartitionData(table.spec().partitionType()); |
| 1959 | + partition.set(0, value); |
| 1960 | + if (keepPartitioned) { |
| 1961 | + partition.set(1, 3); |
| 1962 | + } |
| 1963 | + table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet") |
| 1964 | + .withPartition(partition).withRecordCount(1).withFileSizeInBytes(128).build()).commit(); |
| 1965 | + table.updateSpec().removeField("partition_key").commit(); |
| 1966 | + table.updateSchema().deleteColumn("partition_key").commit(); |
| 1967 | + // Reload metadata so historical specs bind to the schema without the dropped source column. |
| 1968 | + table = tables.load(location); |
| 1969 | + Assert.assertEquals(keepPartitioned, table.spec().isPartitioned()); |
| 1970 | + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); |
| 1971 | + setIcebergTable(node, table); |
| 1972 | + setPrivateField(node, "isPartitionedTable", keepPartitioned); |
| 1973 | + setPrivateField(node, "storagePropertiesMap", Collections.emptyMap()); |
| 1974 | + setPrivateField(node, "formatVersion", 2); |
| 1975 | + setPrivateField(node, "partitionMapInfos", new HashMap<>()); |
| 1976 | + setPrivateField(node, "orderedPathPartitionKeys", Collections.emptyList()); |
| 1977 | + setPrivateField(node, "orderedPartitionMetadataKeys", Collections.emptyList()); |
| 1978 | + node.addSlot(0, new Column("record_key", Type.INT)); |
| 1979 | + if (requireRowId) { |
| 1980 | + node.addSlot(1, IcebergRowId.createHiddenColumn()); |
| 1981 | + } |
| 1982 | + try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) { |
| 1983 | + int fileCount = 0; |
| 1984 | + for (FileScanTask task : tasks) { |
| 1985 | + PartitionData actualPartition = (PartitionData) task.file().partition(); |
| 1986 | + Assert.assertEquals(Types.UnknownType.get(), actualPartition.getPartitionType().fields().get(0).type()); |
| 1987 | + Assert.assertEquals(value, actualPartition.get(0)); |
| 1988 | + if (requireRowId) { |
| 1989 | + InvocationTargetException thrown = Assert.assertThrows(InvocationTargetException.class, |
| 1990 | + () -> createIcebergSplit(node, task)); |
| 1991 | + Assert.assertTrue(thrown.getCause() instanceof UserException); |
| 1992 | + Assert.assertTrue(thrown.getCause().getMessage().contains( |
| 1993 | + "Cannot produce Iceberg row IDs with unsupported partition types")); |
| 1994 | + } else { |
| 1995 | + IcebergSplit split = createIcebergSplit(node, task); |
| 1996 | + TFileRangeDesc range = new TFileRangeDesc(); |
| 1997 | + setIcebergParams(node, range, split); |
| 1998 | + TIcebergFileDesc file = range.getTableFormatParams().getIcebergParams(); |
| 1999 | + Assert.assertEquals(task.file().specId(), file.getPartitionSpecId()); |
| 2000 | + Assert.assertFalse(file.isSetPartitionDataJson()); |
| 2001 | + if (keepPartitioned) { |
| 2002 | + Assert.assertEquals(Collections.singletonMap("retained_key", "3"), |
| 2003 | + split.getIcebergPartitionValues()); |
| 2004 | + } |
| 2005 | + } |
| 2006 | + fileCount++; |
| 2007 | + } |
| 2008 | + Assert.assertEquals(1, fileCount); |
| 2009 | + } |
| 2010 | + } |
| 2011 | + |
1926 | 2012 | @Test |
1927 | 2013 | public void testDeleteSplitKeepsBinaryPartitionAfterDroppingField() throws Exception { |
1928 | 2014 | assertDeleteAfterDroppingPartitionField(Types.BinaryType.get(), |
|
0 commit comments