Skip to content

Commit 5d3cb75

Browse files
committed
[fix](fe) Validate retained Iceberg partition schemas
### What problem does this PR solve? Issue Number: N/A Related PR: #69299, #69983 Problem Summary: Reset-safe lazy partition metadata loading parses a retained Iceberg table directly instead of using the live schema cache. It therefore bypassed SchemaCacheValue.validateSchema(), allowing case-insensitive duplicate columns into partition-range and MV/dictionary snapshot derivation even though normal schema loading rejects them. Reuse the same pure validator before deriving partition columns, inside the captured authenticator and existing table/snapshot AnalysisException wrapper. Preserve the retained table/schema generation and avoid a live catalog/cache lookup after reset. ### Release note Reject case-insensitive duplicate Iceberg columns consistently when deriving lazy partition metadata for materialized views and dictionaries. ### Check List (For Author) - Test: Unit tests and packaging - Two real Iceberg metadata tests reproduced the missing rejection for active and retired catalog projections before repair. - All 173 related schema, snapshot, partition and cache FE tests passed, including existing reset/session success paths. - Full FE packaging and Checkstyle passed. - git diff --check passed; no local full P0 or external-service regression in this cycle. - Behavior changed: Yes. Lazy retained partition loads reject ambiguous schemas consistently and keep table/snapshot error context. Snapshot identity, cache admission and generation-retirement semantics remain unchanged. - Does this need documentation: No
1 parent f80fa3b commit 5d3cb75

2 files changed

Lines changed: 61 additions & 4 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2344,10 +2344,14 @@ static IcebergPartitionInfo loadPartitionInfo(String displayName, Table table,
23442344
icebergPartitions = authenticator.execute(() -> loadIcebergPartition(table, snapshotId));
23452345
// A retained projection may outlive its catalog cache group. Its columns and ranges
23462346
// must use the same frozen table/schema, without looking up a new catalog generation.
2347-
partitionColumns = authenticator.execute(() -> buildTableSchemaCacheValue(table,
2348-
parseSchema(Preconditions.checkNotNull(schemaForTable(table, schemaId),
2349-
"Schema for Iceberg table %s is null", displayName),
2350-
enableMappingVarbinary, enableMappingTimestampTz)).getPartitionColumns());
2347+
partitionColumns = authenticator.execute(() -> {
2348+
List<Column> schema = parseSchema(Preconditions.checkNotNull(schemaForTable(table, schemaId),
2349+
"Schema for Iceberg table %s is null", displayName),
2350+
enableMappingVarbinary, enableMappingTimestampTz);
2351+
// This retained-table path bypasses the schema-cache loader's validation boundary.
2352+
new SchemaCacheValue(schema).validateSchema();
2353+
return buildTableSchemaCacheValue(table, schema).getPartitionColumns();
2354+
});
23512355
} catch (Exception e) {
23522356
String errorMsg = String.format("Failed to get iceberg partition info, table: %s, snapshotId: %s",
23532357
displayName, snapshotId);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,59 @@ private IcebergSnapshotCacheValue partitionProjectionForTest(IcebergExternalMeta
11511151
retainedTable, retainedTable, null, false, authenticator, false, false);
11521152
}
11531153

1154+
@Test
1155+
public void testLazyRetainedPartitionRejectsCaseInsensitiveDuplicateColumns() throws Exception {
1156+
assertDuplicateSchemaRejectedByRetainedPartitionLoad(false);
1157+
}
1158+
1159+
@Test
1160+
public void testRetiredPartitionProjectionRejectsCaseInsensitiveDuplicateColumns() throws Exception {
1161+
assertDuplicateSchemaRejectedByRetainedPartitionLoad(true);
1162+
}
1163+
1164+
private void assertDuplicateSchemaRejectedByRetainedPartitionLoad(boolean resetBeforeLoad) throws Exception {
1165+
ExecutorService executor = Executors.newSingleThreadExecutor();
1166+
IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor);
1167+
NameMapping identity = NameMapping.createForTest(1L, "db", "tbl");
1168+
IcebergExternalTable dorisTable = partitionConsumerForTest(identity);
1169+
Schema schema = new Schema(Types.NestedField.optional(1, "dt", Types.DateType.get()),
1170+
Types.NestedField.optional(2, "DT", Types.IntegerType.get()));
1171+
PartitionSpec spec = PartitionSpec.builderFor(schema).day("dt").build();
1172+
String location = temporaryFolder.newFolder("retained-partition-duplicate-schema").toURI().toString();
1173+
Table table = new HadoopTables(new Configuration()).create(schema, spec, location);
1174+
PartitionData partition = new PartitionData(spec.rawPartitionType());
1175+
partition.set(0, 19724);
1176+
table.newFastAppend().appendFile(DataFiles.builder(spec)
1177+
.withPath(location + "/data/old.parquet").withPartition(partition)
1178+
.withFileSizeInBytes(10).withRecordCount(1).build()).commit();
1179+
IcebergTableCacheValue generation = new IcebergTableCacheValue(table);
1180+
try (IcebergTableCacheValue.Lease retained = generation.tryAcquire();
1181+
MockedStatic<IcebergUtils> utils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
1182+
Assert.assertNotNull(retained);
1183+
utils.when(() -> IcebergUtils.useSessionCatalog(dorisTable)).thenReturn(false);
1184+
cache.initCatalog(1L, Collections.emptyMap());
1185+
cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE,
1186+
NameMapping.class, IcebergTableCacheValue.class).put(identity, generation);
1187+
IcebergSnapshotCacheValue projection = partitionProjectionForTest(
1188+
cache, dorisTable, retained.getIcebergTable(), new ExecutionAuthenticator() { });
1189+
if (resetBeforeLoad) {
1190+
cache.invalidateCatalog(1L);
1191+
}
1192+
RuntimeException error = Assert.assertThrows(RuntimeException.class, projection::getPartitionInfo);
1193+
Assert.assertTrue(org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage(error)
1194+
.contains("Duplicate column name found: DT"));
1195+
Assert.assertTrue(org.apache.commons.lang3.exception.ExceptionUtils.getThrowableList(error).stream()
1196+
.anyMatch(cause -> cause instanceof org.apache.doris.common.AnalysisException
1197+
&& cause.getMessage().contains("ice.db.tbl")
1198+
&& cause.getMessage().contains(Long.toString(table.currentSnapshot().snapshotId()))));
1199+
Mockito.verify(dorisTable.getCatalog(), Mockito.never()).getExecutionAuthenticator();
1200+
} finally {
1201+
cache.close();
1202+
generation.retire();
1203+
executor.shutdownNow();
1204+
}
1205+
}
1206+
11541207
@Test
11551208
public void testLazyPartitionLoadSurvivesCatalogReset() throws Exception {
11561209
assertRetainedPartitionLoadAfterReset(false, false);

0 commit comments

Comments
 (0)