Skip to content

Commit 1385a1e

Browse files
committed
[fix](fe) Keep retained partition loading independent of catalog reset
### What problem does this PR solve? Issue Number: N/A Related PR: #69299 Problem Summary: Iceberg partition views are loaded lazily from a retained table generation. Catalog reset can remove its cache group before projection creation or the first partition read. Looking up current partition and schema entries then fails pinned work despite its live table lease. Capture only an existing shared partition entry when applicable and derive partition columns from the same retained table/schema under captured authentication. Session-catalog and retired-group paths load directly without resolving another catalog generation, while existing cache admission and retirement fences remain in place. ### Release note Keep retained Iceberg partition metadata usable after catalog cache reset. ### Check List (For Author) - Test: Unit Test: 204 related FE tests; full FE packaging and Checkstyle via build.sh --fe - Behavior changed: Yes; retained partition projections can finish lazy metadata loading after catalog reset - Does this need documentation: No
1 parent 350e47d commit 1385a1e

3 files changed

Lines changed: 109 additions & 29 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1207,20 +1207,23 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(
12071207
String displayName = dorisTable.getCatalog().getName() + "." + dorisTable.getDbName()
12081208
+ "." + dorisTable.getName();
12091209
Optional<IcebergSnapshotEntryKey> key = IcebergSnapshotEntryKey.tryCreate(tableIdentity, retainedTable);
1210-
MetaCacheEntry<IcebergSnapshotEntryKey, IcebergPartitionInfo> partitions =
1211-
key.isPresent() ? partitionEntry.get(key.get().getNameMapping().getCtlId()) : null;
1210+
boolean useSharedCache = key.isPresent() && !IcebergUtils.useSessionCatalog(dorisTable);
1211+
// Retained work can start after reset retired the group. Capture an existing entry
1212+
// when applicable; direct loads need only the retained metadata and authenticator.
1213+
MetaCacheEntry<IcebergSnapshotEntryKey, IcebergPartitionInfo> partitions = useSharedCache
1214+
? partitionEntry.getIfInitialized(key.get().getNameMapping().getCtlId()) : null;
12121215
Supplier<IcebergPartitionInfo> loader = () -> {
12131216
try {
12141217
Table table = IcebergSnapshotCacheValue.createQueryScopedTable(
12151218
retainedTable, retainedCurrentSnapshotJson);
1216-
return IcebergUtils.loadPartitionInfo(tableIdentity, this, displayName, table,
1219+
return IcebergUtils.loadPartitionInfo(displayName, table,
12171220
latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId(),
12181221
authenticator, enableMappingVarbinary, enableMappingTimestampTz);
12191222
} catch (AnalysisException e) {
12201223
throw new RuntimeException(e.getMessage(), e);
12211224
}
12221225
};
1223-
value.withPartitionInfoLoader(key.isPresent() && !IcebergUtils.useSessionCatalog(dorisTable)
1226+
value.withPartitionInfoLoader(partitions != null
12241227
? () -> getPartitionInfo(key.get(), partitions, loader) : loader);
12251228
}
12261229
return value;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1751,11 +1751,7 @@ private static List<Column> getSchema(ExternalTable dorisTable, long schemaId, b
17511751
}
17521752
} else {
17531753
Table table = icebergTable != null ? icebergTable : getIcebergTable(dorisTable);
1754-
if (schemaId == NEWEST_SCHEMA_ID || table.currentSnapshot() == null) {
1755-
schema = table.schema();
1756-
} else {
1757-
schema = table.schemas().get((int) schemaId);
1758-
}
1754+
schema = schemaForTable(table, schemaId);
17591755
}
17601756
String type = isView ? "view" : "table";
17611757
Preconditions.checkNotNull(schema,
@@ -1769,6 +1765,11 @@ private static List<Column> getSchema(ExternalTable dorisTable, long schemaId, b
17691765

17701766
}
17711767

1768+
private static Schema schemaForTable(Table table, long schemaId) {
1769+
return schemaId == NEWEST_SCHEMA_ID || table.currentSnapshot() == null
1770+
? table.schema() : table.schemas().get((int) schemaId);
1771+
}
1772+
17721773
/**
17731774
* Parse iceberg schema to doris schema
17741775
*/
@@ -2327,21 +2328,26 @@ static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, Table ta
23272328
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) throws AnalysisException {
23282329
String displayName = dorisTable.getCatalog().getName() + "." + dorisTable.getDbName()
23292330
+ "." + dorisTable.getName();
2330-
return loadPartitionInfo(dorisTable.getOrBuildNameMapping(), icebergExternalMetaCache(dorisTable),
2331-
displayName, table, snapshotId, schemaId, authenticator,
2331+
return loadPartitionInfo(displayName, table, snapshotId, schemaId, authenticator,
23322332
enableMappingVarbinary, enableMappingTimestampTz);
23332333
}
23342334

2335-
static IcebergPartitionInfo loadPartitionInfo(org.apache.doris.datasource.NameMapping nameMapping,
2336-
IcebergExternalMetaCache cache,
2337-
String displayName, Table table, long snapshotId, long schemaId, ExecutionAuthenticator authenticator,
2335+
static IcebergPartitionInfo loadPartitionInfo(String displayName, Table table,
2336+
long snapshotId, long schemaId, ExecutionAuthenticator authenticator,
23382337
boolean enableMappingVarbinary, boolean enableMappingTimestampTz) throws AnalysisException {
23392338
if (snapshotId == IcebergUtils.UNKNOWN_SNAPSHOT_ID) {
23402339
return IcebergPartitionInfo.empty();
23412340
}
23422341
List<IcebergPartition> icebergPartitions;
2342+
List<Column> partitionColumns;
23432343
try {
23442344
icebergPartitions = authenticator.execute(() -> loadIcebergPartition(table, snapshotId));
2345+
// A retained projection may outlive its catalog cache group. Its columns and ranges
2346+
// must use the same frozen table/schema, without looking up a new catalog generation.
2347+
partitionColumns = authenticator.execute(() -> buildTableSchemaCacheValue(table,
2348+
parseSchema(Preconditions.checkNotNull(schemaForTable(table, schemaId),
2349+
"Schema for Iceberg table %s is null", displayName),
2350+
enableMappingVarbinary, enableMappingTimestampTz)).getPartitionColumns());
23452351
} catch (Exception e) {
23462352
String errorMsg = String.format("Failed to get iceberg partition info, table: %s, snapshotId: %s",
23472353
displayName, snapshotId);
@@ -2352,9 +2358,6 @@ static IcebergPartitionInfo loadPartitionInfo(org.apache.doris.datasource.NameMa
23522358
Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
23532359
long retainedPayloadBytes = 0L;
23542360

2355-
List<Column> partitionColumns = cache.getIcebergSchemaCacheValue(
2356-
nameMapping, schemaId, table, authenticator,
2357-
enableMappingVarbinary, enableMappingTimestampTz).getPartitionColumns();
23582361
for (IcebergPartition partition : icebergPartitions) {
23592362
nameToPartition.put(partition.getPartitionName(), partition);
23602363
retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java‎

Lines changed: 86 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,7 @@ public void testSnapshotPartitionLoadUsesCapturedAuthenticatorAndProjectionGener
816816
try (MockedStatic<IcebergUtils> icebergUtils = Mockito.mockStatic(
817817
IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
818818
icebergUtils.when(() -> IcebergUtils.loadPartitionInfo(
819-
tableIdentity, cache, "ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false))
819+
"ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false))
820820
.thenReturn(IcebergPartitionInfo.empty());
821821
icebergUtils.when(() -> IcebergUtils.getNameMapping(projectionTable))
822822
.thenReturn(Optional.empty());
@@ -829,10 +829,10 @@ public void testSnapshotPartitionLoadUsesCapturedAuthenticatorAndProjectionGener
829829
null, false, capturedAuthenticator, true, false);
830830

831831
icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo(
832-
tableIdentity, cache, "ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false), Mockito.never());
832+
"ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false), Mockito.never());
833833
value.getPartitionInfo();
834834
icebergUtils.verify(() -> IcebergUtils.loadPartitionInfo(
835-
tableIdentity, cache, "ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false));
835+
"ice.db.tbl", projectionTable, 11L, 3L, capturedAuthenticator, true, false));
836836
icebergUtils.verify(() -> IcebergUtils.isValidRelatedTable(projectionTable));
837837
Mockito.verify(dorisTable, Mockito.never()).isValidRelatedTable();
838838
} finally {
@@ -867,9 +867,8 @@ private void assertMvBulkPartitionSnapshotsReuseUnadmittedView(Map<String, Strin
867867
ExecutionAuthenticator authenticator = new ExecutionAuthenticator() {};
868868
try (MockedStatic<IcebergUtils> utils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
869869
utils.when(() -> IcebergUtils.isValidRelatedTable(Mockito.any(Table.class))).thenReturn(true);
870-
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq(identity), Mockito.same(cache),
871-
Mockito.eq("ice.db.tbl"), Mockito.any(Table.class), Mockito.anyLong(), Mockito.anyLong(),
872-
Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
870+
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq("ice.db.tbl"), Mockito.any(Table.class),
871+
Mockito.anyLong(), Mockito.anyLong(), Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
873872
.thenAnswer(call -> {
874873
loads.incrementAndGet();
875874
return info;
@@ -957,9 +956,8 @@ public void testOldPartitionLoadsCannotRepublishAfterTableReplacement() throws E
957956
ExecutionAuthenticator authenticator = new ExecutionAuthenticator() {};
958957
try (MockedStatic<IcebergUtils> utils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
959958
utils.when(() -> IcebergUtils.isValidRelatedTable(Mockito.any(Table.class))).thenReturn(true);
960-
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq(identity), Mockito.same(cache),
961-
Mockito.eq("ice.db.tbl"), Mockito.any(Table.class), Mockito.anyLong(), Mockito.anyLong(),
962-
Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
959+
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq("ice.db.tbl"), Mockito.any(Table.class),
960+
Mockito.anyLong(), Mockito.anyLong(), Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
963961
.thenAnswer(call -> {
964962
loads.incrementAndGet();
965963
if (replaceDuringLoad.getAndSet(false)) {
@@ -1058,6 +1056,83 @@ private IcebergSnapshotCacheValue partitionProjectionForTest(IcebergExternalMeta
10581056
retainedTable, retainedTable, null, false, authenticator, false, false);
10591057
}
10601058

1059+
@Test
1060+
public void testLazyPartitionLoadSurvivesCatalogReset() throws Exception {
1061+
assertRetainedPartitionLoadAfterReset(false, false);
1062+
}
1063+
1064+
@Test
1065+
public void testPartitionProjectionCreatedAfterCatalogReset() throws Exception {
1066+
assertRetainedPartitionLoadAfterReset(true, false);
1067+
}
1068+
1069+
@Test
1070+
public void testSessionPartitionProjectionNeedsNoCatalogCacheGroup() throws Exception {
1071+
assertRetainedPartitionLoadAfterReset(true, true);
1072+
}
1073+
1074+
private void assertRetainedPartitionLoadAfterReset(boolean resetBeforeProjection, boolean sessionCatalog)
1075+
throws Exception {
1076+
ExecutorService executor = Executors.newSingleThreadExecutor();
1077+
IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor);
1078+
NameMapping identity = NameMapping.createForTest(1L, "db", "tbl");
1079+
IcebergExternalTable dorisTable = partitionConsumerForTest(identity);
1080+
Schema schema = new Schema(Types.NestedField.optional(1, "dt", Types.DateType.get()));
1081+
PartitionSpec spec = PartitionSpec.builderFor(schema).day("dt").build();
1082+
String location = temporaryFolder.newFolder("retained-partition-reset").toURI().toString();
1083+
Table table = new HadoopTables(new Configuration()).create(schema, spec, location);
1084+
PartitionData partition = new PartitionData(spec.rawPartitionType());
1085+
partition.set(0, 19724);
1086+
table.newFastAppend().appendFile(DataFiles.builder(spec)
1087+
.withPath(location + "/data/old.parquet").withPartition(partition)
1088+
.withFileSizeInBytes(10).withRecordCount(1).build()).commit();
1089+
AtomicInteger authenticatedCalls = new AtomicInteger();
1090+
ExecutionAuthenticator captured = new ExecutionAuthenticator() {
1091+
@Override
1092+
public <T> T execute(Callable<T> action) throws Exception {
1093+
authenticatedCalls.incrementAndGet();
1094+
return action.call();
1095+
}
1096+
};
1097+
IcebergTableCacheValue generation = new IcebergTableCacheValue(table);
1098+
try (IcebergTableCacheValue.Lease retained = generation.tryAcquire();
1099+
MockedStatic<IcebergUtils> utils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
1100+
Assert.assertNotNull(retained);
1101+
utils.when(() -> IcebergUtils.useSessionCatalog(dorisTable)).thenReturn(sessionCatalog);
1102+
if (!sessionCatalog) {
1103+
cache.initCatalog(1L, Collections.emptyMap());
1104+
cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE,
1105+
NameMapping.class, IcebergTableCacheValue.class)
1106+
.put(identity, generation);
1107+
if (resetBeforeProjection) {
1108+
cache.invalidateCatalog(1L);
1109+
}
1110+
}
1111+
IcebergSnapshotCacheValue projection = partitionProjectionForTest(
1112+
cache, dorisTable, retained.getIcebergTable(), captured);
1113+
Assert.assertEquals("partition enumeration remains lazy", 0, authenticatedCalls.get());
1114+
if (!resetBeforeProjection) {
1115+
cache.invalidateCatalog(1L);
1116+
}
1117+
IcebergPartitionInfo info = projection.getPartitionInfo();
1118+
Assert.assertEquals(1, info.getNameToPartitionItem().size());
1119+
Assert.assertEquals(1, info.getNameToIcebergPartition().size());
1120+
Assert.assertEquals(table.currentSnapshot().snapshotId(),
1121+
info.getNameToIcebergPartition().values().iterator().next().getLastSnapshotId());
1122+
Assert.assertTrue(authenticatedCalls.get() > 0);
1123+
Mockito.verify(dorisTable.getCatalog(), Mockito.never()).getExecutionAuthenticator();
1124+
// Retired work must not populate the next catalog generation's budgeted entries.
1125+
cache.initCatalog(1L, Collections.emptyMap());
1126+
Assert.assertNull(cache.entry(1L, IcebergExternalMetaCache.ENTRY_PARTITION,
1127+
IcebergSnapshotEntryKey.class, IcebergPartitionInfo.class)
1128+
.peekIfPresent(IcebergSnapshotEntryKey.tryCreate(identity, table).get()));
1129+
} finally {
1130+
cache.close();
1131+
generation.retire();
1132+
executor.shutdownNow();
1133+
}
1134+
}
1135+
10611136
@Test
10621137
public void testPartitionViewLoadsOncePerPhysicalGenerationAndRefreshInvalidatesIt() throws Exception {
10631138
ExecutorService executor = Executors.newSingleThreadExecutor();
@@ -1083,9 +1158,8 @@ public void testPartitionViewLoadsOncePerPhysicalGenerationAndRefreshInvalidates
10831158
loader.setAccessible(true);
10841159
try (MockedStatic<IcebergUtils> utils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) {
10851160
utils.when(() -> IcebergUtils.isValidRelatedTable(Mockito.any(Table.class))).thenReturn(true);
1086-
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq(identity), Mockito.same(cache),
1087-
Mockito.eq("ice.db.tbl"), Mockito.any(Table.class), Mockito.anyLong(), Mockito.anyLong(),
1088-
Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
1161+
utils.when(() -> IcebergUtils.loadPartitionInfo(Mockito.eq("ice.db.tbl"), Mockito.any(Table.class),
1162+
Mockito.anyLong(), Mockito.anyLong(), Mockito.same(authenticator), Mockito.eq(false), Mockito.eq(false)))
10891163
.thenAnswer(call -> {
10901164
loads.incrementAndGet();
10911165
return IcebergPartitionInfo.empty();

0 commit comments

Comments
 (0)