Skip to content

Commit 350e47d

Browse files
committed
[fix](fe) Reuse and retire lazy Iceberg partition views
### What problem does this PR solve? Issue Number: None Related PR: #69299 Problem Summary: Moving Iceberg partition views to a separate cache caused an MV refresh to enumerate the complete view once per requested partition whenever caching was disabled or weight admission rejected the view. Table-generation replacement also left obsolete partition views behind, and a previously pinned projection could republish an old view after retirement. Load one view for the native/HMS Iceberg bulk MV operation, share the existing version fallback, and preserve per-partition failures. Retire obsolete physical-generation entries during replacement and prevent stale lazy loads from republishing them. Same-generation reuse and cache accounting remain intact. ### Release note Reduce repeated Iceberg metadata loading during materialized-view refresh and retire obsolete partition-cache views after table metadata refresh. ### Check List (For Author) - Test: Unit Test - Reproduced all four lifecycle/load-count regressions before the fix. - All 197 related FE tests pass, including native/HMS bulk reads, disabled/rejected cache admission, replacement and late loads, version fallback and per-partition errors. - Full FE packaging and Checkstyle pass through build.sh --fe -j4 (UI build disabled). - Behavior changed: Yes (reuse operation-local partition views and retire obsolete cached generations) - Does this need documentation: No
1 parent 4471ebb commit 350e47d

7 files changed

Lines changed: 319 additions & 32 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1102,6 +1102,9 @@ public Map<String, MTMVSnapshotIf> getPartitionSnapshots(Set<String> partitionNa
11021102
if (dlaTable instanceof HiveDlaTable) {
11031103
return ((HiveDlaTable) dlaTable).getPartitionSnapshots(partitionNames, snapshot);
11041104
}
1105+
if (dlaTable instanceof IcebergDlaTable) {
1106+
return IcebergUtils.getPartitionSnapshots(this, partitionNames, context, snapshot);
1107+
}
11051108
return MTMVRelatedTableIf.super.getPartitionSnapshots(partitionNames, context, snapshot);
11061109
}
11071110

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -68,21 +68,8 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
6868
@Override
6969
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
7070
Optional<MvccSnapshot> snapshot) throws AnalysisException {
71-
IcebergSnapshotCacheValue snapshotValue = IcebergUtils.getSnapshotCacheValue(snapshot, hmsTable);
72-
long latestSnapshotId = snapshotValue.getPartitionInfo().getLatestSnapshotId(partitionName);
73-
// If partition snapshot ID is unavailable (<= 0), fallback to table snapshot ID
74-
// This can happen when last_updated_snapshot_id is null in Iceberg metadata
75-
if (latestSnapshotId <= 0) {
76-
long tableSnapshotId = snapshotValue.getSnapshot().getSnapshotId();
77-
// If table snapshot ID is also invalid, it means empty table
78-
if (tableSnapshotId <= 0) {
79-
throw new AnalysisException("can not find partition: " + partitionName
80-
+ ", and table snapshot ID is also invalid");
81-
}
82-
// Use table snapshot ID as fallback when partition snapshot ID is unavailable
83-
return new MTMVSnapshotIdSnapshot(tableSnapshotId);
84-
}
85-
return new MTMVSnapshotIdSnapshot(latestSnapshotId);
71+
return IcebergUtils.getPartitionSnapshot(partitionName,
72+
IcebergUtils.getSnapshotCacheValue(snapshot, hmsTable));
8673
}
8774

8875
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -975,6 +975,14 @@ private void retireTableGeneration(NameMapping nameMapping,
975975
snapshots.invalidateIf((key, value) -> key.getNameMapping().equals(nameMapping)
976976
&& (!key.belongsTo(currentValue) || !sharesOperationalResources(currentValue, value)));
977977
}
978+
MetaCacheEntry<IcebergSnapshotEntryKey, IcebergPartitionInfo> partitions =
979+
partitionEntry.getIfInitialized(nameMapping.getCtlId());
980+
if (partitions != null) {
981+
// Partition maps retain no FileIO; preserve reuse across handles of the same
982+
// physical generation, but retire maps no latest reader can reach anymore.
983+
partitions.invalidateIf(key -> key.getNameMapping().equals(nameMapping)
984+
&& !key.belongsTo(currentValue));
985+
}
978986
Optional<String> currentUuid = currentValue.getTableUuid();
979987
MetaCacheEntry<IcebergSchemaCacheKey, SchemaCacheValue> schemas =
980988
schemaEntry.getIfInitialized(nameMapping.getCtlId());
@@ -1213,7 +1221,7 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(
12131221
}
12141222
};
12151223
value.withPartitionInfoLoader(key.isPresent() && !IcebergUtils.useSessionCatalog(dorisTable)
1216-
? () -> partitions.get(key.get(), ignored -> loader.get()) : loader);
1224+
? () -> getPartitionInfo(key.get(), partitions, loader) : loader);
12171225
}
12181226
return value;
12191227
} catch (AnalysisException e) {
@@ -1224,6 +1232,36 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(
12241232
}
12251233
}
12261234

1235+
private IcebergPartitionInfo getPartitionInfo(IcebergSnapshotEntryKey key,
1236+
MetaCacheEntry<IcebergSnapshotEntryKey, IcebergPartitionInfo> partitions,
1237+
Supplier<IcebergPartitionInfo> loader) {
1238+
if (!canCachePartitionInfo(key)) {
1239+
// A pinned old projection can still read its immutable generation after retirement.
1240+
return loader.get();
1241+
}
1242+
IcebergPartitionInfo info = partitions.get(key, ignored -> loader.get());
1243+
if (!canCachePartitionInfo(key)) {
1244+
// Replacement can race the load, including a count-mode contextual miss. Do not leave
1245+
// the superseded map published after returning it to the query that still owns it.
1246+
partitions.invalidateKeyIfSame(key, info);
1247+
}
1248+
return info;
1249+
}
1250+
1251+
private boolean canCachePartitionInfo(IcebergSnapshotEntryKey key) {
1252+
MetaCacheEntry<NameMapping, IcebergTableCacheValue> tables =
1253+
tableEntry.getIfInitialized(key.getNameMapping().getCtlId());
1254+
if (tables == null) {
1255+
return false;
1256+
}
1257+
if (!tables.isEffectivelyEnabled()) {
1258+
// A disabled table entry never publishes handles; physical keys still permit reuse.
1259+
return true;
1260+
}
1261+
IcebergTableCacheValue current = tables.peekIfPresent(key.getNameMapping());
1262+
return current != null && key.belongsTo(current);
1263+
}
1264+
12271265
private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) {
12281266
if (catalog instanceof HMSExternalCatalog) {
12291267
return ((HMSExternalCatalog) catalog).getIcebergMetadataOps();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -221,21 +221,13 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
221221
@Override
222222
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
223223
Optional<MvccSnapshot> snapshot) throws AnalysisException {
224-
IcebergSnapshotCacheValue snapshotValue = IcebergUtils.getSnapshotCacheValue(snapshot, this);
225-
long latestSnapshotId = snapshotValue.getPartitionInfo().getLatestSnapshotId(partitionName);
226-
// If partition snapshot ID is unavailable (<= 0), fallback to table snapshot ID
227-
// This can happen when last_updated_snapshot_id is null in Iceberg metadata
228-
if (latestSnapshotId <= 0) {
229-
long tableSnapshotId = snapshotValue.getSnapshot().getSnapshotId();
230-
// If table snapshot ID is also invalid, it means empty table
231-
if (tableSnapshotId <= 0) {
232-
throw new AnalysisException("can not find partition: " + partitionName
233-
+ ", and table snapshot ID is also invalid");
234-
}
235-
// Use table snapshot ID as fallback when partition snapshot ID is unavailable
236-
return new MTMVSnapshotIdSnapshot(tableSnapshotId);
237-
}
238-
return new MTMVSnapshotIdSnapshot(latestSnapshotId);
224+
return IcebergUtils.getPartitionSnapshot(partitionName, IcebergUtils.getSnapshotCacheValue(snapshot, this));
225+
}
226+
227+
@Override
228+
public Map<String, MTMVSnapshotIf> getPartitionSnapshots(Set<String> partitionNames,
229+
MTMVRefreshContext context, Optional<MvccSnapshot> snapshot) throws AnalysisException {
230+
return IcebergUtils.getPartitionSnapshots(this, partitionNames, context, snapshot);
239231
}
240232

241233
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@
6565
import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
6666
import org.apache.doris.datasource.property.storage.S3Properties;
6767
import org.apache.doris.datasource.property.storage.StorageProperties;
68+
import org.apache.doris.mtmv.MTMVRefreshContext;
69+
import org.apache.doris.mtmv.MTMVRelatedTableIf;
70+
import org.apache.doris.mtmv.MTMVSnapshotIdSnapshot;
71+
import org.apache.doris.mtmv.MTMVSnapshotIf;
6872
import org.apache.doris.nereids.exceptions.NotSupportedException;
6973
import org.apache.doris.nereids.trees.expressions.literal.Result;
7074
import org.apache.doris.nereids.types.VarBinaryType;
@@ -158,6 +162,7 @@
158162
import java.util.Comparator;
159163
import java.util.EnumMap;
160164
import java.util.HashMap;
165+
import java.util.LinkedHashMap;
161166
import java.util.LinkedHashSet;
162167
import java.util.List;
163168
import java.util.Locale;
@@ -2637,6 +2642,45 @@ public static <T> T withSnapshotCacheValue(Optional<MvccSnapshot> snapshot,
26372642
return withLatestSnapshotCacheValue(dorisTable, action);
26382643
}
26392644

2645+
/** Derive all MV partition versions from one view without retaining an unbudgeted map. */
2646+
public static Map<String, MTMVSnapshotIf> getPartitionSnapshots(ExternalTable dorisTable,
2647+
Set<String> partitionNames, MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
2648+
throws AnalysisException {
2649+
Map<String, MTMVSnapshotIf> versions = new LinkedHashMap<>();
2650+
if (partitionNames.isEmpty()) {
2651+
return versions;
2652+
}
2653+
IcebergSnapshotCacheValue value = getSnapshotCacheValue(snapshot, dorisTable);
2654+
IcebergPartitionInfo info = value.getPartitionInfo();
2655+
for (String name : partitionNames) {
2656+
try {
2657+
versions.put(name, partitionSnapshotOf(name, info, value.getSnapshot().getSnapshotId()));
2658+
} catch (AnalysisException e) {
2659+
context.recordPartitionSnapshotFailure((MTMVRelatedTableIf) dorisTable, name, e);
2660+
}
2661+
}
2662+
return versions;
2663+
}
2664+
2665+
public static MTMVSnapshotIf getPartitionSnapshot(String partitionName, IcebergSnapshotCacheValue value)
2666+
throws AnalysisException {
2667+
return partitionSnapshotOf(partitionName, value.getPartitionInfo(), value.getSnapshot().getSnapshotId());
2668+
}
2669+
2670+
private static MTMVSnapshotIf partitionSnapshotOf(String partitionName,
2671+
IcebergPartitionInfo info, long tableSnapshotId) throws AnalysisException {
2672+
long latestSnapshotId = info.getLatestSnapshotId(partitionName);
2673+
// last_updated_snapshot_id can be absent; retain the table-snapshot fallback for MV refresh.
2674+
if (latestSnapshotId <= 0) {
2675+
if (tableSnapshotId <= 0) {
2676+
throw new AnalysisException("can not find partition: " + partitionName
2677+
+ ", and table snapshot ID is also invalid");
2678+
}
2679+
return new MTMVSnapshotIdSnapshot(tableSnapshotId);
2680+
}
2681+
return new MTMVSnapshotIdSnapshot(latestSnapshotId);
2682+
}
2683+
26402684
public static IcebergSnapshotCacheValue getSnapshotCacheValue(Optional<MvccSnapshot> snapshot,
26412685
ExternalTable dorisTable) {
26422686
if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,8 @@ private void loadSnapshots(MTMVRelatedTableIf table, Set<String> partitionNames)
135135
knownMissing.addAll(stillMissing);
136136
}
137137

138-
void recordPartitionSnapshotFailure(
138+
/** Preserve a per-partition failure reported by a related table's bulk loader. */
139+
public void recordPartitionSnapshotFailure(
139140
MTMVRelatedTableIf table, String partitionName, AnalysisException failure) {
140141
partitionSnapshotFailureCache.computeIfAbsent(table, ignored -> new LinkedHashMap<>())
141142
.put(partitionName, failure);

0 commit comments

Comments
 (0)