Skip to content

Commit 914257a

Browse files
authored
branch-4.2: [improvement](datasource) Version Iceberg partition views and SQL cache metadata (apache#69299) (apache#69983)
cherry pick from apache#69299
1 parent fa07b7c commit 914257a

20 files changed

Lines changed: 1182 additions & 93 deletions

‎fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import org.apache.doris.datasource.CatalogIf;
3535
import org.apache.doris.datasource.CatalogMgr;
3636
import org.apache.doris.datasource.hive.HMSExternalTable;
37+
import org.apache.doris.datasource.mvcc.MvccReadSelector;
38+
import org.apache.doris.datasource.mvcc.MvccSnapshot;
3739
import org.apache.doris.mysql.privilege.DataMaskPolicy;
3840
import org.apache.doris.mysql.privilege.RowFilterPolicy;
3941
import org.apache.doris.nereids.CascadesContext;
@@ -42,6 +44,7 @@
4244
import org.apache.doris.nereids.SqlCacheContext.FullColumnName;
4345
import org.apache.doris.nereids.SqlCacheContext.FullTableName;
4446
import org.apache.doris.nereids.SqlCacheContext.ScanTable;
47+
import org.apache.doris.nereids.SqlCacheContext.TableReference;
4548
import org.apache.doris.nereids.SqlCacheContext.TableVersion;
4649
import org.apache.doris.nereids.StatementContext;
4750
import org.apache.doris.nereids.analyzer.UnboundVariable;
@@ -67,6 +70,8 @@
6770
import org.apache.doris.qe.StmtExecutor;
6871
import org.apache.doris.qe.cache.CacheAnalyzer;
6972
import org.apache.doris.qe.cache.SqlCache;
73+
import org.apache.doris.qe.cache.SqlCacheVersion;
74+
import org.apache.doris.qe.cache.SqlCacheVersionProvider;
7075
import org.apache.doris.rpc.RpcException;
7176

7277
import com.github.benmanes.caffeine.cache.Cache;
@@ -126,8 +131,8 @@ public void invalidateAboutTable(TableIf tableIf) {
126131
for (Entry<String, SqlCacheContext> kv : sqlCaches.asMap().entrySet()) {
127132
String key = kv.getKey();
128133
SqlCacheContext context = kv.getValue();
129-
for (Entry<FullTableName, TableVersion> nameToVersion : context.getUsedTables().entrySet()) {
130-
FullTableName tableName = nameToVersion.getKey();
134+
for (Entry<TableReference, TableVersion> nameToVersion : context.getUsedTables().entrySet()) {
135+
FullTableName tableName = nameToVersion.getKey().tableName;
131136
TableVersion tableVersion = nameToVersion.getValue();
132137
if (tableVersion.id == tableIf.getId()) {
133138
invalidateKeys.add(key);
@@ -341,7 +346,7 @@ private Optional<LogicalSqlCache> tryParseSql(
341346
if (privilegeChanged(connectContext, env, sqlCacheContext)) {
342347
return invalidateCache(key);
343348
}
344-
IsChanged tableIsChanged = tablesOrDataChanged(env, sqlCacheContext);
349+
IsChanged tableIsChanged = tablesOrDataChanged(env, sqlCacheContext, connectContext.getStatementContext());
345350
if (tableIsChanged.changed) {
346351
if (tableIsChanged.invalidCache) {
347352
return invalidateCache(key);
@@ -427,21 +432,22 @@ private Optional<LogicalSqlCache> tryParseSql(
427432
}
428433
}
429434

430-
private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) {
435+
private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext, StatementContext statement) {
431436
if (sqlCacheContext.hasUnsupportedTables()) {
432437
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
433438
}
434439

435440
// the query maybe scan empty partition of the table, we should check these table version too,
436441
// but the table not exists in sqlCacheContext.getScanTables(), so we need check here.
437442
// check table type and version
438-
for (Entry<FullTableName, TableVersion> scanTable : sqlCacheContext.getUsedTables().entrySet()) {
443+
for (Entry<TableReference, TableVersion> scanTable : sqlCacheContext.getUsedTables().entrySet()) {
439444
TableVersion tableVersion = scanTable.getValue();
440445
if (tableVersion.type != TableType.OLAP && tableVersion.type != TableType.MATERIALIZED_VIEW
441-
&& tableVersion.type != TableType.HMS_EXTERNAL_TABLE) {
446+
&& tableVersion.type != TableType.HMS_EXTERNAL_TABLE
447+
&& tableVersion.type != TableType.ICEBERG_EXTERNAL_TABLE) {
442448
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
443449
}
444-
TableIf tableIf = findTableIf(env, scanTable.getKey());
450+
TableIf tableIf = findTableIf(env, scanTable.getKey().tableName);
445451
if (tableIf == null) {
446452
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
447453
} else if (tableIf.isTemporary()) {
@@ -471,9 +477,24 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext)
471477
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
472478
}
473479
}
480+
} else if (tableVersion.sqlCacheVersion != null) {
481+
MvccReadSelector selector = scanTable.getKey().selector;
482+
Optional<MvccSnapshot> pin = Optional.empty();
483+
if (!selector.isLatest()) {
484+
// Reuse the analyzer's resolver: snapshot existence, timestamp semantics, ref kind,
485+
// branch/tag movement and schema selection all remain owned by the existing MVCC path.
486+
statement.loadSnapshots(tableIf, selector.getTableSnapshot(), selector.getScanParams());
487+
pin = statement.getSnapshot(tableIf, selector.getTableSnapshot(), selector.getScanParams());
488+
}
489+
Optional<SqlCacheVersion> current = ((SqlCacheVersionProvider) tableIf)
490+
.getSqlCacheVersion(pin);
491+
if (!current.isPresent() || !tableVersion.sqlCacheVersion.getVersion()
492+
.equals(current.get().getVersion())) {
493+
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
494+
}
474495
} else if (tableIf instanceof HMSExternalTable) {
475-
HMSExternalTable hiveTable = (HMSExternalTable) tableIf;
476-
if (tableVersion.version != hiveTable.getUpdateTime()) {
496+
// Preserve the existing Hive update-time validation when no version provider is available.
497+
if (tableVersion.version != ((HMSExternalTable) tableIf).getUpdateTime()) {
477498
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
478499
}
479500
} else {
@@ -494,7 +515,8 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext)
494515
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
495516
}
496517
}
497-
} else if (!(tableIf instanceof HMSExternalTable)) {
518+
} else if (!(tableIf instanceof HMSExternalTable)
519+
&& !(tableIf instanceof SqlCacheVersionProvider)) {
498520
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
499521
}
500522
}
@@ -559,7 +581,8 @@ private boolean dataMaskPoliciesChanged(
559581
*/
560582
private boolean tryLockTables(ConnectContext connectContext, Env env, SqlCacheContext sqlCacheContext) {
561583
StatementContext currentStatementContext = connectContext.getStatementContext();
562-
for (FullTableName fullTableName : sqlCacheContext.getUsedTables().keySet()) {
584+
for (TableReference reference : sqlCacheContext.getUsedTables().keySet()) {
585+
FullTableName fullTableName = reference.tableName;
563586
TableIf tableIf = findTableIf(env, fullTableName);
564587
if (tableIf == null) {
565588
return false;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1102,6 +1102,9 @@ public Map<String, MTMVSnapshotIf> getPartitionSnapshots(Set<String> partitionNa
11021102
if (dlaTable instanceof HiveDlaTable) {
11031103
return ((HiveDlaTable) dlaTable).getPartitionSnapshots(partitionNames, snapshot);
11041104
}
1105+
if (dlaTable instanceof IcebergDlaTable) {
1106+
return IcebergUtils.getPartitionSnapshots(this, partitionNames, context, snapshot);
1107+
}
11051108
return MTMVRelatedTableIf.super.getPartitionSnapshots(partitionNames, context, snapshot);
11061109
}
11071110

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -68,21 +68,8 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
6868
@Override
6969
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
7070
Optional<MvccSnapshot> snapshot) throws AnalysisException {
71-
IcebergSnapshotCacheValue snapshotValue = IcebergUtils.getSnapshotCacheValue(snapshot, hmsTable);
72-
long latestSnapshotId = snapshotValue.getPartitionInfo().getLatestSnapshotId(partitionName);
73-
// If partition snapshot ID is unavailable (<= 0), fallback to table snapshot ID
74-
// This can happen when last_updated_snapshot_id is null in Iceberg metadata
75-
if (latestSnapshotId <= 0) {
76-
long tableSnapshotId = snapshotValue.getSnapshot().getSnapshotId();
77-
// If table snapshot ID is also invalid, it means empty table
78-
if (tableSnapshotId <= 0) {
79-
throw new AnalysisException("can not find partition: " + partitionName
80-
+ ", and table snapshot ID is also invalid");
81-
}
82-
// Use table snapshot ID as fallback when partition snapshot ID is unavailable
83-
return new MTMVSnapshotIdSnapshot(tableSnapshotId);
84-
}
85-
return new MTMVSnapshotIdSnapshot(latestSnapshotId);
71+
return IcebergUtils.getPartitionSnapshot(partitionName,
72+
IcebergUtils.getSnapshotCacheValue(snapshot, hmsTable));
8673
}
8774

8875
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ final class IcebergCacheSizeEstimator {
9696
private static final long METADATA_ENTRY_WEIGHT = 128L;
9797
private static final long SNAPSHOT_WEIGHT = 512L;
9898
private static final long CURRENT_SNAPSHOT_WEIGHT = 1024L;
99+
// Reserve the projection's lazy maximum partition-update timestamp and its loaded flag.
100+
private static final long SNAPSHOT_UPDATE_TIME_STATE_WEIGHT = 32L;
99101
private static final long SNAPSHOT_LOG_WEIGHT = 64L;
100102
private static final long METADATA_LOG_WEIGHT = 160L;
101103
private static final long SNAPSHOT_REF_WEIGHT = 256L;
@@ -161,15 +163,15 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCac
161163

162164
static MetaCacheSizeEstimate estimateSnapshotEntry(
163165
IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
164-
long bytes = KEY_BASE_WEIGHT;
166+
long bytes = KEY_BASE_WEIGHT + SNAPSHOT_UPDATE_TIME_STATE_WEIGHT;
165167
bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
166168
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
167169
bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
168170
MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
169171
bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
170172
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
171173

172-
IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
174+
IcebergPartitionInfo partitionInfo = value.getRetainedPartitionInfo();
173175
bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_WEIGHT);
174176
bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_WEIGHT);
175177
bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(),

0 commit comments

Comments
 (0)