Skip to content

Commit 5c17b74

Browse files
committed
[fix](catalog) gate Cache B partition-ranges reuse on a real (non-sentinel) MVCC snapshot id
PluginDrivenMvccExternalTable#getPartitionMetaVersion keys NereidsSortedPartitionsCacheManager (Cache B) on "<snapshotId>@<schemaId>" from the pinned ConnectorMvccSnapshot. Hive's beginQuerySnapshot always pins snapshotId == -1 (hive has no real MVCC snapshot), so for hive that token is a CONSTANT across queries and cannot detect a partition-set change: a cache hit could serve SortedPartitionRanges built from an older partition set than the current pin's nameToPartitionItem (e.g. after a CachingHmsClient TTL refresh with no invalidate event), pruning against a stale range set and dropping partitions from the result. Override getSortedPartitionRanges to return Optional.empty() when the pinned snapshot's snapshotId == -1, so the caller (PruneFileScanPartition) builds ranges fresh every query for hive (and for an empty iceberg/paimon table, which also pins -1 but is already unreachable via the caller's !nameToPartitionItem.isEmpty() guard); a real snapshot id (iceberg/paimon) keeps delegating to super's cache-manager lookup, unchanged.
1 parent d4b3830 commit 5c17b74

2 files changed

Lines changed: 76 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.doris.mtmv.MTMVSnapshotIdSnapshot;
5656
import org.apache.doris.mtmv.MTMVSnapshotIf;
5757
import org.apache.doris.mtmv.MTMVTimestampSnapshot;
58+
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
5859
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
5960
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
6061

@@ -645,6 +646,29 @@ public long getPartitionMetaLoadTimeMillis(CatalogRelation scan) {
645646
return 0L;
646647
}
647648

649+
@Override
650+
public Optional<SortedPartitionRanges<String>> getSortedPartitionRanges(CatalogRelation scan) {
651+
// Cache B (NereidsSortedPartitionsCacheManager) is keyed by getPartitionMetaVersion(scan), i.e.
652+
// "<snapshotId>@<schemaId>". That token only detects a partition-set change when snapshotId is a
653+
// REAL, monotonically-changing value (iceberg/paimon). Hive's beginQuerySnapshot always pins the
654+
// sentinel snapshotId == -1 (hive has no MVCC snapshot), so the token would be the CONSTANT
655+
// "-1@<schemaId>" across queries -- a cache hit could then serve ranges built from an OLDER
656+
// partition set than the current pin's nameToPartitionItem (e.g. after a CachingHmsClient TTL
657+
// refresh with no invalidate event), silently pruning against a stale range set. Skip the cache
658+
// for a -1 (non-MVCC / sentinel) snapshot: return empty so the caller (PruneFileScanPartition)
659+
// builds ranges fresh from THIS query's pin every time, which is always consistent with it -- the
660+
// pre-cache behavior. A real snapshot id (iceberg/paimon) keeps the cache via super's delegation
661+
// to the cache manager. An EMPTY iceberg table also pins snapshotId == -1, but its
662+
// nameToPartitionItem is then empty too, and PruneFileScanPartition's
663+
// "!nameToPartitionItem.isEmpty()" guard already skips ranges entirely -- so gating on -1 here is
664+
// safe for that case as well.
665+
ConnectorMvccSnapshot cs = getOrMaterialize(pinnedSnapshot(scan)).getConnectorSnapshot();
666+
if (cs.getSnapshotId() == -1L) {
667+
return Optional.empty();
668+
}
669+
return super.getSortedPartitionRanges(scan);
670+
}
671+
648672
private Optional<MvccSnapshot> pinnedSnapshot(CatalogRelation scan) {
649673
if (scan instanceof LogicalFileScan) {
650674
LogicalFileScan fileScan = (LogicalFileScan) scan;

‎fe/fe-core/src/test/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManagerExternalTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,58 @@ public void testPluginDrivenMvccExternalTableRealOriginPartitionsAndVersion() th
297297
}
298298
}
299299

300+
// ──────────────────── Hive sentinel snapshot (-1) must skip Cache B ────────────────────
301+
//
302+
// getPartitionMetaVersion's token is "<snapshotId>@<schemaId>". Hive's beginQuerySnapshot always pins
303+
// snapshotId == -1 (no real MVCC snapshot), so that token is a CONSTANT across queries and cannot
304+
// detect a partition-set change. PluginDrivenMvccExternalTable#getSortedPartitionRanges must therefore
305+
// gate on the pinned connector snapshot's snapshotId and return empty (skip Cache B, build fresh every
306+
// query) rather than delegating to the cache manager.
307+
308+
@Test
309+
public void testGetSortedPartitionRangesEmptyForSentinelSnapshotId() throws Exception {
310+
Map<String, PartitionItem> pinnedParts = Maps.newHashMap();
311+
pinnedParts.put("id=1", listItem(1));
312+
// Hive's sentinel: no real MVCC snapshot id.
313+
ConnectorMvccSnapshot sentinelSnapshot = ConnectorMvccSnapshot.builder()
314+
.snapshotId(-1L).schemaId(0L).build();
315+
PluginDrivenMvccSnapshot pin = new PluginDrivenMvccSnapshot(
316+
sentinelSnapshot, pinnedParts, Maps.newHashMap());
317+
318+
PluginDrivenMvccExternalTable table =
319+
Mockito.mock(PluginDrivenMvccExternalTable.class, Mockito.CALLS_REAL_METHODS);
320+
ExternalDatabase<?> database = Mockito.mock(ExternalDatabase.class);
321+
CatalogIf<?> catalog = Mockito.mock(CatalogIf.class);
322+
Mockito.doReturn(TBL).when(table).getName();
323+
Mockito.doReturn(database).when(table).getDatabase();
324+
Mockito.when(database.getFullName()).thenReturn(DB);
325+
Mockito.when(database.getCatalog()).thenReturn((CatalogIf) catalog);
326+
Mockito.when(catalog.getName()).thenReturn(CTL);
327+
Mockito.doReturn(SelectedPartitions.NOT_PRUNED).when(table).initSelectedPartitions(Mockito.any());
328+
329+
ConnectContext ctx = new ConnectContext();
330+
StatementContext stmtCtx = new StatementContext(ctx, null);
331+
ctx.setStatementContext(stmtCtx);
332+
ctx.setThreadLocalInfo();
333+
try {
334+
// B5a implicit query-begin (latest) pin -- mirrors a plain (no @tag/@branch/time-travel) hive scan.
335+
Mockito.doReturn(pin).when(table).loadSnapshot(Optional.empty(), Optional.empty());
336+
stmtCtx.loadSnapshots(table, Optional.empty(), Optional.empty());
337+
338+
LogicalFileScan scan = new LogicalFileScan(new RelationId(1), table,
339+
Collections.singletonList(DB), Collections.emptyList(),
340+
Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
341+
342+
Optional<SortedPartitionRanges<String>> ranges = table.getSortedPartitionRanges(scan);
343+
344+
Assertions.assertFalse(ranges.isPresent(),
345+
"snapshotId == -1 (hive sentinel) must skip Cache B and yield empty, so the caller "
346+
+ "(PruneFileScanPartition) builds ranges fresh from this query's pin");
347+
} finally {
348+
ConnectContext.remove();
349+
}
350+
}
351+
300352
// ──────────────────── Real production wiring: ExternalMetaCacheMgr.invalidateTable ────────────────────
301353

302354
@Test

0 commit comments

Comments
 (0)