Skip to content

Commit 0976b2c

Browse files
authored
[opt](sql-cache) Preserve SQL cache across unrelated partition writes (#69883)
### What problem does this PR solve? Issue Number: None Problem Summary: SQL cache currently invalidates an OLAP query whenever the table visible version changes. For example, a cached query reading historical partition p1 is discarded after an insert into unrelated partition p2, even though its result is unchanged. Record the partition versions actually assigned to the scan, including selected empty partitions removed during planning. When the table version advances, reuse the cache only if those dependencies and the schema/partition topology still match. Use the same stable dependency identity for the BE cache key. Keep table-wide validation for metadata aggregates, variable-dependent keys and materialized-view rewrites whose physical scans can hide source dependencies. The change is FE-only and preserves the existing BE cache protocol. Regression cases use only 3–5 source rows and cover unrelated/relevant writes, empty partitions and branches, self joins, partition DDL, partial MV rewrites and eliminated empty MVs.
1 parent 38a38dd commit 0976b2c

15 files changed

Lines changed: 984 additions & 13 deletions

‎fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -451,26 +451,28 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext,
451451
TableIf tableIf = findTableIf(env, scanTable.getKey().tableName);
452452
if (tableIf == null) {
453453
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
454-
} else if (tableIf.isTemporary()) {
454+
}
455+
if (tableIf.isTemporary()) {
455456
return IsChanged.CHANGED_BUT_NOT_INVALIDATE_CACHE;
456-
} else if (tableVersion.id != tableIf.getId()) { // should after isTemporary condition
457+
}
458+
if (tableVersion.id != tableIf.getId()) { // should after isTemporary condition
457459
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
458460
}
459461
if (tableIf instanceof OlapTable) {
460462
OlapTable olapTable = (OlapTable) tableIf;
461-
long currentTableVersion = 0L;
462463
try {
463-
currentTableVersion = olapTable.getVisibleVersion();
464+
long currentTableVersion = olapTable.getVisibleVersion();
465+
boolean versionMatches = currentTableVersion == tableVersion.version;
466+
if (tableVersion.olapVersion != null && tableVersion.olapVersion.isUsable()) {
467+
versionMatches = tableVersion.olapVersion.matches(olapTable, currentTableVersion);
468+
}
469+
if (!versionMatches) {
470+
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
471+
}
464472
} catch (RpcException e) {
465473
LOG.warn("table {}, in cloud getVisibleVersion exception", olapTable.getName(), e);
466474
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
467475
}
468-
long cacheTableVersion = tableVersion.version;
469-
// some partitions have been dropped, or delete or updated or replaced,
470-
// or insert rows into new partition?
471-
if (currentTableVersion != cacheTableVersion) {
472-
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
473-
}
474476
if (tableIf instanceof MTMV) {
475477
// mtmv maybe access old data when grace_period > 0, we should disable cache at this case
476478
long gracePeriod = ((MTMV) tableIf).getGracePeriod();

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.analysis.StatementBase;
2323
import org.apache.doris.catalog.Column;
2424
import org.apache.doris.catalog.Env;
25+
import org.apache.doris.catalog.OlapTableWrapper;
2526
import org.apache.doris.catalog.TableIf;
2627
import org.apache.doris.common.Config;
2728
import org.apache.doris.common.FeConstants;
@@ -74,6 +75,7 @@
7475
import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache;
7576
import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
7677
import org.apache.doris.planner.AddLocalExchange;
78+
import org.apache.doris.planner.OlapScanNode;
7779
import org.apache.doris.planner.PlanFragment;
7880
import org.apache.doris.planner.PlanNodeId;
7981
import org.apache.doris.planner.Planner;
@@ -740,6 +742,18 @@ protected void splitFragments(PhysicalPlan resultPlan) {
740742
// update scan nodes visible version at the end of plan phase for cloud mode.
741743
try {
742744
ScanNode.setVisibleVersionForOlapScanNodes(getScanNodes());
745+
statementContext.getSqlCacheContext().ifPresent(cache -> {
746+
for (ScanNode scan : getScanNodes()) {
747+
if (!(scan instanceof OlapScanNode)) {
748+
continue;
749+
}
750+
OlapScanNode olapScan = (OlapScanNode) scan;
751+
if (olapScan.getOlapTable() instanceof OlapTableWrapper) {
752+
continue;
753+
}
754+
cache.addOlapScan(olapScan.getOlapTable(), olapScan.getSqlCachePartitionVersions());
755+
}
756+
});
743757
} catch (UserException ue) {
744758
throw new NereidsException(ue.getMessage(), ue);
745759
}
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids;
19+
20+
import org.apache.doris.catalog.MaterializedIndexMeta;
21+
import org.apache.doris.catalog.OlapTable;
22+
import org.apache.doris.catalog.Partition;
23+
import org.apache.doris.cloud.catalog.CloudPartition;
24+
import org.apache.doris.common.Config;
25+
import org.apache.doris.common.Pair;
26+
import org.apache.doris.rpc.RpcException;
27+
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Set;
32+
import java.util.TreeMap;
33+
34+
/** Metadata and partition dependencies of a partitioned OLAP SQL cache entry. */
35+
public class OlapTableSqlCacheVersion {
36+
private final long tableVersion;
37+
private final long baseIndexId;
38+
private final long topologyVersion;
39+
private final Map<Long, Pair<Integer, Integer>> schemas;
40+
private final Map<Long, String> partitionNames = new TreeMap<>();
41+
// Include empty partitions: pruning can remove their scan nodes before they receive their first rows.
42+
private final Map<Long, Pair<Long, Long>> partitionVersions = new TreeMap<>();
43+
private boolean hasScan;
44+
private boolean usable = true;
45+
private volatile long validatedTableVersion;
46+
47+
/** Capture table metadata while the planner holds the table read lock. */
48+
public OlapTableSqlCacheVersion(OlapTable table, long tableVersion) {
49+
this.tableVersion = tableVersion;
50+
this.validatedTableVersion = tableVersion;
51+
this.baseIndexId = table.getBaseIndexId();
52+
this.topologyVersion = table.getPartitionTopologyVersion();
53+
this.schemas = schemaVersions(table);
54+
}
55+
56+
private static Map<Long, Pair<Integer, Integer>> schemaVersions(OlapTable table) {
57+
Map<Long, Pair<Integer, Integer>> versions = new TreeMap<>();
58+
for (Map.Entry<Long, MaterializedIndexMeta> entry : table.getIndexIdToMeta().entrySet()) {
59+
MaterializedIndexMeta meta = entry.getValue();
60+
versions.put(entry.getKey(), Pair.of(meta.getSchemaVersion(), meta.getSchemaHash()));
61+
}
62+
return versions;
63+
}
64+
65+
private static Pair<Long, Long> version(Partition partition, long version) {
66+
// MS has no update timestamp for an empty partition and FE may synthesize one on every refresh.
67+
return Pair.of(version, version == Partition.PARTITION_INIT_VERSION ? 0L : partition.getVisibleVersionTime());
68+
}
69+
70+
/** Track negative dependencies removed by PruneEmptyPartition, not unrelated empty partitions. */
71+
public void addEmptyPartitions(OlapTable table, Set<Long> partitionIds) {
72+
for (Long id : partitionIds) {
73+
Partition partition = table.getPartition(id);
74+
if (table.isTemporaryPartition(id)
75+
|| partition.getCachedVisibleVersion() != Partition.PARTITION_INIT_VERSION) {
76+
usable = false;
77+
return;
78+
}
79+
partitionNames.put(id, partition.getName());
80+
partitionVersions.put(id, version(partition, Partition.PARTITION_INIT_VERSION));
81+
}
82+
}
83+
84+
private boolean metadataMatches(OlapTable table) {
85+
return table.getBaseIndexId() == baseIndexId
86+
&& table.getPartitionTopologyVersion() == topologyVersion
87+
&& schemas.equals(schemaVersions(table));
88+
}
89+
90+
/** Record the versions actually assigned to the BE scan, before executing the query. */
91+
public void addScan(OlapTable table, Map<Long, Long> scanVersions) throws RpcException {
92+
if (!usable) {
93+
return;
94+
}
95+
long currentVersion = table.getVisibleVersion();
96+
// Native versions are protected by the statement's table read lock; the extra lock is cloud-only.
97+
if (Config.isCloudMode()) {
98+
table.versionReadLock();
99+
}
100+
try {
101+
// Do not attach a newer snapshot to a plan analyzed before a concurrent commit or DDL.
102+
if (currentVersion != tableVersion || !metadataMatches(table)) {
103+
usable = false;
104+
return;
105+
}
106+
for (Map.Entry<Long, Long> entry : scanVersions.entrySet()) {
107+
Partition partition = table.getPartition(entry.getKey());
108+
if (table.isTemporaryPartition(entry.getKey())
109+
|| partition.getCachedVisibleVersion() != entry.getValue()) {
110+
usable = false;
111+
return;
112+
}
113+
partitionNames.put(entry.getKey(), partition.getName());
114+
// Singleton cloud refreshes can advance the cache despite the table's version read lock.
115+
// Keep the version assigned to the scan, rather than reading the mutable cache again.
116+
Pair<Long, Long> scanned = version(partition, entry.getValue());
117+
Pair<Long, Long> previous = partitionVersions.putIfAbsent(entry.getKey(), scanned);
118+
if (previous != null && !previous.equals(scanned)) {
119+
usable = false;
120+
return;
121+
}
122+
hasScan = true;
123+
}
124+
} finally {
125+
if (Config.isCloudMode()) {
126+
table.versionReadUnlock();
127+
}
128+
}
129+
}
130+
131+
/** Keep table-wide invalidation when some dependencies have no ordinary scan ranges. */
132+
public void useTableVersion() {
133+
usable = false;
134+
}
135+
136+
public boolean isUsable() {
137+
return usable && hasScan;
138+
}
139+
140+
private static List<Long> getPartitionVersions(List<Partition> partitions) throws RpcException {
141+
if (!Config.isCloudMode()) {
142+
return Partition.getVisibleVersions(partitions);
143+
}
144+
List<CloudPartition> cloudPartitions = new ArrayList<>(partitions.size());
145+
for (Partition partition : partitions) {
146+
cloudPartitions.add((CloudPartition) partition);
147+
}
148+
// A fresh table version must not be validated against stale FE partition-version caches.
149+
return CloudPartition.getSnapshotVisibleVersionFromMs(cloudPartitions);
150+
}
151+
152+
/** Check metadata first, and refresh partition dependencies when the table's data version changes. */
153+
public boolean matches(OlapTable table, long currentVersion) throws RpcException {
154+
if (!metadataMatches(table)) {
155+
return false;
156+
}
157+
if (currentVersion == validatedTableVersion) {
158+
return true;
159+
}
160+
List<Partition> partitions = new ArrayList<>(partitionVersions.size());
161+
for (Long id : partitionVersions.keySet()) {
162+
partitions.add(table.getPartition(id));
163+
}
164+
List<Long> versions = getPartitionVersions(partitions);
165+
// Native versions are protected by the statement's table read lock; the extra lock is cloud-only.
166+
if (Config.isCloudMode()) {
167+
table.versionReadLock();
168+
}
169+
try {
170+
for (int i = 0; i < partitions.size(); i++) {
171+
Pair<Long, Long> cached = partitionVersions.get(partitions.get(i).getId());
172+
if (!cached.first.equals(versions.get(i))
173+
|| (cached.first != Partition.PARTITION_INIT_VERSION
174+
&& cached.second != partitions.get(i).getVisibleVersionTime())) {
175+
return false;
176+
}
177+
}
178+
validatedTableVersion = currentVersion;
179+
return true;
180+
} finally {
181+
if (Config.isCloudMode()) {
182+
table.versionReadUnlock();
183+
}
184+
}
185+
}
186+
187+
/** A stable BE key for identical metadata and partition dependencies, independent of other writes. */
188+
public String getCacheKey() {
189+
return "partitions-v1:" + baseIndexId + ":" + schemas + ":" + partitionNames + ":" + partitionVersions;
190+
}
191+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.catalog.DatabaseIf;
2525
import org.apache.doris.catalog.MTMV;
2626
import org.apache.doris.catalog.OlapTable;
27+
import org.apache.doris.catalog.OlapTableWrapper;
2728
import org.apache.doris.catalog.TableIf;
2829
import org.apache.doris.catalog.TableIf.TableType;
2930
import org.apache.doris.common.Pair;
@@ -44,6 +45,7 @@
4445
import org.apache.doris.qe.SessionVariable;
4546
import org.apache.doris.qe.cache.CacheProxy;
4647
import org.apache.doris.qe.cache.SqlCacheVersionProvider;
48+
import org.apache.doris.rpc.RpcException;
4749
import org.apache.doris.thrift.TUniqueId;
4850

4951
import com.google.common.collect.ImmutableList;
@@ -244,7 +246,11 @@ public synchronized void addUsedTable(TableIf tableIf, Optional<MvccSnapshot> sn
244246
new TableVersion(
245247
tableIf.getId(),
246248
version,
247-
tableIf.getType(), sqlCacheVersion
249+
tableIf.getType(), sqlCacheVersion,
250+
usedVariables.isEmpty() && tableIf.getType() == TableType.OLAP && tableIf instanceof OlapTable
251+
&& !(tableIf instanceof OlapTableWrapper)
252+
&& ((OlapTable) tableIf).isPartitionedTable()
253+
? new OlapTableSqlCacheVersion((OlapTable) tableIf, version) : null
248254
)
249255
);
250256
}
@@ -318,8 +324,12 @@ public synchronized Map<FullColumnName, Optional<DataMaskSpec>> getDataMaskPolic
318324
return ImmutableMap.copyOf(dataMaskPolicies);
319325
}
320326

327+
/** Record a variable and retain partition-independent lookup keys for its cached value variants. */
321328
public synchronized void addUsedVariable(Variable value) {
322329
usedVariables.add(value);
330+
// Variable-aware FE lookup derives another value's key before planning its partition pruning.
331+
// Keep the existing table-wide key for these queries, including variables resolved after table binding.
332+
useSourceTableVersions();
323333
}
324334

325335
public synchronized List<Variable> getUsedVariables() {
@@ -394,6 +404,57 @@ public Map<TableReference, TableVersion> getUsedTables() {
394404
return Collections.unmodifiableMap(usedTables);
395405
}
396406

407+
/** Record native scan dependencies before the first SQL cache key is computed. */
408+
public synchronized void addOlapScan(OlapTable table, Map<Long, Long> partitionVersions) {
409+
if (table instanceof MTMV) {
410+
// MV substitution can hide source partitions from the final physical scans. Preserve the
411+
// logical source tables' versions before the BE key is computed, including partial UNION rewrites.
412+
useSourceTableVersions();
413+
return;
414+
}
415+
for (TableVersion version : usedTables.values()) {
416+
if (version.id == table.getId() && version.olapVersion != null) {
417+
try {
418+
version.olapVersion.addScan(table, partitionVersions);
419+
} catch (RpcException e) {
420+
setHasUnsupportedTables(true);
421+
LOG.warn("Cannot record SQL cache partitions for {}", table.getName(), e);
422+
}
423+
}
424+
}
425+
}
426+
427+
/** Preserve table-wide validation for metadata aggregates. */
428+
public synchronized void useOlapTableVersion(OlapTable table) {
429+
for (TableVersion version : usedTables.values()) {
430+
if (version.id == table.getId() && version.olapVersion != null) {
431+
version.olapVersion.useTableVersion();
432+
}
433+
}
434+
}
435+
436+
/** Retain empty partitions that were eligible for this query before empty-partition pruning. */
437+
public synchronized void addPrunedOlapPartitions(OlapTable table, Set<Long> partitionIds) {
438+
if (table instanceof MTMV) {
439+
// An empty MV can disappear before physical scan capture, hiding its source dependencies.
440+
useSourceTableVersions();
441+
return;
442+
}
443+
for (TableVersion version : usedTables.values()) {
444+
if (version.id == table.getId() && version.olapVersion != null) {
445+
version.olapVersion.addEmptyPartitions(table, partitionIds);
446+
}
447+
}
448+
}
449+
450+
private void useSourceTableVersions() {
451+
for (TableVersion version : usedTables.values()) {
452+
if (version.olapVersion != null) {
453+
version.olapVersion.useTableVersion();
454+
}
455+
}
456+
}
457+
397458
public Map<FullTableName, String> getUsedViews() {
398459
return ImmutableMap.copyOf(usedViews);
399460
}
@@ -484,8 +545,7 @@ public synchronized PUniqueId doComputeCacheKeyMd5(
484545
.append("=")
485546
.append(tableVersion.id)
486547
.append("@")
487-
.append(tableVersion.sqlCacheVersion == null ? Long.toString(tableVersion.version)
488-
: "sql-cache:" + tableVersion.sqlCacheVersion.getVersion());
548+
.append(tableVersion.getCacheKey());
489549
}
490550
for (Entry<FullTableName, String> entry : usedViews.entrySet()) {
491551
cacheKey.append("|")
@@ -637,10 +697,23 @@ public static class TableVersion {
637697
public final long version;
638698
public final TableType type;
639699
public final ConnectorSqlCacheVersion sqlCacheVersion;
700+
public final OlapTableSqlCacheVersion olapVersion;
701+
702+
public TableVersion(long id, long version, TableType type, ConnectorSqlCacheVersion sqlCacheVersion) {
703+
this(id, version, type, sqlCacheVersion, null);
704+
}
640705

641706
public TableVersion(long id, long version, TableType type) {
642707
this(id, version, type, null);
643708
}
709+
710+
/** Data-version component shared by FE validation and BE cache lookup. */
711+
public String getCacheKey() {
712+
if (olapVersion != null && olapVersion.isUsable()) {
713+
return olapVersion.getCacheKey();
714+
}
715+
return sqlCacheVersion == null ? Long.toString(version) : "sql-cache:" + sqlCacheVersion.getVersion();
716+
}
644717
}
645718

646719
/** CacheKeyType */

0 commit comments

Comments
 (0)