Skip to content

Commit 61c910f

Browse files
authored
[fix](fe) Fix stream scan normalization and snapshot/reset pruning (#65468)
### What problem does this PR solve? Issue Number: close #65469 Problem Summary: This PR fixes two FE issues in the stream scan path. First, when a stream scan appears inside a CTE, `NormalizeOlapTableStreamScan` may not be applied on the reused CTE branch, so the stream virtual columns are not lowered consistently and later rewrite steps see the wrong scan shape. Second, `PruneEmptyPartition` uses `selectNonEmptyPartitionIds` before stream scan normalization. For `stream@snapshot` and `stream@reset`, the old stream-wrapper pruning logic still used incremental stream visibility semantics, so a stream with no new delta could be pruned as empty even though snapshot/reset should still read snapshot data. This change keeps stream read mode as explicit logical scan state, threads that mode through `LogicalCatalogRelation` into partition pruning, preserves stream scan helper predicates on the logical node, and lets snapshot/reset pruning follow the base table path while incremental reads still use stream delta visibility. It also keeps stream scan normalization behavior consistent when the scan is referenced through a CTE. As a follow-up for the alias copy path, this PR also fills the remaining `LogicalOlapTableStreamScan` rebuild methods inherited from `LogicalOlapScan`, so alias / virtual-column / topn rewrite paths do not accidentally rebuild a stream scan as a plain `LogicalOlapScan` and skip stream normalization.
1 parent 9b26dc6 commit 61c910f

16 files changed

Lines changed: 351 additions & 63 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.doris.catalog.Tablet.TabletStatus;
3434
import org.apache.doris.catalog.info.IndexType;
3535
import org.apache.doris.catalog.stream.BaseTableStream;
36+
import org.apache.doris.catalog.stream.StreamReadMode;
3637
import org.apache.doris.clone.TabletScheduler;
3738
import org.apache.doris.cloud.catalog.CloudPartition;
3839
import org.apache.doris.cloud.catalog.CloudReplica;
@@ -1564,7 +1565,8 @@ public void getVersionInBatchForCloudMode(Collection<Long> partitionIds) throws
15641565
// select the non-empty partition ids belonging to this table.
15651566
//
15661567
// ATTN: partitions not belonging to this table will be filtered.
1567-
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds) {
1568+
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds,
1569+
Optional<StreamReadMode> streamReadMode) {
15681570
if (Config.isCloudMode() && Config.enable_cloud_snapshot_version) {
15691571
// Assumption: all partitions are CloudPartition.
15701572
List<CloudPartition> partitions = partitionIds.stream()

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTableWrapper.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import org.apache.doris.catalog.stream.StreamReadMode;
2021
import org.apache.doris.common.Pair;
2122

2223
import java.util.Collection;
2324
import java.util.HashMap;
2425
import java.util.List;
2526
import java.util.Map;
2627
import java.util.Objects;
28+
import java.util.Optional;
2729
import java.util.Set;
2830
import java.util.concurrent.TimeUnit;
2931

@@ -146,8 +148,9 @@ public Collection<Partition> getPartitions() {
146148
}
147149

148150
@Override
149-
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds) {
150-
return originTable.selectNonEmptyPartitionIds(partitionIds);
151+
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds,
152+
Optional<StreamReadMode> streamReadMode) {
153+
return originTable.selectNonEmptyPartitionIds(partitionIds, streamReadMode);
151154
}
152155

153156
@Override

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/OlapTableStreamWrapper.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.util.Collection;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.Optional;
3738
import java.util.stream.Collectors;
3839

3940
// runtime-only class for unified query/insert experience, created when bind relation with OlapTableStream
@@ -174,7 +175,12 @@ public Collection<Partition> getPartitions() {
174175
}
175176

176177
@Override
177-
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds) {
178+
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds,
179+
Optional<StreamReadMode> streamReadMode) {
180+
StreamReadMode readMode = streamReadMode.orElse(StreamReadMode.INCREMENTAL);
181+
if (readMode == StreamReadMode.SNAPSHOT || readMode == StreamReadMode.RESET) {
182+
return baseTable.selectNonEmptyPartitionIds(partitionIds, Optional.of(readMode));
183+
}
178184
List<Long> nonEmptyIds = Lists.newArrayListWithCapacity(partitionIds.size());
179185
for (Long partitionId : partitionIds) {
180186
if (stream.hasData(getPartition(partitionId))) {
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog.stream;
19+
20+
/**
21+
* Read mode of a stream scan after binding.
22+
*/
23+
public enum StreamReadMode {
24+
INCREMENTAL,
25+
SNAPSHOT,
26+
RESET
27+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
package org.apache.doris.nereids.jobs.executor;
1919

20-
import org.apache.doris.common.Config;
2120
import org.apache.doris.nereids.CascadesContext;
2221
import org.apache.doris.nereids.jobs.JobContext;
2322
import org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob;
@@ -408,6 +407,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
408407
topDown(
409408
new PruneOlapScanPartition(),
410409
new PruneEmptyPartition(),
410+
new NormalizeOlapTableStreamScan(),
411411
new PruneFileScanPartition(),
412412
new PushDownFilterIntoSchemaScan()
413413
)
@@ -728,6 +728,9 @@ public class Rewriter extends AbstractBatchJobExecutor {
728728
new LogicalResultSinkToShortCircuitPointQuery(),
729729
new PruneOlapScanPartition(),
730730
new PruneEmptyPartition(),
731+
// Stream lowering needs the pruned partitions and must finish before
732+
// OperativeColumnDerive treats stream virtual columns as scan slots.
733+
new NormalizeOlapTableStreamScan(),
731734
new PruneFileScanPartition(),
732735
new PushDownFilterIntoSchemaScan(),
733736
new PruneOlapScanTablet()
@@ -892,14 +895,6 @@ private static List<RewriteJob> getWholeTreeRewriteJobs(
892895
ImmutableSet.of(LogicalCTEAnchor.class),
893896
() -> {
894897
List<RewriteJob> rewriteJobs = Lists.newArrayListWithExpectedSize(300);
895-
if (Config.enable_table_stream) {
896-
rewriteJobs.addAll(jobs(
897-
topic("normalize olap table stream scan",
898-
topDown(new NormalizeOlapTableStreamScan())
899-
)
900-
)
901-
);
902-
}
903898
rewriteJobs.addAll(jobs(
904899
topic("cte inline and pull up all cte anchor",
905900
custom(RuleType.PULL_UP_CTE_ANCHOR, PullUpCteAnchor::new),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.doris.catalog.stream.BaseTableStream.StreamScanType;
4040
import org.apache.doris.catalog.stream.OlapTableStream;
4141
import org.apache.doris.catalog.stream.OlapTableStreamWrapper;
42+
import org.apache.doris.catalog.stream.StreamReadMode;
4243
import org.apache.doris.common.Config;
4344
import org.apache.doris.common.IdGenerator;
4445
import org.apache.doris.common.Pair;
@@ -341,9 +342,9 @@ private LogicalOlapTableStreamScan makeOlapTableStreamScan(OlapTableStream olapT
341342
if (unboundRelation.getScanParams() != null) {
342343
unboundRelation.getScanParams().validateOlapTableStream();
343344
if (unboundRelation.getScanParams().isSnapshot()) {
344-
scan = scan.withIsSnapshot(true);
345+
scan = scan.withReadMode(StreamReadMode.SNAPSHOT);
345346
} else if (unboundRelation.getScanParams().isReset()) {
346-
scan = scan.withIsReset(true);
347+
scan = scan.withReadMode(StreamReadMode.RESET);
347348
}
348349
}
349350
if (!tabletIds.isEmpty()) {
@@ -1001,7 +1002,7 @@ private LogicalPlan makeTableStreamScan(TableIf table, UnboundRelation unboundRe
10011002
OlapTableStream olapTableStream = (OlapTableStream) table;
10021003
LogicalOlapTableStreamScan scan = makeOlapTableStreamScan(olapTableStream,
10031004
unboundRelation, qualifier);
1004-
if (!scan.isSnapshot() && !scan.isReset() && isScanAppendOnlyTableStream(olapTableStream)) {
1005+
if (scan.isIncremental() && isScanAppendOnlyTableStream(olapTableStream)) {
10051006
LOG.debug("Add append only filter on olap scan if need.");
10061007
return addAppendOnlyFilter(scan);
10071008
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import java.util.List;
5151
import java.util.Map;
5252
import java.util.Objects;
53+
import java.util.Optional;
5354
import java.util.Set;
5455

5556
/**
@@ -176,7 +177,8 @@ private static Pair<Pair<BaseTableInfo, Set<String>>, Pair<BaseColInfo, Set<Stri
176177
Set<String> relatedBaseTablePartitions = partitionMapping.get(mvValidPartition.getName());
177178
if (relatedBaseTablePartitions != null) {
178179
mvValidBaseTablePartitionNameSet.addAll(relatedBaseTablePartitions);
179-
if (!mtmv.selectNonEmptyPartitionIds(ImmutableList.of(mvValidPartition.getId())).isEmpty()) {
180+
if (!mtmv.selectNonEmptyPartitionIds(ImmutableList.of(mvValidPartition.getId()),
181+
Optional.empty()).isEmpty()) {
180182
mvValidHasDataRelatedBaseTableNameSet.addAll(relatedBaseTablePartitions);
181183
}
182184
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public Rule build() {
4949
LogicalOlapScan scan = ctx.root;
5050
OlapTable table = scan.getTable();
5151
List<Long> partitionIdsToPrune = scan.getSelectedPartitionIds();
52-
List<Long> ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune);
52+
List<Long> ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune, scan.getStreamReadMode());
5353
if (ctx.connectContext != null && ctx.connectContext.isTxnModel()) {
5454
// In transaction load, need to add empty partitions which have invisible data of sub transactions
5555
selectNonEmptyPartitionIdsForTxnLoad(ctx.connectContext.getTxnEntry(), table, scan.getSelectedIndexId(),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
330330
continue;
331331
}
332332
if (!((OlapTable) targetTable).selectNonEmptyPartitionIds(
333-
Lists.newArrayList(partition.getId())).isEmpty()) {
333+
Lists.newArrayList(partition.getId()), Optional.empty()).isEmpty()) {
334334
// Add filter only when partition has data when olap table
335335
partitionHasDataItems.add(
336336
((OlapTable) targetTable).getPartitionInfo().getItem(partition.getId()));

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
2626
import org.apache.doris.catalog.constraint.UniqueConstraint;
2727
import org.apache.doris.catalog.info.TableNameInfo;
28+
import org.apache.doris.catalog.stream.StreamReadMode;
2829
import org.apache.doris.common.IdGenerator;
2930
import org.apache.doris.common.util.Util;
3031
import org.apache.doris.datasource.CatalogIf;
@@ -194,6 +195,10 @@ public String getTableAlias() {
194195
return tableAlias;
195196
}
196197

198+
public Optional<StreamReadMode> getStreamReadMode() {
199+
return Optional.empty();
200+
}
201+
197202
@Override
198203
public void computeUnique(DataTrait.Builder builder) {
199204
Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());

0 commit comments

Comments
 (0)