Skip to content

Commit b377f17

Browse files
committed
[fix](cloud) Refresh row-binlog partitions from MetaService
### What problem does this PR solve? Issue Number: None Related PR: #67181 Problem Summary: A cloud row-binlog scan can begin immediately after waiting for its target transaction to become visible. Empty-partition pruning could still use an older cached initial version and remove a partition that already contains newly visible binlog data. The previous implementation placed cloud incremental-read details directly in PruneEmptyPartition. Keep PruneEmptyPartition generic and limit the behavior change to RowBinlogTableWrapper. In cloud mode, a row-binlog wrapper without fixed visible versions refreshes only cached-empty or unknown partitions directly from MetaService before pruning. Cached-known-non-empty partitions still avoid the RPC. Wrappers with fixed visible versions and the OlapTableWrapper base behavior continue to delegate to the origin table unchanged. The existing CloudPartition selection path also retains its previous behavior and ordering. ### Release note Fix cloud incremental reads that could miss newly visible data because stale partition-version cache entries caused premature empty-partition pruning. ### Check List (For Author) - Test: Unit Test - OlapTableWrapperTest - PruneEmptyPartitionTest - Behavior changed: Yes. Only cloud row-binlog scans without fixed visible versions refresh cached-empty or unknown partitions before pruning. - Does this need documentation: No
1 parent e192f75 commit b377f17

5 files changed

Lines changed: 100 additions & 158 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RowBinlogTableWrapper.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,21 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import org.apache.doris.catalog.stream.StreamReadMode;
21+
import org.apache.doris.cloud.catalog.CloudPartition;
22+
import org.apache.doris.common.Config;
2023
import org.apache.doris.common.Pair;
2124

2225
import com.google.common.base.Preconditions;
2326
import com.google.common.collect.Maps;
2427

28+
import java.util.Collection;
2529
import java.util.Collections;
30+
import java.util.List;
2631
import java.util.Map;
2732
import java.util.Objects;
33+
import java.util.Optional;
34+
import java.util.stream.Collectors;
2835

2936
/**
3037
* A lightweight wrapper base for read binlog<Row> of table
@@ -79,6 +86,22 @@ public KeysType getKeysType() {
7986
return KeysType.DUP_KEYS;
8087
}
8188

89+
@Override
90+
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds,
91+
Optional<StreamReadMode> streamReadMode) {
92+
if (Config.isCloudMode() && !hasFixedVisibleVersions()) {
93+
// A row-binlog scan can start immediately after its target transaction becomes visible. Refresh
94+
// cached-empty or unknown partitions so an older cache entry cannot prune newly visible binlog data.
95+
List<CloudPartition> partitions = partitionIds.stream()
96+
.map(this::getPartition)
97+
.filter(Objects::nonNull)
98+
.map(partition -> (CloudPartition) partition)
99+
.collect(Collectors.toList());
100+
return CloudPartition.selectNonEmptyPartitionIdsFromMs(partitions);
101+
}
102+
return super.selectNonEmptyPartitionIds(partitionIds, streamReadMode);
103+
}
104+
82105
@Override
83106
public boolean equals(Object obj) {
84107
if (!super.equals(obj)) {

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,15 @@ private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
213213

214214
// Select the non-empty partitions and return the ids.
215215
public static List<Long> selectNonEmptyPartitionIds(List<CloudPartition> partitions) {
216+
return selectNonEmptyPartitionIds(partitions, false);
217+
}
218+
219+
// Select non-empty partitions while bypassing the version cache for cached-empty or unknown partitions.
220+
public static List<Long> selectNonEmptyPartitionIdsFromMs(List<CloudPartition> partitions) {
221+
return selectNonEmptyPartitionIds(partitions, true);
222+
}
223+
224+
private static List<Long> selectNonEmptyPartitionIds(List<CloudPartition> partitions, boolean forceRefresh) {
216225
List<Long> nonEmptyPartitionIds = partitions.stream()
217226
.filter(CloudPartition::hasDataCached)
218227
.map(CloudPartition::getId)
@@ -231,7 +240,9 @@ public static List<Long> selectNonEmptyPartitionIds(List<CloudPartition> partiti
231240
}
232241

233242
try {
234-
List<Long> versions = CloudPartition.getSnapshotVisibleVersion(unknowns);
243+
List<Long> versions = forceRefresh
244+
? CloudPartition.getSnapshotVisibleVersionFromMs(unknowns, false)
245+
: CloudPartition.getSnapshotVisibleVersion(unknowns);
235246

236247
int size = versions.size();
237248
for (int i = 0; i < size; i++) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java‎

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,18 @@
1818
package org.apache.doris.nereids.rules.rewrite;
1919

2020
import org.apache.doris.catalog.OlapTable;
21-
import org.apache.doris.catalog.OlapTableWrapper;
2221
import org.apache.doris.catalog.Partition;
23-
import org.apache.doris.cloud.catalog.CloudPartition;
24-
import org.apache.doris.common.Config;
2522
import org.apache.doris.nereids.rules.Rule;
2623
import org.apache.doris.nereids.rules.RuleType;
2724
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
2825
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
2926
import org.apache.doris.qe.ConnectContext;
30-
import org.apache.doris.rpc.RpcException;
3127
import org.apache.doris.transaction.TransactionEntry;
3228

3329
import org.apache.logging.log4j.LogManager;
3430
import org.apache.logging.log4j.Logger;
3531

36-
import java.util.ArrayList;
37-
import java.util.HashSet;
3832
import java.util.List;
39-
import java.util.Objects;
40-
import java.util.Set;
41-
import java.util.stream.Collectors;
4233

4334
/**
4435
* Used to prune empty partition.
@@ -58,7 +49,7 @@ public Rule build() {
5849
LogicalOlapScan scan = ctx.root;
5950
OlapTable table = scan.getTable();
6051
List<Long> partitionIdsToPrune = scan.getSelectedPartitionIds();
61-
List<Long> ids = selectNonEmptyPartitionIds(scan, table, partitionIdsToPrune);
52+
List<Long> ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune, scan.getStreamReadMode());
6253
if (ctx.connectContext != null && ctx.connectContext.isTxnModel()) {
6354
// In transaction load, need to add empty partitions which have invisible data of sub transactions
6455
selectNonEmptyPartitionIdsForTxnLoad(ctx.connectContext.getTxnEntry(), table, scan.getSelectedIndexId(),
@@ -76,55 +67,6 @@ public Rule build() {
7667
}).toRule(RuleType.PRUNE_EMPTY_PARTITION);
7768
}
7869

79-
private List<Long> selectNonEmptyPartitionIds(LogicalOlapScan scan, OlapTable table,
80-
List<Long> partitionIds) {
81-
if (shouldRefreshPartitionVersionsFromMs(scan, table)) {
82-
List<CloudPartition> partitions = partitionIds.stream()
83-
.map(table::getPartition)
84-
.filter(Objects::nonNull)
85-
.map(partition -> (CloudPartition) partition)
86-
.collect(Collectors.toList());
87-
List<CloudPartition> partitionsToRefresh = new ArrayList<>();
88-
Set<Long> nonEmptyPartitionIds = new HashSet<>();
89-
for (CloudPartition partition : partitions) {
90-
if (partition.hasDataCached()) {
91-
nonEmptyPartitionIds.add(partition.getId());
92-
} else {
93-
// The incremental read may have just waited for a transaction to become visible. Refresh a
94-
// cached-empty or unknown partition so newly visible data is not pruned before scan planning.
95-
partitionsToRefresh.add(partition);
96-
}
97-
}
98-
if (partitionsToRefresh.isEmpty()) {
99-
return partitions.stream().map(CloudPartition::getId).collect(Collectors.toList());
100-
}
101-
try {
102-
List<Long> versions = CloudPartition.getSnapshotVisibleVersionFromMs(partitionsToRefresh, false);
103-
assert versions.size() == partitionsToRefresh.size()
104-
: "the got num versions is not equals to acquired num versions";
105-
for (int i = 0; i < versions.size(); i++) {
106-
if (versions.get(i) > Partition.PARTITION_INIT_VERSION) {
107-
nonEmptyPartitionIds.add(partitionsToRefresh.get(i).getId());
108-
}
109-
}
110-
} catch (RpcException e) {
111-
throw new RuntimeException("get version from meta service failed", e);
112-
}
113-
return partitions.stream()
114-
.filter(partition -> nonEmptyPartitionIds.contains(partition.getId()))
115-
.map(CloudPartition::getId)
116-
.collect(Collectors.toList());
117-
}
118-
return table.selectNonEmptyPartitionIds(partitionIds, scan.getStreamReadMode());
119-
}
120-
121-
private boolean shouldRefreshPartitionVersionsFromMs(LogicalOlapScan scan, OlapTable table) {
122-
boolean hasFixedVisibleVersions = table instanceof OlapTableWrapper
123-
&& ((OlapTableWrapper) table).hasFixedVisibleVersions();
124-
return Config.isCloudMode() && !hasFixedVisibleVersions && scan.getScanParams().isPresent()
125-
&& scan.getScanParams().get().incrementalRead();
126-
}
127-
12870
private void selectNonEmptyPartitionIdsForTxnLoad(TransactionEntry txnEntry, OlapTable table, long indexId,
12971
List<Long> selectedPartitions, List<Long> nonEmptyPartitionIds) {
13072
for (Long selectedPartitionId : selectedPartitions) {

‎fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableWrapperTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,21 @@
1818
package org.apache.doris.catalog;
1919

2020
import org.apache.doris.binlog.BinlogTestUtils;
21+
import org.apache.doris.cloud.catalog.CloudPartition;
22+
import org.apache.doris.common.Config;
2123
import org.apache.doris.thrift.TStorageType;
2224

25+
import com.google.common.collect.ImmutableList;
26+
import com.google.common.collect.ImmutableMap;
2327
import com.google.common.collect.Lists;
2428
import org.junit.jupiter.api.Assertions;
2529
import org.junit.jupiter.api.Test;
30+
import org.mockito.MockedStatic;
31+
import org.mockito.Mockito;
2632

33+
import java.util.Collections;
2734
import java.util.List;
35+
import java.util.Optional;
2836
import java.util.concurrent.TimeUnit;
2937

3038
public class OlapTableWrapperTest {
@@ -117,4 +125,59 @@ public void testRowBinlogTableWrapper() {
117125
Assertions.assertEquals(table.getSchemaByIndexId(table.getBaseIndexId()), wrapper.getSchemaByIndexId(table.getBaseIndexId()));
118126
Assertions.assertEquals(table.getIndexSchemaVersion(table.getBaseIndexId()), wrapper.getIndexSchemaVersion(table.getBaseIndexId()));
119127
}
128+
129+
@Test
130+
public void testCloudRowBinlogWrapperRefreshesCachedEmptyPartitions() {
131+
long stalePartitionId = 100L;
132+
long cachedNonEmptyPartitionId = 101L;
133+
List<Long> partitionIds = ImmutableList.of(stalePartitionId, cachedNonEmptyPartitionId);
134+
135+
CloudPartition stalePartition = Mockito.mock(CloudPartition.class);
136+
Mockito.when(stalePartition.getId()).thenReturn(stalePartitionId);
137+
CloudPartition cachedNonEmptyPartition = Mockito.mock(CloudPartition.class);
138+
Mockito.when(cachedNonEmptyPartition.getId()).thenReturn(cachedNonEmptyPartitionId);
139+
Mockito.when(cachedNonEmptyPartition.hasDataCached()).thenReturn(true);
140+
141+
OlapTable table = Mockito.spy(newTestTable(BinlogTestUtils.newTestRowBinlogConfig(true, true)));
142+
Mockito.doReturn(stalePartition).when(table).getPartition(stalePartitionId);
143+
Mockito.doReturn(cachedNonEmptyPartition).when(table).getPartition(cachedNonEmptyPartitionId);
144+
RowBinlogTableWrapper wrapper = new RowBinlogTableWrapper(table);
145+
146+
try (MockedStatic<Config> mockedConfig = Mockito.mockStatic(Config.class);
147+
MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(
148+
CloudPartition.class, Mockito.CALLS_REAL_METHODS)) {
149+
mockedConfig.when(Config::isCloudMode).thenReturn(true);
150+
mockedPartition.when(() -> CloudPartition.getSnapshotVisibleVersionFromMs(
151+
ImmutableList.of(stalePartition), false)).thenReturn(ImmutableList.of(2L));
152+
mockedPartition.clearInvocations();
153+
154+
Assertions.assertEquals(ImmutableList.of(cachedNonEmptyPartitionId, stalePartitionId),
155+
wrapper.selectNonEmptyPartitionIds(partitionIds, Optional.empty()));
156+
mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersionFromMs(
157+
ImmutableList.of(stalePartition), false));
158+
}
159+
}
160+
161+
@Test
162+
public void testCloudRowBinlogWrapperWithFixedVisibleVersionsUsesOriginTable() {
163+
long emptyPartitionId = 100L;
164+
long nonEmptyPartitionId = 101L;
165+
List<Long> partitionIds = ImmutableList.of(emptyPartitionId, nonEmptyPartitionId);
166+
167+
OlapTable table = Mockito.spy(newTestTable(BinlogTestUtils.newTestRowBinlogConfig(true, true)));
168+
Mockito.doReturn(ImmutableList.of(nonEmptyPartitionId)).when(table)
169+
.selectNonEmptyPartitionIds(partitionIds, Optional.empty());
170+
RowBinlogTableWrapper wrapper = new RowBinlogTableWrapper(table, Collections.emptyMap(),
171+
ImmutableMap.of(emptyPartitionId, Partition.PARTITION_INIT_VERSION, nonEmptyPartitionId, 2L));
172+
173+
try (MockedStatic<Config> mockedConfig = Mockito.mockStatic(Config.class);
174+
MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(CloudPartition.class)) {
175+
mockedConfig.when(Config::isCloudMode).thenReturn(true);
176+
177+
Assertions.assertEquals(ImmutableList.of(nonEmptyPartitionId),
178+
wrapper.selectNonEmptyPartitionIds(partitionIds, Optional.empty()));
179+
Mockito.verify(table).selectNonEmptyPartitionIds(partitionIds, Optional.empty());
180+
mockedPartition.verifyNoInteractions();
181+
}
182+
}
120183
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartitionTest.java‎

Lines changed: 1 addition & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,8 @@
1717

1818
package org.apache.doris.nereids.rules.rewrite;
1919

20-
import org.apache.doris.analysis.TableScanParams;
2120
import org.apache.doris.catalog.OlapTable;
22-
import org.apache.doris.catalog.OlapTableWrapper;
2321
import org.apache.doris.cloud.catalog.CloudPartition;
24-
import org.apache.doris.common.Config;
2522
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
2623
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
2724
import org.apache.doris.nereids.util.MemoTestUtils;
@@ -30,67 +27,16 @@
3027
import org.apache.doris.qe.ConnectContext;
3128

3229
import com.google.common.collect.ImmutableList;
33-
import com.google.common.collect.ImmutableMap;
3430
import org.junit.jupiter.api.Assertions;
3531
import org.junit.jupiter.api.Test;
3632
import org.mockito.MockedStatic;
3733
import org.mockito.Mockito;
3834

39-
import java.util.Collections;
4035
import java.util.List;
4136
import java.util.Optional;
4237

4338
class PruneEmptyPartitionTest implements MemoPatternMatchSupported {
4439

45-
@Test
46-
void testIncrementalReadGetsFreshVersionsBeforePruning() throws Exception {
47-
long stalePartitionId = 100L;
48-
long nonEmptyPartitionId = 101L;
49-
List<Long> partitionIds = ImmutableList.of(stalePartitionId, nonEmptyPartitionId);
50-
51-
CloudPartition stalePartition = Mockito.mock(CloudPartition.class);
52-
Mockito.when(stalePartition.getId()).thenReturn(stalePartitionId);
53-
CloudPartition nonEmptyPartition = Mockito.mock(CloudPartition.class);
54-
Mockito.when(nonEmptyPartition.getId()).thenReturn(nonEmptyPartitionId);
55-
Mockito.when(nonEmptyPartition.hasDataCached()).thenReturn(true);
56-
57-
OlapTable table = Mockito.spy(PlanConstructor.newOlapTable(10L, "incr_tbl", 0));
58-
Mockito.doReturn(partitionIds).when(table).getPartitionIds();
59-
Mockito.doReturn(stalePartition).when(table).getPartition(stalePartitionId);
60-
Mockito.doReturn(nonEmptyPartition).when(table).getPartition(nonEmptyPartitionId);
61-
Mockito.doReturn(ImmutableList.of(nonEmptyPartitionId)).when(table)
62-
.selectNonEmptyPartitionIds(Mockito.anyCollection(), Mockito.any());
63-
64-
TableScanParams scanParams = new TableScanParams(
65-
TableScanParams.INCREMENTAL_READ, Collections.emptyMap(), Collections.emptyList());
66-
LogicalOlapScan scan = new LogicalOlapScan(
67-
PlanConstructor.getNextRelationId(), table, ImmutableList.of("incr_tbl"),
68-
ImmutableList.of(), ImmutableList.of(), Optional.empty(), ImmutableList.of(),
69-
Optional.of(scanParams));
70-
Assertions.assertEquals(partitionIds, scan.getSelectedPartitionIds());
71-
ConnectContext connectContext = MemoTestUtils.createConnectContext();
72-
73-
try (MockedStatic<Config> mockedConfig = Mockito.mockStatic(Config.class);
74-
MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(CloudPartition.class)) {
75-
mockedConfig.when(Config::isCloudMode).thenReturn(true);
76-
mockedPartition.when(() -> CloudPartition.getSnapshotVisibleVersionFromMs(
77-
Mockito.anyList(), Mockito.eq(false))).thenAnswer(invocation -> {
78-
List<?> partitions = invocation.getArgument(0);
79-
return Collections.nCopies(partitions.size(), 2L);
80-
});
81-
82-
LogicalOlapScan rewritten = (LogicalOlapScan) PlanChecker.from(connectContext, scan)
83-
.applyTopDown(new PruneEmptyPartition())
84-
.getPlan();
85-
86-
Assertions.assertEquals(partitionIds, rewritten.getSelectedPartitionIds());
87-
mockedPartition.verify(() -> CloudPartition.getSnapshotVisibleVersionFromMs(
88-
Mockito.eq(ImmutableList.of(stalePartition)), Mockito.eq(false)));
89-
Mockito.verify(table, Mockito.never()).selectNonEmptyPartitionIds(
90-
Mockito.anyCollection(), Mockito.any());
91-
}
92-
}
93-
9440
@Test
9541
void testNormalReadUsesCachedPartitionVersions() {
9642
long emptyPartitionId = 100L;
@@ -113,50 +59,7 @@ void testNormalReadUsesCachedPartitionVersions() {
11359
PlanConstructor.getNextRelationId(), table, ImmutableList.of("normal_tbl"));
11460
ConnectContext connectContext = MemoTestUtils.createConnectContext();
11561

116-
try (MockedStatic<Config> mockedConfig = Mockito.mockStatic(Config.class);
117-
MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(CloudPartition.class)) {
118-
mockedConfig.when(Config::isCloudMode).thenReturn(true);
119-
120-
LogicalOlapScan rewritten = (LogicalOlapScan) PlanChecker.from(connectContext, scan)
121-
.applyTopDown(new PruneEmptyPartition())
122-
.getPlan();
123-
124-
Assertions.assertEquals(ImmutableList.of(nonEmptyPartitionId), rewritten.getSelectedPartitionIds());
125-
Mockito.verify(table).selectNonEmptyPartitionIds(partitionIds, Optional.empty());
126-
mockedPartition.verifyNoInteractions();
127-
}
128-
}
129-
130-
@Test
131-
void testIncrementalReadWithFixedVersionsKeepsSnapshot() {
132-
long emptyPartitionId = 100L;
133-
long nonEmptyPartitionId = 101L;
134-
List<Long> partitionIds = ImmutableList.of(emptyPartitionId, nonEmptyPartitionId);
135-
136-
CloudPartition emptyPartition = Mockito.mock(CloudPartition.class);
137-
CloudPartition nonEmptyPartition = Mockito.mock(CloudPartition.class);
138-
OlapTable originTable = Mockito.spy(PlanConstructor.newOlapTable(10L, "stream_tbl", 0));
139-
Mockito.doReturn(partitionIds).when(originTable).getPartitionIds();
140-
Mockito.doReturn(emptyPartition).when(originTable).getPartition(emptyPartitionId);
141-
Mockito.doReturn(nonEmptyPartition).when(originTable).getPartition(nonEmptyPartitionId);
142-
143-
OlapTableWrapper table = Mockito.spy(new OlapTableWrapper(originTable, Collections.emptyMap(),
144-
ImmutableMap.of(emptyPartitionId, 1L, nonEmptyPartitionId, 2L)));
145-
Mockito.doReturn(ImmutableList.of(nonEmptyPartitionId)).when(table)
146-
.selectNonEmptyPartitionIds(Mockito.anyCollection(), Mockito.any());
147-
148-
TableScanParams scanParams = new TableScanParams(
149-
TableScanParams.INCREMENTAL_READ, Collections.emptyMap(), Collections.emptyList());
150-
LogicalOlapScan scan = new LogicalOlapScan(
151-
PlanConstructor.getNextRelationId(), table, ImmutableList.of("stream_tbl"),
152-
ImmutableList.of(), ImmutableList.of(), Optional.empty(), ImmutableList.of(),
153-
Optional.of(scanParams));
154-
ConnectContext connectContext = MemoTestUtils.createConnectContext();
155-
156-
try (MockedStatic<Config> mockedConfig = Mockito.mockStatic(Config.class);
157-
MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(CloudPartition.class)) {
158-
mockedConfig.when(Config::isCloudMode).thenReturn(true);
159-
62+
try (MockedStatic<CloudPartition> mockedPartition = Mockito.mockStatic(CloudPartition.class)) {
16063
LogicalOlapScan rewritten = (LogicalOlapScan) PlanChecker.from(connectContext, scan)
16164
.applyTopDown(new PruneEmptyPartition())
16265
.getPlan();

0 commit comments

Comments
 (0)