Skip to content

Commit c1d7629

Browse files
fix fe ut & clould snapshot
1 parent 0008a78 commit c1d7629

2 files changed

Lines changed: 25 additions & 7 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/OlapTableStreamWrapper.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,8 +307,17 @@ public List<Long> filterIncrementalPartitionIds(List<Long> partitionIds) {
307307
public List<Long> filterConsumedPartitionIds(List<Long> partitionIds) {
308308
if (hasCloudReadStates()) {
309309
return partitionIds.stream()
310-
.filter(id -> cloudReadStates.get(id).getOffsetState()
311-
== Cloud.TableStreamOffsetStatePB.TABLE_STREAM_OFFSET_CONSUMED)
310+
.filter(id -> {
311+
Cloud.TableStreamPartitionReadStatePB state = cloudReadStates.get(id);
312+
// A partition empty at stream creation is recorded as CONSUMED with the
313+
// sentinel offset -1 (CloudInternalCatalog: emptyPartition -> commit_tso=-1).
314+
// Such a partition has no real consumption baseline; exclude it so snapshot
315+
// rebuild does not fall back to the live TSO and leak post-snapshot rows.
316+
// Mirrors the non-cloud hasConsumedData() offset > 0 guard.
317+
return state.getOffsetState()
318+
== Cloud.TableStreamOffsetStatePB.TABLE_STREAM_OFFSET_CONSUMED
319+
&& state.hasOffsetTso() && state.getOffsetTso() > 0;
320+
})
312321
.collect(ImmutableList.toImmutableList());
313322
}
314323
return partitionIds.stream()

‎fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -144,11 +144,20 @@ protected void runBeforeAll() throws Exception {
144144
+ "'enable_unique_key_merge_on_write' = 'false',"
145145
+ "'sequence_mapping.s1' = 'v1',"
146146
+ "'sequence_mapping.s2' = 'v2');");
147+
// Bump the base table with a real (positive) tso before creating the stream so that the
148+
// stream records a positive consumption offset at creation time; hasConsumedData() only
149+
// treats a positive offset as a real baseline, otherwise every partition would be pruned out
150+
// of the snapshot scan.
151+
Database database = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException("test_db");
152+
OlapTable binlogScanSchemaTable =
153+
(OlapTable) database.getTableOrMetaException("binlog_scan_schema_t");
154+
bumpPartitionsAndReplicas(binlogScanSchemaTable, 2L, 100L);
147155
createTable("create stream test_db.binlog_scan_schema_stream "
148156
+ "on table test_db.binlog_scan_schema_t properties('type' = 'append_only')");
149-
Database database = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException("test_db");
150-
bumpPartitionsAndReplicas(
151-
(OlapTable) database.getTableOrMetaException("binlog_scan_schema_t"), 2L);
157+
// Advance the base table tso again after the stream is created so the partition has new data
158+
// beyond the recorded consumption offset, driving the snapshot scan down the rebuild path
159+
// (base scan wrapped in OlapTableWrapper unioned with the binlog before-image).
160+
bumpPartitionsAndReplicas(binlogScanSchemaTable, 3L, 200L);
152161
createTable("create table test_db.t_topn_lazy(c1 int, c2 int, c3 int) "
153162
+ "duplicate key(c1) distributed by hash(c1) buckets 1 "
154163
+ "properties('replication_num' = '1', 'light_schema_change' = 'true');");
@@ -544,10 +553,10 @@ private List<OlapScanNode> getOlapScanNodes(String sql) throws Exception {
544553
return scanNodes;
545554
}
546555

547-
private static void bumpPartitionsAndReplicas(OlapTable table, long newVersion) {
556+
private static void bumpPartitionsAndReplicas(OlapTable table, long newVersion, long tso) {
548557
for (Partition partition : table.getPartitions()) {
549558
long timestamp = System.currentTimeMillis();
550-
partition.setVisibleVersionAndTime(newVersion, timestamp, timestamp);
559+
partition.setVisibleVersionAndTime(newVersion, timestamp, tso);
551560
partition.setNextVersion(newVersion + 1);
552561
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE, true)) {
553562
for (Tablet tablet : index.getTablets()) {

0 commit comments

Comments
 (0)