Skip to content

Commit 616b309

Browse files
committed
[fix](paimon) Reuse fixed scan identities without live probes
### What problem does this PR solve? Issue Number: None Related PR: #66530 Problem Summary: Paimon scans whose handles were already pinned to a snapshot, including pinned-empty scans, still read the live latest snapshot before consulting statement-level scan reuse. Equal aliases therefore repeated authenticated metadata I/O, and a concurrent commit could assign different generation keys to aliases that both scan the same pinned snapshot. Use the pinned snapshot as the generation for fixed identities and skip the live probe for pinned-empty identities, while retaining live generation fencing for selectors such as open-ended incremental scans. ### Release note None ### Check List (For Author) - Test: Unit Test - PaimonScanPlanProviderTest (103 tests passed) - ./build.sh --fe - Behavior changed: Yes; fixed Paimon scan identities no longer read the live latest snapshot for reuse-key generation - Does this need documentation: No
1 parent 98f451d commit 616b309

2 files changed

Lines changed: 149 additions & 8 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -556,7 +556,7 @@ public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScan
556556
// this, a no-cache catalog or a schema change between two aliases can give alias A's ranges
557557
// to alias B's generation-B serialized table.
558558
Table table = resolveScanTableConsistent(session, paimonHandle);
559-
long generation = resolvePaimonGeneration(table);
559+
long generation = resolvePaimonGeneration(paimonHandle, table);
560560
// Statement-scoped reuse: within one statement the identical scan (same table, same
561561
// branch/options pin, same generation, same projection, same filter, same limit, same COUNT
562562
// pushdown) plans once and every duplicated relation shares the result. Session variables
@@ -571,7 +571,18 @@ public List<ConnectorScanRange> planScan(ConnectorSession session, ConnectorScan
571571
request.getLimit(), request.isCountPushdown(), table)));
572572
}
573573

574-
private long resolvePaimonGeneration(Table table) {
574+
private long resolvePaimonGeneration(PaimonTableHandle handle, Table table) {
575+
Map<String, String> scanOptions = effectiveScanOptions(handle);
576+
if (PaimonScanParams.isPinnedEmptyScan(scanOptions)) {
577+
return -1L;
578+
}
579+
String pinnedSnapshotId = scanOptions.get(CoreOptions.SCAN_SNAPSHOT_ID.key());
580+
if (pinnedSnapshotId != null) {
581+
// The resolved handle already owns this scan's immutable data identity. Re-reading the
582+
// table's live latest pointer here would both add remote I/O for every alias and make two
583+
// aliases pinned to the same snapshot miss reuse when a later commit advances latest.
584+
return Long.parseLong(pinnedSnapshotId);
585+
}
575586
if (catalogOps == null) {
576587
return -1L;
577588
}
@@ -2646,12 +2657,14 @@ public int hashCode() {
26462657
* Statement-scoped cache key for one Paimon scan.
26472658
*
26482659
* <p>Includes every input that changes the planned split list: table identity, the branch pin,
2649-
* the whole scan-options map (snapshot / tag / incremental / options pins), the resolved table
2650-
* generation (latestSnapshotId), the projected columns in order, the pushed filter, the limit
2651-
* and the COUNT pushdown flag. The generation fences the key against same-path table recreation
2652-
* or schema change between two aliases: without it, alias A's ranges (from generation A) could
2653-
* be paired with alias B's serialized table (from generation B). System tables are excluded
2654-
* upstream, and session variables are statement-constant, so both stay out of the key.
2660+
* the whole scan-options map (snapshot / tag / incremental / options pins), the resolved scan
2661+
* generation (the pinned snapshot for fixed identities, or live latest for latest-dependent
2662+
* selectors), the projected columns in order, the pushed filter, the limit and the COUNT
2663+
* pushdown flag. The generation fences latest-dependent scans against same-path table
2664+
* recreation or schema change between two aliases: without it, alias A's ranges (from
2665+
* generation A) could be paired with alias B's serialized table (from generation B). System
2666+
* tables are excluded upstream, and session variables are statement-constant, so both stay out
2667+
* of the key.
26552668
*/
26562669
private static final class PaimonScanReuseKey {
26572670
private final String databaseName;

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
import java.util.List;
9999
import java.util.Map;
100100
import java.util.Optional;
101+
import java.util.OptionalLong;
101102
import java.util.concurrent.ConcurrentHashMap;
102103
import java.util.function.Supplier;
103104

@@ -324,6 +325,133 @@ public void planScanEnumeratesSplitsInsideAuthScope(@TempDir Path warehouse) thr
324325
}
325326
}
326327

328+
@Test
329+
public void fixedSnapshotAliasesDoNotReReadLiveGeneration(@TempDir Path warehouse) throws Exception {
330+
try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
331+
new org.apache.paimon.fs.Path(warehouse.toUri()))) {
332+
catalog.createDatabase("db", false);
333+
Identifier id = Identifier.create("db", "fixed_reuse");
334+
catalog.createTable(id, Schema.newBuilder()
335+
.column("id", DataTypes.INT())
336+
.option("bucket", "-1")
337+
.build(), false);
338+
Table table = catalog.getTable(id);
339+
BatchWriteBuilder wb = table.newBatchWriteBuilder();
340+
try (BatchTableWrite write = wb.newWrite()) {
341+
write.write(GenericRow.of(1));
342+
List<CommitMessage> messages = write.prepareCommit();
343+
try (BatchTableCommit commit = wb.newCommit()) {
344+
commit.commit(messages);
345+
}
346+
}
347+
348+
long pinnedSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id();
349+
RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
350+
ops.table = table;
351+
PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
352+
PaimonCatalogProperties.of(Collections.emptyMap()), ops);
353+
PaimonTableHandle handle = new PaimonTableHandle(
354+
"db", "fixed_reuse", Collections.emptyList(), Collections.emptyList())
355+
.withScanOptions(Collections.singletonMap(
356+
CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(pinnedSnapshotId)));
357+
ConnectorSession session = sessionWithProps(
358+
Collections.singletonMap("enable_external_scan_task_reuse", "true"),
359+
new TestStatementScope());
360+
ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build();
361+
362+
ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId);
363+
List<ConnectorScanRange> first = provider.planScan(session, request);
364+
ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId + 1);
365+
List<ConnectorScanRange> second = provider.planScan(session, request);
366+
367+
Assertions.assertSame(first, second,
368+
"aliases pinned to one snapshot must reuse even when live latest advances");
369+
Assertions.assertFalse(ops.log.contains("latestSnapshotId"),
370+
"a fixed snapshot identity must not probe the live latest pointer");
371+
}
372+
}
373+
374+
@Test
375+
public void latestDependentIncrementalScanStillFencesOnLiveGeneration(@TempDir Path warehouse)
376+
throws Exception {
377+
try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
378+
new org.apache.paimon.fs.Path(warehouse.toUri()))) {
379+
catalog.createDatabase("db", false);
380+
Identifier id = Identifier.create("db", "incremental_reuse");
381+
catalog.createTable(id, Schema.newBuilder()
382+
.column("id", DataTypes.INT())
383+
.option("bucket", "-1")
384+
.build(), false);
385+
Table table = catalog.getTable(id);
386+
BatchWriteBuilder wb = table.newBatchWriteBuilder();
387+
try (BatchTableWrite write = wb.newWrite()) {
388+
write.write(GenericRow.of(1));
389+
List<CommitMessage> messages = write.prepareCommit();
390+
try (BatchTableCommit commit = wb.newCommit()) {
391+
commit.commit(messages);
392+
}
393+
}
394+
395+
long latestSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id();
396+
RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
397+
ops.table = table;
398+
PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
399+
PaimonCatalogProperties.of(Collections.emptyMap()), ops);
400+
PaimonTableHandle handle = new PaimonTableHandle(
401+
"db", "incremental_reuse", Collections.emptyList(), Collections.emptyList())
402+
.withScanOptions(Collections.singletonMap(
403+
"incremental-between-timestamp", "0," + Long.MAX_VALUE));
404+
ConnectorSession session = sessionWithProps(
405+
Collections.singletonMap("enable_external_scan_task_reuse", "true"),
406+
new TestStatementScope());
407+
ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build();
408+
409+
ops.latestSnapshotId = OptionalLong.of(latestSnapshotId);
410+
List<ConnectorScanRange> first = provider.planScan(session, request);
411+
ops.latestSnapshotId = OptionalLong.of(latestSnapshotId + 1);
412+
List<ConnectorScanRange> second = provider.planScan(session, request);
413+
414+
Assertions.assertNotSame(first, second,
415+
"an open-ended incremental scan must not reuse across a live generation change");
416+
Assertions.assertEquals(2, ops.log.stream().filter("latestSnapshotId"::equals).count(),
417+
"latest-dependent scans must retain one live generation fence per alias");
418+
}
419+
}
420+
421+
@Test
422+
public void pinnedEmptyAliasesDoNotProbeLatest(@TempDir Path warehouse) throws Exception {
423+
try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
424+
new org.apache.paimon.fs.Path(warehouse.toUri()))) {
425+
catalog.createDatabase("db", false);
426+
Identifier id = Identifier.create("db", "empty_reuse");
427+
catalog.createTable(id, Schema.newBuilder()
428+
.column("id", DataTypes.INT())
429+
.option("bucket", "-1")
430+
.build(), false);
431+
Table table = catalog.getTable(id);
432+
RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
433+
ops.table = table;
434+
PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
435+
PaimonCatalogProperties.of(Collections.emptyMap()), ops);
436+
PaimonTableHandle handle = new PaimonTableHandle(
437+
"db", "empty_reuse", Collections.emptyList(), Collections.emptyList())
438+
.withScanOptions(PaimonScanParams.pinOptionsToSnapshot(Collections.emptyMap(), -1L));
439+
ConnectorSession session = sessionWithProps(
440+
Collections.singletonMap("enable_external_scan_task_reuse", "true"),
441+
new TestStatementScope());
442+
ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build();
443+
444+
List<ConnectorScanRange> first = provider.planScan(session, request);
445+
ops.latestSnapshotId = OptionalLong.of(1L);
446+
List<ConnectorScanRange> second = provider.planScan(session, request);
447+
448+
Assertions.assertSame(first, second, "the statement's pinned-empty plan must remain empty and reusable");
449+
Assertions.assertTrue(first.isEmpty());
450+
Assertions.assertFalse(ops.log.contains("latestSnapshotId"),
451+
"a pinned-empty identity must not probe the live latest pointer");
452+
}
453+
}
454+
327455
@Test
328456
public void statementResolutionKeepsAliasesOnOnePhysicalGeneration() {
329457
RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();

0 commit comments

Comments
 (0)