|
98 | 98 | import java.util.List; |
99 | 99 | import java.util.Map; |
100 | 100 | import java.util.Optional; |
| 101 | +import java.util.OptionalLong; |
101 | 102 | import java.util.concurrent.ConcurrentHashMap; |
102 | 103 | import java.util.function.Supplier; |
103 | 104 |
|
@@ -324,6 +325,133 @@ public void planScanEnumeratesSplitsInsideAuthScope(@TempDir Path warehouse) thr |
324 | 325 | } |
325 | 326 | } |
326 | 327 |
|
| 328 | + @Test |
| 329 | + public void fixedSnapshotAliasesDoNotReReadLiveGeneration(@TempDir Path warehouse) throws Exception { |
| 330 | + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), |
| 331 | + new org.apache.paimon.fs.Path(warehouse.toUri()))) { |
| 332 | + catalog.createDatabase("db", false); |
| 333 | + Identifier id = Identifier.create("db", "fixed_reuse"); |
| 334 | + catalog.createTable(id, Schema.newBuilder() |
| 335 | + .column("id", DataTypes.INT()) |
| 336 | + .option("bucket", "-1") |
| 337 | + .build(), false); |
| 338 | + Table table = catalog.getTable(id); |
| 339 | + BatchWriteBuilder wb = table.newBatchWriteBuilder(); |
| 340 | + try (BatchTableWrite write = wb.newWrite()) { |
| 341 | + write.write(GenericRow.of(1)); |
| 342 | + List<CommitMessage> messages = write.prepareCommit(); |
| 343 | + try (BatchTableCommit commit = wb.newCommit()) { |
| 344 | + commit.commit(messages); |
| 345 | + } |
| 346 | + } |
| 347 | + |
| 348 | + long pinnedSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id(); |
| 349 | + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); |
| 350 | + ops.table = table; |
| 351 | + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( |
| 352 | + PaimonCatalogProperties.of(Collections.emptyMap()), ops); |
| 353 | + PaimonTableHandle handle = new PaimonTableHandle( |
| 354 | + "db", "fixed_reuse", Collections.emptyList(), Collections.emptyList()) |
| 355 | + .withScanOptions(Collections.singletonMap( |
| 356 | + CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(pinnedSnapshotId))); |
| 357 | + ConnectorSession session = sessionWithProps( |
| 358 | + Collections.singletonMap("enable_external_scan_task_reuse", "true"), |
| 359 | + new TestStatementScope()); |
| 360 | + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); |
| 361 | + |
| 362 | + ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId); |
| 363 | + List<ConnectorScanRange> first = provider.planScan(session, request); |
| 364 | + ops.latestSnapshotId = OptionalLong.of(pinnedSnapshotId + 1); |
| 365 | + List<ConnectorScanRange> second = provider.planScan(session, request); |
| 366 | + |
| 367 | + Assertions.assertSame(first, second, |
| 368 | + "aliases pinned to one snapshot must reuse even when live latest advances"); |
| 369 | + Assertions.assertFalse(ops.log.contains("latestSnapshotId"), |
| 370 | + "a fixed snapshot identity must not probe the live latest pointer"); |
| 371 | + } |
| 372 | + } |
| 373 | + |
| 374 | + @Test |
| 375 | + public void latestDependentIncrementalScanStillFencesOnLiveGeneration(@TempDir Path warehouse) |
| 376 | + throws Exception { |
| 377 | + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), |
| 378 | + new org.apache.paimon.fs.Path(warehouse.toUri()))) { |
| 379 | + catalog.createDatabase("db", false); |
| 380 | + Identifier id = Identifier.create("db", "incremental_reuse"); |
| 381 | + catalog.createTable(id, Schema.newBuilder() |
| 382 | + .column("id", DataTypes.INT()) |
| 383 | + .option("bucket", "-1") |
| 384 | + .build(), false); |
| 385 | + Table table = catalog.getTable(id); |
| 386 | + BatchWriteBuilder wb = table.newBatchWriteBuilder(); |
| 387 | + try (BatchTableWrite write = wb.newWrite()) { |
| 388 | + write.write(GenericRow.of(1)); |
| 389 | + List<CommitMessage> messages = write.prepareCommit(); |
| 390 | + try (BatchTableCommit commit = wb.newCommit()) { |
| 391 | + commit.commit(messages); |
| 392 | + } |
| 393 | + } |
| 394 | + |
| 395 | + long latestSnapshotId = table.latestSnapshot().orElseThrow(AssertionError::new).id(); |
| 396 | + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); |
| 397 | + ops.table = table; |
| 398 | + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( |
| 399 | + PaimonCatalogProperties.of(Collections.emptyMap()), ops); |
| 400 | + PaimonTableHandle handle = new PaimonTableHandle( |
| 401 | + "db", "incremental_reuse", Collections.emptyList(), Collections.emptyList()) |
| 402 | + .withScanOptions(Collections.singletonMap( |
| 403 | + "incremental-between-timestamp", "0," + Long.MAX_VALUE)); |
| 404 | + ConnectorSession session = sessionWithProps( |
| 405 | + Collections.singletonMap("enable_external_scan_task_reuse", "true"), |
| 406 | + new TestStatementScope()); |
| 407 | + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); |
| 408 | + |
| 409 | + ops.latestSnapshotId = OptionalLong.of(latestSnapshotId); |
| 410 | + List<ConnectorScanRange> first = provider.planScan(session, request); |
| 411 | + ops.latestSnapshotId = OptionalLong.of(latestSnapshotId + 1); |
| 412 | + List<ConnectorScanRange> second = provider.planScan(session, request); |
| 413 | + |
| 414 | + Assertions.assertNotSame(first, second, |
| 415 | + "an open-ended incremental scan must not reuse across a live generation change"); |
| 416 | + Assertions.assertEquals(2, ops.log.stream().filter("latestSnapshotId"::equals).count(), |
| 417 | + "latest-dependent scans must retain one live generation fence per alias"); |
| 418 | + } |
| 419 | + } |
| 420 | + |
| 421 | + @Test |
| 422 | + public void pinnedEmptyAliasesDoNotProbeLatest(@TempDir Path warehouse) throws Exception { |
| 423 | + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), |
| 424 | + new org.apache.paimon.fs.Path(warehouse.toUri()))) { |
| 425 | + catalog.createDatabase("db", false); |
| 426 | + Identifier id = Identifier.create("db", "empty_reuse"); |
| 427 | + catalog.createTable(id, Schema.newBuilder() |
| 428 | + .column("id", DataTypes.INT()) |
| 429 | + .option("bucket", "-1") |
| 430 | + .build(), false); |
| 431 | + Table table = catalog.getTable(id); |
| 432 | + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); |
| 433 | + ops.table = table; |
| 434 | + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( |
| 435 | + PaimonCatalogProperties.of(Collections.emptyMap()), ops); |
| 436 | + PaimonTableHandle handle = new PaimonTableHandle( |
| 437 | + "db", "empty_reuse", Collections.emptyList(), Collections.emptyList()) |
| 438 | + .withScanOptions(PaimonScanParams.pinOptionsToSnapshot(Collections.emptyMap(), -1L)); |
| 439 | + ConnectorSession session = sessionWithProps( |
| 440 | + Collections.singletonMap("enable_external_scan_task_reuse", "true"), |
| 441 | + new TestStatementScope()); |
| 442 | + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); |
| 443 | + |
| 444 | + List<ConnectorScanRange> first = provider.planScan(session, request); |
| 445 | + ops.latestSnapshotId = OptionalLong.of(1L); |
| 446 | + List<ConnectorScanRange> second = provider.planScan(session, request); |
| 447 | + |
| 448 | + Assertions.assertSame(first, second, "the statement's pinned-empty plan must remain empty and reusable"); |
| 449 | + Assertions.assertTrue(first.isEmpty()); |
| 450 | + Assertions.assertFalse(ops.log.contains("latestSnapshotId"), |
| 451 | + "a pinned-empty identity must not probe the live latest pointer"); |
| 452 | + } |
| 453 | + } |
| 454 | + |
327 | 455 | @Test |
328 | 456 | public void statementResolutionKeepsAliasesOnOnePhysicalGeneration() { |
329 | 457 | RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); |
|
0 commit comments