|
24 | 24 | import org.apache.paimon.Snapshot; |
25 | 25 | import org.apache.paimon.options.ConfigOption; |
26 | 26 | import org.apache.paimon.options.FallbackKey; |
| 27 | +import org.apache.paimon.options.Options; |
27 | 28 | import org.apache.paimon.schema.TableSchema; |
| 29 | +import org.apache.paimon.table.ChainGroupReadTable; |
28 | 30 | import org.apache.paimon.table.FallbackReadFileStoreTable; |
29 | 31 | import org.apache.paimon.table.FileStoreTable; |
30 | 32 | import org.apache.paimon.table.Table; |
@@ -103,6 +105,13 @@ public final class PaimonScanParams { |
103 | 105 |
|
104 | 106 | private static final Set<String> INHERITED_READ_STATE_KEYS = buildInheritedReadStateKeys(); |
105 | 107 |
|
| 108 | + private static final Set<String> SCHEMA_INDEPENDENT_READ_KEYS = ImmutableSet.<String>builder() |
| 109 | + .addAll(INHERITED_READ_STATE_KEYS) |
| 110 | + .add(CoreOptions.BRANCH.key()) |
| 111 | + .add(CoreOptions.SCAN_PRIMARY_BRANCH.key()) |
| 112 | + .add(CoreOptions.SCAN_FALLBACK_BRANCH.key()) |
| 113 | + .build(); |
| 114 | + |
106 | 115 | private static final Set<String> TIME_TRAVEL_SELECTOR_KEYS = ImmutableSet.of( |
107 | 116 | CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), |
108 | 117 | CoreOptions.SCAN_WATERMARK.key(), |
@@ -204,13 +213,60 @@ public static Table applyOptions(Table table, Map<String, String> options) { |
204 | 213 | } |
205 | 214 | // Apply the relation override first, then normalize each physical planning branch. A |
206 | 215 | // single combined copy would flatten independent fallback values into one sibling's cap. |
207 | | - Table effectiveTable = PaimonReaderOptions.runtimeSafeTable(table.copy(isolatedOptions)); |
| 216 | + Table effectiveTable = PaimonReaderOptions.runtimeSafeTable( |
| 217 | + copyWithHistoricalSchema(table, isolatedOptions, options)); |
208 | 218 | // Validate after every copy so relation options participate in the documented |
209 | 219 | // relation > catalog > physical precedence before the effective value is judged. |
210 | 220 | PaimonReaderOptions.validateEffectiveTable(effectiveTable); |
211 | 221 | return effectiveTable; |
212 | 222 | } |
213 | 223 |
|
| 224 | + static Table copyWithHistoricalSchema(Table table, Map<String, String> options) { |
| 225 | + return copyWithHistoricalSchema(table, userOptions(options), options); |
| 226 | + } |
| 227 | + |
| 228 | + private static Table copyWithHistoricalSchema( |
| 229 | + Table table, Map<String, String> dynamicOptions, Map<String, String> provenance) { |
| 230 | + if (!(table instanceof FileStoreTable) || table instanceof ChainGroupReadTable |
| 231 | + || PaimonTableDecorators.isChainTable(table)) { |
| 232 | + // Chain groups have their own option filtering and are not ordinary fallback readers. |
| 233 | + return table.copy(dynamicOptions); |
| 234 | + } |
| 235 | + if (table instanceof FallbackReadFileStoreTable) { |
| 236 | + FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) table; |
| 237 | + // Each branch has independent snapshot IDs. Keep the SDK's timestamp translation |
| 238 | + // before restoring either schema; copying one historical schema onto the pair is invalid. |
| 239 | + return PaimonTableDecorators.rebuildFallbackPair(pair, |
| 240 | + (FileStoreTable) copyWithHistoricalSchema(pair.wrapped(), dynamicOptions, provenance), |
| 241 | + (FileStoreTable) copyWithHistoricalSchema(pair.other(), |
| 242 | + PaimonTableDecorators.otherOptions(pair, dynamicOptions), provenance)); |
| 243 | + } |
| 244 | + FileStoreTable dataTable = (FileStoreTable) table; |
| 245 | + Map<String, String> effectiveOptions = new HashMap<>(table.options()); |
| 246 | + dynamicOptions.forEach((key, value) -> { |
| 247 | + if (value == null) { |
| 248 | + effectiveOptions.remove(key); |
| 249 | + } else { |
| 250 | + effectiveOptions.put(key, value); |
| 251 | + } |
| 252 | + }); |
| 253 | + Optional<Snapshot> snapshot; |
| 254 | + try { |
| 255 | + snapshot = TimeTravelUtil.tryTravelToSnapshot(Options.fromMap(effectiveOptions), |
| 256 | + dataTable.snapshotManager(), dataTable.tagManager()); |
| 257 | + } catch (Exception e) { |
| 258 | + // Match the SDK's schema-selection fallback (for example a timestamp on an empty table). |
| 259 | + // Its normal copy below still owns option validation and the eventual scan error. |
| 260 | + return table.copy(dynamicOptions); |
| 261 | + } |
| 262 | + if (snapshot.isPresent() && snapshot.get().schemaId() != dataTable.schema().id()) { |
| 263 | + // Paimon overlays current options onto historical columns. Restore the matching |
| 264 | + // sequence/aggregation options first, or renamed fields fail before scan planning. |
| 265 | + dataTable = restoreBoundSchema(dataTable, snapshot.get().schemaId(), provenance, ""); |
| 266 | + } |
| 267 | + return dataTable.copy(dynamicOptions); |
| 268 | + } |
| 269 | + |
214 | 270 | public static FileStoreTable applyOptionsWithoutTimeTravel( |
215 | 271 | FileStoreTable table, Map<String, String> options) { |
216 | 272 | Map<String, String> tableOptions = userOptions(options); |
@@ -282,6 +338,17 @@ private static FileStoreTable restoreBoundSchema( |
282 | 338 | merged.put(key, value); |
283 | 339 | } |
284 | 340 | }); |
| 341 | + // Read selectors and branch topology belong to the resolved table, not its historical schema. |
| 342 | + // Preserve absent keys too: reviving an old mode can invalidate a snapshot selector, while |
| 343 | + // reviving an old branch policy can change fallback priority when reader caps rebuild the pair. |
| 344 | + SCHEMA_INDEPENDENT_READ_KEYS.forEach(key -> { |
| 345 | + String value = table.options().get(key); |
| 346 | + if (value == null) { |
| 347 | + merged.remove(key); |
| 348 | + } else { |
| 349 | + merged.put(key, value); |
| 350 | + } |
| 351 | + }); |
285 | 352 | // Explicit catalog overrides can equal old physical values; equality is not provenance. |
286 | 353 | options.forEach((key, value) -> { |
287 | 354 | if (key.startsWith(INTERNAL_PREFIX + "catalog-option.")) { |
@@ -353,6 +420,11 @@ public static boolean usesStatementSnapshot(Map<String, String> options) { |
353 | 420 | } |
354 | 421 |
|
355 | 422 | public static Map<String, String> resolveOptions(Table table, Map<String, String> options) { |
| 423 | + return resolveOptions(table, options, java.util.Collections.emptyMap()); |
| 424 | + } |
| 425 | + |
| 426 | + static Map<String, String> resolveOptions( |
| 427 | + Table table, Map<String, String> options, Map<String, String> catalogOptions) { |
356 | 428 | validateOptions(options); |
357 | 429 | if (!hasStartupOptions(options)) { |
358 | 430 | return options; |
@@ -388,7 +460,7 @@ public static Map<String, String> resolveOptions(Table table, Map<String, String |
388 | 460 | return resolveFileCreationTime(options, fileStoreTable, creationTime); |
389 | 461 | } |
390 | 462 |
|
391 | | - Table selectedTable = applyOptions(table, options); |
| 463 | + Table selectedTable = applyOptions(table, withCatalogOptions(options, catalogOptions)); |
392 | 464 | if ("compacted-full".equalsIgnoreCase(options.get(CoreOptions.SCAN_MODE.key()))) { |
393 | 465 | Long snapshotId = compactedFullSnapshotId((FileStoreTable) selectedTable); |
394 | 466 | return snapshotId == null |
@@ -508,9 +580,13 @@ private static Map<String, String> resolvedEmptyOptions(Map<String, String> opti |
508 | 580 | } |
509 | 581 |
|
510 | 582 | private static Map<String, String> userOptions(Map<String, String> options) { |
511 | | - return options.entrySet().stream() |
512 | | - .filter(entry -> !entry.getKey().startsWith(INTERNAL_PREFIX)) |
513 | | - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
| 583 | + Map<String, String> result = new HashMap<>(); |
| 584 | + options.forEach((key, value) -> { |
| 585 | + if (!key.startsWith(INTERNAL_PREFIX)) { |
| 586 | + result.put(key, value); |
| 587 | + } |
| 588 | + }); |
| 589 | + return result; |
514 | 590 | } |
515 | 591 |
|
516 | 592 | /** |
|
0 commit comments