Skip to content

Commit 6a97eb7

Browse files
committed
[fix](paimon) Preserve explicit system table time travel
1 parent 24e6968 commit 6a97eb7

3 files changed

Lines changed: 39 additions & 6 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,10 @@ public static Table runtimeSafeSystemTable(
193193

194194
public static Table runtimeSafeSystemSource(Table sourceTable, Map<String, String> scanOptions) {
195195
if (PaimonScanParams.isOptionsPin(scanOptions)) {
196-
if (sourceTable instanceof FileStoreTable) {
196+
if (sourceTable instanceof FileStoreTable
197+
&& PaimonScanParams.preservesBoundSchema(scanOptions)) {
198+
// Only an internal statement fence already owns its schema generation. A user tag
199+
// or snapshot must still time-travel here so the rebuilt wrapper matches binding.
197200
return PaimonScanParams.applyOptionsWithoutTimeTravel(
198201
(FileStoreTable) sourceTable, scanOptions);
199202
}

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanParams.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public final class PaimonScanParams {
6161
private static final String PINNED_FILE_CREATION_TIME =
6262
"doris.internal.paimon.file-creation-time-millis";
6363
private static final String PINNED_EMPTY_SCAN = "doris.internal.paimon.empty-scan";
64+
private static final String PRESERVE_BOUND_SCHEMA =
65+
"doris.internal.paimon.preserve-bound-schema";
6466
/**
6567
* Marks a pin as originating from {@code @options} rather than from a time-travel / {@code @incr}
6668
* selector. The three families share one carrier (the handle's scan-option map), but only this one
@@ -375,9 +377,17 @@ public static Map<String, String> pinOptionsToSnapshot(
375377
// Statement-fence pinning removes inherited selectors, so validate the raw map first;
376378
// otherwise an unsupported inherited key can disappear before the common validation path.
377379
validateOptions(options);
378-
return snapshotId < 0
380+
Map<String, String> pinned = snapshotId < 0
379381
? resolvedEmptyOptions(options)
380382
: resolvedSnapshotOptions(options, String.valueOf(snapshotId));
383+
// This snapshot selector comes from the statement fence, whose table already owns the
384+
// bound schema generation. Explicit user selectors must not carry this provenance marker.
385+
pinned.put(PRESERVE_BOUND_SCHEMA, Boolean.TRUE.toString());
386+
return pinned;
387+
}
388+
389+
public static boolean preservesBoundSchema(Map<String, String> options) {
390+
return options != null && Boolean.parseBoolean(options.get(PRESERVE_BOUND_SCHEMA));
381391
}
382392

383393
private static Map<String, String> resolvedTagOptions(Map<String, String> options, String tagName) {

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,11 @@ public void systemOptionsDoNotReapplyFenceSnapshotToBoundSchema(@TempDir Path wa
186186
.commitChanges(SchemaChange.addColumn("c2", DataTypes.INT()));
187187
FileStoreTable latestGeneration = FileStoreTableFactory.create(
188188
firstGeneration.fileIO(), firstGeneration.location());
189-
Map<String, String> inheritedOptions = new HashMap<>();
190-
inheritedOptions.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), "1");
191-
inheritedOptions.put(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1");
192-
FileStoreTable statementSource = latestGeneration.copyWithoutTimeTravel(inheritedOptions);
189+
Map<String, String> rawOptions = Collections.singletonMap(
190+
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1");
191+
Map<String, String> inheritedOptions = PaimonScanParams.pinOptionsToSnapshot(rawOptions, 1);
192+
FileStoreTable statementSource = PaimonScanParams.applyOptionsWithoutTimeTravel(
193+
latestGeneration, inheritedOptions);
193194
Table systemTable = SystemTableLoader.load("audit_log", statementSource);
194195

195196
Table safe = PaimonReaderOptions.runtimeSafeSystemTable(
@@ -200,6 +201,25 @@ public void systemOptionsDoNotReapplyFenceSnapshotToBoundSchema(@TempDir Path wa
200201
"relation options must retain the schema generation captured by the statement fence");
201202
}
202203

204+
@Test
205+
public void explicitTagSelectsSystemTableSchema(@TempDir Path warehouse) throws Exception {
206+
FileStoreTable firstGeneration = commit(newRealTable(warehouse, "system_options_tag_schema"), 1);
207+
firstGeneration.createTag("old_schema", 1L);
208+
new SchemaManager(firstGeneration.fileIO(), firstGeneration.location())
209+
.commitChanges(SchemaChange.addColumn("c2", DataTypes.INT()));
210+
FileStoreTable latestGeneration = FileStoreTableFactory.create(
211+
firstGeneration.fileIO(), firstGeneration.location());
212+
Table latestSystemTable = SystemTableLoader.load("audit_log", latestGeneration);
213+
Map<String, String> tagOptions = PaimonScanParams.markAsOptions(
214+
Collections.singletonMap(CoreOptions.SCAN_TAG_NAME.key(), "old_schema"));
215+
216+
Table safe = PaimonReaderOptions.runtimeSafeSystemTable(
217+
"audit_log", latestSystemTable, latestGeneration, tagOptions);
218+
219+
Assertions.assertFalse(safe.rowType().getFieldNames().contains("c2"),
220+
"an explicit tag must select the system table's historical schema");
221+
}
222+
203223
@Test
204224
public void systemTableWithoutACapturedBaseIsHandedOverUntouched(@TempDir Path warehouse) {
205225
// A format / object table has no FileStoreTable base, so getSysTableHandle falls back to the

0 commit comments

Comments
 (0)