Skip to content

Commit 4872fed

Browse files
committed
[chore](paimon) Upgrade Paimon to 1.4.2
Issue Number: close apache#65086 Related PR: apache#66346 Problem Summary: Upgrade the master Paimon runtime from 1.3.1 to 1.4.2 and adapt the JNI scanner and Connector SPI read path to the updated fallback-table, catalog, partition, scan-statistics, and split-count APIs. This establishes the dependency baseline required by the branch-4.1 table-write forward-port without importing unrelated write implementations. Upgrade the Paimon connector runtime to 1.4.2. - Test: Unit Test - PaimonJniScannerTest: 33 tests passed - Connector Paimon targeted tests: 182 tests passed - Maven validate passed - Behavior changed: Yes (Paimon runtime is upgraded from 1.3.1 to 1.4.2) - Does this need documentation: No
1 parent 0c2c2d5 commit 4872fed

15 files changed

Lines changed: 182 additions & 83 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
import org.apache.paimon.table.system.SystemTableLoader;
4141
import org.apache.paimon.types.DataType;
4242
import org.apache.paimon.types.TimestampType;
43+
import org.apache.paimon.utils.ChainTableUtils;
44+
import org.apache.paimon.utils.StringUtils;
4345
import org.slf4j.Logger;
4446
import org.slf4j.LoggerFactory;
4547

@@ -697,14 +699,14 @@ private static Table applyManifestParallelismBound(
697699
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) table;
698700
FileStoreTable main = applyManifestParallelismBound(
699701
pair.wrapped(), safeBound, materializeAbsent);
700-
FileStoreTable fallback = applyManifestParallelismBound(
701-
pair.fallback(), safeBound, materializeAbsent);
702-
if (main == pair.wrapped() && fallback == pair.fallback()) {
702+
FileStoreTable other = applyManifestParallelismBound(
703+
pair.other(), safeBound, materializeAbsent);
704+
if (main == pair.wrapped() && other == pair.other()) {
703705
return table;
704706
}
705707
// Each branch owns an independent planner setting; a smaller sibling is not an
706708
// execution ceiling and must never throttle the other branch.
707-
return new FallbackReadFileStoreTable(main, fallback);
709+
return new FallbackReadFileStoreTable(main, other, isWrappedFirst(pair));
708710
}
709711

710712
if (table instanceof DelegatedFileStoreTable) {
@@ -767,6 +769,21 @@ private static FileStoreTable applyManifestParallelismBound(
767769
(Table) table, safeBound, materializeAbsent);
768770
}
769771

772+
static boolean isWrappedFirst(FallbackReadFileStoreTable table) {
773+
Map<String, String> options = table.options();
774+
// Mirror Paimon 1.4.2 FileStoreTableFactory. There is no public accessor for the wrapper's
775+
// private ordering flag, so reconstruction must recover the factory decision from options.
776+
if (ChainTableUtils.isChainTable(options)) {
777+
return true;
778+
}
779+
if (!StringUtils.isNullOrWhitespaceOnly(
780+
options.get(CoreOptions.SCAN_FALLBACK_BRANCH.key()))) {
781+
return true;
782+
}
783+
return StringUtils.isNullOrWhitespaceOnly(
784+
options.get(CoreOptions.SCAN_PRIMARY_BRANCH.key()));
785+
}
786+
770787
private static FileStoreTable unwrapSystemPlanningSource(FileStoreTable table) {
771788
FileStoreTable current = table;
772789
// System wrappers dispatch fallback reads only when the fallback pair is their direct
@@ -795,7 +812,7 @@ private static void validateSerializedReaderOptions(Table table) {
795812
validateSerializedAsyncThreshold(table.options().get(CoreOptions.FILE_READER_ASYNC_THRESHOLD.key()));
796813
validateSerializedSplitTargetSize(table.options().get(CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key()));
797814
if (table instanceof FallbackReadFileStoreTable) {
798-
validateSerializedReaderOptions(((FallbackReadFileStoreTable) table).fallback());
815+
validateSerializedReaderOptions(((FallbackReadFileStoreTable) table).other());
799816
}
800817
if (table instanceof DelegatedFileStoreTable) {
801818
validateSerializedReaderOptions(((DelegatedFileStoreTable) table).wrapped());

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ public void testOldFeSerializedFallbackZeroReadBatchIsRejected() throws Exceptio
153153
Collections.singletonMap(CoreOptions.READ_BATCH_SIZE.key(), "0"));
154154
Map<String, String> params = createBaseParams();
155155
params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString(
156-
InstantiationUtil.serializeObject(new FallbackReadFileStoreTable(main, fallback))));
156+
InstantiationUtil.serializeObject(new FallbackReadFileStoreTable(main, fallback, true))));
157157
PaimonJniScanner scanner = new PaimonJniScanner(1024, params);
158158
Method initTable = PaimonJniScanner.class.getDeclaredMethod("initTable");
159159
initTable.setAccessible(true);
@@ -175,7 +175,8 @@ public void testOldFeSerializedAsyncThresholdIsRejectedInEveryChild() throws Exc
175175
FileStoreTable main = serializableFileStoreTable(Collections.emptyMap());
176176
FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap(
177177
CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), "2 GB"));
178-
for (Table configuredTable : Arrays.asList(visible, new FallbackReadFileStoreTable(main, fallback))) {
178+
for (Table configuredTable : Arrays.asList(
179+
visible, new FallbackReadFileStoreTable(main, fallback, true))) {
179180
Map<String, String> params = createBaseParams();
180181
params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString(
181182
InstantiationUtil.serializeObject(configuredTable)));
@@ -197,7 +198,7 @@ public void testOldFeSerializedSystemSourceRejectsZeroSplitTarget() throws Excep
197198
FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap(
198199
CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "0 B"));
199200
Table filesTable = SystemTableLoader.load(
200-
"files", new FallbackReadFileStoreTable(main, fallback));
201+
"files", new FallbackReadFileStoreTable(main, fallback, true));
201202
Map<String, String> params = createBaseParams();
202203
params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString(
203204
InstantiationUtil.serializeObject(filesTable)));
@@ -218,7 +219,7 @@ public void testOldFeSerializedSystemSourceRejectsFallbackZeroReadBatch() throws
218219
FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap(
219220
CoreOptions.READ_BATCH_SIZE.key(), "0"));
220221
Table readerBackedSystemTable = SystemTableLoader.load(
221-
"audit_log", new FallbackReadFileStoreTable(main, fallback));
222+
"audit_log", new FallbackReadFileStoreTable(main, fallback, true));
222223
Map<String, String> params = createBaseParams();
223224
params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString(
224225
InstantiationUtil.serializeObject(readerBackedSystemTable)));
@@ -240,10 +241,10 @@ public void testBackendManifestCapReachesHiddenFallbackPlanner() {
240241
Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "8"));
241242

242243
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
243-
new FallbackReadFileStoreTable(main, fallback), "8", 4);
244+
new FallbackReadFileStoreTable(main, fallback, true), "8", 4);
244245

245246
Assertions.assertTrue(safe instanceof FallbackReadFileStoreTable);
246-
Assertions.assertEquals("4", ((FallbackReadFileStoreTable) safe).fallback()
247+
Assertions.assertEquals("4", ((FallbackReadFileStoreTable) safe).other()
247248
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
248249
}
249250

@@ -254,9 +255,9 @@ public void testAdvertisedFeCapStillChecksSerializedChildren() {
254255
Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "200"));
255256

256257
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
257-
new FallbackReadFileStoreTable(main, fallback), "32", 64);
258+
new FallbackReadFileStoreTable(main, fallback, true), "32", 64);
258259

259-
Assertions.assertEquals("32", ((FallbackReadFileStoreTable) safe).fallback()
260+
Assertions.assertEquals("32", ((FallbackReadFileStoreTable) safe).other()
260261
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
261262
}
262263

@@ -271,11 +272,11 @@ public void testOldFeManifestBackstopKeepsStableMaximum() {
271272
Table safeVisible = PaimonJniScanner.applyBackendManifestParallelism(
272273
visible, null, 512);
273274
Table safeFallback = PaimonJniScanner.applyBackendManifestParallelism(
274-
new FallbackReadFileStoreTable(main, fallback), null, 512);
275+
new FallbackReadFileStoreTable(main, fallback, true), null, 512);
275276

276277
Assertions.assertEquals("256", safeVisible.options()
277278
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
278-
Assertions.assertEquals("256", ((FallbackReadFileStoreTable) safeFallback).fallback()
279+
Assertions.assertEquals("256", ((FallbackReadFileStoreTable) safeFallback).other()
279280
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
280281
}
281282

@@ -295,20 +296,20 @@ public void testFallbackManifestParallelismIsCappedPerBranch() {
295296
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1"));
296297
FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap(
297298
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128"));
298-
Table pair = new FallbackReadFileStoreTable(main, fallback);
299+
Table pair = new FallbackReadFileStoreTable(main, fallback, true);
299300

300301
FallbackReadFileStoreTable unchanged = (FallbackReadFileStoreTable)
301302
PaimonJniScanner.applyBackendManifestParallelism(pair, "128", 128);
302303
Assertions.assertEquals("1", unchanged.wrapped().options()
303304
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
304-
Assertions.assertEquals("128", unchanged.fallback().options()
305+
Assertions.assertEquals("128", unchanged.other().options()
305306
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
306307

307308
FallbackReadFileStoreTable capped = (FallbackReadFileStoreTable)
308309
PaimonJniScanner.applyBackendManifestParallelism(pair, "128", 64);
309310
Assertions.assertEquals("1", capped.wrapped().options()
310311
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
311-
Assertions.assertEquals("64", capped.fallback().options()
312+
Assertions.assertEquals("64", capped.other().options()
312313
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
313314
}
314315

@@ -323,7 +324,7 @@ public void testBackendCapTraversesPrivilegeDelegate() {
323324
new Class<?>[] {PrivilegeChecker.class},
324325
(proxy, method, args) -> null);
325326
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
326-
new FallbackReadFileStoreTable(main, fallback), checker,
327+
new FallbackReadFileStoreTable(main, fallback, true), checker,
327328
Identifier.create("db", "table"));
328329

329330
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
@@ -336,7 +337,7 @@ public void testBackendCapTraversesPrivilegeDelegate() {
336337
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) planningTable;
337338
Assertions.assertEquals("1", pair.wrapped().options()
338339
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
339-
Assertions.assertEquals("64", pair.fallback().options()
340+
Assertions.assertEquals("64", pair.other().options()
340341
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
341342
}
342343

@@ -347,7 +348,7 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex
347348
FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap(
348349
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128"));
349350
Table wrapper = SystemTableLoader.load(
350-
"partitions", new FallbackReadFileStoreTable(main, fallback));
351+
"partitions", new FallbackReadFileStoreTable(main, fallback, true));
351352

352353
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
353354
wrapper, null, 64);
@@ -357,7 +358,7 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex
357358

358359
Assertions.assertEquals("1", pair.wrapped().options()
359360
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
360-
Assertions.assertEquals("64", pair.fallback().options()
361+
Assertions.assertEquals("64", pair.other().options()
361362
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
362363
}
363364

@@ -372,7 +373,7 @@ public void testSystemWrapperExposesSafeFallbackBehindPrivilegeDelegate() throws
372373
new Class<?>[] {PrivilegeChecker.class},
373374
(proxy, method, args) -> null);
374375
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
375-
new FallbackReadFileStoreTable(main, fallback), checker,
376+
new FallbackReadFileStoreTable(main, fallback, true), checker,
376377
Identifier.create("db", "table"));
377378
Table wrapper = SystemTableLoader.load("partitions", privileged);
378379

@@ -863,6 +864,11 @@ public String[] tempDirs() {
863864
return tempDirs;
864865
}
865866

867+
@Override
868+
public String pickTempDir() {
869+
return tempDirs.length == 0 ? null : tempDirs[0];
870+
}
871+
866872
@Override
867873
public FileIOChannel.Enumerator createChannelEnumerator() {
868874
throw new UnsupportedOperationException();

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -576,9 +576,11 @@ static Catalog createHmsCatalog(CatalogContext catalogContext, HadoopAuthenticat
576576
fileIO.checkOrMkdirs(warehousePath);
577577
String clientClass = options.get(HiveCatalogOptions.METASTORE_CLIENT_CLASS);
578578
Catalog catalog = hmsAuth == null
579-
? new HiveCatalog(fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString())
579+
? new HiveCatalog(fileIO, hiveConf, clientClass, catalogContext,
580+
warehousePath.toUri().toString())
580581
: hmsAuth.doAs(() -> new HiveCatalog(
581-
fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString()));
582+
fileIO, hiveConf, clientClass, catalogContext,
583+
warehousePath.toUri().toString()));
582584
catalog = PaimonHmsClientPool.install(catalog, hmsAuth);
583585
catalog = PaimonHmsCatalog.install(catalog, properties, storageHadoopConfig);
584586
return catalog;

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import org.apache.paimon.table.FileStoreTable;
2929
import org.apache.paimon.table.Table;
3030
import org.apache.paimon.table.system.SystemTableLoader;
31+
import org.apache.paimon.utils.ChainTableUtils;
32+
import org.apache.paimon.utils.StringUtils;
3133

3234
import java.util.Collections;
3335
import java.util.LinkedHashMap;
@@ -218,10 +220,11 @@ private static Table normalizeManifestParallelism(
218220
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) table;
219221
FileStoreTable main = normalizeManifestParallelism(
220222
pair.wrapped(), safeBound, materializeAbsent);
221-
FileStoreTable fallback = normalizeManifestParallelism(
222-
pair.fallback(), safeBound, materializeAbsent);
223-
return main == pair.wrapped() && fallback == pair.fallback()
224-
? table : new FallbackReadFileStoreTable(main, fallback);
223+
FileStoreTable other = normalizeManifestParallelism(
224+
pair.other(), safeBound, materializeAbsent);
225+
return main == pair.wrapped() && other == pair.other()
226+
? table : new FallbackReadFileStoreTable(
227+
main, other, isWrappedFirst(pair));
225228
}
226229

227230
if (table instanceof DelegatedFileStoreTable) {
@@ -270,12 +273,26 @@ private static FileStoreTable normalizeManifestParallelism(
270273
(Table) table, safeBound, materializeAbsent);
271274
}
272275

276+
static boolean isWrappedFirst(FallbackReadFileStoreTable table) {
277+
Map<String, String> options = table.options();
278+
// Match FileStoreTableFactory's construction order. Paimon does not expose wrappedFirst.
279+
if (ChainTableUtils.isChainTable(options)) {
280+
return true;
281+
}
282+
if (!StringUtils.isNullOrWhitespaceOnly(
283+
options.get(CoreOptions.SCAN_FALLBACK_BRANCH.key()))) {
284+
return true;
285+
}
286+
return StringUtils.isNullOrWhitespaceOnly(
287+
options.get(CoreOptions.SCAN_PRIMARY_BRANCH.key()));
288+
}
289+
273290
public static void validateEffectiveTable(Table table) {
274291
validateEffectiveTableOptions(table.options());
275292
if (table instanceof FallbackReadFileStoreTable) {
276293
// The fallback scan plans its private child independently, so the visible main options
277294
// cannot prove that every manifest executor input is safe.
278-
validateEffectiveTable(((FallbackReadFileStoreTable) table).fallback());
295+
validateEffectiveTable(((FallbackReadFileStoreTable) table).other());
279296
}
280297
if (table instanceof DelegatedFileStoreTable) {
281298
// Privilege and other supported delegates can hide a fallback planner behind their
@@ -290,7 +307,7 @@ public static void validateEffectivePlanningTable(Table table) {
290307
validateIfPresentForRuntime(table.options(), CoreOptions.SCAN_MANIFEST_PARALLELISM.key());
291308
validateIfPresent(table.options(), CoreOptions.SCAN_PLAN_SORT_PARTITION.key());
292309
if (table instanceof FallbackReadFileStoreTable) {
293-
validateEffectivePlanningTable(((FallbackReadFileStoreTable) table).fallback());
310+
validateEffectivePlanningTable(((FallbackReadFileStoreTable) table).other());
294311
}
295312
if (table instanceof DelegatedFileStoreTable) {
296313
validateEffectivePlanningTable(((DelegatedFileStoreTable) table).wrapped());

0 commit comments

Comments
 (0)