Skip to content

Commit 5bfd030

Browse files
committed
[fix](paimon) Adapt Rust reader compatibility to master
### What problem does this PR solve? Issue Number: None Related PR: #66227 Problem Summary: The master connector framework did not propagate name-based nested projections when no stable nested field IDs were available. Paimon could therefore select the Rust reader for a projected nested value and BE decoded the projected Arrow struct as the full Doris struct. Master also differs from branch-4.1 in credential selection, branch resolution, and TIMESTAMP_LTZ mapping. Propagate nested projection state to connector handles, make Paimon fall back to JNI for nested projections, and align the regression fixtures with master semantics. ### Release note None ### Check List (For Author) - Test: Unit Test, FE build, and Regression test - PaimonRustReaderSelectorTest - PluginDrivenScanNodeProjectionTest - test_paimon_rust_reader_compatibility - test_paimon_rust_reader_eq_for_null - Behavior changed: No. The Rust reader remains disabled by default and unsupported nested projections fall back to JNI. - Does this need documentation: No
1 parent a4064a0 commit 5bfd030

7 files changed

Lines changed: 59 additions & 21 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonColumnHandle.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919

2020
import org.apache.doris.connector.spi.handle.ConnectorColumnHandle;
2121

22-
import java.util.Collections;
23-
import java.util.LinkedHashSet;
2422
import java.util.Objects;
2523
import java.util.Set;
2624

@@ -34,16 +32,16 @@ public class PaimonColumnHandle implements ConnectorColumnHandle {
3432

3533
private final String name;
3634
private final int fieldIndex;
37-
private final Set<Integer> projectedFieldIds;
35+
private final boolean nestedProjection;
3836

3937
public PaimonColumnHandle(String name, int fieldIndex) {
40-
this(name, fieldIndex, Collections.emptySet());
38+
this(name, fieldIndex, false);
4139
}
4240

43-
private PaimonColumnHandle(String name, int fieldIndex, Set<Integer> projectedFieldIds) {
41+
private PaimonColumnHandle(String name, int fieldIndex, boolean nestedProjection) {
4442
this.name = Objects.requireNonNull(name, "name");
4543
this.fieldIndex = fieldIndex;
46-
this.projectedFieldIds = Collections.unmodifiableSet(new LinkedHashSet<>(projectedFieldIds));
44+
this.nestedProjection = nestedProjection;
4745
}
4846

4947
public String getName() {
@@ -59,12 +57,12 @@ public boolean isMetadataColumn() {
5957
}
6058

6159
public boolean hasNestedProjection() {
62-
return !projectedFieldIds.isEmpty();
60+
return nestedProjection;
6361
}
6462

6563
@Override
6664
public ConnectorColumnHandle withProjectedFieldIds(Set<Integer> fieldIds) {
67-
return new PaimonColumnHandle(name, fieldIndex, fieldIds);
65+
return new PaimonColumnHandle(name, fieldIndex, true);
6866
}
6967

7068
@Override

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonRustReaderSelectorTest.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,16 @@ public void compatibleSplitUsesRustOnlyWhenEveryPlanGateIsEnabled() throws Excep
7171
false, false, true).canRead(split));
7272
}
7373

74+
@Test
75+
public void nonDefaultScanModeUsesJni() throws Exception {
76+
FileStoreTable table = table(fileLocation("latest-full"),
77+
Collections.singletonMap(CoreOptions.SCAN_MODE.key(), "latest-full"), DataTypes.INT());
78+
DataSplit split = split(file("data.parquet", table.schema().id(), null));
79+
80+
Assertions.assertFalse(selector(enabledSession(), true, table, columns(), Collections.emptyMap(),
81+
false, false, false).canRead(split));
82+
}
83+
7484
@Test
7585
public void locationSchemeAndCredentialsStayOnTheVerifiedAllowlist() {
7686
Assertions.assertTrue(PaimonRustReaderSelector.isRustVerifiedLocationScheme("file:///warehouse/t"));
@@ -102,6 +112,16 @@ public void s3RequiresTranslatableCredentials() throws Exception {
102112
conflicting.put("AWS_CREDENTIALS_PROVIDER_TYPE", "ANONYMOUS");
103113
Assertions.assertFalse(selector(enabledSession(), true, table, columns(), conflicting,
104114
false, false, false).canRead(split));
115+
116+
conflicting = new HashMap<>(staticKeys);
117+
conflicting.put("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/example");
118+
Assertions.assertFalse(selector(enabledSession(), true, table, columns(), conflicting,
119+
false, false, false).canRead(split));
120+
121+
conflicting = new HashMap<>(staticKeys);
122+
conflicting.put("AWS_TOKEN", "session-token");
123+
Assertions.assertFalse(selector(enabledSession(), true, table, columns(), conflicting,
124+
false, false, false).canRead(split));
105125
}
106126

107127
@Test
@@ -137,7 +157,7 @@ public void nestedProjectionUsesJni() throws Exception {
137157
FileStoreTable table = table(fileLocation("nested-projection"), Collections.emptyMap(),
138158
DataTypes.ROW(DataTypes.FIELD(1, "nested", DataTypes.INT())));
139159
ConnectorColumnHandle projected = new PaimonColumnHandle("v", 0)
140-
.withProjectedFieldIds(Collections.singleton(1));
160+
.withProjectedFieldIds(Collections.emptySet());
141161

142162
Assertions.assertFalse(selector(enabledSession(), true, table, Collections.singletonList(projected),
143163
Collections.emptyMap(), false, false, false)

‎fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorColumnHandle.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ public interface ConnectorColumnHandle extends Serializable {
2828

2929
/**
3030
* Returns a handle scoped to the stable field IDs that the engine actually projected below this
31-
* top-level column. Connectors that do not use stable nested IDs keep the original opaque handle.
31+
* top-level column. The engine also invokes this method with an empty set for a name-based nested
32+
* projection whose schema does not publish stable field IDs. Connectors that do not need nested
33+
* projection information keep the original opaque handle.
3234
*/
3335
default ConnectorColumnHandle withProjectedFieldIds(Set<Integer> projectedFieldIds) {
3436
return this;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2573,10 +2573,12 @@ static boolean requiresPinnedColumnHandle(Column column, Set<String> pinnedNames
25732573
static ConnectorColumnHandle withProjectedFieldIds(
25742574
ConnectorColumnHandle handle, SlotDescriptor slot) {
25752575
Set<Integer> projectedFieldIds = new HashSet<>();
2576+
boolean hasNestedProjection = false;
25762577
for (ColumnAccessPath accessPath : slot.getAllAccessPaths()) {
2578+
hasNestedProjection |= accessPath.getPath().size() > 1;
25772579
collectProjectedFieldIds(accessPath.getPath(), slot.getColumn(), projectedFieldIds);
25782580
}
2579-
return projectedFieldIds.isEmpty()
2581+
return projectedFieldIds.isEmpty() && !hasNestedProjection
25802582
? handle : handle.withProjectedFieldIds(projectedFieldIds);
25812583
}
25822584

‎fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeProjectionTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.junit.jupiter.api.Assertions;
3131
import org.junit.jupiter.api.Test;
3232

33+
import java.util.Collections;
3334
import java.util.Set;
3435
import java.util.concurrent.atomic.AtomicReference;
3536

@@ -79,6 +80,18 @@ public void numericVariantSelectorsAreNotReportedAsSchemaFieldIds() {
7980
Assertions.assertEquals(ImmutableSet.of(10), observed.get());
8081
}
8182

83+
@Test
84+
public void nameBasedNestedProjectionIsCarriedWithoutFieldIds() {
85+
AtomicReference<Set<Integer>> observed = new AtomicReference<>();
86+
ConnectorColumnHandle handle = recordingHandle(observed);
87+
SlotDescriptor slot = new SlotDescriptor(new SlotId(3), new TupleId(3));
88+
slot.setAllAccessPaths(ImmutableList.of(
89+
ColumnAccessPath.data(ImmutableList.of("payload", "nested"))));
90+
91+
Assertions.assertSame(handle, PluginDrivenScanNode.withProjectedFieldIds(handle, slot));
92+
Assertions.assertEquals(Collections.emptySet(), observed.get());
93+
}
94+
8295
private static ConnectorColumnHandle recordingHandle(AtomicReference<Set<Integer>> observed) {
8396
return new ConnectorColumnHandle() {
8497
@Override

‎regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_compatibility.groovy‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,14 +72,15 @@ suite("test_paimon_rust_reader_compatibility", "p0,external,paimon") {
7272
createPk("credential_values", "v INT", "")
7373
sql "INSERT INTO credential_values VALUES (1,11)"
7474
check("select id,v from credential_values", [[1,11]], true)
75-
["'s3.credentials_provider_type'='ANONYMOUS'",
76-
"'s3.role_arn'='arn:aws:iam::123456789012:role/example'",
77-
"'s3.session_token'='expired-example-token'"].eachWithIndex { extra, index ->
75+
["'s3.credentials_provider_type'='ANONYMOUS'"].eachWithIndex { extra, index ->
7876
def crossedCatalog = "${catalog}_credentials_${index}"
7977
sql "DROP CATALOG IF EXISTS ${crossedCatalog}"
8078
try {
8179
// Select S3 explicitly: endpoint guessing would choose MinIO and ignore S3 auth modes.
82-
// Hadoop chooses static keys first; conflicting settings must retain that identity.
80+
// Static keys keep the catalog readable, while the conflicting provider mode must
81+
// conservatively keep the scan on JNI. Role ARN and session-token conflicts are
82+
// selector unit tests: on master they change the catalog's effective credentials,
83+
// so deliberately invalid values cannot form a readable regression fixture.
8384
sql """CREATE CATALOG ${crossedCatalog} PROPERTIES (
8485
'type'='paimon', 'paimon.catalog.type'='filesystem', 'fs.s3.support'='true',
8586
'warehouse'='s3://warehouse/wh', 's3.endpoint'='http://${endpoint}:${port}',
@@ -91,13 +92,14 @@ suite("test_paimon_rust_reader_compatibility", "p0,external,paimon") {
9192
}
9293
}
9394

94-
createPk("branch_scan_mode", "v INT", ", 'scan.mode'='latest-full'")
95-
sql "INSERT INTO branch_scan_mode VALUES (1,11)"
96-
spark_paimon """CALL paimon.sys.create_tag(table => '${database}.branch_scan_mode', tag => 'audit_tag')"""
97-
spark_paimon """CALL paimon.sys.create_branch(table => '${database}.branch_scan_mode',
95+
createPk("branch_values", "v INT", "")
96+
sql "INSERT INTO branch_values VALUES (1,11)"
97+
spark_paimon """CALL paimon.sys.create_tag(table => '${database}.branch_values', tag => 'audit_tag')"""
98+
spark_paimon """CALL paimon.sys.create_branch(table => '${database}.branch_values',
9899
branch => 'audit', tag => 'audit_tag')"""
99-
// A branch preserves the persisted scan mode instead of deriving one from a snapshot selector.
100-
check("select id,v from branch_scan_mode@branch(audit)", [[1,11]], false)
100+
// Master resolves a branch as its own table and binds its latest snapshot. The Rust
101+
// reader consumes that resolved table and split rather than re-evaluating the selector.
102+
check("select id,v from branch_values@branch(audit)", [[1,11]], true)
101103

102104
sql """CREATE TABLE footer_aggregates (v INT NULL) ENGINE=paimon
103105
PROPERTIES ('bucket'='-1', 'file.format'='parquet')"""

‎regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_eq_for_null.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ suite("test_paimon_rust_reader_eq_for_null", "p0,external,paimon") {
187187
's3.access_key' = 'admin',
188188
's3.secret_key' = 'password',
189189
's3.region' = 'us-east-1',
190+
'enable.mapping.timestamp_tz' = 'true',
190191
'use_path_style' = 'true'
191192
);
192193
"""

0 commit comments

Comments
 (0)