Skip to content

Commit 5ede4b6

Browse files
authored
[fix](paimon) Correct pruned struct field reads (apache#66223)
## Proposed changes - Preserve the full Paimon `RowType` arity when reading projected struct fields. - Add unit coverage for non-leading, non-contiguous, and complete struct projections backed by a binary row. ## Problem summary Struct projection entries use original child indexes, but the scanner initialized the nested binary row with the number of projected fields. A projection such as the third child therefore created a one-field row and then accessed index two, producing incorrect reads or an out-of-bounds failure. ## Regression coverage - `testUnpackNonLeadingStructField` constructs a three-field binary row and projects only original field index 2. The previous implementation fails with `index (2) should < 1`. - `testUnpackNonContiguousStructFields` projects original field indexes 0 and 2. The previous implementation fails with `index (2) should < 2`. - `testUnpackCompleteStruct` protects the existing full-struct behavior. - With this fix restored, all three tests pass with Maven build caching disabled. ## Validation - Reverted the row-arity fix locally and confirmed that the two pruned-field regression tests fail. - Restored the fix and reran the same test class: 3 tests, 0 failures, 0 errors. - Ran all Paimon scanner test classes with Maven build caching disabled: 15 tests, 0 failures, 0 errors. - Checkstyle passed for the affected reactor modules.
1 parent efb70b3 commit 5ede4b6

2 files changed

Lines changed: 93 additions & 3 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,12 @@ public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
197197

198198
@Override
199199
public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue> values) {
200-
// todo: support pruned struct fields
201-
InternalRow row = record.getRow(idx, structFieldIndex.size());
200+
RowType rowType = (RowType) dataType;
201+
// Projection entries are original child indexes, so the binary row must keep the full RowType arity.
202+
InternalRow row = record.getRow(idx, rowType.getFieldCount());
202203
for (int i : structFieldIndex) {
203204
values.add(new PaimonColumnValue(row, i, dorisType.getChildTypes().get(i),
204-
((RowType) dataType).getFields().get(i).type(), timeZone));
205+
rowType.getFields().get(i).type(), timeZone));
205206
}
206207
}
207208
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.paimon;
19+
20+
import org.apache.doris.common.jni.vec.ColumnType;
21+
import org.apache.doris.common.jni.vec.ColumnValue;
22+
23+
import org.apache.paimon.data.BinaryString;
24+
import org.apache.paimon.data.GenericRow;
25+
import org.apache.paimon.data.InternalRow;
26+
import org.apache.paimon.data.serializer.InternalRowSerializer;
27+
import org.apache.paimon.types.BigIntType;
28+
import org.apache.paimon.types.DataType;
29+
import org.apache.paimon.types.IntType;
30+
import org.apache.paimon.types.RowType;
31+
import org.apache.paimon.types.VarCharType;
32+
import org.junit.Assert;
33+
import org.junit.Test;
34+
35+
import java.util.ArrayList;
36+
import java.util.Arrays;
37+
import java.util.Collections;
38+
import java.util.List;
39+
40+
public class PaimonColumnValueTest {
41+
private final RowType paimonStructType = RowType.of(
42+
new DataType[] {new IntType(), new VarCharType(), new BigIntType()},
43+
new String[] {"a", "b", "c"});
44+
private final ColumnType dorisStructType =
45+
ColumnType.parseType("s", "struct<a:int,b:string,c:bigint>");
46+
47+
@Test
48+
public void testUnpackNonLeadingStructField() {
49+
PaimonColumnValue structValue = createStructValue();
50+
List<ColumnValue> values = new ArrayList<>();
51+
52+
structValue.unpackStruct(Collections.singletonList(2), values);
53+
54+
Assert.assertEquals(1, values.size());
55+
Assert.assertEquals(100L, values.get(0).getLong());
56+
}
57+
58+
@Test
59+
public void testUnpackNonContiguousStructFields() {
60+
PaimonColumnValue structValue = createStructValue();
61+
List<ColumnValue> values = new ArrayList<>();
62+
63+
structValue.unpackStruct(Arrays.asList(0, 2), values);
64+
65+
Assert.assertEquals(2, values.size());
66+
Assert.assertEquals(10, values.get(0).getInt());
67+
Assert.assertEquals(100L, values.get(1).getLong());
68+
}
69+
70+
@Test
71+
public void testUnpackCompleteStruct() {
72+
PaimonColumnValue structValue = createStructValue();
73+
List<ColumnValue> values = new ArrayList<>();
74+
75+
structValue.unpackStruct(Arrays.asList(0, 1, 2), values);
76+
77+
Assert.assertEquals(3, values.size());
78+
Assert.assertEquals(10, values.get(0).getInt());
79+
Assert.assertEquals("x", values.get(1).getString());
80+
Assert.assertEquals(100L, values.get(2).getLong());
81+
}
82+
83+
private PaimonColumnValue createStructValue() {
84+
GenericRow nestedRow = GenericRow.of(10, BinaryString.fromString("x"), 100L);
85+
RowType outerType = RowType.of(new DataType[] {paimonStructType}, new String[] {"s"});
86+
InternalRow outerRow = new InternalRowSerializer(outerType).toBinaryRow(GenericRow.of(nestedRow));
87+
return new PaimonColumnValue(outerRow, 0, dorisStructType, paimonStructType, "UTC");
88+
}
89+
}

0 commit comments

Comments
 (0)