Skip to content

Commit 96b43d1

Browse files
committed
[fix](variant) Adapt Iceberg Variant reader to branch-4.1
1 parent f1bc202 commit 96b43d1

7 files changed

Lines changed: 20 additions & 12 deletions

File tree

‎be/src/core/column/variant_v2/column_variant_v2.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,9 @@ class VariantShreddedState {
6565
virtual size_t byte_size() const = 0;
6666
virtual size_t allocated_bytes() const = 0;
6767
virtual void sanity_check() const = 0;
68-
virtual void for_each_subcolumn(const IColumn::ColumnCallback& callback) const = 0;
68+
// The 4.1 ColumnCallback is mutable COW traversal, while shredded columns are immutable and
69+
// shared. Expose the physical tree only through the immutable callback contract.
70+
virtual void for_each_subcolumn(const IColumn::ImutableColumnCallback& callback) const = 0;
6971
// Row selection must remain in the native shredded representation. A scanner may compact a
7072
// predicate column before every logical Variant value is available for materialization.
7173
virtual std::shared_ptr<VariantShreddedState> filter(const IColumn::Filter& filter,

‎be/src/format_v2/parquet/reader/variant_column_reader.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,7 @@ class ParquetVariantShreddedState final : public VariantShreddedState {
615615
}
616616
void sanity_check() const override { _physical->sanity_check(); }
617617

618-
void for_each_subcolumn(const IColumn::ColumnCallback& callback) const override {
618+
void for_each_subcolumn(const IColumn::ImutableColumnCallback& callback) const override {
619619
callback(*_physical);
620620
}
621621

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con
127127
int slotId = slotReference.getExprId().asInt();
128128
slotToAccessPaths.put(slotId, new CollectAccessPathResult(
129129
ImmutableList.of(slotReference.getName()),
130-
context.bottomFilter, ColumnAccessPathType.DATA));
130+
context.bottomFilter, TAccessPathType.DATA));
131131
return null;
132132
}
133133
if (dataType instanceof NestedColumnPrunable) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.nereids.trees.expressions.Expression;
2626
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2727
import org.apache.doris.nereids.trees.expressions.Slot;
28+
import org.apache.doris.nereids.trees.expressions.SlotReference;
2829
import org.apache.doris.nereids.trees.expressions.functions.Function;
2930
import org.apache.doris.nereids.trees.expressions.functions.generator.Explode;
3031
import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap;
@@ -240,6 +241,12 @@ public Void visitLogicalProject(LogicalProject<? extends Plan> project, Statemen
240241
List<String> outerPath = outerSlotAccessPath.getPath();
241242
List<String> replaceSlotNamePath = new ArrayList<>();
242243
replaceSlotNamePath.add(innerSlot.getName());
244+
if (outerPath.size() == 1 && innerSlot instanceof SlotReference
245+
&& ((SlotReference) innerSlot).hasSubColPath()) {
246+
// A whole access to a derived subcolumn slot is whole only relative to that
247+
// slot; preserve its physical leaf path when propagating to the scan slot.
248+
replaceSlotNamePath.addAll(((SlotReference) innerSlot).getSubPath());
249+
}
243250
replaceSlotNamePath.addAll(outerPath.subList(1, outerPath.size()));
244251
allSlotToAccessPaths.put(
245252
innerSlot.getExprId().asInt(),

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,13 @@
2020
import org.apache.doris.analysis.TableScanParams;
2121
import org.apache.doris.analysis.TableSnapshot;
2222
import org.apache.doris.catalog.Column;
23-
import org.apache.doris.catalog.Env;
2423
import org.apache.doris.catalog.StructField;
2524
import org.apache.doris.catalog.Type;
2625
import org.apache.doris.common.UserException;
2726
import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo;
2827
import org.apache.doris.nereids.exceptions.AnalysisException;
29-
import org.apache.doris.qe.ConnectContext;
3028

29+
import com.google.common.collect.ImmutableList;
3130
import com.google.common.collect.ImmutableMap;
3231
import org.apache.iceberg.GenericPartitionFieldSummary;
3332
import org.apache.iceberg.HistoryEntry;
@@ -255,12 +254,12 @@ public void testIcebergVariantUsesComputeV2Representation() {
255254
@Test
256255
public void testIcebergWriteRejectsRootAndNestedVariant() {
257256
Type variant = IcebergUtils.icebergTypeToDorisType(Types.VariantType.get(), false, false);
258-
for (Column column : List.of(
257+
for (Column column : ImmutableList.of(
259258
new Column("payload", variant),
260259
new Column("nested", new org.apache.doris.catalog.StructType(
261-
new ArrayList<>(List.of(new StructField("payload", variant))))))) {
260+
new ArrayList<>(ImmutableList.of(new StructField("payload", variant))))))) {
262261
try {
263-
IcebergUtils.validateWriteSchema(List.of(column));
262+
IcebergUtils.validateWriteSchema(ImmutableList.of(column));
264263
Assert.fail("Iceberg writes must be rejected until the writer supports Variant");
265264
} catch (AnalysisException e) {
266265
Assert.assertTrue(e.getMessage().contains("VARIANT"));

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,7 @@ public void testRejectSmoothUpgradeSourceBackendForVariantProjection() throws Ex
776776
Backend smoothUpgradeSource = Mockito.mock(Backend.class);
777777
Mockito.when(smoothUpgradeSource.isSmoothUpgradeSrc()).thenReturn(true);
778778
Mockito.when(smoothUpgradeSource.getId()).thenReturn(10002L);
779-
List<Backend> backends = List.of(currentBackend, smoothUpgradeSource);
779+
List<Backend> backends = ImmutableList.of(currentBackend, smoothUpgradeSource);
780780

781781
IcebergScanNode.checkVariantBackendCompatibility(false, backends);
782782
try {
@@ -802,7 +802,7 @@ public void testBatchVariantProjectionUsesSharedCompatibilityGate() throws Excep
802802
Assert.assertTrue(node.isBatchMode());
803803
try {
804804
node.checkVariantBackendCompatibilityForCurrentScan(
805-
List.of(currentBackend, smoothUpgradeSource));
805+
ImmutableList.of(currentBackend, smoothUpgradeSource));
806806
Assert.fail("batch Variant projection must use the shared backend compatibility gate");
807807
} catch (UserException e) {
808808
Assert.assertTrue(e.getMessage().contains("backend 10003 is a smooth upgrade source"));

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -898,10 +898,10 @@ public void testDataTypeAccessTreeKeepsVariantTerminalPath() {
898898
StructType type = new StructType(ImmutableList.of(
899899
new StructField("payload", VariantType.INSTANCE, true, "")));
900900
SlotReference slot = new SlotReference("info", type);
901-
DataTypeAccessTree tree = DataTypeAccessTree.ofRoot(slot, ColumnAccessPathType.DATA);
901+
DataTypeAccessTree tree = DataTypeAccessTree.ofRoot(slot, TAccessPathType.DATA);
902902

903903
tree.setAccessByPath(ImmutableList.of("info", "payload", "typed_col"), 0,
904-
ColumnAccessPathType.DATA);
904+
TAccessPathType.DATA);
905905

906906
DataType prunedType = tree.pruneDataType().get();
907907
Assertions.assertInstanceOf(StructType.class, prunedType);

0 commit comments

Comments
 (0)