Skip to content

Commit ae194a4

Browse files
morningmanclaude
andcommitted
[fix](nereids) Keep every Variant argument of a multi-argument explode whole
A multi-argument explode generates as many rows as its longest array argument, so every argument shapes the output even when its generated field is never read. AccessPathPlanCollector kept the whole container only for the Variant arguments whose generated field the query referenced. In SELECT x1['k'], v2['k'] FROM t LATERAL VIEW explode(v1, v2) e AS x1, x2 v2 was left with the narrow path [v2, k]; a native Parquet read could then project only that shredded leaf of v2, and explode, which casts the whole v2 to an array, fails on the partial value. Collect the whole container of every ordinary Variant argument of a multi-argument explode, as for the single-argument form. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 78b5d95 commit ae194a4

2 files changed

Lines changed: 41 additions & 3 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,18 @@ public Void visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, State
110110
// cannot encode. Preserve the whole input container so residual values stay visible.
111111
exprCollector.collect(function.child(0));
112112
} else {
113+
if (function.arity() > 1) {
114+
// Every argument of a multi-argument explode determines how many rows it
115+
// generates, including one whose generated field is never read: in
116+
// explode(v1, v2) AS (x1, x2), a row with v1 = [{"k":1}] and v2 = [1, 2]
117+
// yields two rows even if only x1 is used. Keep the whole container of each
118+
// ordinary Variant argument, not only of those whose field is referenced.
119+
for (Expression argument : function.children()) {
120+
if (argument.getDataType().isVariantType() && !isNestedGroupVariant(argument)) {
121+
exprCollector.collect(argument);
122+
}
123+
}
124+
}
113125
for (CollectAccessPathResult accessPath : accessPaths) {
114126
List<String> path = accessPath.getPath();
115127
if (function.arity() == 1) {
@@ -131,9 +143,8 @@ public Void visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, State
131143
argumentContext.setType(accessPath.getType());
132144
if (function.child(colIndex).getDataType().isVariantType()
133145
&& !isNestedGroupVariant(function.child(colIndex))) {
134-
// Every Variant argument determines the generated array shape, so
135-
// an output-field path must not narrow the input container.
136-
exprCollector.collect(function.child(colIndex));
146+
// Its whole container is collected above, so an output-field path
147+
// must not narrow it.
137148
continue;
138149
} else if (function.child(colIndex).getDataType().isVariantType()) {
139150
argumentContext.getAccessPathBuilder().addSuffix(path.subList(2, path.size()));

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ public void createTable() throws Exception {
111111
+ " v variant\n"
112112
+ ") properties ('replication_num'='1')");
113113

114+
createTable("create table two_variant_tbl(\n"
115+
+ " id int,\n"
116+
+ " v1 variant,\n"
117+
+ " v2 variant\n"
118+
+ ") properties ('replication_num'='1')");
119+
114120
// Table for string-length offset-only optimization tests
115121
createTable("create table str_tbl(\n"
116122
+ " id int,\n"
@@ -567,6 +573,27 @@ public void testMultiArgumentExplodePreservesVariantContainers() throws Exceptio
567573
ImmutableList.of());
568574
}
569575

576+
@Test
577+
public void testMultiArgumentExplodeKeepsUnreadVariantArgumentContainer() throws Exception {
578+
// x2 is never read, but v2 still decides how many rows explode generates, so the narrow
579+
// v2['k'] projection must not leave v2 as a partial read. Sub-path pruning, which gives
580+
// v2['k'] its own slot, only rewrites OLAP scans; with it off, v2['k'] and the explode
581+
// argument share one slot as they do in an external file scan.
582+
withVariantSubPathPruningDisabled(() -> {
583+
assertColumns("select x1['k'], v2['k'] from two_variant_tbl "
584+
+ "lateral view explode(v1, v2) tmp as x1, x2",
585+
ImmutableList.of(
586+
Triple.of("variant", ImmutableList.of(path("v1")), ImmutableList.of()),
587+
Triple.of("variant", ImmutableList.of(path("v2")), ImmutableList.of())));
588+
589+
assertColumns("select x2['k'], v1['k'] from two_variant_tbl "
590+
+ "lateral view explode_outer(v1, v2) tmp as x1, x2",
591+
ImmutableList.of(
592+
Triple.of("variant", ImmutableList.of(path("v1")), ImmutableList.of()),
593+
Triple.of("variant", ImmutableList.of(path("v2")), ImmutableList.of())));
594+
});
595+
}
596+
570597
@Test
571598
public void testStruct() throws Throwable {
572599
assertColumn("select element_at(s, 1) from tbl",

0 commit comments

Comments
 (0)