Skip to content

Commit eb2567d

Browse files
authored
[opt](nereids) optimize length(str_col) by only read offset sub column (#62205)
### What problem does this PR solve? Optimized the calculation of length(str_col). Treat the string column as a combination of an offset sub column and a chars sub column. Prune the string column via NestedColumnPruning so that the BE only needs to read the offset sub column, thereby saving I/O for reading the chars sub column.
1 parent b6e5f22 commit eb2567d

36 files changed

Lines changed: 1046 additions & 46 deletions

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ public enum TableFrom {
137137

138138
private boolean isDpHyp = false;
139139

140+
private boolean isDelete = false;
140141
private boolean hasNondeterministic = false;
141142

142143
// hasUnknownColStats true if any column stats in the tables used by this sql is
@@ -303,8 +304,6 @@ public enum TableFrom {
303304
private final Set<List<String>> materializationRewrittenSuccessSet = new HashSet<>();
304305

305306
private boolean isInsert = false;
306-
private boolean skipPrunePredicate = false;
307-
308307
private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = Optional.empty();
309308

310309
// For Iceberg rewrite operations: store file scan tasks to be used by
@@ -1179,14 +1178,6 @@ public boolean isUseGatherForIcebergRewrite() {
11791178
return this.useGatherForIcebergRewrite;
11801179
}
11811180

1182-
public boolean isSkipPrunePredicate() {
1183-
return skipPrunePredicate;
1184-
}
1185-
1186-
public void setSkipPrunePredicate(boolean skipPrunePredicate) {
1187-
this.skipPrunePredicate = skipPrunePredicate;
1188-
}
1189-
11901181
public boolean hasNestedColumns() {
11911182
return hasNestedColumns;
11921183
}
@@ -1216,4 +1207,12 @@ public int getLowerCaseDatabaseNames(String catalogName) {
12161207
}
12171208
return lowerCaseDatabaseNamesCache.computeIfAbsent(catalogName, Env::getLowerCaseDatabaseNames);
12181209
}
1210+
1211+
public boolean isDelete() {
1212+
return isDelete;
1213+
}
1214+
1215+
public void setIsDelete(boolean del) {
1216+
isDelete = del;
1217+
}
12191218
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.doris.nereids.rules.expression.rules.SimplifyRange;
3737
import org.apache.doris.nereids.rules.expression.rules.SimplifySelfComparison;
3838
import org.apache.doris.nereids.rules.expression.rules.SimplifyTimeFieldFromUnixtime;
39+
import org.apache.doris.nereids.rules.expression.rules.StringEmptyToLengthRule;
3940
import org.apache.doris.nereids.rules.expression.rules.TopnToMax;
4041

4142
import com.google.common.collect.ImmutableList;
@@ -71,7 +72,8 @@ public class ExpressionOptimization extends ExpressionRewrite {
7172
TopnToMax.INSTANCE,
7273
NullSafeEqualToEqual.INSTANCE,
7374
LikeToEqualRewrite.INSTANCE,
74-
BetweenToEqual.INSTANCE
75+
BetweenToEqual.INSTANCE,
76+
StringEmptyToLengthRule.INSTANCE
7577
)
7678
);
7779

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public enum ExpressionRuleType {
6161
SIMPLIFY_EQUAL_BOOLEAN_LITERAL,
6262
SIMPLIFY_IN_PREDICATE,
6363
SIMPLIFY_NOT_EXPR,
64+
STRING_EMPTY_TO_LENGTH,
6465
SIMPLIFY_RANGE,
6566
SIMPLIFY_SELF_COMPARISON,
6667
SUPPORT_JAVA_DATE_FORMATTER,
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.expression.rules;
19+
20+
import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
21+
import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
22+
import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
23+
import org.apache.doris.nereids.trees.expressions.EqualTo;
24+
import org.apache.doris.nereids.trees.expressions.Expression;
25+
import org.apache.doris.nereids.trees.expressions.Not;
26+
import org.apache.doris.nereids.trees.expressions.SlotReference;
27+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Length;
28+
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
29+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
30+
import org.apache.doris.qe.ConnectContext;
31+
32+
import com.google.common.collect.ImmutableList;
33+
34+
import java.util.List;
35+
36+
/**
37+
* Rewrites comparisons with empty strings to equivalent length()-based expressions so that
38+
* the NestedColumnPruning OFFSET optimization can apply.
39+
*
40+
* <ul>
41+
* <li>{@code str_col = ''} → {@code length(str_col) = 0}</li>
42+
* <li>{@code str_col <> ''} → {@code length(str_col) != 0}
43+
* (represented as {@code NOT(length(str_col) = 0)})</li>
44+
* </ul>
45+
*
46+
* This is a semantics-preserving rewrite: for any non-NULL string, {@code s = ''} is equivalent
47+
* to {@code length(s) = 0}; for NULL, both sides evaluate to NULL.
48+
*
49+
* Only applies when the compared expression is a direct {@link SlotReference} of string-like type,
50+
* so that the resulting {@code length(slot)} call can benefit from OFFSET-only column reading.
51+
*/
52+
public class StringEmptyToLengthRule implements ExpressionPatternRuleFactory {
53+
public static final StringEmptyToLengthRule INSTANCE = new StringEmptyToLengthRule();
54+
55+
@Override
56+
public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
57+
return ImmutableList.of(
58+
// str_col = '' → length(str_col) = 0
59+
matchesType(EqualTo.class)
60+
.thenApply(ctx -> rewriteEqualToEmpty(ctx.expr))
61+
.toRule(ExpressionRuleType.STRING_EMPTY_TO_LENGTH),
62+
63+
// NOT(str_col = '') → NOT(length(str_col) = 0) (i.e. str_col <> '')
64+
matchesType(Not.class)
65+
.thenApply(ctx -> {
66+
Not not = ctx.expr;
67+
if (!(not.child() instanceof EqualTo)) {
68+
return not;
69+
}
70+
Expression rewritten = rewriteEqualToEmpty((EqualTo) not.child());
71+
if (rewritten == not.child()) {
72+
return not;
73+
}
74+
return new Not(rewritten);
75+
})
76+
.toRule(ExpressionRuleType.STRING_EMPTY_TO_LENGTH)
77+
);
78+
}
79+
80+
/**
81+
* If {@code equalTo} compares a string-typed SlotReference against an empty-string literal,
82+
* rewrites it to {@code length(slot) = 0}. Returns the original expression unchanged otherwise.
83+
*/
84+
private static Expression rewriteEqualToEmpty(EqualTo equalTo) {
85+
if (ConnectContext.get().getStatementContext().isDelete()) {
86+
return equalTo;
87+
}
88+
Expression left = equalTo.left();
89+
Expression right = equalTo.right();
90+
91+
SlotReference slot = null;
92+
if (isStringSlot(left) && isEmptyStringLiteral(right)) {
93+
slot = (SlotReference) left;
94+
} else if (isStringSlot(right) && isEmptyStringLiteral(left)) {
95+
slot = (SlotReference) right;
96+
}
97+
98+
if (slot == null) {
99+
return equalTo;
100+
}
101+
return new EqualTo(new Length(slot), new IntegerLiteral(0));
102+
}
103+
104+
private static boolean isStringSlot(Expression expr) {
105+
return expr instanceof SlotReference && expr.getDataType().isStringLikeType();
106+
}
107+
108+
private static boolean isEmptyStringLiteral(Expression expr) {
109+
return expr instanceof Literal
110+
&& expr.getDataType().isStringLikeType()
111+
&& ((Literal) expr).getStringValue().isEmpty();
112+
}
113+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,15 @@
4040
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySort;
4141
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
4242
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
43+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality;
4344
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
4445
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
46+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Length;
4547
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
4648
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey;
4749
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue;
4850
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
51+
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapSize;
4952
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
5053
import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
5154
import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -127,9 +130,75 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con
127130
int slotId = slotReference.getExprId().asInt();
128131
slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type));
129132
}
133+
if (dataType.isStringLikeType()) {
134+
int slotId = slotReference.getExprId().asInt();
135+
if (!context.accessPathBuilder.isEmpty()) {
136+
// Accessed via an offset-only function (e.g. length()).
137+
// Builder already has "offset" at the tail; add the column name as prefix.
138+
context.accessPathBuilder.addPrefix(slotReference.getName());
139+
ImmutableList<String> path = ImmutableList.copyOf(context.accessPathBuilder.accessPath);
140+
slotToAccessPaths.put(slotId,
141+
new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA));
142+
} else {
143+
// Direct access to the string column → record a DATA path so that any
144+
// concurrent offset-only path for the same slot is suppressed.
145+
List<String> path = ImmutableList.of(slotReference.getName());
146+
slotToAccessPaths.put(slotId,
147+
new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA));
148+
}
149+
}
130150
return null;
131151
}
132152

153+
@Override
154+
public Void visitLength(Length length, CollectorContext context) {
155+
Expression arg = length.child(0);
156+
// length() only needs the offset array, not the chars data.
157+
// Add ACCESS_STRING_OFFSET as a suffix so the path builder accumulates
158+
// e.g. ["str_col", "OFFSET"] or ["c_struct", "f3", "OFFSET"].
159+
if (arg.getDataType().isStringLikeType() && context.accessPathBuilder.isEmpty()) {
160+
CollectorContext offsetContext =
161+
new CollectorContext(context.statementContext, context.bottomFilter);
162+
offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET);
163+
return arg.accept(this, offsetContext);
164+
}
165+
// fall through to default (recurse into children with fresh contexts)
166+
return visit(length, context);
167+
}
168+
169+
@Override
170+
public Void visitMapSize(MapSize mapSize, CollectorContext context) {
171+
Expression arg = mapSize.child();
172+
DataType argType = arg.getDataType();
173+
if (argType.isMapType() && context.accessPathBuilder.isEmpty()) {
174+
CollectorContext offsetContext =
175+
new CollectorContext(context.statementContext, context.bottomFilter);
176+
offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET);
177+
return arg.accept(this, offsetContext);
178+
}
179+
return visit(mapSize, context);
180+
}
181+
182+
@Override
183+
public Void visitCardinality(Cardinality cardinality, CollectorContext context) {
184+
Expression arg = cardinality.child(0);
185+
// cardinality(arr) / cardinality(map) only needs the offset array, not element data.
186+
// Arrays and maps share the same offset-array + data storage layout as strings on the BE.
187+
DataType argType = arg.getDataType();
188+
if ((argType.isArrayType() || argType.isMapType()) && context.accessPathBuilder.isEmpty()) {
189+
CollectorContext offsetContext =
190+
new CollectorContext(context.statementContext, context.bottomFilter);
191+
offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET);
192+
// cardinality(map_keys(m)) == cardinality(m) == cardinality(map_values(m)):
193+
// all three count map entries, so emit the same [map_col, OFFSET] path.
194+
Expression effectiveArg = (arg instanceof MapKeys || arg instanceof MapValues)
195+
? arg.child(0) : arg;
196+
return effectiveArg.accept(this, offsetContext);
197+
}
198+
// fall through to default
199+
return visit(cardinality, context);
200+
}
201+
133202
@Override
134203
public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) {
135204
if (nameToLambdaArguments.isEmpty()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ public class AccessPathInfo {
2727
public static final String ACCESS_ALL = "*";
2828
public static final String ACCESS_MAP_KEYS = "KEYS";
2929
public static final String ACCESS_MAP_VALUES = "VALUES";
30+
// Suffix appended to a string-column path to indicate that only the offset array
31+
// (not the char data) is needed — agreed with BE as the special path component name.
32+
public static final String ACCESS_STRING_OFFSET = "OFFSET";
3033

3134
private DataType prunedType;
3235
// allAccessPaths is used to record all access path include predicate access path and non-predicate access path,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter;
3434
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
3535
import org.apache.doris.nereids.trees.plans.Plan;
36+
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
3637
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
3738
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
3839
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
@@ -69,7 +70,9 @@ public Map<Slot, List<CollectAccessPathResult>> collect(Plan root, StatementCont
6970
}
7071

7172
private boolean shouldCollectAccessPath(Slot slot) {
72-
return slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType();
73+
return slot.getDataType() instanceof NestedColumnPrunable
74+
|| slot.getDataType().isVariantType()
75+
|| slot.getDataType().isStringLikeType();
7376
}
7477

7578
@Override
@@ -263,6 +266,15 @@ public Void visitLogicalFilter(LogicalFilter<? extends Plan> filter, StatementCo
263266
return filter.child().accept(this, context);
264267
}
265268

269+
@Override
270+
public Void visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, StatementContext context) {
271+
// Collect access paths from aggregate expressions (e.g. sum(length(str_col))) before
272+
// visiting children so that when the bottom project is processed next, str_col's offset
273+
// path is already recorded and the direct-DATA suppression guard can fire correctly.
274+
collectByExpressions(aggregate, context);
275+
return aggregate.child().accept(this, context);
276+
}
277+
266278
@Override
267279
public Void visitLogicalCTEAnchor(
268280
LogicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, StatementContext context) {

0 commit comments

Comments
 (0)