Skip to content

Commit 6228602

Browse files
committed
[fix](fe) Preserve NULL folding and verify condition cache behavior
### What problem does this PR solve? Issue Number: None Related PR: #68672 Problem Summary: The new volatility checks prevent known-NULL TIME casts and array_shuffle calls from folding, unnecessarily excluding stable queries from SQL/query caches. Allow these NULL cases through the existing folding and type legality checks while retaining volatility for non-NULL inputs. The regression previously compared counts without proving cache behavior and recorded a seeded-shuffle count that depends on input block boundaries. Check isolated BE lookup metrics for volatile predicates at batch sizes 1 and 1024, and assert a profile cache hit on the second deterministic execution. Temporarily disable the global cache default so internal statistics queries cannot pollute the metric; restore it after the test. Generate only stable expected results. ### Release note Preserve constant folding and cache reuse for known-NULL TIME conversions and array shuffle expressions. ### Check List (For Author) - Test: 93 FE unit tests passed; FE build and Checkstyle passed. Three regression suites passed: condition_cache_eligibility (generated output and independent verification), test_cast_time_to_datetime, test_array_functions_by_literal. FE-only build used a temporary BE Rust dependency preflight workaround; the build script was restored. Runtime tests used the existing ASAN BE with FE be_exec_version=13; no BE source changed in this follow-up. - Behavior changed: Yes. Known-NULL expressions remain foldable and cacheable. - Does this need documentation: No
1 parent 07f16ef commit 6228602

6 files changed

Lines changed: 165 additions & 86 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ protected Cast(List<Expression> child, DataType targetType, boolean isExplicitTy
9696
@Override
9797
public boolean isDeterministic() {
9898
// TIME has no date. These casts supply the date from the current query's clock.
99-
return !(child().getDataType().isTimeType() && targetType.isDateLikeType());
99+
return child().isNullLiteral() || !(child().getDataType().isTimeType() && targetType.isDateLikeType());
100100
}
101101

102102
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayShuffle.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.nereids.types.ArrayType;
2828
import org.apache.doris.nereids.types.BigIntType;
2929
import org.apache.doris.nereids.types.coercion.AnyDataType;
30+
import org.apache.doris.nereids.util.ExpressionUtils;
3031

3132
import com.google.common.base.Preconditions;
3233
import com.google.common.collect.ImmutableList;
@@ -79,6 +80,12 @@ public ArrayShuffle withChildren(List<Expression> children) {
7980
return new ArrayShuffle(getFunctionParams(children));
8081
}
8182

83+
@Override
84+
public boolean foldable() {
85+
// Only NULL propagation is safe; non-NULL permutations depend on the input block.
86+
return ExpressionUtils.hasNullLiteral(getArguments());
87+
}
88+
8289
@Override
8390
public ArrayShuffle withIgnoreUniqueId(boolean ignoreUniqueId) {
8491
return new ArrayShuffle(volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), children);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.catalog.Function;
2121
import org.apache.doris.catalog.FunctionSignature;
2222
import org.apache.doris.catalog.FunctionVolatility;
23+
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
2324
import org.apache.doris.nereids.trees.expressions.Add;
2425
import org.apache.doris.nereids.trees.expressions.Cast;
2526
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -34,7 +35,9 @@
3435
import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdf;
3536
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
3637
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
38+
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
3739
import org.apache.doris.nereids.types.ArrayType;
40+
import org.apache.doris.nereids.types.BigIntType;
3841
import org.apache.doris.nereids.types.DataType;
3942
import org.apache.doris.nereids.types.DateTimeType;
4043
import org.apache.doris.nereids.types.DateTimeV2Type;
@@ -112,6 +115,35 @@ void testTimeCastsDependOnQueryDate() {
112115
new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT), DateV2Type.INSTANCE)));
113116
}
114117

118+
@Test
119+
void testNullTimeCastsFoldBeforeCacheEligibility() {
120+
for (DataType target : ImmutableList.of(DateType.INSTANCE, DateV2Type.INSTANCE,
121+
DateTimeType.INSTANCE, DateTimeV2Type.SYSTEM_DEFAULT, DateTimeV2Type.of(6))) {
122+
Expression cast = new Cast(new Cast(NullLiteral.INSTANCE, TimeV2Type.SYSTEM_DEFAULT), target);
123+
Expression folded = FoldConstantRuleOnFE.evaluateWithoutContext(cast);
124+
Assertions.assertEquals(new NullLiteral(target), folded);
125+
Assertions.assertFalse(uncacheable(folded));
126+
}
127+
// NULL folding must not bypass the legality check for unsupported cast pairs.
128+
Expression unsupported = new Cast(new NullLiteral(TimeV2Type.SYSTEM_DEFAULT),
129+
TimeStampTzType.SYSTEM_DEFAULT);
130+
Assertions.assertSame(unsupported, FoldConstantRuleOnFE.evaluateWithoutContext(unsupported));
131+
}
132+
133+
@Test
134+
void testNullShuffleArgumentsFoldBeforeCacheEligibility() {
135+
ArrayType arrayType = ArrayType.of(IntegerType.INSTANCE);
136+
Expression nullArray = new Cast(NullLiteral.INSTANCE, arrayType);
137+
Expression array = new SlotReference("a", arrayType);
138+
for (Expression shuffle : ImmutableList.of(new ArrayShuffle(nullArray),
139+
new ArrayShuffle(nullArray, new BigIntLiteral(1)),
140+
new ArrayShuffle(array, new NullLiteral(BigIntType.INSTANCE)))) {
141+
Expression folded = FoldConstantRuleOnFE.evaluateWithoutContext(shuffle);
142+
Assertions.assertEquals(new NullLiteral(arrayType), folded);
143+
Assertions.assertFalse(uncacheable(folded));
144+
}
145+
}
146+
115147
private boolean uncacheable(Expression expression) {
116148
return ExpressionUtils.containsNonCacheableExpression(ImmutableList.of(expression));
117149
}

‎fe/fe-core/src/test/java/org/apache/doris/planner/ConditionCacheEligibilityTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,30 @@ void testSqlCacheRejectsClockDependentCastsAndShuffle() throws Exception {
109109
}
110110
}
111111

112+
@Test
113+
void testNullExpressionsRemainCacheable() throws Exception {
114+
boolean enabled = connectContext.getSessionVariable().isEnableSqlCache();
115+
connectContext.getSessionVariable().setEnableSqlCache(true);
116+
try {
117+
for (String expression : ImmutableList.of("cast(cast(null as time) as date)",
118+
"cast(cast(null as time) as datetime)",
119+
"array_shuffle(cast(null as array<int>))",
120+
"array_shuffle(cast(null as array<int>), 1)",
121+
"shuffle(cast(null as array<int>), 1)", "array_shuffle(a, null)")) {
122+
NereidsPlanner planner = (NereidsPlanner) getSqlStmtExecutor(
123+
"select k, sum(k) from t where " + expression + " is null group by k").planner();
124+
Assertions.assertTrue(planner.getStatementContext().getSqlCacheContext().orElseThrow()
125+
.supportSqlCache(), expression);
126+
Assertions.assertTrue(scans(planner).get(0).isEnableConditionCache(), expression);
127+
Assertions.assertTrue(planner.getFragments().stream().anyMatch(fragment ->
128+
new QueryCacheNormalizer(fragment, planner.getDescTable())
129+
.normalize(connectContext).isPresent()), expression);
130+
}
131+
} finally {
132+
connectContext.getSessionVariable().setEnableSqlCache(enabled);
133+
}
134+
}
135+
112136
private List<TPlanNode> scans(Planner planner) {
113137
List<TPlanNode> scans = new ArrayList<>();
114138
for (PlanFragment fragment : planner.getFragments()) {
Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,10 @@
11
-- This file is automatically generated. You should know what you did if you want to edit this
2-
-- !rand_baseline --
3-
11
4-
5-
-- !shuffle_baseline --
6-
13
7-
8-
-- !rand_first --
9-
11
10-
11-
-- !rand_second --
12-
11
13-
14-
-- !shuffle_first --
15-
13
16-
17-
-- !shuffle_second --
18-
13
19-
20-
-- !shuffle_alias --
21-
13
22-
23-
-- !time_date --
24-
100000
25-
26-
-- !time_date_second --
27-
100000
28-
292
-- !deterministic_first --
303
10
314

325
-- !deterministic_second --
336
10
347

8+
-- !null_expressions --
9+
100000
10+

‎regression-test/suites/query_p0/cache/condition_cache_eligibility.groovy‎

Lines changed: 98 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -15,64 +15,104 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
suite("condition_cache_eligibility") {
19-
sql "DROP TABLE IF EXISTS condition_cache_eligibility"
20-
sql """
21-
CREATE TABLE condition_cache_eligibility (
22-
k BIGINT,
23-
a ARRAY<INT>,
24-
dt1 DATETIME,
25-
dt2 DATETIME
26-
) DUPLICATE KEY(k)
27-
DISTRIBUTED BY HASH(k) BUCKETS 1
28-
PROPERTIES ("replication_num" = "1")
29-
"""
30-
sql """
31-
INSERT INTO condition_cache_eligibility
32-
SELECT number, [1,2,3,4,5,6,7], '2026-09-30 12:00:00', '2026-09-30 12:00:00'
33-
FROM numbers("number" = "100000")
34-
"""
18+
import org.apache.doris.regression.action.ProfileAction
3519

36-
// Keep seeded random sequences stable across scans; isolate condition cache from result caches.
37-
sql "set enable_parallel_scan = false"
38-
sql "set enable_sql_cache = false"
39-
sql "set enable_query_cache = false"
40-
sql "set enable_condition_cache = false"
41-
order_qt_rand_baseline """
42-
SELECT count(*) FROM condition_cache_eligibility WHERE k < 0 OR rand(1) < 0.0001
43-
"""
44-
order_qt_shuffle_baseline """
45-
SELECT count(*) FROM condition_cache_eligibility WHERE array_shuffle(a, 1) = [1,2,3,4,5,6,7]
46-
"""
20+
// Isolate the global lookup metric from other condition-cache queries.
21+
suite("condition_cache_eligibility", "nonConcurrent") {
22+
// Internal statistics queries use new sessions and would otherwise pollute the global metric.
23+
setGlobalVarTemporary([enable_condition_cache: false]) {
24+
sql "DROP TABLE IF EXISTS condition_cache_eligibility"
25+
sql """
26+
CREATE TABLE condition_cache_eligibility (
27+
k BIGINT,
28+
a ARRAY<INT>,
29+
dt1 DATETIME,
30+
dt2 DATETIME
31+
) DUPLICATE KEY(k)
32+
DISTRIBUTED BY HASH(k) BUCKETS 1
33+
PROPERTIES ("replication_num" = "1")
34+
"""
35+
sql """
36+
INSERT INTO condition_cache_eligibility
37+
SELECT number, [1,2,3,4,5,6,7], '2026-09-30 12:00:00', '2026-09-30 12:00:00'
38+
FROM numbers("number" = "100000")
39+
"""
4740

48-
sql "set enable_condition_cache = true"
49-
order_qt_rand_first """
50-
SELECT count(*) FROM condition_cache_eligibility WHERE k < 0 OR rand(1) < 0.0001
51-
"""
52-
order_qt_rand_second """
53-
SELECT count(*) FROM condition_cache_eligibility WHERE k < 0 OR rand(1) < 0.0001
54-
"""
55-
order_qt_shuffle_first """
56-
SELECT count(*) FROM condition_cache_eligibility WHERE array_shuffle(a, 1) = [1,2,3,4,5,6,7]
57-
"""
58-
order_qt_shuffle_second """
59-
SELECT count(*) FROM condition_cache_eligibility WHERE array_shuffle(a, 1) = [1,2,3,4,5,6,7]
60-
"""
61-
order_qt_shuffle_alias """
62-
SELECT count(*) FROM condition_cache_eligibility WHERE shuffle(a, 1) = [1,2,3,4,5,6,7]
63-
"""
64-
order_qt_time_date """
65-
SELECT count(*) FROM condition_cache_eligibility
66-
WHERE CAST(TIMEDIFF(dt1, dt2) AS DATE) = CURRENT_DATE()
67-
"""
68-
order_qt_time_date_second """
69-
SELECT count(*) FROM condition_cache_eligibility
70-
WHERE CAST(TIMEDIFF(dt1, dt2) AS DATE) = CURRENT_DATE()
71-
"""
72-
order_qt_deterministic_first """
73-
SELECT count(*) FROM condition_cache_eligibility WHERE k % 10000 = 0
74-
"""
75-
order_qt_deterministic_second """
76-
SELECT count(*) FROM condition_cache_eligibility WHERE k % 10000 = 0
77-
"""
41+
sql "set enable_parallel_scan = false"
42+
sql "set enable_sql_cache = false"
43+
sql "set enable_query_cache = false"
44+
sql "set enable_condition_cache = true"
45+
46+
def backendIdToIp = [:]
47+
def backendIdToHttpPort = [:]
48+
getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort)
49+
def cacheLookups = {
50+
long total = 0
51+
backendIdToIp.each { backendId, ip ->
52+
httpTest {
53+
endpoint "${ip}:${backendIdToHttpPort[backendId]}"
54+
uri "/metrics?type=json"
55+
op "get"
56+
printResponse false
57+
check { code, body ->
58+
assertEquals(200, code)
59+
def metric = parseJson(body).find {
60+
it.tags.metric == "condition_cache" && it.tags.type == "condition_cache_search"
61+
}
62+
assertNotNull(metric, "Missing condition-cache lookup metric")
63+
total += metric.value as long
64+
}
65+
}
66+
}
67+
return total
68+
}
69+
70+
// Seeded shuffle is repeatable within a block, but its count changes with block boundaries.
71+
// Check that volatile scans never look up the cache instead of recording their random counts.
72+
for (int batchSize : [1, 1024]) {
73+
sql "set batch_size = ${batchSize}"
74+
for (String predicate : [
75+
"k < 0 OR rand(1) < 0.0001",
76+
"k < 0 OR rand() < 0.0001",
77+
"array_shuffle(a) = [1,2,3,4,5,6,7]",
78+
"array_shuffle(a, 1) = [1,2,3,4,5,6,7]",
79+
"shuffle(a, 1) = [1,2,3,4,5,6,7]",
80+
"CAST(TIMEDIFF(dt1, dt2) AS DATE) = CURRENT_DATE()",
81+
"CAST(TIMEDIFF(dt1, dt2) AS DATETIME) = CAST(CURRENT_DATE() AS DATETIME)"]) {
82+
long before = cacheLookups()
83+
2.times {
84+
// Keep one-row batches inexpensive under ASAN while retaining the volatile scan predicate.
85+
sql "SELECT count(*) FROM condition_cache_eligibility WHERE k < 1000 AND (${predicate})"
86+
}
87+
assertEquals(before, cacheLookups(),
88+
"Volatile scan looked up condition cache: batch_size=${batchSize}, ${predicate}")
89+
}
90+
}
91+
92+
sql "set enable_profile = false"
93+
def token = UUID.randomUUID().toString()
94+
def deterministic = """
95+
SELECT '${token}', count(*) FROM condition_cache_eligibility WHERE k % 10000 = 0
96+
"""
97+
long before = cacheLookups()
98+
// Execute the same SQL twice so both scans have the same digest; omit the profile token from output.
99+
quickRunTest("deterministic_first", deterministic, true, { row -> [row[1]] })
100+
// Only the second execution records a profile, so the token cannot select the warm-up query.
101+
sql "set enable_profile = true"
102+
sql "set profile_level = 2"
103+
quickRunTest("deterministic_second", deterministic, true, { row -> [row[1]] })
104+
assertTrue(cacheLookups() > before, "Deterministic scans did not look up condition cache")
105+
def profile = new ProfileAction(context).getProfileBySql(token, ["ConditionCacheHit"])
106+
def hits = (profile =~ /ConditionCacheHit: (\d+)/)
107+
assertTrue(hits.findAll().any { (it[1] as long) > 0 },
108+
"Second deterministic scan did not hit condition cache: ${profile}")
109+
110+
order_qt_null_expressions """
111+
SELECT count(*) FROM condition_cache_eligibility
112+
WHERE CAST(CAST(NULL AS TIME) AS DATETIME) IS NULL
113+
AND array_shuffle(CAST(NULL AS ARRAY<INT>), 1) IS NULL
114+
AND shuffle(CAST(NULL AS ARRAY<INT>)) IS NULL
115+
AND array_shuffle(a, NULL) IS NULL
116+
"""
117+
}
78118
}

0 commit comments

Comments
 (0)