Skip to content

Commit d78bb71

Browse files
committed
[fix](fe) Check nested casts and runtime filter cache eligibility
### What problem does this PR solve? Issue Number: None Related PR: #68672 Problem Summary: TIME-to-date conversions inside ARRAY, MAP and STRUCT casts use the query date, but the outer container cast was classified as deterministic. SQL, query and condition caches could therefore reuse results across dates. Runtime-filter probe expressions such as CAST(CAST(p.s AS TIME) AS DATETIME) also bypassed scan predicate eligibility checks and could reuse stale all-false ranges. Recursively inspect source/target container types and apply shared expression eligibility to singleton and grouped runtime-filter targets, including implicit probe casts. Preserve known-NULL folding, cast legality checks and cache use for deterministic expressions. Add planner, expression and translator unit coverage and verify zero BE cache lookups for the affected scans. ### Release note Prevent incorrect cached results for nested TIME-to-date conversions and query-date-dependent runtime-filter probes. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - 102 FE unit tests across seven targeted classes passed. - FE build with Checkstyle passed. - condition_cache_eligibility output generation and verification passed. - test_cast_time_to_datetime and test_array_functions_by_literal passed. - Existing ASAN BE, rebuilt FE; deterministic ConditionCacheHit=1 and ConditionCacheFilteredRows=79872, volatile scans have zero lookup deltas. - Behavior changed: Yes, unsafe nested casts and runtime-filter probes bypass caching. - Does this need documentation: No
1 parent 6228602 commit d78bb71

7 files changed

Lines changed: 213 additions & 5 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.doris.nereids.trees.expressions.SlotReference;
3232
import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter;
3333
import org.apache.doris.nereids.types.DataType;
34+
import org.apache.doris.nereids.util.ExpressionUtils;
3435
import org.apache.doris.planner.CTEScanNode;
3536
import org.apache.doris.planner.DataStreamSink;
3637
import org.apache.doris.planner.DistributionMode;
@@ -232,6 +233,7 @@ private void createLegacyRuntimeFilterFromGroup(List<RuntimeFilter> group,
232233
Expr targetExpr = targetExprList.get(i);
233234
origFilter.addTarget(new RuntimeFilterTarget(
234235
scanNode, targetExpr, true, isLocalTarget));
236+
updateConditionCacheEligibility(scanNode, group.get(i));
235237
// TRuntimeFilterDesc keys target expressions and pruning metadata by scan node ID.
236238
// If one grouped RF has different targets on the same scan, BE cannot match
237239
// metadata back to a specific target expression, so skip pruning metadata.
@@ -331,6 +333,7 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT
331333
Expr targetExpr = targetExprList.get(i);
332334
origFilter.addTarget(new RuntimeFilterTarget(
333335
scanNode, targetExpr, true, isLocalTarget));
336+
updateConditionCacheEligibility(scanNode, filter);
334337
setPruningMetadata(origFilter, scanNode, filter);
335338
}
336339
origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv());
@@ -361,6 +364,19 @@ private org.apache.doris.planner.RuntimeFilter finalize(org.apache.doris.planner
361364
return origFilter;
362365
}
363366

367+
private void updateConditionCacheEligibility(ScanNode scanNode, RuntimeFilter filter) {
368+
Expression target = filter.getTargetExpression();
369+
DataType sourceType = filter.getSrcExpr().getDataType();
370+
// Include the implicit cast inserted by castTargetToSourceTypeIfNeeded.
371+
if (!target.getDataType().equals(sourceType)) {
372+
target = new Cast(target, sourceType);
373+
}
374+
// RF probes become scan predicates even when no PhysicalFilter is attached to the scan.
375+
if (ExpressionUtils.containsNonCacheableExpression(ImmutableList.of(target))) {
376+
scanNode.setEnableConditionCache(false);
377+
}
378+
}
379+
364380
private void setPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter,
365381
ScanNode scanNode, RuntimeFilter nereidsFilter) {
366382
if (nereidsFilter.canPrunePartitions()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,38 @@ protected Cast(List<Expression> child, DataType targetType, boolean isExplicitTy
9696
@Override
9797
public boolean isDeterministic() {
9898
// TIME has no date. These casts supply the date from the current query's clock.
99-
return child().isNullLiteral() || !(child().getDataType().isTimeType() && targetType.isDateLikeType());
99+
return child().isNullLiteral() || !containsTimeToDateCast(child().getDataType(), targetType);
100+
}
101+
102+
private static boolean containsTimeToDateCast(DataType sourceType, DataType targetType) {
103+
if (sourceType.equals(targetType)) {
104+
return false;
105+
}
106+
if (sourceType instanceof ArrayType && targetType instanceof ArrayType) {
107+
return containsTimeToDateCast(((ArrayType) sourceType).getItemType(),
108+
((ArrayType) targetType).getItemType());
109+
}
110+
if (sourceType instanceof MapType && targetType instanceof MapType) {
111+
MapType sourceMap = (MapType) sourceType;
112+
MapType targetMap = (MapType) targetType;
113+
return containsTimeToDateCast(sourceMap.getKeyType(), targetMap.getKeyType())
114+
|| containsTimeToDateCast(sourceMap.getValueType(), targetMap.getValueType());
115+
}
116+
if (sourceType instanceof StructType && targetType instanceof StructType) {
117+
List<StructField> sourceFields = ((StructType) sourceType).getFields();
118+
List<StructField> targetFields = ((StructType) targetType).getFields();
119+
// foldable() is called before CheckCast rejects incompatible struct shapes.
120+
if (sourceFields.size() != targetFields.size()) {
121+
return false;
122+
}
123+
for (int i = 0; i < sourceFields.size(); i++) {
124+
if (containsTimeToDateCast(sourceFields.get(i).getDataType(), targetFields.get(i).getDataType())) {
125+
return true;
126+
}
127+
}
128+
return false;
129+
}
130+
return sourceType.isTimeType() && targetType.isDateLikeType();
100131
}
101132

102133
@Override

‎fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.common.IdGenerator;
3030
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
3131
import org.apache.doris.nereids.trees.expressions.Add;
32+
import org.apache.doris.nereids.trees.expressions.Cast;
3233
import org.apache.doris.nereids.trees.expressions.ExprId;
3334
import org.apache.doris.nereids.trees.expressions.Expression;
3435
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -37,7 +38,10 @@
3738
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
3839
import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter;
3940
import org.apache.doris.nereids.types.DataType;
41+
import org.apache.doris.nereids.types.DateTimeV2Type;
4042
import org.apache.doris.nereids.types.IntegerType;
43+
import org.apache.doris.nereids.types.StringType;
44+
import org.apache.doris.nereids.types.TimeV2Type;
4145
import org.apache.doris.planner.OlapScanNode;
4246
import org.apache.doris.planner.PlanFragment;
4347
import org.apache.doris.planner.PlanFragmentId;
@@ -150,6 +154,40 @@ void testDisabledFeatureSkipsClassificationAndSerialization() {
150154
Assertions.assertFalse(desc.isSetBucketPruningTargetIds());
151155
}
152156

157+
@Test
158+
void testVolatileProbeDisablesCacheForSingleAndGroupedFilters() {
159+
for (boolean grouped : ImmutableList.of(false, true)) {
160+
for (boolean implicitCast : ImmutableList.of(false, true)) {
161+
TranslatorHarness harness = new TranslatorHarness(PrimitiveType.DATETIMEV2);
162+
SlotReference target = harness.addTargetSlot("s", new Column("s", PrimitiveType.STRING),
163+
StringType.INSTANCE);
164+
Expression time = new Cast(target, TimeV2Type.SYSTEM_DEFAULT);
165+
Expression probe = implicitCast ? time : new Cast(time, DateTimeV2Type.SYSTEM_DEFAULT);
166+
RuntimeFilter filter = harness.newFilter(target, probe);
167+
List<RuntimeFilter> filters = grouped
168+
? ImmutableList.of(harness.newFilter(target, new Cast(target, DateTimeV2Type.SYSTEM_DEFAULT)),
169+
filter)
170+
: ImmutableList.of(filter);
171+
TRuntimeFilterDesc desc = harness.translate(filters);
172+
Assertions.assertTrue(desc.planId_to_target_expr.containsKey(SCAN_NODE_ID));
173+
Mockito.verify(harness.scanNode).setEnableConditionCache(false);
174+
Mockito.verify(harness.scanNode, Mockito.never()).setEnableConditionCache(true);
175+
}
176+
}
177+
}
178+
179+
@Test
180+
void testDeterministicProbeDoesNotChangeCacheEligibility() {
181+
for (boolean grouped : ImmutableList.of(false, true)) {
182+
TranslatorHarness harness = new TranslatorHarness(PrimitiveType.BIGINT);
183+
SlotReference target = harness.addTargetSlot("k", harness.distributionColumn, IntegerType.INSTANCE);
184+
RuntimeFilter filter = harness.newFilter(target, new Add(target, new IntegerLiteral(1)));
185+
harness.translate(grouped ? ImmutableList.of(filter, harness.newFilter(target, target))
186+
: ImmutableList.of(filter));
187+
Mockito.verify(harness.scanNode, Mockito.never()).setEnableConditionCache(Mockito.anyBoolean());
188+
}
189+
}
190+
153191
private static int firstLegacySlotId(TranslatorHarness harness, SlotReference target) {
154192
return harness.translatorContext.findSlotRef(target.getExprId()).getSlotId().asInt();
155193
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@
4444
import org.apache.doris.nereids.types.DateType;
4545
import org.apache.doris.nereids.types.DateV2Type;
4646
import org.apache.doris.nereids.types.IntegerType;
47+
import org.apache.doris.nereids.types.MapType;
4748
import org.apache.doris.nereids.types.StringType;
49+
import org.apache.doris.nereids.types.StructField;
50+
import org.apache.doris.nereids.types.StructType;
4851
import org.apache.doris.nereids.types.TimeStampTzType;
4952
import org.apache.doris.nereids.types.TimeV2Type;
5053

@@ -115,6 +118,52 @@ void testTimeCastsDependOnQueryDate() {
115118
new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT), DateV2Type.INSTANCE)));
116119
}
117120

121+
@Test
122+
void testNestedTimeCastsDependOnQueryDate() {
123+
for (DataType date : ImmutableList.of(DateV2Type.INSTANCE, DateTimeV2Type.SYSTEM_DEFAULT)) {
124+
DataType time = TimeV2Type.SYSTEM_DEFAULT;
125+
for (DataType[] types : ImmutableList.of(
126+
new DataType[] {ArrayType.of(time), ArrayType.of(date)},
127+
new DataType[] {MapType.of(time, IntegerType.INSTANCE), MapType.of(date, IntegerType.INSTANCE)},
128+
new DataType[] {MapType.of(IntegerType.INSTANCE, time), MapType.of(IntegerType.INSTANCE, date)},
129+
new DataType[] {structWithSecondField(time), structWithSecondField(date)},
130+
new DataType[] {ArrayType.of(MapType.of(IntegerType.INSTANCE, structWithSecondField(time))),
131+
ArrayType.of(MapType.of(IntegerType.INSTANCE, structWithSecondField(date)))})) {
132+
Cast cast = new Cast(new SlotReference("nested", types[0]), types[1]);
133+
Assertions.assertFalse(cast.isDeterministic(), cast.toSql());
134+
Assertions.assertFalse(cast.foldable(), cast.toSql());
135+
Assertions.assertTrue(uncacheable(cast), cast.toSql());
136+
Expression nullCast = new Cast(new NullLiteral(types[0]), types[1]);
137+
Assertions.assertEquals(new NullLiteral(types[1]),
138+
FoldConstantRuleOnFE.evaluateWithoutContext(nullCast));
139+
}
140+
}
141+
}
142+
143+
@Test
144+
void testDeterministicNestedCastsRemainCacheable() {
145+
DataType time = TimeV2Type.SYSTEM_DEFAULT;
146+
for (DataType[] types : ImmutableList.of(
147+
new DataType[] {ArrayType.of(time), ArrayType.of(StringType.INSTANCE)},
148+
new DataType[] {MapType.of(IntegerType.INSTANCE, time), MapType.of(BigIntType.INSTANCE, time)},
149+
new DataType[] {structWithSecondField(time), structWithSecondField(TimeV2Type.of(6))},
150+
new DataType[] {ArrayType.of(DateTimeV2Type.SYSTEM_DEFAULT), ArrayType.of(DateV2Type.INSTANCE)})) {
151+
Cast cast = new Cast(new SlotReference("nested", types[0]), types[1]);
152+
Assertions.assertTrue(cast.isDeterministic(), cast.toSql());
153+
Assertions.assertTrue(cast.foldable(), cast.toSql());
154+
Assertions.assertFalse(uncacheable(cast), cast.toSql());
155+
}
156+
// An invalid shape must still reach the existing cast-legality check before NULL folding.
157+
Expression invalid = new Cast(new NullLiteral(structWithSecondField(time)),
158+
new StructType(ImmutableList.of(new StructField("x", DateV2Type.INSTANCE, true, ""))));
159+
Assertions.assertSame(invalid, FoldConstantRuleOnFE.evaluateWithoutContext(invalid));
160+
}
161+
162+
private DataType structWithSecondField(DataType type) {
163+
return new StructType(ImmutableList.of(new StructField("stable", IntegerType.INSTANCE, true, ""),
164+
new StructField("value", type, true, "")));
165+
}
166+
118167
@Test
119168
void testNullTimeCastsFoldBeforeCacheEligibility() {
120169
for (DataType target : ImmutableList.of(DateType.INSTANCE, DateV2Type.INSTANCE,

‎fe/fe-core/src/test/java/org/apache/doris/planner/ConditionCacheEligibilityTest.java‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class ConditionCacheEligibilityTest extends TestWithFeService {
3636
protected void runBeforeAll() throws Exception {
3737
createDatabase("cache_eligibility");
3838
useDatabase("cache_eligibility");
39-
createTable("create table t(k int, a array<int>, dt1 datetime, dt2 datetime) "
39+
createTable("create table t(k int, a array<int>, dt1 datetime, dt2 datetime, s string) "
4040
+ "duplicate key(k) distributed by hash(k) buckets 3 properties('replication_num'='1')");
4141
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
4242
connectContext.getSessionVariable().setEnableQueryCache(true);
@@ -54,7 +54,10 @@ void testBothCachesRejectUnstableFilters() throws Exception {
5454
"array_shuffle(a, 1) = [1,2,3]",
5555
"shuffle(a, 1) = [1,2,3]",
5656
"cast(timediff(dt1, dt2) as date) = date '2026-10-01'",
57-
"cast(timediff(dt1, dt2) as datetime) = cast(dt1 as datetime)")) {
57+
"cast(timediff(dt1, dt2) as datetime) = cast(dt1 as datetime)",
58+
"cast(array(timediff(dt1, dt2)) as array<datetime>)[1] = dt1",
59+
"cast(map(1, timediff(dt1, dt2)) as map<int, datetime>)[1] = dt1",
60+
"struct_element(cast(struct(k, timediff(dt1, dt2)) as struct<id:int, d:datetime>), 2) = dt1")) {
5861
Planner planner = getSqlStmtExecutor("select k, sum(k) from t where " + predicate + " group by k")
5962
.planner();
6063
List<TPlanNode> scans = scans(planner);
@@ -98,7 +101,10 @@ void testSqlCacheRejectsClockDependentCastsAndShuffle() throws Exception {
98101
.supportSqlCache(), expression);
99102
}
100103
for (String expression : ImmutableList.of("cast(timediff(dt1, dt2) as date)",
101-
"cast(timediff(dt1, dt2) as datetime)", "array_shuffle(a)", "shuffle(a, 1)")) {
104+
"cast(timediff(dt1, dt2) as datetime)", "array_shuffle(a)", "shuffle(a, 1)",
105+
"cast(array(timediff(dt1, dt2)) as array<datetime>)",
106+
"cast(map(1, timediff(dt1, dt2)) as map<int, datetime>)",
107+
"cast(struct(k, timediff(dt1, dt2)) as struct<id:int, d:datetime>)")) {
102108
NereidsPlanner planner = (NereidsPlanner) getSqlStmtExecutor("select " + expression + " from t")
103109
.planner();
104110
SqlCacheContext cache = planner.getStatementContext().getSqlCacheContext().orElseThrow();
@@ -109,6 +115,25 @@ void testSqlCacheRejectsClockDependentCastsAndShuffle() throws Exception {
109115
}
110116
}
111117

118+
@Test
119+
void testRuntimeFilterProbeEligibility() throws Exception {
120+
for (boolean clockDependent : ImmutableList.of(true, false)) {
121+
String probe = clockDependent ? "cast(cast(p.s as time) as datetime)" : "cast(p.s as datetime)";
122+
Planner planner = getSqlStmtExecutor("select /*+ SET_VAR(disable_join_reorder=true, "
123+
+ "enable_runtime_filter_prune=false) */ p.k from t p join [broadcast] t b on "
124+
+ probe + " = b.dt1").planner();
125+
List<TPlanNode> targets = new ArrayList<>();
126+
for (TPlanNode scan : scans(planner)) {
127+
if (scan.isSetRuntimeFilters() && scan.getRuntimeFilters().stream().anyMatch(filter ->
128+
filter.planId_to_target_expr.containsKey(scan.getNodeId()))) {
129+
targets.add(scan);
130+
Assertions.assertEquals(!clockDependent, scan.isEnableConditionCache(), probe);
131+
}
132+
}
133+
Assertions.assertFalse(targets.isEmpty(), "Expected a runtime filter target for " + probe);
134+
}
135+
}
136+
112137
@Test
113138
void testNullExpressionsRemainCacheable() throws Exception {
114139
boolean enabled = connectContext.getSessionVariable().isEnableSqlCache();

‎regression-test/data/query_p0/cache/condition_cache_eligibility.out‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !volatile_rf_first --
3+
10000
4+
5+
-- !volatile_rf_second --
6+
10000
7+
28
-- !deterministic_first --
39
10
410

‎regression-test/suites/query_p0/cache/condition_cache_eligibility.groovy‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,11 @@ suite("condition_cache_eligibility", "nonConcurrent") {
7878
"array_shuffle(a, 1) = [1,2,3,4,5,6,7]",
7979
"shuffle(a, 1) = [1,2,3,4,5,6,7]",
8080
"CAST(TIMEDIFF(dt1, dt2) AS DATE) = CURRENT_DATE()",
81-
"CAST(TIMEDIFF(dt1, dt2) AS DATETIME) = CAST(CURRENT_DATE() AS DATETIME)"]) {
81+
"CAST(TIMEDIFF(dt1, dt2) AS DATETIME) = CAST(CURRENT_DATE() AS DATETIME)",
82+
"CAST(ARRAY(TIMEDIFF(dt1, dt2)) AS ARRAY<DATETIME>)[1] = CAST(CURRENT_DATE() AS DATETIME)",
83+
"CAST(MAP(1, TIMEDIFF(dt1, dt2)) AS MAP<INT, DATETIME>)[1] = CAST(CURRENT_DATE() AS DATETIME)",
84+
"STRUCT_ELEMENT(CAST(STRUCT(k, TIMEDIFF(dt1, dt2)) AS STRUCT<id:BIGINT, d:DATETIME>), 2)"
85+
+ " = CAST(CURRENT_DATE() AS DATETIME)"]) {
8286
long before = cacheLookups()
8387
2.times {
8488
// Keep one-row batches inexpensive under ASAN while retaining the volatile scan predicate.
@@ -89,6 +93,45 @@ suite("condition_cache_eligibility", "nonConcurrent") {
8993
}
9094
}
9195

96+
sql "DROP TABLE IF EXISTS condition_cache_rf_probe"
97+
sql """
98+
CREATE TABLE condition_cache_rf_probe (k BIGINT, s STRING NOT NULL)
99+
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
100+
PROPERTIES ("replication_num" = "1")
101+
"""
102+
sql """
103+
INSERT INTO condition_cache_rf_probe
104+
SELECT number, '00:00:00' FROM numbers("number" = "10000")
105+
"""
106+
sql "DROP TABLE IF EXISTS condition_cache_rf_build"
107+
sql """
108+
CREATE TABLE condition_cache_rf_build (d DATETIME NOT NULL)
109+
DUPLICATE KEY(d) DISTRIBUTED BY HASH(d) BUCKETS 1
110+
PROPERTIES ("replication_num" = "1")
111+
"""
112+
// Keep the expected join count stable even if these executions cross midnight.
113+
sql """
114+
INSERT INTO condition_cache_rf_build VALUES
115+
(CURRENT_DATE() - INTERVAL 1 DAY), (CURRENT_DATE()), (CURRENT_DATE() + INTERVAL 1 DAY)
116+
"""
117+
sql "set enable_runtime_filter_prune = false"
118+
sql "set runtime_filter_wait_infinitely = true"
119+
sql "set disable_join_reorder = true"
120+
def rfQuery = """
121+
SELECT count(*) FROM condition_cache_rf_probe p
122+
JOIN [broadcast] condition_cache_rf_build b
123+
ON CAST(CAST(p.s AS TIME) AS DATETIME) = b.d
124+
"""
125+
explain {
126+
sql rfQuery
127+
contains "RF"
128+
contains "CAST(CAST"
129+
}
130+
long beforeRuntimeFilter = cacheLookups()
131+
order_qt_volatile_rf_first rfQuery
132+
order_qt_volatile_rf_second rfQuery
133+
assertEquals(beforeRuntimeFilter, cacheLookups(), "Volatile runtime-filter probe looked up condition cache")
134+
92135
sql "set enable_profile = false"
93136
def token = UUID.randomUUID().toString()
94137
def deterministic = """

0 commit comments

Comments
 (0)