Skip to content

Commit 14efad6

Browse files
adriangbclaude
andcommitted
test: sqllogictests for adaptive filter placement
The same queries give the same results with and without `datafusion.execution.adaptive_filter_placement`: a scattered filter (moves to the post-scan filter), a clustered filter (stays a row filter), a mix of both with a string filter, a `LIMIT` with `ORDER BY` and a `LIMIT` without it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 68309fe commit 14efad6

1 file changed

Lines changed: 118 additions & 0 deletions

File tree

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Tests for `datafusion.execution.adaptive_filter_placement`: the Parquet scan
19+
# places each filter conjunct in the row filter or in the post-scan filter,
20+
# and changes the placement at row group boundaries. The results must be the
21+
# same as without adaptive placement.
22+
23+
statement ok
24+
set datafusion.execution.parquet.max_row_group_size = 1024;
25+
26+
# `a` is scattered (every second row), `b` is clustered (increases every 100
27+
# rows), `s` is a wide string column that the filters do not read.
28+
query I
29+
COPY (
30+
SELECT i % 2 AS a, i / 100 AS b, repeat('x', 50) || i AS s, i AS v
31+
FROM generate_series(0, 9999) AS t(i)
32+
) TO 'test_files/scratch/parquet_adaptive_filter_placement/t.parquet'
33+
STORED AS PARQUET;
34+
----
35+
10000
36+
37+
statement ok
38+
reset datafusion.execution.parquet.max_row_group_size;
39+
40+
statement ok
41+
CREATE EXTERNAL TABLE t STORED AS PARQUET
42+
LOCATION 'test_files/scratch/parquet_adaptive_filter_placement/t.parquet';
43+
44+
statement ok
45+
set datafusion.execution.parquet.pushdown_filters = true;
46+
47+
statement ok
48+
set datafusion.execution.batch_size = 256;
49+
50+
statement ok
51+
set datafusion.execution.adaptive_filter_placement = true;
52+
53+
query IIII
54+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE a = 0;
55+
----
56+
5000 0 9998 24995000
57+
58+
query IIII
59+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE b < 5 OR b BETWEEN 50 AND 54;
60+
----
61+
1000 0 5499 2749500
62+
63+
query IIII
64+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE a = 0 AND b >= 90 AND s LIKE '%8';
65+
----
66+
100 9008 9998 950300
67+
68+
query IT
69+
SELECT v, s FROM t WHERE a = 1 AND b = 42 ORDER BY v LIMIT 3;
70+
----
71+
4201 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4201
72+
4203 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4203
73+
4205 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4205
74+
75+
# A limit without an order: the placement can move `a = 0` to the post-scan
76+
# filter in the middle of the file.
77+
query I
78+
SELECT count(*) FROM (SELECT v FROM t WHERE a = 0 LIMIT 3000);
79+
----
80+
3000
81+
82+
# The same queries without adaptive placement give the same results.
83+
statement ok
84+
set datafusion.execution.adaptive_filter_placement = false;
85+
86+
query IIII
87+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE a = 0;
88+
----
89+
5000 0 9998 24995000
90+
91+
query IIII
92+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE b < 5 OR b BETWEEN 50 AND 54;
93+
----
94+
1000 0 5499 2749500
95+
96+
query IIII
97+
SELECT count(*), min(v), max(v), sum(v) FROM t WHERE a = 0 AND b >= 90 AND s LIKE '%8';
98+
----
99+
100 9008 9998 950300
100+
101+
query IT
102+
SELECT v, s FROM t WHERE a = 1 AND b = 42 ORDER BY v LIMIT 3;
103+
----
104+
4201 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4201
105+
4203 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4203
106+
4205 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx4205
107+
108+
statement ok
109+
reset datafusion.execution.batch_size;
110+
111+
statement ok
112+
reset datafusion.execution.parquet.pushdown_filters;
113+
114+
statement ok
115+
reset datafusion.execution.adaptive_filter_placement;
116+
117+
statement ok
118+
DROP TABLE t;

0 commit comments

Comments
 (0)