Skip to content

Commit cd7235f

Browse files
committed
test: compare dictionary functions on aggregate and filter output in dictionary.slt
1 parent b2a62af commit cd7235f

1 file changed

Lines changed: 47 additions & 17 deletions

File tree

‎datafusion/sqllogictest/test_files/dictionary.slt‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -715,48 +715,48 @@ NULL
715715
statement ok
716716
DROP TABLE dict_nulls_order;
717717

718-
# Aggregate output is emitted in `batch_size` slices that keep every group's
719-
# dictionary value, so these batches have many more dictionary values than
720-
# rows. Hashing them and running string functions on them must give the same
721-
# results as on the decoded values.
718+
# Aggregate output is emitted in `batch_size` slices of one array that holds
719+
# every group's dictionary value. Hashing the slices and running string
720+
# functions on them must give the same results as on the decoded values.
722721
statement ok
723-
set datafusion.execution.batch_size = 1;
722+
set datafusion.execution.batch_size = 2;
724723

725724
statement ok
726725
set datafusion.execution.target_partitions = 2;
727726

728727
statement ok
729728
create table padded_groups as
730729
select arrow_cast(
731-
case when value % 13 = 0 then null else ' Grp' || (value % 100) || ' ' end,
730+
case when value % 13 = 0 then null else ' Grp' || (value % 400) || ' ' end,
732731
'Dictionary(Int32, Utf8)'
733732
) as p
734733
from generate_series(1, 1000);
735734

736735
query I
737736
select count(*) from (select p from padded_groups group by p);
738737
----
739-
101
738+
401
740739

741740
# Each function on the grouped dictionary must match the same function on the
742-
# decoded value
743-
query I
744-
select count(*)
745-
from (
746-
select p, arrow_cast(p, 'Utf8') as u
747-
from (select p from padded_groups group by p)
748-
)
749-
where lower(p) is distinct from lower(u)
741+
# decoded value. These are projected, not filtered on: a filter on the group
742+
# key is pushed below the aggregate, where a batch has very few values.
743+
query B
744+
select distinct
745+
lower(p) is distinct from lower(u)
750746
or upper(p) is distinct from upper(u)
751747
or reverse(p) is distinct from reverse(u)
752748
or character_length(p) is distinct from character_length(u)
753749
or ascii(p) is distinct from ascii(u)
754750
or initcap(p) is distinct from initcap(u)
755751
or btrim(p) is distinct from btrim(u)
756752
or ltrim(p) is distinct from ltrim(u)
757-
or rtrim(p) is distinct from rtrim(u);
753+
or rtrim(p) is distinct from rtrim(u)
754+
from (
755+
select p, arrow_cast(p, 'Utf8') as u
756+
from (select p from padded_groups group by p)
757+
);
758758
----
759-
0
759+
false
760760

761761
query T
762762
select distinct arrow_typeof(lower(p)) from (select p from padded_groups group by p);
@@ -766,6 +766,36 @@ Dictionary(Int32, Utf8)
766766
statement ok
767767
drop table padded_groups;
768768

769+
# A filter keeps the whole dictionary, so these batches have many more
770+
# dictionary values than rows.
771+
statement ok
772+
set datafusion.execution.batch_size = 8192;
773+
774+
statement ok
775+
create table filtered_groups as
776+
select value as id, arrow_cast(' Grp' || (value % 400) || ' ', 'Dictionary(Int32, Utf8)') as p
777+
from generate_series(1, 1000);
778+
779+
query B
780+
select distinct
781+
lower(p) is distinct from lower(u)
782+
or upper(p) is distinct from upper(u)
783+
or reverse(p) is distinct from reverse(u)
784+
or character_length(p) is distinct from character_length(u)
785+
or ascii(p) is distinct from ascii(u)
786+
or initcap(p) is distinct from initcap(u)
787+
or btrim(p) is distinct from btrim(u)
788+
or ltrim(p) is distinct from ltrim(u)
789+
or rtrim(p) is distinct from rtrim(u)
790+
from (
791+
select p, arrow_cast(p, 'Utf8') as u from filtered_groups where id % 200 = 0
792+
);
793+
----
794+
false
795+
796+
statement ok
797+
drop table filtered_groups;
798+
769799
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
770800
# reset it explicitly.
771801
statement ok

0 commit comments

Comments
 (0)