Skip to content

Commit 8262c8a

Browse files
masonh22claude
masonh22
and
claude
committed
test: add failing tests for projected literal distinct counts
A projected literal reports `distinct_count` as `Exact(1)` whatever its input row count, and a NULL literal reports `Exact(1)` as well. `AggregateStatistics` answers `COUNT(DISTINCT)` from an exact `distinct_count`, so queries over empty or filtered input return wrong results, for example: SELECT count(DISTINCT x), count(DISTINCT y) FROM (SELECT 1 AS x, 2 AS y FROM t WHERE a = 2); returns 1, 1 when no row has a = 2, and `SELECT NULL AS x, ...` makes count(DISTINCT x) return 1 instead of 0. Add a sqllogictest in parquet_statistics.slt covering a filtered input with an estimated row count, an estimated zero row count, an exactly empty file, an exact positive row count and a NULL literal, with EXPLAIN plans. Add a unit test for `project_statistics` over `Exact(0)`, `Exact(n)`, `Inexact(0)`, `Inexact(n)` and `Absent` row counts for non-null and NULL literals. These tests fail until the fix in the next commit. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0d79252 commit 8262c8a

2 files changed

Lines changed: 145 additions & 0 deletions

File tree

‎datafusion/physical-expr/src/projection.rs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3717,6 +3717,53 @@ pub(crate) mod tests {
37173717
Ok(())
37183718
}
37193719

3720+
// A literal's distinct count depends on whether any row is present: an
3721+
// estimated row count cannot prove the value appears. NULLs are not
3722+
// counted as distinct values, matching `COUNT(DISTINCT)`.
3723+
#[test]
3724+
fn test_project_statistics_literal_distinct_count_by_row_count() -> Result<()> {
3725+
let input_schema = get_schema();
3726+
let projection = ProjectionExprs::new(vec![
3727+
ProjectionExpr {
3728+
expr: Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),
3729+
alias: "constant".to_string(),
3730+
},
3731+
ProjectionExpr {
3732+
expr: Arc::new(Literal::new(ScalarValue::Int64(None))),
3733+
alias: "null_col".to_string(),
3734+
},
3735+
]);
3736+
let output_schema = projection.project_schema(&input_schema)?;
3737+
3738+
let cases = [
3739+
(Precision::Exact(0), Precision::Exact(0)),
3740+
(Precision::Exact(5), Precision::Exact(1)),
3741+
(Precision::Inexact(0), Precision::Inexact(0)),
3742+
(Precision::Inexact(5), Precision::Inexact(1)),
3743+
(Precision::Absent, Precision::Inexact(1)),
3744+
];
3745+
for (num_rows, expected) in cases {
3746+
let input_stats = Statistics {
3747+
num_rows,
3748+
total_byte_size: Precision::Absent,
3749+
column_statistics: Statistics::unknown_column(&input_schema),
3750+
};
3751+
let output_stats =
3752+
projection.project_statistics(input_stats, &output_schema)?;
3753+
assert_eq!(
3754+
output_stats.column_statistics[0].distinct_count, expected,
3755+
"non-null literal, num_rows={num_rows:?}"
3756+
);
3757+
assert_eq!(
3758+
output_stats.column_statistics[1].distinct_count,
3759+
Precision::Exact(0),
3760+
"NULL literal, num_rows={num_rows:?}"
3761+
);
3762+
}
3763+
3764+
Ok(())
3765+
}
3766+
37203767
// Test statistics calculation for complex type literal (e.g., Utf8 string)
37213768
#[test]
37223769
fn test_project_statistics_with_complex_type_literal() -> Result<()> {

‎datafusion/sqllogictest/test_files/parquet_statistics.slt‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,104 @@ physical_plan
337337
statement ok
338338
DROP TABLE cast_stats_ints_overflow;
339339

340+
######
341+
# Aggregate statistics through projected literals
342+
######
343+
344+
# A projected literal has one distinct value only when some row is present.
345+
# Two COUNT(DISTINCT) aggregates keep SingleDistinctToGroupBy from rewriting
346+
# the query, so it can be answered from statistics.
347+
statement ok
348+
COPY (SELECT * FROM (VALUES (1, 10), (3, 30)) AS t(a, b))
349+
TO 'test_files/scratch/parquet_statistics/literal_ndv.parquet' STORED AS PARQUET;
350+
351+
statement ok
352+
CREATE EXTERNAL TABLE literal_ndv STORED AS PARQUET
353+
LOCATION 'test_files/scratch/parquet_statistics/literal_ndv.parquet';
354+
355+
# No row has a = 2, but the filter only estimates its row count.
356+
query II
357+
SELECT count(DISTINCT x), count(DISTINCT y)
358+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a = 2);
359+
----
360+
0 0
361+
362+
query TT
363+
EXPLAIN SELECT count(DISTINCT x), count(DISTINCT y)
364+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a = 2);
365+
----
366+
physical_plan
367+
01)AggregateExec: mode=Single, gby=[], aggr=[count(DISTINCT x), count(DISTINCT y)]
368+
02)--ProjectionExec: expr=[1 as x, 2 as y]
369+
03)----FilterExec: a@0 = 2
370+
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/literal_ndv.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 = 2, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 2 AND 2 <= a_max@1, required_guarantees=[a in (2)]
371+
372+
# The filter estimates zero rows, but not exactly.
373+
query II
374+
SELECT count(DISTINCT x), count(DISTINCT y)
375+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a > 100);
376+
----
377+
0 0
378+
379+
# An exact positive row count proves each literal is present once.
380+
query II
381+
SELECT count(DISTINCT x), count(DISTINCT y)
382+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv);
383+
----
384+
1 1
385+
386+
query TT
387+
EXPLAIN SELECT count(DISTINCT x), count(DISTINCT y)
388+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv);
389+
----
390+
physical_plan
391+
01)ProjectionExec: expr=[1 as count(DISTINCT x), 1 as count(DISTINCT y)]
392+
02)--PlaceholderRowExec
393+
394+
# COUNT(DISTINCT) does not count NULLs.
395+
query II
396+
SELECT count(DISTINCT x), count(DISTINCT y)
397+
FROM (SELECT NULL AS x, 2 AS y FROM literal_ndv);
398+
----
399+
0 1
400+
401+
query TT
402+
EXPLAIN SELECT count(DISTINCT x), count(DISTINCT y)
403+
FROM (SELECT NULL AS x, 2 AS y FROM literal_ndv);
404+
----
405+
physical_plan
406+
01)ProjectionExec: expr=[0 as count(DISTINCT x), 1 as count(DISTINCT y)]
407+
02)--PlaceholderRowExec
408+
409+
statement ok
410+
DROP TABLE literal_ndv;
411+
412+
# An exactly empty file has no distinct values.
413+
statement ok
414+
COPY (SELECT * FROM (VALUES (1, 10)) AS t(a, b) WHERE a > 100)
415+
TO 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet' STORED AS PARQUET;
416+
417+
statement ok
418+
CREATE EXTERNAL TABLE literal_ndv_empty STORED AS PARQUET
419+
LOCATION 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet';
420+
421+
query II
422+
SELECT count(DISTINCT x), count(DISTINCT y)
423+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);
424+
----
425+
0 0
426+
427+
query TT
428+
EXPLAIN SELECT count(DISTINCT x), count(DISTINCT y)
429+
FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);
430+
----
431+
physical_plan
432+
01)ProjectionExec: expr=[0 as count(DISTINCT x), 0 as count(DISTINCT y)]
433+
02)--PlaceholderRowExec
434+
435+
statement ok
436+
DROP TABLE literal_ndv_empty;
437+
340438
statement ok
341439
SET datafusion.execution.target_partitions = 4;
342440

0 commit comments

Comments
 (0)