Skip to content

Commit 14ae4a8

Browse files
committed
Try and explain what is going ion better in comments
1 parent f11e19d commit 14ae4a8

1 file changed

Lines changed: 11 additions & 13 deletions

File tree

‎datafusion/optimizer/src/replace_distinct_aggregate.rs‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -102,27 +102,25 @@ impl OptimizerRule for ReplaceDistinctWithAggregate {
102102
let schema = input.schema();
103103
let field_count = schema.fields().len();
104104
for dep in schema.functional_dependencies().iter() {
105-
// If the input is already unique on all of its columns (e.g.
106-
// it is a GROUP BY over exactly these columns), the DISTINCT
107-
// is a no-op and we can simply remove it.
108-
//
109-
// The dependency mode must be `Single`: a `Multi`
110-
// dependence (e.g. a former key downgraded by a join) means
111-
// equal rows may occur multiple times, so the DISTINCT
112-
// still has work to do.
113-
//
114-
// The grouping columns must also not contain NULLs because
115-
// a nullable UNIQUE constraint permits multiple NULL keys,
116-
// but DISTINCT treats NULLs as equal and must still
117-
// collapse them.
105+
// We can remove the DISTINCT entirely if functional
106+
// dependencies prove it is already unique and therefore a
107+
// DISTINCT would be a no op.
118108
let source_indices = &dep.source_indices;
119109
let any_source_field_nullable = source_indices
120110
.iter()
121111
.any(|&idx| schema.field(idx).is_nullable());
122112
let nullable = dep.nullable && any_source_field_nullable;
123113
if !nullable
114+
// The dependency mode must be `Single` because a `Multi`
115+
// dependence means equal rows may occur multiple times
124116
&& dep.mode == Dependency::Single
125117
&& source_indices.len() >= field_count
118+
// The grouping columns must not contain NULLs because a
119+
// nullable dependency allows multiple rows with NULL in
120+
// the same column (because NULL != NULL). However,
121+
// DISTINCT treats NULLs as equal and collapses them, so
122+
// we can only remove the DISTINCT if we know that no
123+
// source column can actually be NULL.
126124
&& source_indices[..field_count]
127125
.iter()
128126
.enumerate()

0 commit comments

Comments
 (0)