Skip to content

Commit bd90343

Browse files
mohitgurav20mohitgurav20
committed
test(sqllogictest): assert spilled_rows=100.0 K in aggregate_memory_spill.slt
The previous pattern used <slt:ignore> K, to verify a positive spill count, but AggregateExec emits output_rows=100.0 K, before spill_count in its metrics string. That makes the leading wildcard capable of consuming output_rows=100.0 K, and the test would pass even when spill_count=0 -- the false-positive @kosiew identified. Fix: match spilled_bytes=<slt:ignore>, between spill_count and spilled_rows, and assert the literal spilled_rows=100.0 K, for Cases A-F. The workload writes 100k groups to disk on every spill, so the formatter always produces 100.0 K when spilling occurred and 0 (no K suffix) when it did not.
1 parent 7e3dd7c commit bd90343

1 file changed

Lines changed: 15 additions & 21 deletions

File tree

‎datafusion/sqllogictest/test_files/aggregate_memory_spill.slt‎

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,9 @@ FROM (
5050
----
5151
100000 5000050000
5252

53-
# Assert spill happened: anchor on `spill_count=` so the leading wildcard
54-
# cannot skip to a later metric, then verify `spilled_rows` has a K-suffix
55-
# (only emitted for counts >= 1000). `spilled_rows=0` has no K-suffix, so the
56-
# pattern fails when spilling does not occur.
53+
# Assert spill happened: the aggregate spills all 100k rows, so
54+
# spilled_rows is formatted as "100.0 K". This value cannot appear
55+
# when spill_count=0 because spilled_rows would then be 0 (no K suffix).
5756
query TT
5857
EXPLAIN ANALYZE
5958
SELECT count(*), sum(total)
@@ -64,7 +63,7 @@ FROM (
6463
)
6564
----
6665
<slt:ignore>
67-
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
66+
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
6867
<slt:ignore>
6968

7069
# --- Case B: multi-column GROUP BY (is_single() = false) ---
@@ -79,8 +78,7 @@ FROM (
7978
----
8079
100000 5000050000
8180

82-
# Assert spill happened: spill_count anchors the search; spilled_rows K-suffix
83-
# only appears for counts >= 1000, failing when no spill occurs.
81+
# Assert spill happened (same reasoning as Case A).
8482
query TT
8583
EXPLAIN ANALYZE
8684
SELECT count(*), sum(total)
@@ -91,7 +89,7 @@ FROM (
9189
)
9290
----
9391
<slt:ignore>
94-
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000), v@0 * 13 % 100000 as t.v * Int64(13) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
92+
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000), v@0 * 13 % 100000 as t.v * Int64(13) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
9593
<slt:ignore>
9694

9795
# --- Case C: DISTINCT aggregate under memory limit ---
@@ -106,8 +104,7 @@ FROM (
106104
----
107105
100000 100000
108106

109-
# Assert spill happened: spill_count anchors the search; spilled_rows K-suffix
110-
# only appears for counts >= 1000, failing when no spill occurs.
107+
# Assert spill happened (same reasoning as Case A).
111108
query TT
112109
EXPLAIN ANALYZE
113110
SELECT count(*), sum(d)
@@ -118,7 +115,7 @@ FROM (
118115
)
119116
----
120117
<slt:ignore>
121-
04)------AggregateExec: mode=Single, gby=[group_alias_0@0 as group_alias_0], aggr=[count(alias1)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
118+
04)------AggregateExec: mode=Single, gby=[group_alias_0@0 as group_alias_0], aggr=[count(alias1)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
122119
<slt:ignore>
123120

124121
# --- Case D: multiple aggregates (sum/min/max) under memory limit ---
@@ -133,8 +130,7 @@ FROM (
133130
----
134131
100000 5000050000 1 100000
135132

136-
# Assert spill happened: spill_count anchors the search; spilled_rows K-suffix
137-
# only appears for counts >= 1000, failing when no spill occurs.
133+
# Assert spill happened (same reasoning as Case A).
138134
query TT
139135
EXPLAIN ANALYZE
140136
SELECT count(*), sum(s), min(mn), max(mx)
@@ -145,7 +141,7 @@ FROM (
145141
)
146142
----
147143
<slt:ignore>
148-
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v), min(t.v), max(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
144+
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v), min(t.v), max(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
149145
<slt:ignore>
150146

151147
# --- Case E: avg() aggregate (Float64 output) under memory limit ---
@@ -160,8 +156,7 @@ FROM (
160156
----
161157
100000 1 100000
162158

163-
# Assert spill happened: spill_count anchors the search; spilled_rows K-suffix
164-
# only appears for counts >= 1000, failing when no spill occurs.
159+
# Assert spill happened (same reasoning as Case A).
165160
query TT
166161
EXPLAIN ANALYZE
167162
SELECT count(*), min(a), max(a)
@@ -172,7 +167,7 @@ FROM (
172167
)
173168
----
174169
<slt:ignore>
175-
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[avg(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
170+
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[avg(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
176171
<slt:ignore>
177172

178173
# --- Case F: array_agg() aggregate (growable state) under memory limit ---
@@ -187,8 +182,7 @@ FROM (
187182
----
188183
100000 100000
189184

190-
# Assert spill happened: spill_count anchors the search; spilled_rows K-suffix
191-
# only appears for counts >= 1000, failing when no spill occurs.
185+
# Assert spill happened (same reasoning as Case A).
192186
query TT
193187
EXPLAIN ANALYZE
194188
SELECT count(*), sum(l)
@@ -199,7 +193,7 @@ FROM (
199193
)
200194
----
201195
<slt:ignore>
202-
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[array_agg(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
196+
04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[array_agg(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=100.0 K,<slt:ignore>]
203197
<slt:ignore>
204198

205199
# --- Case G: partial/final aggregation under memory limit ---
@@ -240,7 +234,7 @@ FROM (
240234
)
241235
----
242236
<slt:ignore>
243-
06)----------AggregateExec: mode=FinalPartitioned, gby=[t.v * Int64(7) % Int64(100000)@0 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
237+
06)----------AggregateExec: mode=FinalPartitioned, gby=[t.v * Int64(7) % Int64(100000)@0 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=<slt:ignore>, spilled_bytes=<slt:ignore>, spilled_rows=<slt:ignore> K,<slt:ignore>]
244238
<slt:ignore>
245239
08)--------------AggregateExec: mode=Partial, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>early_emit_count=<slt:ignore>]
246240
<slt:ignore>

0 commit comments

Comments
 (0)