|
2 | 2 | # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership |
3 | 3 | # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements |
4 | 4 |
|
5 | | -# Spilled sorts over wide rows. |
| 5 | +# Spilled sorts and aggregates over wide rows. |
6 | 6 | # |
7 | 7 | # Each query below spills rows much wider than `batch_size` rows of narrow |
8 | 8 | # data, under a memory limit that fits each row but not a whole batch of them. |
@@ -61,6 +61,81 @@ RESET datafusion.execution.sort_spill_reservation_bytes |
61 | 61 | statement ok |
62 | 62 | RESET datafusion.execution.batch_size |
63 | 63 |
|
| 64 | +# --- Aggregate: a few groups with large states (#25851) --- |
| 65 | + |
| 66 | +statement ok |
| 67 | +SET datafusion.execution.target_partitions = 2 |
| 68 | + |
| 69 | +statement ok |
| 70 | +SET datafusion.runtime.memory_limit = '12M' |
| 71 | + |
| 72 | +query II |
| 73 | +SELECT count(*) AS groups, sum(n) AS values_collected FROM ( |
| 74 | + SELECT k, cardinality(array_agg(v)) AS n |
| 75 | + FROM ( |
| 76 | + SELECT value % 4 AS k, concat(CAST(value AS VARCHAR), repeat('x', 100)) AS v |
| 77 | + FROM generate_series(1, 200000) |
| 78 | + ) |
| 79 | + GROUP BY k |
| 80 | +) |
| 81 | +---- |
| 82 | +4 200000 |
| 83 | + |
| 84 | +statement ok |
| 85 | +SET datafusion.execution.target_partitions = 1 |
| 86 | + |
| 87 | +# --- Aggregate: many groups whose states are wide relative to the pool --- |
| 88 | + |
| 89 | +statement ok |
| 90 | +SET datafusion.runtime.memory_limit = '4M' |
| 91 | + |
| 92 | +query II |
| 93 | +SELECT count(*), sum(length(s)) FROM ( |
| 94 | + SELECT k, string_agg(v, ',') AS s |
| 95 | + FROM (SELECT value % 40000 AS k, repeat('x', 600) AS v FROM generate_series(1, 160000)) |
| 96 | + GROUP BY k |
| 97 | +) |
| 98 | +---- |
| 99 | +40000 96120000 |
| 100 | + |
| 101 | +# The same groups under a 2 MB limit, where the smallest merge of two runs of |
| 102 | +# 1 MiB batches does not fit in what the aggregate held. |
| 103 | +statement ok |
| 104 | +SET datafusion.runtime.memory_limit = '2M' |
| 105 | + |
| 106 | +query error Resources exhausted |
| 107 | +SELECT count(*), sum(length(s)) FROM ( |
| 108 | + SELECT k, string_agg(v, ',') AS s |
| 109 | + FROM (SELECT value % 40000 AS k, repeat('x', 200) AS v FROM generate_series(1, 160000)) |
| 110 | + GROUP BY k |
| 111 | +) |
| 112 | + |
| 113 | +query error Resources exhausted |
| 114 | +SELECT count(*), sum(length(s)) FROM ( |
| 115 | + SELECT k, string_agg(v, ',') AS s |
| 116 | + FROM (SELECT value % 40000 AS k, repeat('x', 600) AS v FROM generate_series(1, 160000)) |
| 117 | + GROUP BY k |
| 118 | +) |
| 119 | + |
| 120 | +# The legacy aggregate stream. |
| 121 | +statement ok |
| 122 | +SET datafusion.execution.enable_migration_aggregate = false |
| 123 | + |
| 124 | +statement ok |
| 125 | +SET datafusion.runtime.memory_limit = '4M' |
| 126 | + |
| 127 | +query II |
| 128 | +SELECT count(*), sum(length(s)) FROM ( |
| 129 | + SELECT k, string_agg(v, ',') AS s |
| 130 | + FROM (SELECT value % 40000 AS k, repeat('x', 200) AS v FROM generate_series(1, 160000)) |
| 131 | + GROUP BY k |
| 132 | +) |
| 133 | +---- |
| 134 | +40000 32120000 |
| 135 | + |
| 136 | +statement ok |
| 137 | +RESET datafusion.execution.enable_migration_aggregate |
| 138 | + |
64 | 139 | # Restore settings to slt runner defaults |
65 | 140 | statement ok |
66 | 141 | SET datafusion.execution.target_partitions = 4 |
|
0 commit comments