@@ -787,59 +787,101 @@ async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> {
787787 Ok ( ( ) )
788788}
789789
790+ /// Number of groups the `covar_samp` queries below produce.
791+ const ADAPTER_GROUPS : i64 = 128 ;
792+ /// Rows per input batch for the `covar_samp` queries below.
793+ const ADAPTER_BATCH_SIZE : i64 = 8192 ;
794+ /// Bytes of scratch row indices `GroupsAccumulatorAdapter` retains for the
795+ /// `covar_samp` queries below: one `u32` per row of the largest batch each
796+ /// group has ever received, kept for the lifetime of the group.
797+ const ADAPTER_RETAINED_BYTES : usize =
798+ ( ADAPTER_GROUPS * ADAPTER_BATCH_SIZE ) as usize * size_of :: < u32 > ( ) ;
799+
800+ /// Runs a `GROUP BY` `covar_samp` query under `memory_limit` and returns the
801+ /// executed plan so the caller can inspect its metrics.
802+ ///
790803/// `covar_samp` has no native [`GroupsAccumulator`], so its per-group state is
791- /// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch
792- /// `Vec<u32>` of row indices per group, grown to the largest number of rows
793- /// that group has ever taken from a single input batch and retained (cleared,
794- /// but not deallocated) for the lifetime of the group.
804+ /// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch `Vec<u32>`
805+ /// of row indices per group, grown to the largest number of rows that group
806+ /// has ever taken from a single input batch and retained (cleared, but not
807+ /// deallocated) for the lifetime of the group.
795808///
796- /// This query hands the aggregate one batch of 8192 rows per group, so 128
797- /// groups retain `128 * 8192 * 4` bytes = 4 MiB of scratch capacity, four
798- /// times the 1 MiB limit. Everything else the aggregate holds is two orders of
799- /// magnitude smaller: without the scratch capacity the reported size peaks
800- /// around 85 KB, twelve times under the limit, and the query runs to
801- /// completion without ever asking the pool for what it is really using.
809+ /// The query hands each of the [`ADAPTER_GROUPS`] groups `batches_per_group`
810+ /// consecutive full batches of [`ADAPTER_BATCH_SIZE`] rows, so the adapter
811+ /// retains [`ADAPTER_RETAINED_BYTES`] (4 MiB) of scratch capacity however many
812+ /// batches each group receives. Everything else the aggregate holds is two
813+ /// orders of magnitude smaller.
802814///
803815/// `target_partitions = 1` puts the aggregate in `Single` mode, which spills
804816/// under memory pressure instead of emitting groups early, so the accounting
805817/// is observable as a spill.
806818///
807819/// [`GroupsAccumulator`]: datafusion_expr::GroupsAccumulator
808- #[ tokio:: test]
809- async fn aggregate_adapter_spills_on_retained_indices ( ) -> Result < ( ) > {
810- const GROUPS : i64 = 128 ;
811- const BATCH_SIZE : i64 = 8192 ;
812-
820+ async fn run_adapter_query (
821+ memory_limit : usize ,
822+ batches_per_group : i64 ,
823+ ) -> Result < Arc < dyn ExecutionPlan > > {
813824 let runtime = RuntimeEnvBuilder :: new ( )
814- . with_memory_pool ( Arc :: new ( GreedyMemoryPool :: new ( 1024 * 1024 ) ) )
825+ . with_memory_pool ( Arc :: new ( GreedyMemoryPool :: new ( memory_limit ) ) )
815826 . with_disk_manager_builder (
816827 DiskManagerBuilder :: default ( ) . with_mode ( DiskManagerMode :: OsTmpDirectory ) ,
817828 )
818829 . build_arc ( ) ?;
819830
820831 let config = SessionConfig :: new ( )
821832 . with_target_partitions ( 1 )
822- . with_batch_size ( BATCH_SIZE as usize ) ;
833+ . with_batch_size ( ADAPTER_BATCH_SIZE as usize ) ;
823834 let ctx = SessionContext :: new_with_config_rt ( config, runtime) ;
824835
836+ let rows_per_group = ADAPTER_BATCH_SIZE * batches_per_group;
825837 let sql = format ! (
826- "SELECT v / {BATCH_SIZE } AS g, covar_samp(v, v) AS c \
838+ "SELECT v / {rows_per_group } AS g, covar_samp(v, v) AS c \
827839 FROM generate_series(0, {}) AS t(v) \
828- GROUP BY v / {BATCH_SIZE }",
829- GROUPS * BATCH_SIZE - 1
840+ GROUP BY v / {rows_per_group }",
841+ ADAPTER_GROUPS * rows_per_group - 1
830842 ) ;
831843
832844 let plan = ctx. sql ( & sql) . await ?. create_physical_plan ( ) . await ?;
833845 let batches = collect_batches ( Arc :: clone ( & plan) , ctx. task_ctx ( ) ) . await ?;
834846
835847 let rows: usize = batches. iter ( ) . map ( |batch| batch. num_rows ( ) ) . sum ( ) ;
836- assert_eq ! ( rows, GROUPS as usize ) ;
848+ assert_eq ! ( rows, ADAPTER_GROUPS as usize ) ;
849+
850+ Ok ( plan)
851+ }
852+
853+ /// The scratch capacity `GroupsAccumulatorAdapter` retains is four times the
854+ /// memory limit, so the aggregate must spill. Without the capacity everything
855+ /// the aggregate reports is far under the limit, and the query runs to
856+ /// completion without ever asking the pool for what it is really using.
857+ #[ tokio:: test]
858+ async fn aggregate_adapter_spills_on_retained_indices ( ) -> Result < ( ) > {
859+ let plan = run_adapter_query ( ADAPTER_RETAINED_BYTES / 4 , 1 ) . await ?;
837860
838861 let spill_count = plan_spill_count ( plan. as_ref ( ) ) ;
839862 assert ! (
840863 spill_count > 0 ,
841- "the aggregate retains 4 MiB of scratch indices against a 1 MiB limit, \
842- so it must spill, but spill_count was {spill_count}"
864+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
865+ against a limit of a quarter of that, so it must spill, \
866+ but spill_count was {spill_count}"
867+ ) ;
868+
869+ Ok ( ( ) )
870+ }
871+
872+ /// The retained scratch capacity is charged once, not once per batch. Every
873+ /// group receives four batches, so charging the capacity per batch would
874+ /// report four times the retained bytes, and the memory limit of twice the
875+ /// retained bytes fits the aggregate only if it is charged once.
876+ #[ tokio:: test]
877+ async fn aggregate_adapter_charges_retained_indices_once ( ) -> Result < ( ) > {
878+ let plan = run_adapter_query ( ADAPTER_RETAINED_BYTES * 2 , 4 ) . await ?;
879+
880+ let spill_count = plan_spill_count ( plan. as_ref ( ) ) ;
881+ assert_eq ! (
882+ spill_count, 0 ,
883+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
884+ under a limit of twice that, so it must not spill"
843885 ) ;
844886
845887 Ok ( ( ) )
0 commit comments