@@ -23,7 +23,6 @@ use futures_util::{StreamExt, TryFutureExt};
2323use mz_dyncfg:: Config ;
2424use mz_ore:: cast:: CastFrom ;
2525use mz_ore:: error:: ErrorExt ;
26- use mz_persist:: indexed:: columnar:: ColumnarRecords ;
2726use mz_persist:: indexed:: encoding:: { BatchColumnarFormat , BlobTraceUpdates } ;
2827use mz_persist:: location:: Blob ;
2928use mz_persist_types:: { Codec , Codec64 } ;
@@ -48,7 +47,7 @@ use crate::internal::maintenance::RoutineMaintenance;
4847use crate :: internal:: metrics:: ShardMetrics ;
4948use crate :: internal:: state:: { HollowBatch , RunMeta , RunOrder , RunPart } ;
5049use crate :: internal:: trace:: { ApplyMergeResult , FueledMergeRes } ;
51- use crate :: iter:: { CodecSort , Consolidator , StructuredSort } ;
50+ use crate :: iter:: { Consolidator , StructuredSort } ;
5251use crate :: { Metrics , PersistConfig , ShardId } ;
5352
5453/// A request for compaction.
@@ -807,151 +806,76 @@ where
807806 cfg. version . clone ( ) ,
808807 ) ;
809808
810- // Duplicating a large codepath here during the migration.
811- // TODO(database-issues#7188): dedup once the migration is complete.
812- if cfg. batch . preferred_order == RunOrder :: Structured {
813- // If we're not writing down the record metadata, we must always use the old compaction
814- // order. (Since that's the default when the metadata's not present.)
815- let mut consolidator = Consolidator :: new (
816- format ! (
817- "{}[lower={:?},upper={:?}]" ,
818- shard_id,
819- desc. lower( ) . elements( ) ,
820- desc. upper( ) . elements( )
821- ) ,
822- * shard_id,
823- StructuredSort :: < K , V , T , D > :: new ( write_schemas. clone ( ) ) ,
824- blob,
825- Arc :: clone ( & metrics) ,
826- shard_metrics,
827- metrics. read . compaction . clone ( ) ,
828- FetchBatchFilter :: Compaction {
829- since : desc. since ( ) . clone ( ) ,
830- } ,
831- prefetch_budget_bytes,
832- ) ;
833-
834- for ( desc, meta, parts) in runs {
835- consolidator. enqueue_run ( desc, meta, parts. iter ( ) . cloned ( ) ) ;
836- }
809+ let mut consolidator = Consolidator :: new (
810+ format ! (
811+ "{}[lower={:?},upper={:?}]" ,
812+ shard_id,
813+ desc. lower( ) . elements( ) ,
814+ desc. upper( ) . elements( )
815+ ) ,
816+ * shard_id,
817+ StructuredSort :: < K , V , T , D > :: new ( write_schemas. clone ( ) ) ,
818+ blob,
819+ Arc :: clone ( & metrics) ,
820+ shard_metrics,
821+ metrics. read . compaction . clone ( ) ,
822+ FetchBatchFilter :: Compaction {
823+ since : desc. since ( ) . clone ( ) ,
824+ } ,
825+ prefetch_budget_bytes,
826+ ) ;
837827
838- let remaining_budget = consolidator. start_prefetches ( ) ;
839- if remaining_budget. is_none ( ) {
840- metrics. compaction . not_all_prefetched . inc ( ) ;
841- }
828+ for ( desc, meta, parts) in runs {
829+ consolidator. enqueue_run ( desc, meta, parts. iter ( ) . cloned ( ) ) ;
830+ }
842831
843- loop {
844- let mut chunks = vec ! [ ] ;
845- let mut total_bytes = 0 ;
846- // We attempt to pull chunks out of the consolidator that match our target size,
847- // but it's possible that we may get smaller chunks... for example, if not all
848- // parts have been fetched yet. Loop until we've got enough data to justify flushing
849- // it out to blob (or we run out of data.)
850- while total_bytes < cfg. batch . blob_target_size {
851- let fetch_start = Instant :: now ( ) ;
852- let Some ( chunk) = consolidator
853- . next_chunk (
854- cfg. compaction_yield_after_n_updates ,
855- cfg. batch . blob_target_size - total_bytes,
856- )
857- . await ?
858- else {
859- break ;
860- } ;
861- timings. part_fetching += fetch_start. elapsed ( ) ;
862- total_bytes += chunk. goodbytes ( ) ;
863- chunks. push ( chunk) ;
864- tokio:: task:: yield_now ( ) . await ;
865- }
832+ let remaining_budget = consolidator. start_prefetches ( ) ;
833+ if remaining_budget. is_none ( ) {
834+ metrics. compaction . not_all_prefetched . inc ( ) ;
835+ }
866836
867- if chunks. is_empty ( ) {
837+ loop {
838+ let mut chunks = vec ! [ ] ;
839+ let mut total_bytes = 0 ;
840+ // We attempt to pull chunks out of the consolidator that match our target size,
841+ // but it's possible that we may get smaller chunks... for example, if not all
842+ // parts have been fetched yet. Loop until we've got enough data to justify flushing
843+ // it out to blob (or we run out of data.)
844+ while total_bytes < cfg. batch . blob_target_size {
845+ let fetch_start = Instant :: now ( ) ;
846+ let Some ( chunk) = consolidator
847+ . next_chunk (
848+ cfg. compaction_yield_after_n_updates ,
849+ cfg. batch . blob_target_size - total_bytes,
850+ )
851+ . await ?
852+ else {
868853 break ;
869- }
870-
871- // If we're going to write this Batch in a structured format, and our CYA
872- // dyncfg is enabled, then don't worry about making sure Codec data exists.
873- let ensure_codec = !( cfg. compaction_output_structured_only
874- && cfg. batch . batch_columnar_format == BatchColumnarFormat :: Structured ) ;
875- // In the hopefully-common case of a single chunk, this will not copy.
876- let updates = BlobTraceUpdates :: concat :: < K , V > (
877- chunks,
878- write_schemas. key . as_ref ( ) ,
879- write_schemas. val . as_ref ( ) ,
880- & metrics. columnar ,
881- ensure_codec,
882- ) ?;
883- batch. flush_part ( desc. clone ( ) , updates) . await ;
884- }
885- } else {
886- let mut consolidator = Consolidator :: < T , D , CodecSort < K , V , T , D > > :: new (
887- format ! (
888- "{}[lower={:?},upper={:?}]" ,
889- shard_id,
890- desc. lower( ) . elements( ) ,
891- desc. upper( ) . elements( )
892- ) ,
893- * shard_id,
894- CodecSort :: new ( write_schemas. clone ( ) ) ,
895- blob,
896- Arc :: clone ( & metrics) ,
897- shard_metrics,
898- metrics. read . compaction . clone ( ) ,
899- FetchBatchFilter :: Compaction {
900- since : desc. since ( ) . clone ( ) ,
901- } ,
902- prefetch_budget_bytes,
903- ) ;
904-
905- for ( desc, meta, parts) in runs {
906- consolidator. enqueue_run ( desc, meta, parts. iter ( ) . cloned ( ) ) ;
854+ } ;
855+ timings. part_fetching += fetch_start. elapsed ( ) ;
856+ total_bytes += chunk. goodbytes ( ) ;
857+ chunks. push ( chunk) ;
858+ tokio:: task:: yield_now ( ) . await ;
907859 }
908860
909- let remaining_budget = consolidator. start_prefetches ( ) ;
910- if remaining_budget. is_none ( ) {
911- metrics. compaction . not_all_prefetched . inc ( ) ;
861+ if chunks. is_empty ( ) {
862+ break ;
912863 }
913864
914- loop {
915- let mut chunks = vec ! [ ] ;
916- let mut total_bytes = 0 ;
917- // We attempt to pull chunks out of the consolidator that match our target size,
918- // but it's possible that we may get smaller chunks... for example, if not all
919- // parts have been fetched yet. Loop until we've got enough data to justify flushing
920- // it out to blob (or we run out of data.)
921- while total_bytes < cfg. batch . blob_target_size {
922- let fetch_start = Instant :: now ( ) ;
923- let Some ( mut chunk) = consolidator
924- . next_chunk (
925- cfg. compaction_yield_after_n_updates ,
926- cfg. batch . blob_target_size - total_bytes,
927- )
928- . await ?
929- else {
930- break ;
931- } ;
932- timings. part_fetching += fetch_start. elapsed ( ) ;
933- total_bytes += chunk. goodbytes ( ) ;
934- chunks. push (
935- chunk
936- . get_or_make_codec :: < K , V > (
937- write_schemas. key . as_ref ( ) ,
938- write_schemas. val . as_ref ( ) ,
939- )
940- . clone ( ) ,
941- ) ;
942- tokio:: task:: yield_now ( ) . await ;
943- }
944-
945- if chunks. is_empty ( ) {
946- break ;
947- }
948-
949- // In the hopefully-common case of a single chunk, this will not copy.
950- let updates = ColumnarRecords :: concat ( & chunks, & metrics. columnar ) ;
951- batch
952- . flush_part ( desc. clone ( ) , BlobTraceUpdates :: Row ( updates) )
953- . await ;
954- }
865+ // If we're going to write this Batch in a structured format, and our CYA
866+ // dyncfg is enabled, then don't worry about making sure Codec data exists.
867+ let ensure_codec = !( cfg. compaction_output_structured_only
868+ && cfg. batch . batch_columnar_format == BatchColumnarFormat :: Structured ) ;
869+
870+ // In the hopefully-common case of a single chunk, this will not copy.
871+ let updates = BlobTraceUpdates :: concat :: < K , V > (
872+ chunks,
873+ write_schemas. key . as_ref ( ) ,
874+ write_schemas. val . as_ref ( ) ,
875+ & metrics. columnar ,
876+ ensure_codec,
877+ ) ?;
878+ batch. flush_part ( desc. clone ( ) , updates) . await ;
955879 }
956880 let mut batch = batch. finish ( desc. clone ( ) ) . await ?;
957881
0 commit comments