@@ -52,12 +52,19 @@ use std::rc::Rc;
5252use std:: sync:: Arc ;
5353
5454use differential_dataflow:: { Hashable , VecCollection } ;
55- use mz_compute_types:: dyncfgs:: MV_SINK_ADVANCE_PERSIST_FRONTIERS ;
55+ use mz_compute_types:: dyncfgs:: {
56+ ENABLE_SYNC_MV_SINK_SHARED_BATCHES , MV_SINK_ADVANCE_PERSIST_FRONTIERS ,
57+ } ;
5658use mz_dyncfg:: ConfigSet ;
5759use mz_ore:: cast:: CastFrom ;
5860use mz_persist_client:: batch:: { Batch , ProtoBatch } ;
5961use mz_persist_client:: write:: WriteHandle ;
62+ use mz_persist_client:: { PersistClient , Schemas } ;
63+ use mz_persist_types:: ShardId ;
64+ use mz_persist_types:: codec_impls:: UnitSchema ;
65+ use mz_persist_types:: part:: PartBuilder ;
6066use mz_repr:: { Diff , GlobalId , Row , Timestamp } ;
67+ use mz_storage_operators:: persist:: SharedBatches ;
6168use mz_storage_types:: StorageDiff ;
6269use mz_storage_types:: sources:: SourceData ;
6370use timely:: PartialOrder ;
@@ -102,6 +109,7 @@ pub(super) fn persist_sink<'s>(
102109
103110 let persist_api = PersistApi {
104111 persist_clients : Arc :: clone ( & compute_state. persist_clients ) ,
112+ persist_batches : compute_state. persist_batches . clone ( ) ,
105113 collection : target. clone ( ) ,
106114 shard_name : sink_id. to_string ( ) ,
107115 purpose : format ! ( "MV sink {sink_id}" ) ,
@@ -655,6 +663,10 @@ mod write {
655663 let advance_persist_frontiers_at_startup =
656664 MV_SINK_ADVANCE_PERSIST_FRONTIERS . get ( & worker_config) ;
657665
666+ // Read the shared-batch flag on the Timely thread. `worker_config` is an `Rc<ConfigSet>`
667+ // and thus not `Send`, so the value must be captured before it moves into the Tokio task.
668+ let shared_batches_enabled = ENABLE_SYNC_MV_SINK_SHARED_BATCHES . get ( & worker_config) ;
669+
658670 // Mirror the persist-frontier initialization performed by `State::new` below. With the
659671 // flag enabled, `State` advances its Timely-side `persist_frontiers` to `as_of`, opening
660672 // the `maybe_start_batch` write gate (`desc.lower <= persist_frontiers.frontier()`)
@@ -679,17 +691,17 @@ mod write {
679691 let ( cmd_tx, mut cmd_rx) = mpsc:: unbounded_channel :: < WriteCommand > ( ) ;
680692 let ( resp_tx, mut resp_rx) = mpsc:: unbounded_channel :: < WriteResponse > ( ) ;
681693
682- // Spawn Tokio task that owns the WriteHandle and corrections buffer.
694+ // Spawn Tokio task that owns the batch-writing context and corrections buffer.
683695 let ( activator, activation_ack) = ArcActivator :: new ( scope, & info) ;
684696 let write_task_handle = {
685697 mz_ore:: task:: spawn (
686698 || operator_name ( sink_id, "write::batch_writer" ) ,
687699 async move {
688- let writer = persist_api . open_writer ( ) . await ;
700+ let batch_writer = BatchWriter :: new ( & persist_api , shared_batches_enabled ) . await ;
689701
690702 while let Some ( cmd) = cmd_rx. recv ( ) . await {
691703 corrections =
692- apply_command ( sink_id, corrections, & writer , cmd, & resp_tx) . await ;
704+ apply_command ( sink_id, corrections, & batch_writer , cmd, & resp_tx) . await ;
693705 // Activate the operator to drain logging events and process batch responses.
694706 // ArcActivator suppresses redundant activations, so this is cheap.
695707 activator. activate ( ) ;
@@ -841,6 +853,52 @@ mod write {
841853 /// batch builder. Together with [`READ_BACK_CHUNK`] this bounds the handoff to a few thousand
842854 /// buffered updates.
843855 const READ_BACK_CHUNKS_IN_FLIGHT : usize = 4 ;
856+ /// How many updates a worker accumulates before handing a part to the shared batch.
857+ ///
858+ /// The shared batch concatenates the parts it receives until they reach persist's blob target
859+ /// size, so this only sets the granularity at which a worker's updates become visible to that
860+ /// coalescing, not the size of the parts persist writes. Keeping it near [`READ_BACK_CHUNK`]
861+ /// costs one part per read-back chunk.
862+ const SHARED_PART_LEN : usize = 1024 ;
863+
864+ /// The batch-writing context owned by the Tokio write task.
865+ ///
866+ /// Batches are built through the process-global [`SharedBatches`], so that the workers running
867+ /// in one process coalesce their parts for a given batch interval into a single, larger batch
868+ /// instead of each writing its own small batch. The `writer` is retained only to clean up a
869+ /// finished batch when the response channel has already gone away.
870+ struct BatchWriter {
871+ persist_client : PersistClient ,
872+ shard_id : ShardId ,
873+ shard_name : String ,
874+ schemas : Schemas < SourceData , ( ) > ,
875+ shared_batches : SharedBatches ,
876+ shared_batches_enabled : bool ,
877+ writer : WriteHandle < SourceData , ( ) , Timestamp , StorageDiff > ,
878+ }
879+
880+ impl BatchWriter {
881+ async fn new ( persist_api : & PersistApi , shared_batches_enabled : bool ) -> Self {
882+ let writer = persist_api. open_writer ( ) . await ;
883+ Self {
884+ persist_client : persist_api. open_client ( ) . await ,
885+ shard_id : persist_api. collection . data_shard ,
886+ shard_name : persist_api. shard_name . clone ( ) ,
887+ // Parts carry the id of the schema they were written under, and a part written
888+ // under none cannot be decoded through the shard's schema registry. The handle
889+ // resolves the registered id for this sink's schema, so take it from there rather
890+ // than assembling an unregistered `Schemas`.
891+ schemas : Schemas {
892+ id : writer. schema_id ( ) ,
893+ key : Arc :: new ( persist_api. collection . relation_desc . clone ( ) ) ,
894+ val : Arc :: new ( UnitSchema ) ,
895+ } ,
896+ shared_batches : persist_api. persist_batches . clone ( ) ,
897+ shared_batches_enabled,
898+ writer,
899+ }
900+ }
901+ }
844902
845903 /// Apply a single command to the task state, returning the correction buffers.
846904 ///
@@ -857,7 +915,7 @@ mod write {
857915 async fn apply_command (
858916 sink_id : GlobalId ,
859917 mut corrections : Corrections ,
860- writer : & WriteHandle < SourceData , ( ) , Timestamp , StorageDiff > ,
918+ batch_writer : & BatchWriter ,
861919 cmd : WriteCommand ,
862920 resp_tx : & mpsc:: UnboundedSender < WriteResponse > ,
863921 ) -> Corrections {
@@ -914,30 +972,39 @@ mod write {
914972 } ,
915973 ) ;
916974
917- // Create the builder lazily: an idle sink's descriptions find no corrections.
918- let mut builder = None ;
919- while let Some ( chunk) = updates_rx. recv ( ) . await {
920- let builder = builder. get_or_insert_with ( || writer. builder ( desc. lower . clone ( ) ) ) ;
921- for ( ( k, v) , t, d) in & chunk {
922- builder. add ( k, v, t, d) . await . expect ( "valid usage" ) ;
975+ let proto_batch = if batch_writer. shared_batches_enabled {
976+ write_shared_batch ( batch_writer, & desc, & mut updates_rx) . await
977+ } else {
978+ // Per-worker path: each worker writes its own batch for the interval. Create
979+ // the builder lazily: an idle sink's descriptions find no corrections.
980+ let mut builder = None ;
981+ while let Some ( chunk) = updates_rx. recv ( ) . await {
982+ let builder = builder
983+ . get_or_insert_with ( || batch_writer. writer . builder ( desc. lower . clone ( ) ) ) ;
984+ for ( ( k, v) , t, d) in & chunk {
985+ builder. add ( k, v, t, d) . await . expect ( "valid usage" ) ;
986+ }
987+ }
988+ match builder {
989+ Some ( builder) => {
990+ let batch = builder
991+ . finish ( desc. upper . clone ( ) )
992+ . await
993+ . expect ( "valid usage" ) ;
994+ Some ( batch. into_transmittable_batch ( ) )
995+ }
996+ None => None ,
923997 }
924- }
925- let corrections = read_back. await ;
926-
927- let Some ( builder) = builder else {
928- // No corrections to write.
929- let _ = resp_tx. send ( WriteResponse { batch : None } ) ;
930- return corrections;
931998 } ;
999+ let corrections = read_back. await ;
9321000
933- let batch = builder. finish ( desc. upper ) . await . expect ( "valid usage" ) ;
934- let proto_batch = batch. into_transmittable_batch ( ) ;
935- if let Err ( err) = resp_tx. send ( WriteResponse {
936- batch : Some ( proto_batch) ,
937- } ) {
938- let batch =
939- writer. batch_from_transmittable_batch ( err. 0 . batch . expect ( "just sent" ) ) ;
940- batch. delete ( ) . await ;
1001+ if let Err ( err) = resp_tx. send ( WriteResponse { batch : proto_batch } ) {
1002+ if let Some ( proto_batch) = err. 0 . batch {
1003+ let batch = batch_writer
1004+ . writer
1005+ . batch_from_transmittable_batch ( proto_batch) ;
1006+ batch. delete ( ) . await ;
1007+ }
9411008 }
9421009
9431010 corrections
@@ -998,6 +1065,47 @@ mod write {
9981065 . await
9991066 }
10001067
1068+ /// Build the batch for `desc` through the process-global [`SharedBatches`], coalescing this
1069+ /// worker's parts with those of the other workers in the process that build the same interval.
1070+ ///
1071+ /// All workers building an interval share `desc.shared_id` (the `mint` operator broadcasts one
1072+ /// description), so their parts land in a single shared batch. Only the last worker to `finish`
1073+ /// receives that batch; the rest get `None`, which maps to the empty-batch response the write
1074+ /// operator already handles. Every worker must `finish` even when it pushed nothing, since any
1075+ /// one of them may be the last holder and thus responsible for delivering all workers' data.
1076+ async fn write_shared_batch (
1077+ batch_writer : & BatchWriter ,
1078+ desc : & BatchDescription ,
1079+ updates_rx : & mut mpsc:: Receiver < Vec < ( ( SourceData , ( ) ) , Timestamp , StorageDiff ) > > ,
1080+ ) -> Option < ProtoBatch > {
1081+ let schemas = & batch_writer. schemas ;
1082+ let shared = batch_writer. shared_batches . builder (
1083+ desc. shared_id ,
1084+ batch_writer. persist_client . clone ( ) ,
1085+ batch_writer. shard_id ,
1086+ batch_writer. shard_name . clone ( ) ,
1087+ schemas. clone ( ) ,
1088+ desc. lower . clone ( ) ,
1089+ desc. upper . clone ( ) ,
1090+ ) ;
1091+
1092+ let mut builder = PartBuilder :: new ( & * schemas. key , & * schemas. val ) ;
1093+ while let Some ( chunk) = updates_rx. recv ( ) . await {
1094+ for ( ( k, v) , t, d) in & chunk {
1095+ builder. push ( k, v, * t, * d) ;
1096+ if builder. len ( ) >= SHARED_PART_LEN {
1097+ let part = builder. finish_and_replace ( & * schemas. key , & * schemas. val ) ;
1098+ shared. push ( part) . await ;
1099+ }
1100+ }
1101+ }
1102+ let part = builder. finish ( ) ;
1103+ shared. push ( part) . await ;
1104+
1105+ let batch = shared. finish ( ) . await ;
1106+ batch. map ( |batch| batch. into_transmittable_batch ( ) )
1107+ }
1108+
10011109 /// State maintained by the `write` operator on the Timely thread.
10021110 struct State {
10031111 sink_id : GlobalId ,
0 commit comments