@@ -55,7 +55,7 @@ use differential_dataflow::{Hashable, VecCollection};
5555use mz_compute_types:: dyncfgs:: {
5656 ENABLE_SYNC_MV_SINK_SHARED_BATCHES , MV_SINK_ADVANCE_PERSIST_FRONTIERS ,
5757} ;
58- use mz_dyncfg:: ConfigSet ;
58+ use mz_dyncfg:: { ConfigSet , ConfigValHandle } ;
5959use mz_ore:: cast:: CastFrom ;
6060use mz_persist_client:: batch:: { Batch , ProtoBatch } ;
6161use mz_persist_client:: write:: WriteHandle ;
@@ -81,7 +81,7 @@ use crate::sink::materialized_view::{
8181 BatchDescription , BatchesStream , DescsStream , DesiredStreams , OkErr , PersistApi ,
8282 PersistStreams , SharedSinkFrontier , advance, operator_name, persist_source,
8383} ;
84- use crate :: sink:: shared_batch:: SharedBatchWriter ;
84+ use crate :: sink:: shared_batch:: { SharedBatchWrite , SharedBatchWriter } ;
8585
8686/// Renders an MV sink writing the given desired collection into the `target` persist collection.
8787///
@@ -659,9 +659,7 @@ mod write {
659659 let advance_persist_frontiers_at_startup =
660660 MV_SINK_ADVANCE_PERSIST_FRONTIERS . get ( & worker_config) ;
661661
662- // Read the shared-batch flag on the Timely thread. `worker_config` is an `Rc<ConfigSet>`
663- // and thus not `Send`, so the value must be captured before it moves into the Tokio task.
664- let shared_batches_enabled = ENABLE_SYNC_MV_SINK_SHARED_BATCHES . get ( & worker_config) ;
662+ let shared_batches_enabled = ENABLE_SYNC_MV_SINK_SHARED_BATCHES . handle ( & worker_config) ;
665663
666664 // Mirror the persist-frontier initialization performed by `State::new` below. With the
667665 // flag enabled, `State` advances its Timely-side `persist_frontiers` to `as_of`, opening
@@ -851,22 +849,20 @@ mod write {
851849 const READ_BACK_CHUNKS_IN_FLIGHT : usize = 4 ;
852850 /// The batch-writing context owned by the Tokio write task.
853851 ///
854- /// With `shared` set, batches are built through the process-global shared batches. The
855- /// `writer` then serves only to clean up a finished batch when the response channel has
856- /// already gone away.
852+ /// The `writer` writes per-worker batches and cleans up a finished batch when the response
853+ /// channel has already gone away.
857854 struct BatchWriter {
858855 writer : WriteHandle < SourceData , ( ) , Timestamp , StorageDiff > ,
859- shared : Option < SharedBatchWriter > ,
856+ shared : SharedBatchWriter ,
860857 }
861858
862859 impl BatchWriter {
863- async fn new ( persist_api : & PersistApi , shared_batches_enabled : bool ) -> Self {
860+ async fn new (
861+ persist_api : & PersistApi ,
862+ shared_batches_enabled : ConfigValHandle < bool > ,
863+ ) -> Self {
864864 let writer = persist_api. open_writer ( ) . await ;
865- let shared = if shared_batches_enabled {
866- Some ( SharedBatchWriter :: new ( persist_api, & writer) . await )
867- } else {
868- None
869- } ;
865+ let shared = SharedBatchWriter :: new ( persist_api, & writer, shared_batches_enabled) . await ;
870866 Self { writer, shared }
871867 }
872868 }
@@ -943,35 +939,9 @@ mod write {
943939 } ,
944940 ) ;
945941
946- let proto_batch = if let Some ( shared) = & batch_writer. shared {
947- let mut write = shared. start ( & desc) ;
948- while let Some ( chunk) = updates_rx. recv ( ) . await {
949- for ( kv, t, d) in & chunk {
950- write. push ( kv, * t, * d) . await ;
951- }
952- }
953- write. finish ( ) . await
954- } else {
955- // Per-worker path: each worker writes its own batch for the interval. Create
956- // the builder lazily: an idle sink's descriptions find no corrections.
957- let mut builder = None ;
958- while let Some ( chunk) = updates_rx. recv ( ) . await {
959- let builder = builder
960- . get_or_insert_with ( || batch_writer. writer . builder ( desc. lower . clone ( ) ) ) ;
961- for ( ( k, v) , t, d) in & chunk {
962- builder. add ( k, v, t, d) . await . expect ( "valid usage" ) ;
963- }
964- }
965- match builder {
966- Some ( builder) => {
967- let batch = builder
968- . finish ( desc. upper . clone ( ) )
969- . await
970- . expect ( "valid usage" ) ;
971- Some ( batch. into_transmittable_batch ( ) )
972- }
973- None => None ,
974- }
942+ let proto_batch = match batch_writer. shared . start ( & desc) {
943+ Some ( write) => write_shared_batch ( write, & mut updates_rx) . await ,
944+ None => write_worker_batch ( & batch_writer. writer , & desc, & mut updates_rx) . await ,
975945 } ;
976946 let corrections = read_back. await ;
977947
@@ -989,6 +959,41 @@ mod write {
989959 }
990960 }
991961
962+ /// Hand the updates for one interval to the shared batch, returning the batch if this worker
963+ /// delivers it.
964+ async fn write_shared_batch (
965+ mut write : SharedBatchWrite ,
966+ updates_rx : & mut mpsc:: Receiver < Vec < ( ( SourceData , ( ) ) , Timestamp , StorageDiff ) > > ,
967+ ) -> Option < ProtoBatch > {
968+ while let Some ( chunk) = updates_rx. recv ( ) . await {
969+ for ( kv, t, d) in & chunk {
970+ write. push ( kv, * t, * d) . await ;
971+ }
972+ }
973+ write. finish ( ) . await
974+ }
975+
976+ /// Write this worker's own batch for `desc`, or nothing if it has no updates.
977+ async fn write_worker_batch (
978+ writer : & WriteHandle < SourceData , ( ) , Timestamp , StorageDiff > ,
979+ desc : & BatchDescription ,
980+ updates_rx : & mut mpsc:: Receiver < Vec < ( ( SourceData , ( ) ) , Timestamp , StorageDiff ) > > ,
981+ ) -> Option < ProtoBatch > {
982+ // Create the builder lazily: an idle sink's descriptions find no corrections.
983+ let mut builder = None ;
984+ while let Some ( chunk) = updates_rx. recv ( ) . await {
985+ let builder = builder. get_or_insert_with ( || writer. builder ( desc. lower . clone ( ) ) ) ;
986+ for ( ( k, v) , t, d) in & chunk {
987+ builder. add ( k, v, t, d) . await . expect ( "valid usage" ) ;
988+ }
989+ }
990+ let batch = builder?
991+ . finish ( desc. upper . clone ( ) )
992+ . await
993+ . expect ( "valid usage" ) ;
994+ Some ( batch. into_transmittable_batch ( ) )
995+ }
996+
992997 /// Apply a coalesced batch of updates to the correction buffers, on a blocking thread.
993998 ///
994999 /// See [`apply_command`] for why this work must not run on a Tokio worker thread.
0 commit comments