Skip to content

Commit ae3b03b

Browse files
antiguruclaude
andcommitted
compute: coalesce MV sink batches across workers in the sync sink
Wire the shared-batch builder into the sync (v2) materialized-view sink so the workers in one process coalesce their parts for a given batch interval into a single, larger batch instead of each writing its own small batch. All workers building an interval share the batch description broadcast by the mint operator, so they share a batch id and their parts land in one process-global shared batch. Only the last worker to finish receives that batch; the rest get nothing, which maps to the empty-batch response the write operator already handles. Every worker still finishes its builder even when it pushed no data, since any one of them may be the last holder responsible for delivering all workers' parts. The behavior is gated behind the new enable_compute_sync_mv_sink_shared_batches dyncfg, default off. The prior per-worker path is retained unchanged for the off case. Measured locally on an 8-worker replica with a churning view: blob PUTs down ~67% and consensus state-diff bytes down ~54%, at unchanged consensus command count. Adds a testdrive test asserting the view equals the equivalent one-shot aggregation across a multi-worker cluster with churn, enables the flag in the CI system-parameter defaults, and registers it with parallel-workload's flag flipper. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 0ff833a commit ae3b03b

6 files changed

Lines changed: 240 additions & 26 deletions

File tree

‎misc/python/materialize/mzcompose/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,11 @@ def get_variable_system_parameters(
305305
"true",
306306
["true", "false"],
307307
),
308+
VariableSystemParameter(
309+
"enable_compute_sync_mv_sink_shared_batches",
310+
"true",
311+
["true", "false"],
312+
),
308313
VariableSystemParameter(
309314
"enable_password_auth",
310315
"true",

‎misc/python/materialize/parallel_workload/action.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2882,6 +2882,9 @@ def __init__(
28822882
BOOLEAN_FLAG_VALUES
28832883
)
28842884
self.flags_with_values["enable_eager_delta_joins"] = BOOLEAN_FLAG_VALUES
2885+
self.flags_with_values["enable_compute_sync_mv_sink_shared_batches"] = (
2886+
BOOLEAN_FLAG_VALUES
2887+
)
28852888
self.flags_with_values["enable_public_metrics_endpoint"] = BOOLEAN_FLAG_VALUES
28862889
self.flags_with_values["persist_batch_structured_key_lower_len"] = [
28872890
"0",

‎src/compute-types/src/dyncfgs.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,16 @@ pub const ENABLE_SYNC_MV_SINK: Config<bool> = Config::new(
262262
ParameterScope::Environment,
263263
);
264264

265+
/// Coalesce the parts written by the workers in one process into a single shared batch per batch
266+
/// interval, instead of each worker writing its own batch.
267+
pub const ENABLE_SYNC_MV_SINK_SHARED_BATCHES: Config<bool> = Config::new(
268+
"enable_compute_sync_mv_sink_shared_batches",
269+
false,
270+
"Coalesce the parts written by the workers in one process into a single shared batch per \
271+
batch interval in the MV sink.",
272+
ParameterScope::Environment,
273+
);
274+
265275
/// Whether rendering should use the new MV sink correction buffer implementation.
266276
pub const ENABLE_CORRECTION_V2: Config<bool> = Config::new(
267277
"enable_compute_correction_v2",
@@ -688,6 +698,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
688698
.add(&ENABLE_ERROR_DISTINCT)
689699
.add(&ENABLE_MZ_JOIN_CORE)
690700
.add(&ENABLE_SYNC_MV_SINK)
701+
.add(&ENABLE_SYNC_MV_SINK_SHARED_BATCHES)
691702
.add(&ENABLE_CORRECTION_V2)
692703
.add(&CORRECTION_V2_CHAIN_PROPORTIONALITY)
693704
.add(&CORRECTION_V2_CHUNK_SIZE)

‎src/compute/src/sink/materialized_view.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ use mz_persist_client::write::WriteHandle;
134134
use mz_persist_client::{Diagnostics, PersistClient};
135135
use mz_persist_types::codec_impls::UnitSchema;
136136
use mz_repr::{Diff, GlobalId, Row, Timestamp};
137+
use mz_storage_operators::persist::{SharedBatchId, SharedBatches};
137138
use mz_storage_types::StorageDiff;
138139
use mz_storage_types::controller::CollectionMetadata;
139140
use mz_storage_types::sources::SourceData;
@@ -267,6 +268,7 @@ where
267268

268269
let persist_api = PersistApi {
269270
persist_clients: Arc::clone(&compute_state.persist_clients),
271+
persist_batches: compute_state.persist_batches.clone(),
270272
collection: target.clone(),
271273
shard_name: sink_id.to_string(),
272274
purpose: format!("MV sink {sink_id}"),
@@ -351,6 +353,7 @@ pub(super) fn advance(
351353
#[derive(Clone)]
352354
pub(super) struct PersistApi {
353355
pub(super) persist_clients: Arc<PersistClientCache>,
356+
pub(super) persist_batches: SharedBatches,
354357
pub(super) collection: CollectionMetadata,
355358
pub(super) shard_name: String,
356359
pub(super) purpose: String,
@@ -442,6 +445,8 @@ pub(super) struct BatchDescription {
442445
pub(super) lower: Antichain<Timestamp>,
443446
pub(super) upper: Antichain<Timestamp>,
444447
pub(super) append_worker: usize,
448+
/// Identifies the shared batch all workers building this description contribute to.
449+
pub(super) shared_id: SharedBatchId,
445450
}
446451

447452
impl BatchDescription {
@@ -455,6 +460,7 @@ impl BatchDescription {
455460
lower,
456461
upper,
457462
append_worker,
463+
shared_id: SharedBatchId::new(),
458464
}
459465
}
460466
}

‎src/compute/src/sink/materialized_view_v2.rs‎

Lines changed: 134 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,19 @@ use std::rc::Rc;
5252
use std::sync::Arc;
5353

5454
use differential_dataflow::{Hashable, VecCollection};
55-
use mz_compute_types::dyncfgs::MV_SINK_ADVANCE_PERSIST_FRONTIERS;
55+
use mz_compute_types::dyncfgs::{
56+
ENABLE_SYNC_MV_SINK_SHARED_BATCHES, MV_SINK_ADVANCE_PERSIST_FRONTIERS,
57+
};
5658
use mz_dyncfg::ConfigSet;
5759
use mz_ore::cast::CastFrom;
5860
use mz_persist_client::batch::{Batch, ProtoBatch};
5961
use mz_persist_client::write::WriteHandle;
62+
use mz_persist_client::{PersistClient, Schemas};
63+
use mz_persist_types::ShardId;
64+
use mz_persist_types::codec_impls::UnitSchema;
65+
use mz_persist_types::part::PartBuilder;
6066
use mz_repr::{Diff, GlobalId, Row, Timestamp};
67+
use mz_storage_operators::persist::SharedBatches;
6168
use mz_storage_types::StorageDiff;
6269
use mz_storage_types::sources::SourceData;
6370
use timely::PartialOrder;
@@ -102,6 +109,7 @@ pub(super) fn persist_sink<'s>(
102109

103110
let persist_api = PersistApi {
104111
persist_clients: Arc::clone(&compute_state.persist_clients),
112+
persist_batches: compute_state.persist_batches.clone(),
105113
collection: target.clone(),
106114
shard_name: sink_id.to_string(),
107115
purpose: format!("MV sink {sink_id}"),
@@ -655,6 +663,10 @@ mod write {
655663
let advance_persist_frontiers_at_startup =
656664
MV_SINK_ADVANCE_PERSIST_FRONTIERS.get(&worker_config);
657665

666+
// Read the shared-batch flag on the Timely thread. `worker_config` is an `Rc<ConfigSet>`
667+
// and thus not `Send`, so the value must be captured before it moves into the Tokio task.
668+
let shared_batches_enabled = ENABLE_SYNC_MV_SINK_SHARED_BATCHES.get(&worker_config);
669+
658670
// Mirror the persist-frontier initialization performed by `State::new` below. With the
659671
// flag enabled, `State` advances its Timely-side `persist_frontiers` to `as_of`, opening
660672
// the `maybe_start_batch` write gate (`desc.lower <= persist_frontiers.frontier()`)
@@ -679,17 +691,17 @@ mod write {
679691
let (cmd_tx, mut cmd_rx) = mpsc::unbounded_channel::<WriteCommand>();
680692
let (resp_tx, mut resp_rx) = mpsc::unbounded_channel::<WriteResponse>();
681693

682-
// Spawn Tokio task that owns the WriteHandle and corrections buffer.
694+
// Spawn Tokio task that owns the batch-writing context and corrections buffer.
683695
let (activator, activation_ack) = ArcActivator::new(scope, &info);
684696
let write_task_handle = {
685697
mz_ore::task::spawn(
686698
|| operator_name(sink_id, "write::batch_writer"),
687699
async move {
688-
let writer = persist_api.open_writer().await;
700+
let batch_writer = BatchWriter::new(&persist_api, shared_batches_enabled).await;
689701

690702
while let Some(cmd) = cmd_rx.recv().await {
691703
corrections =
692-
apply_command(sink_id, corrections, &writer, cmd, &resp_tx).await;
704+
apply_command(sink_id, corrections, &batch_writer, cmd, &resp_tx).await;
693705
// Activate the operator to drain logging events and process batch responses.
694706
// ArcActivator suppresses redundant activations, so this is cheap.
695707
activator.activate();
@@ -841,6 +853,52 @@ mod write {
841853
/// batch builder. Together with [`READ_BACK_CHUNK`] this bounds the handoff to a few thousand
842854
/// buffered updates.
843855
const READ_BACK_CHUNKS_IN_FLIGHT: usize = 4;
856+
/// How many updates a worker accumulates before handing a part to the shared batch.
857+
///
858+
/// The shared batch concatenates the parts it receives until they reach persist's blob target
859+
/// size, so this only sets the granularity at which a worker's updates become visible to that
860+
/// coalescing, not the size of the parts persist writes. Keeping it near [`READ_BACK_CHUNK`]
861+
/// costs one part per read-back chunk.
862+
const SHARED_PART_LEN: usize = 1024;
863+
864+
/// The batch-writing context owned by the Tokio write task.
865+
///
866+
/// Batches are built through the process-global [`SharedBatches`], so that the workers running
867+
/// in one process coalesce their parts for a given batch interval into a single, larger batch
868+
/// instead of each writing its own small batch. The `writer` is retained only to clean up a
869+
/// finished batch when the response channel has already gone away.
870+
struct BatchWriter {
871+
persist_client: PersistClient,
872+
shard_id: ShardId,
873+
shard_name: String,
874+
schemas: Schemas<SourceData, ()>,
875+
shared_batches: SharedBatches,
876+
shared_batches_enabled: bool,
877+
writer: WriteHandle<SourceData, (), Timestamp, StorageDiff>,
878+
}
879+
880+
impl BatchWriter {
881+
async fn new(persist_api: &PersistApi, shared_batches_enabled: bool) -> Self {
882+
let writer = persist_api.open_writer().await;
883+
Self {
884+
persist_client: persist_api.open_client().await,
885+
shard_id: persist_api.collection.data_shard,
886+
shard_name: persist_api.shard_name.clone(),
887+
// Parts carry the id of the schema they were written under, and a part written
888+
// under none cannot be decoded through the shard's schema registry. The handle
889+
// resolves the registered id for this sink's schema, so take it from there rather
890+
// than assembling an unregistered `Schemas`.
891+
schemas: Schemas {
892+
id: writer.schema_id(),
893+
key: Arc::new(persist_api.collection.relation_desc.clone()),
894+
val: Arc::new(UnitSchema),
895+
},
896+
shared_batches: persist_api.persist_batches.clone(),
897+
shared_batches_enabled,
898+
writer,
899+
}
900+
}
901+
}
844902

845903
/// Apply a single command to the task state, returning the correction buffers.
846904
///
@@ -857,7 +915,7 @@ mod write {
857915
async fn apply_command(
858916
sink_id: GlobalId,
859917
mut corrections: Corrections,
860-
writer: &WriteHandle<SourceData, (), Timestamp, StorageDiff>,
918+
batch_writer: &BatchWriter,
861919
cmd: WriteCommand,
862920
resp_tx: &mpsc::UnboundedSender<WriteResponse>,
863921
) -> Corrections {
@@ -914,30 +972,39 @@ mod write {
914972
},
915973
);
916974

917-
// Create the builder lazily: an idle sink's descriptions find no corrections.
918-
let mut builder = None;
919-
while let Some(chunk) = updates_rx.recv().await {
920-
let builder = builder.get_or_insert_with(|| writer.builder(desc.lower.clone()));
921-
for ((k, v), t, d) in &chunk {
922-
builder.add(k, v, t, d).await.expect("valid usage");
975+
let proto_batch = if batch_writer.shared_batches_enabled {
976+
write_shared_batch(batch_writer, &desc, &mut updates_rx).await
977+
} else {
978+
// Per-worker path: each worker writes its own batch for the interval. Create
979+
// the builder lazily: an idle sink's descriptions find no corrections.
980+
let mut builder = None;
981+
while let Some(chunk) = updates_rx.recv().await {
982+
let builder = builder
983+
.get_or_insert_with(|| batch_writer.writer.builder(desc.lower.clone()));
984+
for ((k, v), t, d) in &chunk {
985+
builder.add(k, v, t, d).await.expect("valid usage");
986+
}
987+
}
988+
match builder {
989+
Some(builder) => {
990+
let batch = builder
991+
.finish(desc.upper.clone())
992+
.await
993+
.expect("valid usage");
994+
Some(batch.into_transmittable_batch())
995+
}
996+
None => None,
923997
}
924-
}
925-
let corrections = read_back.await;
926-
927-
let Some(builder) = builder else {
928-
// No corrections to write.
929-
let _ = resp_tx.send(WriteResponse { batch: None });
930-
return corrections;
931998
};
999+
let corrections = read_back.await;
9321000

933-
let batch = builder.finish(desc.upper).await.expect("valid usage");
934-
let proto_batch = batch.into_transmittable_batch();
935-
if let Err(err) = resp_tx.send(WriteResponse {
936-
batch: Some(proto_batch),
937-
}) {
938-
let batch =
939-
writer.batch_from_transmittable_batch(err.0.batch.expect("just sent"));
940-
batch.delete().await;
1001+
if let Err(err) = resp_tx.send(WriteResponse { batch: proto_batch }) {
1002+
if let Some(proto_batch) = err.0.batch {
1003+
let batch = batch_writer
1004+
.writer
1005+
.batch_from_transmittable_batch(proto_batch);
1006+
batch.delete().await;
1007+
}
9411008
}
9421009

9431010
corrections
@@ -998,6 +1065,47 @@ mod write {
9981065
.await
9991066
}
10001067

1068+
/// Build the batch for `desc` through the process-global [`SharedBatches`], coalescing this
1069+
/// worker's parts with those of the other workers in the process that build the same interval.
1070+
///
1071+
/// All workers building an interval share `desc.shared_id` (the `mint` operator broadcasts one
1072+
/// description), so their parts land in a single shared batch. Only the last worker to `finish`
1073+
/// receives that batch; the rest get `None`, which maps to the empty-batch response the write
1074+
/// operator already handles. Every worker must `finish` even when it pushed nothing, since any
1075+
/// one of them may be the last holder and thus responsible for delivering all workers' data.
1076+
async fn write_shared_batch(
1077+
batch_writer: &BatchWriter,
1078+
desc: &BatchDescription,
1079+
updates_rx: &mut mpsc::Receiver<Vec<((SourceData, ()), Timestamp, StorageDiff)>>,
1080+
) -> Option<ProtoBatch> {
1081+
let schemas = &batch_writer.schemas;
1082+
let shared = batch_writer.shared_batches.builder(
1083+
desc.shared_id,
1084+
batch_writer.persist_client.clone(),
1085+
batch_writer.shard_id,
1086+
batch_writer.shard_name.clone(),
1087+
schemas.clone(),
1088+
desc.lower.clone(),
1089+
desc.upper.clone(),
1090+
);
1091+
1092+
let mut builder = PartBuilder::new(&*schemas.key, &*schemas.val);
1093+
while let Some(chunk) = updates_rx.recv().await {
1094+
for ((k, v), t, d) in &chunk {
1095+
builder.push(k, v, *t, *d);
1096+
if builder.len() >= SHARED_PART_LEN {
1097+
let part = builder.finish_and_replace(&*schemas.key, &*schemas.val);
1098+
shared.push(part).await;
1099+
}
1100+
}
1101+
}
1102+
let part = builder.finish();
1103+
shared.push(part).await;
1104+
1105+
let batch = shared.finish().await;
1106+
batch.map(|batch| batch.into_transmittable_batch())
1107+
}
1108+
10011109
/// State maintained by the `write` operator on the Timely thread.
10021110
struct State {
10031111
sink_id: GlobalId,

0 commit comments

Comments
 (0)