Skip to content

Commit f02f202

Browse files
committed
Remove codec-order consolidation
1 parent 815bed7 commit f02f202

8 files changed

Lines changed: 157 additions & 349 deletions

File tree

‎misc/python/materialize/parallel_workload/action.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1059,7 +1059,6 @@ def __init__(
10591059
"both_v2",
10601060
"structured",
10611061
]
1062-
self.flags_with_values["persist_batch_structured_order"] = BOOLEAN_FLAG_VALUES
10631062
self.flags_with_values["persist_batch_builder_structured"] = BOOLEAN_FLAG_VALUES
10641063
self.flags_with_values["persist_batch_structured_key_lower_len"] = [
10651064
"0",

‎src/persist-client/src/batch.rs‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -386,21 +386,6 @@ pub(crate) const ENCODING_COMPRESSION_FORMAT: Config<&'static str> = Config::new
386386
"A feature flag to enable compression of Parquet data (Materialize).",
387387
);
388388

389-
pub(crate) const STRUCTURED_ORDER: Config<bool> = Config::new(
390-
"persist_batch_structured_order",
391-
true,
392-
"If enabled, output compaction batches in structured-data order.",
393-
);
394-
395-
pub(crate) const STRUCTURED_ORDER_UNTIL_SHARD: Config<&'static str> = Config::new(
396-
"persist_batch_structured_order_from_shard",
397-
"sz",
398-
"Restrict shards using structured ordering to those shards with formatted ids less than \
399-
the given string. (For example, `s0` will disable it for all shards, `s8` will enable it for \
400-
half of all shards, `s8888` will enable it for slightly more shards, and `sz` will enable it \
401-
for everyone.)",
402-
);
403-
404389
pub(crate) const STRUCTURED_KEY_LOWER_LEN: Config<usize> = Config::new(
405390
"persist_batch_structured_key_lower_len",
406391
256,
@@ -458,20 +443,13 @@ pub(crate) const INLINE_WRITES_TOTAL_MAX_BYTES: Config<usize> = Config::new(
458443

459444
impl BatchBuilderConfig {
460445
/// Initialize a batch builder config based on a snapshot of the Persist config.
461-
pub fn new(value: &PersistConfig, shard_id: ShardId) -> Self {
446+
pub fn new(value: &PersistConfig, _shard_id: ShardId) -> Self {
462447
let writer_key = WriterKey::for_version(&value.build_version);
463448

464449
let batch_columnar_format =
465450
BatchColumnarFormat::from_str(&BATCH_COLUMNAR_FORMAT.get(value));
466451

467-
let structured_order = STRUCTURED_ORDER.get(value) && {
468-
shard_id.to_string() < STRUCTURED_ORDER_UNTIL_SHARD.get(value)
469-
};
470-
let preferred_order = if structured_order {
471-
RunOrder::Structured
472-
} else {
473-
RunOrder::Codec
474-
};
452+
let preferred_order = RunOrder::Structured;
475453

476454
BatchBuilderConfig {
477455
writer_key,

‎src/persist-client/src/cfg.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,6 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
289289
.add(&crate::batch::INLINE_WRITES_SINGLE_MAX_BYTES)
290290
.add(&crate::batch::ENCODING_ENABLE_DICTIONARY)
291291
.add(&crate::batch::ENCODING_COMPRESSION_FORMAT)
292-
.add(&crate::batch::STRUCTURED_ORDER)
293-
.add(&crate::batch::STRUCTURED_ORDER_UNTIL_SHARD)
294292
.add(&crate::batch::STRUCTURED_KEY_LOWER_LEN)
295293
.add(&crate::batch::MAX_RUN_LEN)
296294
.add(&crate::batch::MAX_RUNS)

‎src/persist-client/src/internal/compact.rs‎

Lines changed: 63 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ use futures_util::{StreamExt, TryFutureExt};
2323
use mz_dyncfg::Config;
2424
use mz_ore::cast::CastFrom;
2525
use mz_ore::error::ErrorExt;
26-
use mz_persist::indexed::columnar::ColumnarRecords;
2726
use mz_persist::indexed::encoding::{BatchColumnarFormat, BlobTraceUpdates};
2827
use mz_persist::location::Blob;
2928
use mz_persist_types::{Codec, Codec64};
@@ -48,7 +47,7 @@ use crate::internal::maintenance::RoutineMaintenance;
4847
use crate::internal::metrics::ShardMetrics;
4948
use crate::internal::state::{HollowBatch, RunMeta, RunOrder, RunPart};
5049
use crate::internal::trace::{ApplyMergeResult, FueledMergeRes};
51-
use crate::iter::{CodecSort, Consolidator, StructuredSort};
50+
use crate::iter::{Consolidator, StructuredSort};
5251
use crate::{Metrics, PersistConfig, ShardId};
5352

5453
/// A request for compaction.
@@ -807,151 +806,76 @@ where
807806
cfg.version.clone(),
808807
);
809808

810-
// Duplicating a large codepath here during the migration.
811-
// TODO(database-issues#7188): dedup once the migration is complete.
812-
if cfg.batch.preferred_order == RunOrder::Structured {
813-
// If we're not writing down the record metadata, we must always use the old compaction
814-
// order. (Since that's the default when the metadata's not present.)
815-
let mut consolidator = Consolidator::new(
816-
format!(
817-
"{}[lower={:?},upper={:?}]",
818-
shard_id,
819-
desc.lower().elements(),
820-
desc.upper().elements()
821-
),
822-
*shard_id,
823-
StructuredSort::<K, V, T, D>::new(write_schemas.clone()),
824-
blob,
825-
Arc::clone(&metrics),
826-
shard_metrics,
827-
metrics.read.compaction.clone(),
828-
FetchBatchFilter::Compaction {
829-
since: desc.since().clone(),
830-
},
831-
prefetch_budget_bytes,
832-
);
833-
834-
for (desc, meta, parts) in runs {
835-
consolidator.enqueue_run(desc, meta, parts.iter().cloned());
836-
}
809+
let mut consolidator = Consolidator::new(
810+
format!(
811+
"{}[lower={:?},upper={:?}]",
812+
shard_id,
813+
desc.lower().elements(),
814+
desc.upper().elements()
815+
),
816+
*shard_id,
817+
StructuredSort::<K, V, T, D>::new(write_schemas.clone()),
818+
blob,
819+
Arc::clone(&metrics),
820+
shard_metrics,
821+
metrics.read.compaction.clone(),
822+
FetchBatchFilter::Compaction {
823+
since: desc.since().clone(),
824+
},
825+
prefetch_budget_bytes,
826+
);
837827

838-
let remaining_budget = consolidator.start_prefetches();
839-
if remaining_budget.is_none() {
840-
metrics.compaction.not_all_prefetched.inc();
841-
}
828+
for (desc, meta, parts) in runs {
829+
consolidator.enqueue_run(desc, meta, parts.iter().cloned());
830+
}
842831

843-
loop {
844-
let mut chunks = vec![];
845-
let mut total_bytes = 0;
846-
// We attempt to pull chunks out of the consolidator that match our target size,
847-
// but it's possible that we may get smaller chunks... for example, if not all
848-
// parts have been fetched yet. Loop until we've got enough data to justify flushing
849-
// it out to blob (or we run out of data.)
850-
while total_bytes < cfg.batch.blob_target_size {
851-
let fetch_start = Instant::now();
852-
let Some(chunk) = consolidator
853-
.next_chunk(
854-
cfg.compaction_yield_after_n_updates,
855-
cfg.batch.blob_target_size - total_bytes,
856-
)
857-
.await?
858-
else {
859-
break;
860-
};
861-
timings.part_fetching += fetch_start.elapsed();
862-
total_bytes += chunk.goodbytes();
863-
chunks.push(chunk);
864-
tokio::task::yield_now().await;
865-
}
832+
let remaining_budget = consolidator.start_prefetches();
833+
if remaining_budget.is_none() {
834+
metrics.compaction.not_all_prefetched.inc();
835+
}
866836

867-
if chunks.is_empty() {
837+
loop {
838+
let mut chunks = vec![];
839+
let mut total_bytes = 0;
840+
// We attempt to pull chunks out of the consolidator that match our target size,
841+
// but it's possible that we may get smaller chunks... for example, if not all
842+
// parts have been fetched yet. Loop until we've got enough data to justify flushing
843+
// it out to blob (or we run out of data.)
844+
while total_bytes < cfg.batch.blob_target_size {
845+
let fetch_start = Instant::now();
846+
let Some(chunk) = consolidator
847+
.next_chunk(
848+
cfg.compaction_yield_after_n_updates,
849+
cfg.batch.blob_target_size - total_bytes,
850+
)
851+
.await?
852+
else {
868853
break;
869-
}
870-
871-
// If we're going to write this Batch in a structured format, and our CYA
872-
// dyncfg is enabled, then don't worry about making sure Codec data exists.
873-
let ensure_codec = !(cfg.compaction_output_structured_only
874-
&& cfg.batch.batch_columnar_format == BatchColumnarFormat::Structured);
875-
// In the hopefully-common case of a single chunk, this will not copy.
876-
let updates = BlobTraceUpdates::concat::<K, V>(
877-
chunks,
878-
write_schemas.key.as_ref(),
879-
write_schemas.val.as_ref(),
880-
&metrics.columnar,
881-
ensure_codec,
882-
)?;
883-
batch.flush_part(desc.clone(), updates).await;
884-
}
885-
} else {
886-
let mut consolidator = Consolidator::<T, D, CodecSort<K, V, T, D>>::new(
887-
format!(
888-
"{}[lower={:?},upper={:?}]",
889-
shard_id,
890-
desc.lower().elements(),
891-
desc.upper().elements()
892-
),
893-
*shard_id,
894-
CodecSort::new(write_schemas.clone()),
895-
blob,
896-
Arc::clone(&metrics),
897-
shard_metrics,
898-
metrics.read.compaction.clone(),
899-
FetchBatchFilter::Compaction {
900-
since: desc.since().clone(),
901-
},
902-
prefetch_budget_bytes,
903-
);
904-
905-
for (desc, meta, parts) in runs {
906-
consolidator.enqueue_run(desc, meta, parts.iter().cloned());
854+
};
855+
timings.part_fetching += fetch_start.elapsed();
856+
total_bytes += chunk.goodbytes();
857+
chunks.push(chunk);
858+
tokio::task::yield_now().await;
907859
}
908860

909-
let remaining_budget = consolidator.start_prefetches();
910-
if remaining_budget.is_none() {
911-
metrics.compaction.not_all_prefetched.inc();
861+
if chunks.is_empty() {
862+
break;
912863
}
913864

914-
loop {
915-
let mut chunks = vec![];
916-
let mut total_bytes = 0;
917-
// We attempt to pull chunks out of the consolidator that match our target size,
918-
// but it's possible that we may get smaller chunks... for example, if not all
919-
// parts have been fetched yet. Loop until we've got enough data to justify flushing
920-
// it out to blob (or we run out of data.)
921-
while total_bytes < cfg.batch.blob_target_size {
922-
let fetch_start = Instant::now();
923-
let Some(mut chunk) = consolidator
924-
.next_chunk(
925-
cfg.compaction_yield_after_n_updates,
926-
cfg.batch.blob_target_size - total_bytes,
927-
)
928-
.await?
929-
else {
930-
break;
931-
};
932-
timings.part_fetching += fetch_start.elapsed();
933-
total_bytes += chunk.goodbytes();
934-
chunks.push(
935-
chunk
936-
.get_or_make_codec::<K, V>(
937-
write_schemas.key.as_ref(),
938-
write_schemas.val.as_ref(),
939-
)
940-
.clone(),
941-
);
942-
tokio::task::yield_now().await;
943-
}
944-
945-
if chunks.is_empty() {
946-
break;
947-
}
948-
949-
// In the hopefully-common case of a single chunk, this will not copy.
950-
let updates = ColumnarRecords::concat(&chunks, &metrics.columnar);
951-
batch
952-
.flush_part(desc.clone(), BlobTraceUpdates::Row(updates))
953-
.await;
954-
}
865+
// If we're going to write this Batch in a structured format, and our CYA
866+
// dyncfg is enabled, then don't worry about making sure Codec data exists.
867+
let ensure_codec = !(cfg.compaction_output_structured_only
868+
&& cfg.batch.batch_columnar_format == BatchColumnarFormat::Structured);
869+
870+
// In the hopefully-common case of a single chunk, this will not copy.
871+
let updates = BlobTraceUpdates::concat::<K, V>(
872+
chunks,
873+
write_schemas.key.as_ref(),
874+
write_schemas.val.as_ref(),
875+
&metrics.columnar,
876+
ensure_codec,
877+
)?;
878+
batch.flush_part(desc.clone(), updates).await;
955879
}
956880
let mut batch = batch.finish(desc.clone()).await?;
957881

‎src/persist-client/src/internal/machine.rs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1403,7 +1403,7 @@ pub mod datadriven {
14031403

14041404
use crate::batch::{
14051405
validate_truncate_batch, Batch, BatchBuilder, BatchBuilderConfig, BatchBuilderInternal,
1406-
BatchParts, BLOB_TARGET_SIZE, BUILDER_STRUCTURED, STRUCTURED_ORDER,
1406+
BatchParts, BLOB_TARGET_SIZE, BUILDER_STRUCTURED,
14071407
};
14081408
use crate::cfg::COMPACTION_MEMORY_BOUND_BYTES;
14091409
use crate::fetch::EncodedPart;
@@ -1448,9 +1448,6 @@ pub mod datadriven {
14481448
// Our structured compaction code uses slightly different estimates
14491449
// for array size than the old path, which can affect the results of
14501450
// some compaction tests.
1451-
client
1452-
.cfg
1453-
.set_config(&STRUCTURED_ORDER, *STRUCTURED_ORDER.default());
14541451
client.cfg.set_config(&BUILDER_STRUCTURED, true);
14551452
client.cfg.set_config(&COMBINE_INLINE_WRITES, false);
14561453
let state_versions = Arc::new(StateVersions::new(

0 commit comments

Comments
 (0)