Skip to content

Commit 273b3e4

Browse files
committed
Make sinks a variant of collections
1 parent fad0bd3 commit 273b3e4

8 files changed

Lines changed: 373 additions & 412 deletions

File tree

‎src/adapter/src/coord.rs‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,12 @@ use mz_sql::session::vars::SystemVars;
146146
use mz_sql_parser::ast::display::AstDisplay;
147147
use mz_sql_parser::ast::ExplainStage;
148148
use mz_storage_client::client::TableData;
149-
use mz_storage_client::controller::{CollectionDescription, DataSource};
149+
use mz_storage_client::controller::{CollectionDescription, DataSource, ExportDescription};
150150
use mz_storage_types::connections::inline::{IntoInlineConnection, ReferencedConnection};
151151
use mz_storage_types::connections::Connection as StorageConnection;
152152
use mz_storage_types::connections::ConnectionContext;
153153
use mz_storage_types::read_holds::ReadHold;
154-
use mz_storage_types::sinks::S3SinkFormat;
154+
use mz_storage_types::sinks::{S3SinkFormat, StorageSinkDesc};
155155
use mz_storage_types::sources::kafka::KAFKA_PROGRESS_DESC;
156156
use mz_storage_types::sources::Timeline;
157157
use mz_timestamp_oracle::postgres_oracle::{
@@ -2076,9 +2076,11 @@ impl Coordinator {
20762076
self.ship_dataflow(df_desc, mview.cluster_id, None).await;
20772077
}
20782078
CatalogItem::Sink(sink) => {
2079-
self.create_storage_export(sink.global_id(), sink)
2080-
.await
2081-
.unwrap_or_terminate("cannot fail to create exports");
2079+
policies_to_set
2080+
.entry(CompactionWindow::Default)
2081+
.or_insert_with(Default::default)
2082+
.storage_ids
2083+
.insert(sink.global_id());
20822084
}
20832085
CatalogItem::Connection(catalog_connection) => {
20842086
if let ConnectionDetails::AwsPrivatelink(conn) = &catalog_connection.details {
@@ -2736,7 +2738,26 @@ impl Coordinator {
27362738
let collection_desc = CollectionDescription {
27372739
// TODO(sinks): make generic once we have more than one sink type.
27382740
desc: KAFKA_PROGRESS_DESC.clone(),
2739-
data_source: DataSource::Other,
2741+
data_source: DataSource::Sink {
2742+
desc: ExportDescription {
2743+
sink: StorageSinkDesc {
2744+
from: sink.from,
2745+
from_desc: KAFKA_PROGRESS_DESC.clone(),
2746+
connection: sink
2747+
.connection
2748+
.clone()
2749+
.into_inline_connection(self.catalog().state()),
2750+
partition_strategy: sink.partition_strategy.clone(),
2751+
envelope: sink.envelope,
2752+
as_of: Antichain::from_elem(Timestamp::minimum()),
2753+
with_snapshot: sink.with_snapshot,
2754+
version: sink.version,
2755+
from_storage_metadata: (),
2756+
to_storage_metadata: (),
2757+
},
2758+
instance_id: sink.cluster_id,
2759+
},
2760+
},
27402761
since: None,
27412762
status_collection_id: None,
27422763
timeline: None,

‎src/adapter/src/coord/ddl.rs‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,11 @@ use mz_sql::session::vars::{
4646
MAX_REPLICAS_PER_CLUSTER, MAX_ROLES, MAX_SCHEMAS_PER_DATABASE, MAX_SECRETS, MAX_SINKS,
4747
MAX_SOURCES, MAX_TABLES,
4848
};
49-
use mz_storage_client::controller::ExportDescription;
49+
use mz_storage_client::controller::{CollectionDescription, DataSource, ExportDescription};
5050
use mz_storage_types::connections::inline::IntoInlineConnection;
5151
use mz_storage_types::connections::PostgresConnection;
5252
use mz_storage_types::read_policy::ReadPolicy;
53+
use mz_storage_types::sources::kafka::KAFKA_PROGRESS_DESC;
5354
use mz_storage_types::sources::GenericSourceConnection;
5455
use serde_json::json;
5556
use tracing::{event, info_span, warn, Instrument, Level};
@@ -1056,9 +1057,10 @@ impl Coordinator {
10561057
}
10571058

10581059
pub(crate) fn drop_storage_sinks(&mut self, sink_gids: Vec<GlobalId>) {
1060+
let storage_metadata = self.catalog.state().storage_metadata();
10591061
self.controller
10601062
.storage
1061-
.drop_sinks(sink_gids)
1063+
.drop_sinks(storage_metadata, sink_gids)
10621064
.unwrap_or_terminate("cannot fail to drop sinks");
10631065
}
10641066

@@ -1333,16 +1335,27 @@ impl Coordinator {
13331335
to_storage_metadata: (),
13341336
};
13351337

1336-
let res = self
1337-
.controller
1338-
.storage
1339-
.create_exports(vec![(
1340-
id,
1341-
ExportDescription {
1338+
let collection_desc = CollectionDescription {
1339+
// TODO(sinks): make generic once we have more than one sink type.
1340+
desc: KAFKA_PROGRESS_DESC.clone(),
1341+
data_source: DataSource::Sink {
1342+
desc: ExportDescription {
13421343
sink: storage_sink_desc,
13431344
instance_id: sink.cluster_id,
13441345
},
1345-
)])
1346+
},
1347+
since: None,
1348+
status_collection_id: None,
1349+
timeline: None,
1350+
};
1351+
let collections = vec![(id, collection_desc)];
1352+
1353+
// Create the collections.
1354+
let storage_metadata = self.catalog.state().storage_metadata();
1355+
let res = self
1356+
.controller
1357+
.storage
1358+
.create_collections(storage_metadata, None, collections)
13461359
.await;
13471360

13481361
// Drop read holds after the export has been created, at which point

‎src/adapter/src/coord/sequencer/inner.rs‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ use mz_ssh_util::keys::SshKeyPairSet;
8383
use mz_storage_client::controller::{CollectionDescription, DataSource, ExportDescription};
8484
use mz_storage_types::connections::inline::IntoInlineConnection;
8585
use mz_storage_types::controller::StorageError;
86-
use mz_storage_types::sources::kafka::KAFKA_PROGRESS_DESC;
8786
use mz_storage_types::stats::RelationPartStats;
8887
use mz_storage_types::AlterCompatible;
8988
use mz_transform::dataflow::DataflowMetainfo;
@@ -1346,28 +1345,13 @@ impl Coordinator {
13461345
}
13471346
};
13481347

1349-
let collection_desc = CollectionDescription {
1350-
// TODO(sinks): make generic once we have more than one sink type.
1351-
desc: KAFKA_PROGRESS_DESC.clone(),
1352-
data_source: DataSource::Other,
1353-
since: None,
1354-
status_collection_id: None,
1355-
timeline: None,
1356-
};
1357-
let collections = vec![(global_id, collection_desc)];
1358-
1359-
// Create the collections.
1360-
let storage_metadata = self.catalog.state().storage_metadata();
1361-
self.controller
1362-
.storage
1363-
.create_collections(storage_metadata, None, collections)
1364-
.await
1365-
.unwrap_or_terminate("cannot fail to create collections");
1366-
13671348
self.create_storage_export(global_id, &catalog_sink)
13681349
.await
13691350
.unwrap_or_terminate("cannot fail to create exports");
13701351

1352+
self.initialize_storage_read_policies([item_id].into(), CompactionWindow::Default)
1353+
.await;
1354+
13711355
ctx.retire(Ok(ExecuteResponse::CreatedSink))
13721356
}
13731357

‎src/adapter/src/util.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@ impl<T> ShouldTerminateGracefully for StorageError<T> {
373373
| StorageError::ReadBeforeSince(_)
374374
| StorageError::InvalidUppers(_)
375375
| StorageError::InvalidUsage(_)
376-
| StorageError::SourceIdReused(_)
376+
| StorageError::CollectionIdReused(_)
377377
| StorageError::SinkIdReused(_)
378378
| StorageError::IdentifierMissing(_)
379379
| StorageError::IdentifierInvalid(_)

‎src/storage-client/src/controller.rs‎

Lines changed: 50 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ use std::sync::Arc;
2727
use std::time::Duration;
2828

2929
use async_trait::async_trait;
30+
use differential_dataflow::lattice::Lattice;
3031
use mz_cluster_client::client::ClusterReplicaLocation;
31-
use mz_cluster_client::metrics::WallclockLagMetrics;
3232
use mz_cluster_client::ReplicaId;
3333
use mz_persist_client::batch::ProtoBatch;
3434
use mz_persist_types::{Codec64, Opaque, ShardId};
@@ -47,6 +47,7 @@ use mz_storage_types::sources::{
4747
SourceExportDetails, Timeline,
4848
};
4949
use serde::{Deserialize, Serialize};
50+
use timely::progress::frontier::MutableAntichain;
5051
use timely::progress::Timestamp as TimelyTimestamp;
5152
use timely::progress::{Antichain, Timestamp};
5253
use tokio::sync::{mpsc, oneshot};
@@ -96,7 +97,7 @@ pub enum IntrospectionType {
9697

9798
/// Describes how data is written to the collection.
9899
#[derive(Clone, Debug, Eq, PartialEq)]
99-
pub enum DataSource {
100+
pub enum DataSource<T> {
100101
/// Ingest data from some external source.
101102
Ingestion(IngestionDescription),
102103
/// This source receives its data from the identified ingestion,
@@ -122,9 +123,11 @@ pub enum DataSource {
122123
/// `storage-controller` we the primary as a dependency.
123124
primary: Option<GlobalId>,
124125
},
125-
/// This source's data is does not need to be managed by the storage
126+
/// This source's data does not need to be managed by the storage
126127
/// controller, e.g. it's a materialized view or the catalog collection.
127128
Other,
129+
/// This collection is the output collection of a sink.
130+
Sink { desc: ExportDescription<T> },
128131
}
129132

130133
/// Describes a request to create a source.
@@ -133,7 +136,7 @@ pub struct CollectionDescription<T> {
133136
/// The schema of this collection
134137
pub desc: RelationDesc,
135138
/// The source of this collection's data.
136-
pub data_source: DataSource,
139+
pub data_source: DataSource<T>,
137140
/// An optional frontier to which the collection's `since` should be advanced.
138141
pub since: Option<Antichain<T>>,
139142
/// A GlobalId to use for this collection to use for the status collection.
@@ -168,7 +171,7 @@ pub enum Response<T> {
168171
}
169172

170173
/// Metadata that the storage controller must know to properly handle the life
171-
/// cycle of creating and dropping collections.j
174+
/// cycle of creating and dropping collections.
172175
///
173176
/// This data should be kept consistent with the state modified using
174177
/// [`StorageTxn`].
@@ -485,12 +488,6 @@ pub trait StorageController: Debug {
485488
id: GlobalId,
486489
) -> Result<&mut ExportState<Self::Timestamp>, StorageError<Self::Timestamp>>;
487490

488-
/// Create the sinks described by the `ExportDescription`.
489-
async fn create_exports(
490-
&mut self,
491-
exports: Vec<(GlobalId, ExportDescription<Self::Timestamp>)>,
492-
) -> Result<(), StorageError<Self::Timestamp>>;
493-
494491
/// Create a oneshot ingestion.
495492
async fn create_oneshot_ingestion(
496493
&mut self,
@@ -532,6 +529,7 @@ pub trait StorageController: Debug {
532529
/// Drops the read capability for the sinks and allows their resources to be reclaimed.
533530
fn drop_sinks(
534531
&mut self,
532+
storage_metadata: &StorageMetadata,
535533
identifiers: Vec<GlobalId>,
536534
) -> Result<(), StorageError<Self::Timestamp>>;
537535

@@ -545,7 +543,11 @@ pub trait StorageController: Debug {
545543
/// created, but have been forgotten by the controller due to a restart.
546544
/// Once command history becomes durable we can remove this method and use the normal
547545
/// `drop_sinks`.
548-
fn drop_sinks_unvalidated(&mut self, identifiers: Vec<GlobalId>);
546+
fn drop_sinks_unvalidated(
547+
&mut self,
548+
storage_metadata: &StorageMetadata,
549+
identifiers: Vec<GlobalId>,
550+
);
549551

550552
/// Drops the read capability for the sources and allows their resources to be reclaimed.
551553
///
@@ -659,7 +661,7 @@ pub trait StorageController: Debug {
659661
>;
660662
}
661663

662-
impl DataSource {
664+
impl<T> DataSource<T> {
663665
/// Returns true if the storage controller manages the data shard for this
664666
/// source using txn-wal.
665667
pub fn in_txns(&self) -> bool {
@@ -671,6 +673,7 @@ impl DataSource {
671673
| DataSource::Introspection(_)
672674
| DataSource::Progress
673675
| DataSource::Webhook => false,
676+
DataSource::Sink { .. } => false,
674677
}
675678
}
676679
}
@@ -710,51 +713,66 @@ impl From<NonZeroI64> for PersistEpoch {
710713
/// State maintained about individual exports.
711714
#[derive(Debug)]
712715
pub struct ExportState<T: TimelyTimestamp> {
713-
/// Description with which the export was created
714-
pub description: ExportDescription<T>,
716+
/// Really only for keeping track of changes to the `derived_since`.
717+
pub read_capabilities: MutableAntichain<T>,
715718

716-
/// The read hold that this export has on its dependencies (inputs). When
719+
/// The cluster this export is associated with.
720+
pub cluster_id: StorageInstanceId,
721+
722+
/// The current since frontier, derived from `write_frontier` using
723+
/// `hold_policy`.
724+
pub derived_since: Antichain<T>,
725+
726+
/// The read holds that this export has on its dependencies (its input and itself). When
717727
/// the upper of the export changes, we downgrade this, which in turn
718728
/// downgrades holds we have on our dependencies' sinces.
719-
pub read_hold: ReadHold<T>,
729+
pub read_holds: [ReadHold<T>; 2],
720730

721731
/// The policy to use to downgrade `self.read_capability`.
722732
pub read_policy: ReadPolicy<T>,
723733

724734
/// Reported write frontier.
725735
pub write_frontier: Antichain<T>,
726-
727-
/// Maximum frontier wallclock lag since the last introspection update.
728-
pub wallclock_lag_max: Duration,
729-
/// Frontier wallclock lag metrics tracked for this collection.
730-
pub wallclock_lag_metrics: WallclockLagMetrics,
731736
}
732737

733738
impl<T: Timestamp> ExportState<T> {
734739
pub fn new(
735-
description: ExportDescription<T>,
740+
cluster_id: StorageInstanceId,
736741
read_hold: ReadHold<T>,
742+
self_hold: ReadHold<T>,
743+
write_frontier: Antichain<T>,
737744
read_policy: ReadPolicy<T>,
738-
wallclock_lag_metrics: WallclockLagMetrics,
739-
) -> Self {
745+
) -> Self
746+
where
747+
T: Lattice,
748+
{
749+
let mut dependency_since = Antichain::from_elem(T::minimum());
750+
for read_hold in [&read_hold, &self_hold] {
751+
dependency_since.join_assign(read_hold.since());
752+
}
740753
Self {
741-
description,
742-
read_hold,
754+
read_capabilities: MutableAntichain::from(dependency_since.borrow()),
755+
cluster_id,
756+
derived_since: dependency_since,
757+
read_holds: [read_hold, self_hold],
743758
read_policy,
744-
write_frontier: Antichain::from_elem(Timestamp::minimum()),
745-
wallclock_lag_max: Default::default(),
746-
wallclock_lag_metrics,
759+
write_frontier,
747760
}
748761
}
749762

750763
/// Returns the cluster to which the export is bound.
751764
pub fn cluster_id(&self) -> StorageInstanceId {
752-
self.description.instance_id
765+
self.cluster_id
766+
}
767+
768+
/// Returns the cluster to which the export is bound.
769+
pub fn input_hold(&self) -> &ReadHold<T> {
770+
&self.read_holds[0]
753771
}
754772

755773
/// Returns whether the export was dropped.
756774
pub fn is_dropped(&self) -> bool {
757-
self.read_hold.since().is_empty()
775+
self.read_holds.iter().all(|h| h.since().is_empty())
758776
}
759777
}
760778
/// A channel that allows you to append a set of updates to a pre-defined [`GlobalId`].

0 commit comments

Comments
 (0)