@@ -27,8 +27,8 @@ use std::sync::Arc;
2727use std:: time:: Duration ;
2828
2929use async_trait:: async_trait;
30+ use differential_dataflow:: lattice:: Lattice ;
3031use mz_cluster_client:: client:: ClusterReplicaLocation ;
31- use mz_cluster_client:: metrics:: WallclockLagMetrics ;
3232use mz_cluster_client:: ReplicaId ;
3333use mz_persist_client:: batch:: ProtoBatch ;
3434use mz_persist_types:: { Codec64 , Opaque , ShardId } ;
@@ -47,6 +47,7 @@ use mz_storage_types::sources::{
4747 SourceExportDetails , Timeline ,
4848} ;
4949use serde:: { Deserialize , Serialize } ;
50+ use timely:: progress:: frontier:: MutableAntichain ;
5051use timely:: progress:: Timestamp as TimelyTimestamp ;
5152use timely:: progress:: { Antichain , Timestamp } ;
5253use tokio:: sync:: { mpsc, oneshot} ;
@@ -96,7 +97,7 @@ pub enum IntrospectionType {
9697
9798/// Describes how data is written to the collection.
9899#[ derive( Clone , Debug , Eq , PartialEq ) ]
99- pub enum DataSource {
100+ pub enum DataSource < T > {
100101 /// Ingest data from some external source.
101102 Ingestion ( IngestionDescription ) ,
102103 /// This source receives its data from the identified ingestion,
@@ -122,9 +123,11 @@ pub enum DataSource {
122123 /// `storage-controller` we the primary as a dependency.
123124 primary : Option < GlobalId > ,
124125 } ,
125- /// This source's data is does not need to be managed by the storage
126+ /// This source's data does not need to be managed by the storage
126127 /// controller, e.g. it's a materialized view or the catalog collection.
127128 Other ,
129+ /// This collection is the output collection of a sink.
130+ Sink { desc : ExportDescription < T > } ,
128131}
129132
130133/// Describes a request to create a source.
@@ -133,7 +136,7 @@ pub struct CollectionDescription<T> {
133136 /// The schema of this collection
134137 pub desc : RelationDesc ,
135138 /// The source of this collection's data.
136- pub data_source : DataSource ,
139+ pub data_source : DataSource < T > ,
137140 /// An optional frontier to which the collection's `since` should be advanced.
138141 pub since : Option < Antichain < T > > ,
139142 /// A GlobalId to use for this collection to use for the status collection.
@@ -168,7 +171,7 @@ pub enum Response<T> {
168171}
169172
170173/// Metadata that the storage controller must know to properly handle the life
171- /// cycle of creating and dropping collections.j
174+ /// cycle of creating and dropping collections.
172175///
173176/// This data should be kept consistent with the state modified using
174177/// [`StorageTxn`].
@@ -485,12 +488,6 @@ pub trait StorageController: Debug {
485488 id : GlobalId ,
486489 ) -> Result < & mut ExportState < Self :: Timestamp > , StorageError < Self :: Timestamp > > ;
487490
488- /// Create the sinks described by the `ExportDescription`.
489- async fn create_exports (
490- & mut self ,
491- exports : Vec < ( GlobalId , ExportDescription < Self :: Timestamp > ) > ,
492- ) -> Result < ( ) , StorageError < Self :: Timestamp > > ;
493-
494491 /// Create a oneshot ingestion.
495492 async fn create_oneshot_ingestion (
496493 & mut self ,
@@ -532,6 +529,7 @@ pub trait StorageController: Debug {
532529 /// Drops the read capability for the sinks and allows their resources to be reclaimed.
533530 fn drop_sinks (
534531 & mut self ,
532+ storage_metadata : & StorageMetadata ,
535533 identifiers : Vec < GlobalId > ,
536534 ) -> Result < ( ) , StorageError < Self :: Timestamp > > ;
537535
@@ -545,7 +543,11 @@ pub trait StorageController: Debug {
545543 /// created, but have been forgotten by the controller due to a restart.
546544 /// Once command history becomes durable we can remove this method and use the normal
547545 /// `drop_sinks`.
548- fn drop_sinks_unvalidated ( & mut self , identifiers : Vec < GlobalId > ) ;
546+ fn drop_sinks_unvalidated (
547+ & mut self ,
548+ storage_metadata : & StorageMetadata ,
549+ identifiers : Vec < GlobalId > ,
550+ ) ;
549551
550552 /// Drops the read capability for the sources and allows their resources to be reclaimed.
551553 ///
@@ -659,7 +661,7 @@ pub trait StorageController: Debug {
659661 > ;
660662}
661663
662- impl DataSource {
664+ impl < T > DataSource < T > {
663665 /// Returns true if the storage controller manages the data shard for this
664666 /// source using txn-wal.
665667 pub fn in_txns ( & self ) -> bool {
@@ -671,6 +673,7 @@ impl DataSource {
671673 | DataSource :: Introspection ( _)
672674 | DataSource :: Progress
673675 | DataSource :: Webhook => false ,
676+ DataSource :: Sink { .. } => false ,
674677 }
675678 }
676679}
@@ -710,51 +713,66 @@ impl From<NonZeroI64> for PersistEpoch {
710713/// State maintained about individual exports.
711714#[ derive( Debug ) ]
712715pub struct ExportState < T : TimelyTimestamp > {
713- /// Description with which the export was created
714- pub description : ExportDescription < T > ,
716+ /// Really only for keeping track of changes to the `derived_since`.
717+ pub read_capabilities : MutableAntichain < T > ,
715718
716- /// The read hold that this export has on its dependencies (inputs). When
719+ /// The cluster this export is associated with.
720+ pub cluster_id : StorageInstanceId ,
721+
722+ /// The current since frontier, derived from `write_frontier` using
723+ /// `hold_policy`.
724+ pub derived_since : Antichain < T > ,
725+
726+ /// The read holds that this export has on its dependencies (its input and itself). When
717727 /// the upper of the export changes, we downgrade this, which in turn
718728 /// downgrades holds we have on our dependencies' sinces.
719- pub read_hold : ReadHold < T > ,
729+ pub read_holds : [ ReadHold < T > ; 2 ] ,
720730
721731 /// The policy to use to downgrade `self.read_capability`.
722732 pub read_policy : ReadPolicy < T > ,
723733
724734 /// Reported write frontier.
725735 pub write_frontier : Antichain < T > ,
726-
727- /// Maximum frontier wallclock lag since the last introspection update.
728- pub wallclock_lag_max : Duration ,
729- /// Frontier wallclock lag metrics tracked for this collection.
730- pub wallclock_lag_metrics : WallclockLagMetrics ,
731736}
732737
733738impl < T : Timestamp > ExportState < T > {
734739 pub fn new (
735- description : ExportDescription < T > ,
740+ cluster_id : StorageInstanceId ,
736741 read_hold : ReadHold < T > ,
742+ self_hold : ReadHold < T > ,
743+ write_frontier : Antichain < T > ,
737744 read_policy : ReadPolicy < T > ,
738- wallclock_lag_metrics : WallclockLagMetrics ,
739- ) -> Self {
745+ ) -> Self
746+ where
747+ T : Lattice ,
748+ {
749+ let mut dependency_since = Antichain :: from_elem ( T :: minimum ( ) ) ;
750+ for read_hold in [ & read_hold, & self_hold] {
751+ dependency_since. join_assign ( read_hold. since ( ) ) ;
752+ }
740753 Self {
741- description,
742- read_hold,
754+ read_capabilities : MutableAntichain :: from ( dependency_since. borrow ( ) ) ,
755+ cluster_id,
756+ derived_since : dependency_since,
757+ read_holds : [ read_hold, self_hold] ,
743758 read_policy,
744- write_frontier : Antichain :: from_elem ( Timestamp :: minimum ( ) ) ,
745- wallclock_lag_max : Default :: default ( ) ,
746- wallclock_lag_metrics,
759+ write_frontier,
747760 }
748761 }
749762
750763 /// Returns the cluster to which the export is bound.
751764 pub fn cluster_id ( & self ) -> StorageInstanceId {
752- self . description . instance_id
765+ self . cluster_id
766+ }
767+
768+ /// Returns the cluster to which the export is bound.
769+ pub fn input_hold ( & self ) -> & ReadHold < T > {
770+ & self . read_holds [ 0 ]
753771 }
754772
755773 /// Returns whether the export was dropped.
756774 pub fn is_dropped ( & self ) -> bool {
757- self . read_hold . since ( ) . is_empty ( )
775+ self . read_holds . iter ( ) . all ( |h| h . since ( ) . is_empty ( ) )
758776 }
759777}
760778/// A channel that allows you to append a set of updates to a pre-defined [`GlobalId`].
0 commit comments