Skip to content

Commit 9f0e58e

Browse files
authored
fix(meta): share peer discovery with plugin hooks (#9977)
Signed-off-by: WenyXu <[email protected]>
1 parent 0dab3ae commit 9f0e58e

2 files changed

Lines changed: 54 additions & 5 deletions

File tree

‎src/meta-srv/src/metasrv.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ use tokio::sync::broadcast::error::RecvError;
6767

6868
use crate::cluster::MetaPeerClientRef;
6969
use crate::discovery;
70+
use crate::discovery::node_info::NodeInfoAccessor;
7071
use crate::error::{
7172
self, InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu,
7273
StartTelemetryTaskSnafu, StopProcedureManagerSnafu,
@@ -515,9 +516,13 @@ pub trait SelectorFactory: Send + Sync {
515516
fn build(&self, ctx: SelectorFactoryContext) -> SelectorRef;
516517

517518
/// Builds the resolver shared by allocation and failover callers.
519+
///
520+
/// The accessor includes last-known records of inactive source nodes; discovery
521+
/// is the shared scheduling-aware view of active nodes.
518522
fn build_affinity_resolver(
519523
&self,
520-
_meta_peer_client: MetaPeerClientRef,
524+
_node_info_accessor: Arc<dyn NodeInfoAccessor>,
525+
_peer_discovery: PeerDiscoveryRef,
521526
) -> Option<AffinityResolverRef> {
522527
None
523528
}

‎src/meta-srv/src/metasrv/builder.rs‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ use common_meta::key::runtime_switch::{RuntimeSwitchManager, RuntimeSwitchManage
3939
use common_meta::kv_backend::memory::MemoryKvBackend;
4040
use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
4141
use common_meta::node_manager::NodeManagerRef;
42+
use common_meta::peer::PeerDiscoveryRef;
4243
use common_meta::reconciliation::manager::ReconciliationManager;
4344
use common_meta::region_keeper::MemoryRegionKeeper;
4445
use common_meta::region_registry::LeaderRegionRegistry;
@@ -412,7 +413,12 @@ impl MetasrvBuilder {
412413
let affinity_resolver = plugins
413414
.as_ref()
414415
.and_then(|plugins| plugins.get::<SelectorFactoryRef>())
415-
.and_then(|factory| factory.build_affinity_resolver(meta_peer_client.clone()));
416+
.and_then(|factory| {
417+
factory.build_affinity_resolver(
418+
meta_peer_client.clone(),
419+
selector_ctx.peer_discovery.clone(),
420+
)
421+
});
416422
let region_supervisor_selector = plugins
417423
.as_ref()
418424
.and_then(|plugins| plugins.get::<RegionStatAwareSelectorRef>());
@@ -497,7 +503,7 @@ impl MetasrvBuilder {
497503
{
498504
let ctx = DdlManagerConfigureContext {
499505
kv_backend: kv_backend.clone(),
500-
meta_peer_client: meta_peer_client.clone(),
506+
peer_discovery: selector_ctx.peer_discovery.clone(),
501507
};
502508
configurator
503509
.configure(ddl_manager, ctx)
@@ -759,7 +765,7 @@ impl Default for MetasrvBuilder {
759765
/// The context for [`DdlManagerConfiguratorRef`].
760766
pub struct DdlManagerConfigureContext {
761767
pub kv_backend: KvBackendRef,
762-
pub meta_peer_client: MetaPeerClientRef,
768+
pub peer_discovery: PeerDiscoveryRef,
763769
}
764770

765771
#[cfg(test)]
@@ -854,6 +860,9 @@ mod tests {
854860
use common_meta::cluster::NodeInfo;
855861
use common_meta::error::Result as MetaResult;
856862

863+
use common_meta::ddl_manager::DdlManagerConfigurator;
864+
865+
use crate::discovery::node_info::NodeInfoAccessor;
857866
use crate::metasrv::{SelectorFactory, SelectorFactoryContext};
858867

859868
struct TestAffinity;
@@ -879,10 +888,32 @@ mod tests {
879888
struct RecordingFactory {
880889
calls: Arc<AtomicUsize>,
881890
resolver: AffinityResolverRef,
891+
discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
892+
}
893+
894+
struct RecordingConfigurator {
895+
discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
896+
}
897+
898+
#[async_trait::async_trait]
899+
impl DdlManagerConfigurator<DdlManagerConfigureContext> for RecordingConfigurator {
900+
async fn configure(
901+
&self,
902+
ddl_manager: DdlManager,
903+
ctx: DdlManagerConfigureContext,
904+
) -> std::result::Result<DdlManager, common_error::ext::BoxedError> {
905+
*self.discovery.lock().unwrap() = Some(ctx.peer_discovery);
906+
Ok(ddl_manager)
907+
}
882908
}
883909

884910
impl SelectorFactory for RecordingFactory {
885-
fn build_affinity_resolver(&self, _: MetaPeerClientRef) -> Option<AffinityResolverRef> {
911+
fn build_affinity_resolver(
912+
&self,
913+
_: Arc<dyn NodeInfoAccessor>,
914+
peer_discovery: PeerDiscoveryRef,
915+
) -> Option<AffinityResolverRef> {
916+
*self.discovery.lock().unwrap() = Some(peer_discovery);
886917
self.calls.fetch_add(1, Ordering::Relaxed);
887918
Some(self.resolver.clone())
888919
}
@@ -892,6 +923,8 @@ mod tests {
892923
}
893924
}
894925

926+
let factory_discovery = Arc::new(std::sync::Mutex::new(None));
927+
let configurator_discovery = Arc::new(std::sync::Mutex::new(None));
895928
let calls = Arc::new(AtomicUsize::new(0));
896929
let resolver: AffinityResolverRef = Arc::new(TestResolver);
897930
let affinity: AffinityRef = Arc::new(TestAffinity);
@@ -901,7 +934,12 @@ mod tests {
901934
plugins.insert(Arc::new(RecordingFactory {
902935
calls: calls.clone(),
903936
resolver: resolver.clone(),
937+
discovery: factory_discovery.clone(),
904938
}) as SelectorFactoryRef);
939+
plugins.insert(Arc::new(RecordingConfigurator {
940+
discovery: configurator_discovery.clone(),
941+
})
942+
as DdlManagerConfiguratorRef<DdlManagerConfigureContext>);
905943
let metasrv = MetasrvBuilder::new()
906944
.options(MetasrvOptions {
907945
enable_telemetry: false,
@@ -921,6 +959,12 @@ mod tests {
921959
AffinityFallback::Unconstrained
922960
);
923961
assert_eq!(calls.load(Ordering::Relaxed), 1);
962+
for discovery in [factory_discovery, configurator_discovery] {
963+
assert!(Arc::ptr_eq(
964+
discovery.lock().unwrap().as_ref().unwrap(),
965+
&metasrv.selector_ctx().peer_discovery,
966+
));
967+
}
924968
assert!(Arc::ptr_eq(&metasrv.selector_ctx().affinity, &affinity));
925969
assert!(Arc::ptr_eq(
926970
metasrv

0 commit comments

Comments
 (0)