|
1 | 1 | //! `buzz-sandbox-reaper`: the in-cluster sandbox reaper for Buzz Agent |
2 | 2 | //! Sandbox sessions (`docs/remote-agents.md` §Sandbox lifecycle). |
3 | | -// Stack glue: these modules are exercised by their tests only until the |
4 | | -// `reconcile` subcommand wires them up. |
5 | | -#![allow(dead_code)] |
| 3 | +//! |
| 4 | +//! Phase 1 has one subcommand, `reconcile`: classify every managed Sandbox in |
| 5 | +//! the configured developer namespaces, tombstone sessions that have ended, |
| 6 | +//! and delete tombstones once their 30-day checkpoint retention has passed. |
| 7 | +//! It never starts or resumes a session; only an explicit desktop recovery |
| 8 | +//! does that. |
6 | 9 |
|
7 | 10 | mod config; |
8 | 11 | mod evidence; |
9 | 12 | mod plan; |
10 | 13 | mod reconcile; |
11 | 14 | mod server; |
12 | 15 |
|
13 | | -fn main() {} |
| 16 | +use std::collections::BTreeMap; |
| 17 | +use std::net::SocketAddr; |
| 18 | +use std::path::PathBuf; |
| 19 | +use std::sync::Arc; |
| 20 | +use std::time::Duration; |
| 21 | + |
| 22 | +use anyhow::{Context, Result}; |
| 23 | +use clap::{Args, Parser, Subcommand}; |
| 24 | +use tokio_util::sync::CancellationToken; |
| 25 | + |
| 26 | +use crate::config::Config; |
| 27 | +use crate::evidence::S3Store; |
| 28 | +use crate::reconcile::{run_namespace, Memory, Reconciler}; |
| 29 | +use crate::server::Health; |
| 30 | + |
| 31 | +/// How long in-flight passes get to finish after SIGTERM before the process |
| 32 | +/// exits anyway (inside the default 30s Pod termination grace). |
| 33 | +const SHUTDOWN_GRACE: Duration = Duration::from_secs(20); |
| 34 | + |
| 35 | +#[derive(Parser)] |
| 36 | +#[command(name = "buzz-sandbox-reaper", version, about)] |
| 37 | +struct Cli { |
| 38 | + #[command(subcommand)] |
| 39 | + command: Command, |
| 40 | +} |
| 41 | + |
| 42 | +#[derive(Subcommand)] |
| 43 | +enum Command { |
| 44 | + /// Reconcile Agent Sandbox session lifecycles. |
| 45 | + Reconcile(ReconcileArgs), |
| 46 | +} |
| 47 | + |
| 48 | +#[derive(Args)] |
| 49 | +struct ReconcileArgs { |
| 50 | + /// Path to the JSON config (ConfigMap `sandbox-reaper-config`, key `config.json`). |
| 51 | + #[arg( |
| 52 | + long, |
| 53 | + env = "BUZZ_SANDBOX_REAPER_CONFIG", |
| 54 | + default_value = "/etc/buzz-sandbox-reaper/config.json" |
| 55 | + )] |
| 56 | + config: PathBuf, |
| 57 | + /// Address for /metrics, /healthz, and /readyz (the only listener). |
| 58 | + #[arg( |
| 59 | + long, |
| 60 | + env = "BUZZ_SANDBOX_REAPER_LISTEN", |
| 61 | + default_value = "0.0.0.0:9090" |
| 62 | + )] |
| 63 | + listen: SocketAddr, |
| 64 | + /// Seconds between passes over each namespace. |
| 65 | + #[arg( |
| 66 | + long, |
| 67 | + env = "BUZZ_SANDBOX_REAPER_INTERVAL_SECONDS", |
| 68 | + default_value_t = 30, |
| 69 | + value_parser = clap::value_parser!(u64).range(5..=3600) |
| 70 | + )] |
| 71 | + interval_seconds: u64, |
| 72 | + /// Log decisions without writing anything to the cluster. |
| 73 | + #[arg(long, env = "BUZZ_SANDBOX_REAPER_DRY_RUN")] |
| 74 | + dry_run: bool, |
| 75 | + /// Run one pass per namespace, print a summary, and exit (no listener). |
| 76 | + #[arg(long)] |
| 77 | + once: bool, |
| 78 | +} |
| 79 | + |
| 80 | +#[tokio::main] |
| 81 | +async fn main() -> Result<()> { |
| 82 | + // Explicit, as in buzz-backend-kubernetes: the workspace unifies ring and |
| 83 | + // aws-lc-rs, which leaves rustls unable to auto-select a provider. |
| 84 | + let _ = rustls::crypto::ring::default_provider().install_default(); |
| 85 | + tracing_subscriber::fmt() |
| 86 | + .json() |
| 87 | + .with_env_filter( |
| 88 | + tracing_subscriber::EnvFilter::try_from_default_env() |
| 89 | + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), |
| 90 | + ) |
| 91 | + .init(); |
| 92 | + match Cli::parse().command { |
| 93 | + Command::Reconcile(args) => reconcile(args).await, |
| 94 | + } |
| 95 | +} |
| 96 | + |
| 97 | +async fn reconcile(args: ReconcileArgs) -> Result<()> { |
| 98 | + let text = std::fs::read_to_string(&args.config) |
| 99 | + .with_context(|| format!("read config {}", args.config.display()))?; |
| 100 | + let config = Config::parse(&text).map_err(anyhow::Error::msg)?; |
| 101 | + let client = kube::Client::try_default() |
| 102 | + .await |
| 103 | + .context("build in-cluster Kubernetes client")?; |
| 104 | + let store = S3Store::new(&config.checkpoint_bucket, &config.checkpoint_region).await; |
| 105 | + let reconciler = Arc::new(Reconciler { |
| 106 | + client, |
| 107 | + store, |
| 108 | + dry_run: args.dry_run, |
| 109 | + }); |
| 110 | + if args.once { |
| 111 | + return once(&reconciler, &config).await; |
| 112 | + } |
| 113 | + |
| 114 | + let metrics = metrics_exporter_prometheus::PrometheusBuilder::new() |
| 115 | + .install_recorder() |
| 116 | + .context("install metrics recorder")?; |
| 117 | + let interval = Duration::from_secs(args.interval_seconds); |
| 118 | + let health = Arc::new(Health::new(config.namespaces.keys().cloned(), interval)); |
| 119 | + let listener = tokio::net::TcpListener::bind(args.listen) |
| 120 | + .await |
| 121 | + .with_context(|| format!("bind {}", args.listen))?; |
| 122 | + tracing::info!(listen = %args.listen, namespaces = config.namespaces.len(), |
| 123 | + dry_run = args.dry_run, "buzz-sandbox-reaper reconcile started"); |
| 124 | + |
| 125 | + let cancel = CancellationToken::new(); |
| 126 | + let mut loops = tokio::task::JoinSet::new(); |
| 127 | + for (name, namespace) in config.namespaces.clone() { |
| 128 | + loops.spawn(run_namespace( |
| 129 | + Arc::clone(&reconciler), |
| 130 | + name, |
| 131 | + namespace, |
| 132 | + interval, |
| 133 | + Arc::clone(&health), |
| 134 | + cancel.clone(), |
| 135 | + )); |
| 136 | + } |
| 137 | + let server = axum::serve(listener, server::router(health, metrics)) |
| 138 | + .with_graceful_shutdown(cancel.clone().cancelled_owned()); |
| 139 | + let mut server = tokio::spawn(std::future::IntoFuture::into_future(server)); |
| 140 | + // Any component ending on its own is a failure: the loops run until |
| 141 | + // cancelled and the listener until shutdown. |
| 142 | + let outcome = tokio::select! { |
| 143 | + () = shutdown() => { |
| 144 | + tracing::info!("shutting down"); |
| 145 | + Ok(()) |
| 146 | + } |
| 147 | + result = &mut server => match result { |
| 148 | + Ok(Ok(())) => Err(anyhow::anyhow!("metrics listener stopped")), |
| 149 | + Ok(Err(error)) => Err(anyhow::Error::new(error).context("metrics listener failed")), |
| 150 | + Err(error) => Err(anyhow::Error::new(error).context("metrics listener panicked")), |
| 151 | + }, |
| 152 | + Some(result) = loops.join_next() => match result { |
| 153 | + Ok(()) => Err(anyhow::anyhow!("a namespace loop stopped")), |
| 154 | + Err(error) => Err(anyhow::Error::new(error).context("a namespace loop panicked")), |
| 155 | + }, |
| 156 | + }; |
| 157 | + cancel.cancel(); |
| 158 | + let drained = tokio::time::timeout(SHUTDOWN_GRACE, async { |
| 159 | + while loops.join_next().await.is_some() {} |
| 160 | + let _ = (&mut server).await; |
| 161 | + }) |
| 162 | + .await; |
| 163 | + if drained.is_err() { |
| 164 | + tracing::warn!("shutdown grace elapsed; abandoning in-flight passes"); |
| 165 | + loops.abort_all(); |
| 166 | + server.abort(); |
| 167 | + } |
| 168 | + outcome |
| 169 | +} |
| 170 | + |
| 171 | +async fn once(reconciler: &Reconciler<S3Store>, config: &Config) -> Result<()> { |
| 172 | + let mut failed = false; |
| 173 | + let mut report = BTreeMap::new(); |
| 174 | + for (name, namespace) in &config.namespaces { |
| 175 | + let mut memory = Memory::default(); |
| 176 | + match reconciler.pass(name, namespace, &mut memory).await { |
| 177 | + Ok(summary) => { |
| 178 | + report.insert(name.clone(), format!("{summary:?}")); |
| 179 | + } |
| 180 | + Err(error) => { |
| 181 | + failed = true; |
| 182 | + report.insert(name.clone(), format!("error: {error}")); |
| 183 | + } |
| 184 | + } |
| 185 | + } |
| 186 | + println!("{}", serde_json::to_string_pretty(&report)?); |
| 187 | + anyhow::ensure!(!failed, "at least one namespace pass failed"); |
| 188 | + Ok(()) |
| 189 | +} |
| 190 | + |
| 191 | +async fn shutdown() { |
| 192 | + #[cfg(unix)] |
| 193 | + { |
| 194 | + if let Ok(mut term) = |
| 195 | + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) |
| 196 | + { |
| 197 | + tokio::select! { |
| 198 | + _ = term.recv() => {} |
| 199 | + _ = tokio::signal::ctrl_c() => {} |
| 200 | + } |
| 201 | + return; |
| 202 | + } |
| 203 | + } |
| 204 | + let _ = tokio::signal::ctrl_c().await; |
| 205 | +} |
| 206 | + |
| 207 | +#[cfg(test)] |
| 208 | +mod tests { |
| 209 | + use super::*; |
| 210 | + use clap::CommandFactory; |
| 211 | + |
| 212 | + #[test] |
| 213 | + fn cli_is_well_formed_and_defaults_match_the_deployment_contract() { |
| 214 | + Cli::command().debug_assert(); |
| 215 | + let cli = Cli::try_parse_from(["buzz-sandbox-reaper", "reconcile"]).unwrap(); |
| 216 | + let Command::Reconcile(args) = cli.command; |
| 217 | + assert_eq!( |
| 218 | + args.config, |
| 219 | + PathBuf::from("/etc/buzz-sandbox-reaper/config.json") |
| 220 | + ); |
| 221 | + assert_eq!(args.listen, "0.0.0.0:9090".parse().unwrap()); |
| 222 | + assert_eq!(args.interval_seconds, 30); |
| 223 | + assert!(!args.dry_run && !args.once); |
| 224 | + assert!(Cli::try_parse_from([ |
| 225 | + "buzz-sandbox-reaper", |
| 226 | + "reconcile", |
| 227 | + "--interval-seconds", |
| 228 | + "1" |
| 229 | + ]) |
| 230 | + .is_err()); |
| 231 | + assert!(Cli::try_parse_from(["buzz-sandbox-reaper", "api"]).is_err()); |
| 232 | + } |
| 233 | +} |
0 commit comments