Skip to content

Commit 9e5e4ca

Browse files
committed
feat(agents): run buzz-sandbox-reaper reconcile as an in-cluster service
Wire the reaper modules into the `reconcile` subcommand: load the JSON config, build the in-cluster client and S3 store, run one loop per namespace beside the /metrics, /healthz, and /readyz listener, and drain in-flight passes on SIGTERM. `--once` runs a single pass per namespace and prints a summary; `--dry-run` decides without writing. Document the flags, configuration, RBAC, and in-memory limits in docs/remote-agents.md. Signed-off-by: Tee8z <[email protected]>
1 parent 730df43 commit 9e5e4ca

4 files changed

Lines changed: 274 additions & 4 deletions

File tree

‎Cargo.lock‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/buzz-sandbox-reaper/Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,20 @@ futures-util = { workspace = true }
2222
serde = { workspace = true }
2323
serde_json = { workspace = true }
2424
chrono = { workspace = true }
25+
anyhow = { workspace = true }
26+
clap = { version = "4", features = ["derive", "env"] }
2527
axum = { workspace = true }
2628
http = "1"
2729
http-body-util = "0.1"
2830
metrics = { workspace = true }
2931
metrics-exporter-prometheus = { workspace = true }
3032
tracing = { workspace = true }
33+
tracing-subscriber = { workspace = true }
3134
aws-config = { version = "1", default-features = false, features = ["rt-tokio", "default-https-client"] }
3235
aws-sdk-s3 = { version = "1", default-features = false, features = ["rt-tokio", "default-https-client"] }
36+
# Same reason as buzz-backend-kubernetes: the process-level CryptoProvider is
37+
# installed explicitly because the workspace unifies ring and aws-lc-rs.
38+
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
3339

3440
[dev-dependencies]
3541
tower = { workspace = true }
Lines changed: 224 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,233 @@
11
//! `buzz-sandbox-reaper`: the in-cluster sandbox reaper for Buzz Agent
22
//! Sandbox sessions (`docs/remote-agents.md` §Sandbox lifecycle).
3-
// Stack glue: these modules are exercised by their tests only until the
4-
// `reconcile` subcommand wires them up.
5-
#![allow(dead_code)]
3+
//!
4+
//! Phase 1 has one subcommand, `reconcile`: classify every managed Sandbox in
5+
//! the configured developer namespaces, tombstone sessions that have ended,
6+
//! and delete tombstones once their 30-day checkpoint retention has passed.
7+
//! It never starts or resumes a session; only an explicit desktop recovery
8+
//! does that.
69
710
mod config;
811
mod evidence;
912
mod plan;
1013
mod reconcile;
1114
mod server;
1215

13-
fn main() {}
16+
use std::collections::BTreeMap;
17+
use std::net::SocketAddr;
18+
use std::path::PathBuf;
19+
use std::sync::Arc;
20+
use std::time::Duration;
21+
22+
use anyhow::{Context, Result};
23+
use clap::{Args, Parser, Subcommand};
24+
use tokio_util::sync::CancellationToken;
25+
26+
use crate::config::Config;
27+
use crate::evidence::S3Store;
28+
use crate::reconcile::{run_namespace, Memory, Reconciler};
29+
use crate::server::Health;
30+
31+
/// How long in-flight passes get to finish after SIGTERM before the process
32+
/// exits anyway (inside the default 30s Pod termination grace).
33+
const SHUTDOWN_GRACE: Duration = Duration::from_secs(20);
34+
35+
#[derive(Parser)]
36+
#[command(name = "buzz-sandbox-reaper", version, about)]
37+
struct Cli {
38+
#[command(subcommand)]
39+
command: Command,
40+
}
41+
42+
#[derive(Subcommand)]
43+
enum Command {
44+
/// Reconcile Agent Sandbox session lifecycles.
45+
Reconcile(ReconcileArgs),
46+
}
47+
48+
#[derive(Args)]
49+
struct ReconcileArgs {
50+
/// Path to the JSON config (ConfigMap `sandbox-reaper-config`, key `config.json`).
51+
#[arg(
52+
long,
53+
env = "BUZZ_SANDBOX_REAPER_CONFIG",
54+
default_value = "/etc/buzz-sandbox-reaper/config.json"
55+
)]
56+
config: PathBuf,
57+
/// Address for /metrics, /healthz, and /readyz (the only listener).
58+
#[arg(
59+
long,
60+
env = "BUZZ_SANDBOX_REAPER_LISTEN",
61+
default_value = "0.0.0.0:9090"
62+
)]
63+
listen: SocketAddr,
64+
/// Seconds between passes over each namespace.
65+
#[arg(
66+
long,
67+
env = "BUZZ_SANDBOX_REAPER_INTERVAL_SECONDS",
68+
default_value_t = 30,
69+
value_parser = clap::value_parser!(u64).range(5..=3600)
70+
)]
71+
interval_seconds: u64,
72+
/// Log decisions without writing anything to the cluster.
73+
#[arg(long, env = "BUZZ_SANDBOX_REAPER_DRY_RUN")]
74+
dry_run: bool,
75+
/// Run one pass per namespace, print a summary, and exit (no listener).
76+
#[arg(long)]
77+
once: bool,
78+
}
79+
80+
#[tokio::main]
81+
async fn main() -> Result<()> {
82+
// Explicit, as in buzz-backend-kubernetes: the workspace unifies ring and
83+
// aws-lc-rs, which leaves rustls unable to auto-select a provider.
84+
let _ = rustls::crypto::ring::default_provider().install_default();
85+
tracing_subscriber::fmt()
86+
.json()
87+
.with_env_filter(
88+
tracing_subscriber::EnvFilter::try_from_default_env()
89+
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
90+
)
91+
.init();
92+
match Cli::parse().command {
93+
Command::Reconcile(args) => reconcile(args).await,
94+
}
95+
}
96+
97+
async fn reconcile(args: ReconcileArgs) -> Result<()> {
98+
let text = std::fs::read_to_string(&args.config)
99+
.with_context(|| format!("read config {}", args.config.display()))?;
100+
let config = Config::parse(&text).map_err(anyhow::Error::msg)?;
101+
let client = kube::Client::try_default()
102+
.await
103+
.context("build in-cluster Kubernetes client")?;
104+
let store = S3Store::new(&config.checkpoint_bucket, &config.checkpoint_region).await;
105+
let reconciler = Arc::new(Reconciler {
106+
client,
107+
store,
108+
dry_run: args.dry_run,
109+
});
110+
if args.once {
111+
return once(&reconciler, &config).await;
112+
}
113+
114+
let metrics = metrics_exporter_prometheus::PrometheusBuilder::new()
115+
.install_recorder()
116+
.context("install metrics recorder")?;
117+
let interval = Duration::from_secs(args.interval_seconds);
118+
let health = Arc::new(Health::new(config.namespaces.keys().cloned(), interval));
119+
let listener = tokio::net::TcpListener::bind(args.listen)
120+
.await
121+
.with_context(|| format!("bind {}", args.listen))?;
122+
tracing::info!(listen = %args.listen, namespaces = config.namespaces.len(),
123+
dry_run = args.dry_run, "buzz-sandbox-reaper reconcile started");
124+
125+
let cancel = CancellationToken::new();
126+
let mut loops = tokio::task::JoinSet::new();
127+
for (name, namespace) in config.namespaces.clone() {
128+
loops.spawn(run_namespace(
129+
Arc::clone(&reconciler),
130+
name,
131+
namespace,
132+
interval,
133+
Arc::clone(&health),
134+
cancel.clone(),
135+
));
136+
}
137+
let server = axum::serve(listener, server::router(health, metrics))
138+
.with_graceful_shutdown(cancel.clone().cancelled_owned());
139+
let mut server = tokio::spawn(std::future::IntoFuture::into_future(server));
140+
// Any component ending on its own is a failure: the loops run until
141+
// cancelled and the listener until shutdown.
142+
let outcome = tokio::select! {
143+
() = shutdown() => {
144+
tracing::info!("shutting down");
145+
Ok(())
146+
}
147+
result = &mut server => match result {
148+
Ok(Ok(())) => Err(anyhow::anyhow!("metrics listener stopped")),
149+
Ok(Err(error)) => Err(anyhow::Error::new(error).context("metrics listener failed")),
150+
Err(error) => Err(anyhow::Error::new(error).context("metrics listener panicked")),
151+
},
152+
Some(result) = loops.join_next() => match result {
153+
Ok(()) => Err(anyhow::anyhow!("a namespace loop stopped")),
154+
Err(error) => Err(anyhow::Error::new(error).context("a namespace loop panicked")),
155+
},
156+
};
157+
cancel.cancel();
158+
let drained = tokio::time::timeout(SHUTDOWN_GRACE, async {
159+
while loops.join_next().await.is_some() {}
160+
let _ = (&mut server).await;
161+
})
162+
.await;
163+
if drained.is_err() {
164+
tracing::warn!("shutdown grace elapsed; abandoning in-flight passes");
165+
loops.abort_all();
166+
server.abort();
167+
}
168+
outcome
169+
}
170+
171+
async fn once(reconciler: &Reconciler<S3Store>, config: &Config) -> Result<()> {
172+
let mut failed = false;
173+
let mut report = BTreeMap::new();
174+
for (name, namespace) in &config.namespaces {
175+
let mut memory = Memory::default();
176+
match reconciler.pass(name, namespace, &mut memory).await {
177+
Ok(summary) => {
178+
report.insert(name.clone(), format!("{summary:?}"));
179+
}
180+
Err(error) => {
181+
failed = true;
182+
report.insert(name.clone(), format!("error: {error}"));
183+
}
184+
}
185+
}
186+
println!("{}", serde_json::to_string_pretty(&report)?);
187+
anyhow::ensure!(!failed, "at least one namespace pass failed");
188+
Ok(())
189+
}
190+
191+
async fn shutdown() {
192+
#[cfg(unix)]
193+
{
194+
if let Ok(mut term) =
195+
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
196+
{
197+
tokio::select! {
198+
_ = term.recv() => {}
199+
_ = tokio::signal::ctrl_c() => {}
200+
}
201+
return;
202+
}
203+
}
204+
let _ = tokio::signal::ctrl_c().await;
205+
}
206+
207+
#[cfg(test)]
208+
mod tests {
209+
use super::*;
210+
use clap::CommandFactory;
211+
212+
#[test]
213+
fn cli_is_well_formed_and_defaults_match_the_deployment_contract() {
214+
Cli::command().debug_assert();
215+
let cli = Cli::try_parse_from(["buzz-sandbox-reaper", "reconcile"]).unwrap();
216+
let Command::Reconcile(args) = cli.command;
217+
assert_eq!(
218+
args.config,
219+
PathBuf::from("/etc/buzz-sandbox-reaper/config.json")
220+
);
221+
assert_eq!(args.listen, "0.0.0.0:9090".parse().unwrap());
222+
assert_eq!(args.interval_seconds, 30);
223+
assert!(!args.dry_run && !args.once);
224+
assert!(Cli::try_parse_from([
225+
"buzz-sandbox-reaper",
226+
"reconcile",
227+
"--interval-seconds",
228+
"1"
229+
])
230+
.is_err());
231+
assert!(Cli::try_parse_from(["buzz-sandbox-reaper", "api"]).is_err());
232+
}
233+
}

‎docs/remote-agents.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1575,6 +1575,46 @@ started. The provider writes the Secret with the Sandbox's generation and
15751575
an owner reference to that Pod. When the Secret write fails, the provider
15761576
also tries to remove its first-Pod binding (CAS).
15771577

1578+
**Running the reaper.** One replica, Deployment strategy `Recreate`:
1579+
1580+
```
1581+
buzz-sandbox-reaper reconcile --config /etc/buzz-sandbox-reaper/config.json
1582+
```
1583+
1584+
| Flag | Environment | Default |
1585+
| --- | --- | --- |
1586+
| `--config <path>` | `BUZZ_SANDBOX_REAPER_CONFIG` | `/etc/buzz-sandbox-reaper/config.json` |
1587+
| `--listen <addr>` | `BUZZ_SANDBOX_REAPER_LISTEN` | `0.0.0.0:9090` |
1588+
| `--interval-seconds <5..3600>` | `BUZZ_SANDBOX_REAPER_INTERVAL_SECONDS` | `30` |
1589+
| `--dry-run` | `BUZZ_SANDBOX_REAPER_DRY_RUN` | off: classify and log, write nothing |
1590+
| `--once` | — | off: run one pass per namespace, print a summary, exit, no listener |
1591+
1592+
The listener is the only one and serves `/metrics` (Prometheus), `/healthz`
1593+
(every namespace loop attempted a pass recently), and `/readyz` (every
1594+
namespace had a successful pass recently). `RUST_LOG` sets the log filter;
1595+
logs are JSON. S3 credentials come from the ambient AWS chain (IRSA).
1596+
1597+
The configuration file is the only source of namespaces and their owners;
1598+
the reaper has no cluster-scoped access:
1599+
1600+
```json
1601+
{"version":1,
1602+
"checkpoint_bucket":"<bucket>","checkpoint_region":"<region>",
1603+
"namespaces":{"<namespace>":{"developer":"<name>",
1604+
"owner_pubkey":"<64 lowercase hex>","checkpoint_prefix":"<name>/"}}}
1605+
```
1606+
1607+
Each listed namespace needs a Role for the reaper's service account:
1608+
`sandboxes.agents.x-k8s.io` get/list/watch/patch/delete, `pods`
1609+
get/list/watch, and `events` (core and `events.k8s.io`) create/patch. The
1610+
reaper needs no Secret access and no `pods/exec`. Its S3 permission is
1611+
`s3:ListBucket` on the checkpoint bucket.
1612+
1613+
Limits: the reaper keeps its timers and captured receipts in memory. A
1614+
restart restarts the 60s lost grace and the 24h hold, which only delays a
1615+
tombstone. A receipt from a Pod that was deleted while the reaper was not
1616+
running is lost, and that session is recorded with `checkpoint=missing`.
1617+
15781618
### Distribution
15791619

15801620
Its own release workflow (macOS arm64/x64 + Linux musl; the sprig workflow's

0 commit comments

Comments
 (0)