Skip to content

Commit 82577c6

Browse files
committed
feat(agents): add the sandbox reaper reconcile engine and health server
Add the per-namespace reconcile pass: classify each managed Sandbox from its annotations and bound Pod, gather checkpoint evidence, and apply the planned tombstone or delete with resourceVersion/UID preconditions, re-reading on CAS conflicts and recording Kubernetes Events and metrics. A Pod watch keeps termination-log receipts after Pods are deleted. Add the /metrics, /healthz, and /readyz router whose health state each namespace loop records into. Scenario and property tests for the pass follow in the next layer. Signed-off-by: Tee8z <[email protected]>
1 parent db54d45 commit 82577c6

7 files changed

Lines changed: 1015 additions & 13 deletions

File tree

‎Cargo.lock‎

Lines changed: 128 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/buzz-sandbox-reaper/Cargo.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,23 @@ path = "src/main.rs"
1414
# never in buzz-backend-kubernetes, whose binary ships inside the desktop app.
1515
[dependencies]
1616
buzz-backend-kubernetes = { path = "../buzz-backend-kubernetes" }
17+
kube = { workspace = true, features = ["runtime"] }
18+
k8s-openapi = { workspace = true }
1719
tokio = { workspace = true }
20+
tokio-util = { workspace = true }
21+
futures-util = { workspace = true }
1822
serde = { workspace = true }
1923
serde_json = { workspace = true }
2024
chrono = { workspace = true }
25+
axum = { workspace = true }
26+
http = "1"
27+
http-body-util = "0.1"
2128
metrics = { workspace = true }
29+
metrics-exporter-prometheus = { workspace = true }
2230
tracing = { workspace = true }
2331
aws-config = { version = "1", default-features = false, features = ["rt-tokio", "default-https-client"] }
2432
aws-sdk-s3 = { version = "1", default-features = false, features = ["rt-tokio", "default-https-client"] }
2533

2634
[dev-dependencies]
35+
tower = { workspace = true }
2736
proptest = { workspace = true }

‎crates/buzz-sandbox-reaper/src/evidence.rs‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -108,20 +108,35 @@ pub async fn from_receipt<S: CheckpointStore>(
108108

109109
/// Evidence for a recovered session that left no verified checkpoint of its
110110
/// own: the checkpoint it was restored from, if that object still exists
111-
/// under the previous generation's prefix. `restore` comes from the
112-
/// provider-written Sandbox annotation, never the Pod, so carrying it forward
111+
/// under the configured developer prefix. The key can belong to an older
112+
/// ancestor than `recovered-from` after repeated failed recovery. `restore`
113+
/// comes from the provider-written Sandbox annotation, never the Pod, so carrying it forward
113114
/// keeps the original work recoverable after a failed restore or stop.
114115
pub async fn inherited<S: CheckpointStore>(
115116
store: &S,
116117
restore: &str,
117-
previous_prefix: &str,
118+
developer_prefix: &str,
118119
) -> Evidence {
119-
if !is_session_key(restore, previous_prefix) {
120+
let Some((generation, _)) = restore
121+
.strip_prefix(developer_prefix)
122+
.and_then(|key| key.split_once('/'))
123+
else {
124+
return Evidence::Missing;
125+
};
126+
if generation.len() != 32
127+
|| !generation
128+
.bytes()
129+
.all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
130+
{
131+
return Evidence::Missing;
132+
}
133+
let session_prefix = format!("{developer_prefix}{generation}/");
134+
if !is_session_key(restore, &session_prefix) {
120135
return Evidence::Missing;
121136
}
122137
probe(
123138
store,
124-
previous_prefix,
139+
&session_prefix,
125140
restore.to_string(),
126141
Evidence::Inherited,
127142
)
@@ -231,30 +246,30 @@ pub mod tests {
231246
}
232247

233248
#[tokio::test]
234-
async fn a_restore_key_is_inherited_only_from_its_previous_generation() {
249+
async fn a_restore_key_is_inherited_only_within_its_developer_prefix() {
235250
let key = format!("{PREFIX}6f9619ff-8b86-d011-b42d-00cf4fc964ff.tar.gz");
236251
let store = FakeStore {
237252
keys: vec![key.clone()],
238253
..Default::default()
239254
};
240255
assert_eq!(
241-
inherited(&store, &key, PREFIX).await,
256+
inherited(&store, &key, "dev/").await,
242257
Evidence::Inherited(key.clone())
243258
);
244259
// Expired or deleted since the recovery.
245260
let gone = format!("{PREFIX}00000000-0000-0000-0000-000000000000.tar.gz");
246-
assert_eq!(inherited(&store, &gone, PREFIX).await, Evidence::Missing);
247-
// Annotation pointing outside the recovered-from generation.
261+
assert_eq!(inherited(&store, &gone, "dev/").await, Evidence::Missing);
262+
// A different developer cannot supply the inherited object.
248263
let probes = store.probes.lock().unwrap().len();
249-
let other = "dev/ffffffffffffffffffffffffffffffff/";
264+
let other = "another-developer/";
250265
assert_eq!(inherited(&store, &key, other).await, Evidence::Missing);
251266
assert_eq!(store.probes.lock().unwrap().len(), probes);
252267
let outage = FakeStore {
253268
fail: true,
254269
..Default::default()
255270
};
256271
assert_eq!(
257-
inherited(&outage, &key, PREFIX).await,
272+
inherited(&outage, &key, "dev/").await,
258273
Evidence::Unavailable
259274
);
260275
}

‎crates/buzz-sandbox-reaper/src/main.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,7 @@
77
mod config;
88
mod evidence;
99
mod plan;
10+
mod reconcile;
11+
mod server;
1012

1113
fn main() {}

0 commit comments

Comments
 (0)