Skip to content

Commit 730df43

Browse files
committed
test(agents): cover the sandbox reaper pass against a fake apiserver
Add an in-memory fake Kubernetes apiserver (Sandboxes, Pods, Events, merge patches with resourceVersion and UID preconditions, Date header) and reconcile scenario tests over it: tombstoning with verified, missing, and inherited checkpoints, the 24-hour hold, lost-Pod grace, CAS conflicts, expired tombstone deletion, foreign owners, and dry run. A property test checks that no pass resurrects or discards a live session; its proptest regression seeds are checked in. Signed-off-by: Tee8z <[email protected]>
1 parent e7b839e commit 730df43

5 files changed

Lines changed: 1045 additions & 0 deletions

File tree

‎crates/buzz-sandbox-reaper/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,5 @@ aws-sdk-s3 = { version = "1", default-features = false, features = ["rt-tokio",
3333

3434
[dev-dependencies]
3535
tower = { workspace = true }
36+
tokio = { workspace = true, features = ["test-util"] }
3637
proptest = { workspace = true }
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Seeds for failure cases proptest has generated in the past. It is
2+
# automatically read and these particular cases re-run before any
3+
# novel cases are generated.
4+
#
5+
# It is recommended to check this file in to source control so that
6+
# everyone who runs the test benefits from these saved cases.
7+
cc 39b358909b33e906e9d30b9f9685a9607b75646de7a4ac32f7430253618d03dc # shrinks to sessions = [Session { owner_matches: false, mode: "Running", lifecycle: None, age: 0, ended_ago: None, bound: false, pod: None, checkpointing: false, receipt: false }], verified = false
8+
cc de5db39b9b6a9f2b561ccb02a7d68e1e1aedc7fd2f35060480fcb34d9587b05b # shrinks to sessions = [Session { owner_matches: true, mode: "Running", lifecycle: Some("ending"), age: 0, ended_ago: None, bound: false, pod: None, checkpointing: false, receipt: false }], verified = false
Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
//! A minimal in-memory apiserver for one namespace, driving the real
2+
//! `kube::Client` request path (`tower::service_fn`, as the provider's
3+
//! `cluster.rs` tests do). It models only what the reconciler touches and
4+
//! records every request so tests can assert what was (never) written.
5+
6+
use std::collections::BTreeMap;
7+
use std::sync::{Arc, Mutex};
8+
9+
use chrono::{DateTime, Utc};
10+
use http::{Method, Request, Response};
11+
use kube::client::Body;
12+
use kube::Client;
13+
use serde_json::{json, Value};
14+
15+
/// One recorded request.
16+
#[derive(Debug, Clone)]
17+
pub struct Call {
18+
pub method: Method,
19+
pub path: String,
20+
pub body: Value,
21+
}
22+
23+
pub struct State {
24+
/// Apiserver clock, served as the `Date` header.
25+
pub now: DateTime<Utc>,
26+
/// Omit the `Date` header.
27+
pub no_date: bool,
28+
pub reject_writes: bool,
29+
pub sandboxes: BTreeMap<String, Value>,
30+
pub pods: BTreeMap<String, Value>,
31+
pub rv: u64,
32+
/// Applied to the named Sandbox (bumping its resourceVersion) just
33+
/// before the next PATCH or DELETE is evaluated: a concurrent writer.
34+
pub race: Option<(String, Value)>,
35+
pub calls: Vec<Call>,
36+
}
37+
38+
impl Default for State {
39+
fn default() -> Self {
40+
Self {
41+
now: DateTime::from_timestamp(1_800_000_000, 0).unwrap(),
42+
no_date: false,
43+
reject_writes: false,
44+
sandboxes: BTreeMap::new(),
45+
pods: BTreeMap::new(),
46+
rv: 100,
47+
race: None,
48+
calls: Vec::new(),
49+
}
50+
}
51+
}
52+
53+
pub type Shared = Arc<Mutex<State>>;
54+
55+
fn status(code: u16, reason: &str) -> Response<Body> {
56+
let body = json!({"kind":"Status","apiVersion":"v1","status":"Failure",
57+
"message":reason,"reason":reason,"code":code});
58+
Response::builder()
59+
.status(code)
60+
.body(Body::from(serde_json::to_vec(&body).unwrap()))
61+
.unwrap()
62+
}
63+
64+
/// JSON merge patch (RFC 7386).
65+
pub fn merge(target: &mut Value, patch: &Value) {
66+
match (target, patch) {
67+
(Value::Object(target), Value::Object(patch)) => {
68+
for (key, value) in patch {
69+
if value.is_null() {
70+
target.remove(key);
71+
} else {
72+
merge(target.entry(key.clone()).or_insert(Value::Null), value);
73+
}
74+
}
75+
}
76+
(target, patch) => *target = patch.clone(),
77+
}
78+
}
79+
80+
fn matches_selector(object: &Value, query: &str) -> bool {
81+
let Some(selector) = query
82+
.split('&')
83+
.find_map(|pair| pair.strip_prefix("labelSelector="))
84+
else {
85+
return true;
86+
};
87+
let selector = selector
88+
.replace("%3D", "=")
89+
.replace("%2C", ",")
90+
.replace("%2F", "/");
91+
selector.split(',').all(|term| {
92+
let (key, value) = term.split_once('=').unwrap_or((term, ""));
93+
object["metadata"]["labels"][key] == value
94+
})
95+
}
96+
97+
impl State {
98+
fn ok(&self, value: &Value) -> Response<Body> {
99+
let mut response = Response::builder().status(200);
100+
if !self.no_date {
101+
response = response.header(
102+
http::header::DATE,
103+
self.now.format("%a, %d %b %Y %H:%M:%S GMT").to_string(),
104+
);
105+
}
106+
response
107+
.body(Body::from(serde_json::to_vec(value).unwrap()))
108+
.unwrap()
109+
}
110+
111+
fn list(&self, items: &BTreeMap<String, Value>, query: &str) -> Response<Body> {
112+
let items: Vec<&Value> = items
113+
.values()
114+
.filter(|o| matches_selector(o, query))
115+
.collect();
116+
self.ok(&json!({"apiVersion":"v1","kind":"List",
117+
"metadata":{"resourceVersion": self.rv.to_string()},"items":items}))
118+
}
119+
120+
fn apply_race(&mut self) {
121+
if let Some((name, mutation)) = self.race.take() {
122+
self.rv += 1;
123+
let rv = self.rv.to_string();
124+
if let Some(current) = self.sandboxes.get_mut(&name) {
125+
merge(current, &mutation);
126+
current["metadata"]["resourceVersion"] = rv.into();
127+
}
128+
}
129+
}
130+
131+
fn handle(&mut self, method: &Method, uri: &str, body: &[u8]) -> Response<Body> {
132+
let (path, query) = uri.split_once('?').unwrap_or((uri, ""));
133+
let body: Value = serde_json::from_slice(body).unwrap_or(Value::Null);
134+
self.calls.push(Call {
135+
method: method.clone(),
136+
path: path.to_string(),
137+
body: body.clone(),
138+
});
139+
if self.reject_writes && *method != Method::GET {
140+
return status(403, "Forbidden");
141+
}
142+
let segments: Vec<&str> = path.trim_start_matches('/').split('/').collect();
143+
match (method.clone(), segments.as_slice()) {
144+
(Method::GET, ["apis", "agents.x-k8s.io", "v1beta1", "namespaces", _, "sandboxes"]) => {
145+
self.list(&self.sandboxes, query)
146+
}
147+
(
148+
Method::GET,
149+
["apis", "agents.x-k8s.io", "v1beta1", "namespaces", _, "sandboxes", name],
150+
) => match self.sandboxes.get(*name) {
151+
Some(sandbox) => self.ok(sandbox),
152+
None => status(404, "NotFound"),
153+
},
154+
(
155+
Method::PATCH,
156+
["apis", "agents.x-k8s.io", "v1beta1", "namespaces", _, "sandboxes", name],
157+
) => {
158+
self.apply_race();
159+
self.rv += 1;
160+
let rv = self.rv.to_string();
161+
let Some(current) = self.sandboxes.get_mut(*name) else {
162+
return status(404, "NotFound");
163+
};
164+
let expected = &body["metadata"]["resourceVersion"];
165+
if !expected.is_null() && *expected != current["metadata"]["resourceVersion"] {
166+
return status(409, "Conflict");
167+
}
168+
merge(current, &body);
169+
current["metadata"]["resourceVersion"] = rv.into();
170+
let updated = current.clone();
171+
self.ok(&updated)
172+
}
173+
(
174+
Method::DELETE,
175+
["apis", "agents.x-k8s.io", "v1beta1", "namespaces", _, "sandboxes", name],
176+
) => {
177+
self.apply_race();
178+
let Some(current) = self.sandboxes.get(*name) else {
179+
return status(404, "NotFound");
180+
};
181+
let pre = &body["preconditions"];
182+
for field in ["uid", "resourceVersion"] {
183+
if pre[field].is_null() || pre[field] != current["metadata"][field] {
184+
return status(409, "Conflict");
185+
}
186+
}
187+
let removed = self.sandboxes.remove(*name).unwrap();
188+
self.ok(&removed)
189+
}
190+
(Method::GET, ["api", "v1", "namespaces", _, "pods"]) => self.list(&self.pods, query),
191+
(Method::GET, ["api", "v1", "namespaces", _, "pods", name]) => {
192+
match self.pods.get(*name) {
193+
Some(pod) => self.ok(pod),
194+
None => status(404, "NotFound"),
195+
}
196+
}
197+
(Method::POST, ["api", "v1", "namespaces", _, "events"]) => self.ok(&body),
198+
_ => status(405, "MethodNotAllowed"),
199+
}
200+
}
201+
}
202+
203+
/// A client backed by `state`.
204+
pub fn client(state: &Shared) -> Client {
205+
let state = Arc::clone(state);
206+
let service = tower::service_fn(move |request: Request<Body>| {
207+
let state = Arc::clone(&state);
208+
async move {
209+
let (parts, body) = request.into_parts();
210+
let bytes = http_body_util::BodyExt::collect(body)
211+
.await
212+
.map(|b| b.to_bytes().to_vec())
213+
.unwrap_or_default();
214+
let response =
215+
state
216+
.lock()
217+
.unwrap()
218+
.handle(&parts.method, &parts.uri.to_string(), &bytes);
219+
Ok::<_, std::convert::Infallible>(response)
220+
}
221+
});
222+
Client::new(service, "agents")
223+
}

‎crates/buzz-sandbox-reaper/src/reconcile.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,3 +685,7 @@ pub fn tombstone_patch(
685685
"spec": {"operatingMode": "Suspended"},
686686
})
687687
}
688+
689+
#[cfg(test)]
690+
#[path = "reconcile_tests.rs"]
691+
mod tests;

0 commit comments

Comments
 (0)