Skip to content

Commit 4d98c30

Browse files
committed
feat(agents): add an explicit stop op for Sandbox sessions
Add a provider `stop` op for a wedged Agent Sandbox session. It takes the deploy payload, requires provider_config.sandbox and an approved identity_policy before any cluster contact, and sets lifecycle=ending, reason stopped, and operatingMode Suspended in one resourceVersion CAS merge patch. A conflict is retried with a jittered backoff, at most five times. A tombstone or an unknown lifecycle is never patched. The controller then deletes the Pod with SIGTERM, so the harness keeps its checkpoint window; the normal stop remains !shutdown. `info` does not advertise `ops` yet, so desktops that predate the field keep accepting it; a later change adds it. The spec marks it as planned. Adds wire tests, a refusal fixture, and fake-apiserver coverage for the CAS patch, its backoff, and the tombstone guard. Signed-off-by: Tee8z <[email protected]>
1 parent 0397d34 commit 4d98c30

10 files changed

Lines changed: 513 additions & 21 deletions

File tree

‎crates/buzz-backend-kubernetes/src/main.rs‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -99,22 +99,49 @@ fn respond(input: &str) -> Response {
9999

100100
match request {
101101
Request::Info => Response::info(),
102-
Request::Deploy(deploy) => {
103-
let runtime = match tokio::runtime::Builder::new_current_thread()
104-
.enable_all()
105-
.build()
106-
{
107-
Ok(runtime) => runtime,
108-
Err(e) => return Response::error(format!("could not start the runtime: {e}")),
109-
};
110-
match runtime.block_on(deploy_agent(&deploy)) {
102+
Request::Deploy(deploy) => match runtime() {
103+
Ok(runtime) => match runtime.block_on(deploy_agent(&deploy)) {
111104
Ok(agent_id) => Response::deployed(agent_id),
112105
Err(e) => Response::failed(e),
113-
}
114-
}
106+
},
107+
Err(e) => Response::error(e),
108+
},
109+
Request::Stop(stop) => match runtime() {
110+
Ok(runtime) => match runtime.block_on(stop_session(&stop)) {
111+
Ok((agent_id, state)) => Response::stopped(agent_id, state),
112+
Err(e) => Response::failed(e),
113+
},
114+
Err(e) => Response::error(e),
115+
},
115116
}
116117
}
117118

119+
fn runtime() -> Result<tokio::runtime::Runtime, String> {
120+
tokio::runtime::Builder::new_current_thread()
121+
.enable_all()
122+
.build()
123+
.map_err(|e| format!("could not start the runtime: {e}"))
124+
}
125+
126+
/// End one Sandbox session. Refusals (identity, scope) precede any cluster
127+
/// contact, exactly as for `deploy`.
128+
async fn stop_session(
129+
request: &wire::DeployRequest,
130+
) -> Result<(String, sandbox::StopState), wire::ProviderError> {
131+
let cfg = config::parse(&request.provider_config)?;
132+
let scope = sandbox::Options::parse(&request.provider_config)?
133+
.ok_or("stop requires provider_config.sandbox")?;
134+
let identity = identity::check(request, true)?;
135+
let owner = request
136+
.agent
137+
.launch
138+
.as_ref()
139+
.and_then(|launch| launch.owner_pubkey.as_deref())
140+
.ok_or("Sandbox requires a verified launch owner")?;
141+
let client = client::connect(cfg.context.as_deref()).await?;
142+
sandbox::stop(client, &identity, &cfg.namespace, &scope, owner).await
143+
}
144+
118145
/// Refuse a shared-compute agent, reading the **raw wire value**.
119146
///
120147
/// Trimmed before comparing: the desktop's own layers disagree about padding
@@ -226,6 +253,27 @@ mod tests {
226253
assert_eq!(json["ok"], true);
227254
assert_eq!(json["protocol_version"], wire::PROTOCOL_VERSION);
228255
assert!(json["config_schema"]["properties"]["namespace"].is_object());
256+
assert!(json.get("ops").is_none());
257+
}
258+
259+
/// `stop` is gated like a preflight: no approved identity policy, no
260+
/// cluster contact.
261+
#[test]
262+
fn stop_requires_sandbox_scope_and_identity_policy_before_cluster_contact() {
263+
use nostr::nips::nip19::ToBech32;
264+
let agent = nostr::Keys::generate();
265+
let mut request = serde_json::json!({
266+
"op": "stop",
267+
"agent": {"relay_url": "wss://example.invalid",
268+
"private_key_nsec": agent.secret_key().to_bech32().unwrap()},
269+
"provider_config": {"namespace": "test",
270+
"image": format!("example.invalid/pilot@sha256:{}", "a".repeat(64))}
271+
});
272+
assert!(error_of(&respond(&request.to_string())).contains("provider_config.sandbox"));
273+
request["provider_config"]["sandbox"] = serde_json::json!({
274+
"channel_id": uuid::Uuid::new_v4().to_string(), "thread_root": "a".repeat(64)
275+
});
276+
assert!(error_of(&respond(&request.to_string())).contains("identity_policy"));
229277
}
230278

231279
#[test]

‎crates/buzz-backend-kubernetes/src/sandbox.rs‎

Lines changed: 85 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,17 @@ use std::time::Duration;
1212
use buzz_backend_kubernetes::lifecycle::{
1313
annotation, owned_pod, sandbox_resource, EndedReason, Lifecycle, OperatingMode,
1414
BIND_TIMEOUT_SECS, CHANNEL, CHECKPOINTING, CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED,
15-
ENDED_REASON, GENERATION, INITIAL_POD, OWNER, RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION,
16-
SESSION, THREAD,
15+
ENDED_REASON, GENERATION, INITIAL_POD, LIFECYCLE, LIFECYCLE_ENDING, OWNER, RESTORE_CHECKPOINT,
16+
SANDBOX_BINDING_VERSION, SESSION, THREAD,
1717
};
1818
use chrono::{DateTime, Utc};
1919
use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, Pod, Secret};
2020
use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
21-
use kube::api::{Api, ListParams, PostParams};
21+
use kube::api::{Api, ListParams, Patch, PatchParams, PostParams};
2222
use kube::core::DynamicObject;
2323
use kube::{Client, ResourceExt};
24-
use serde::Deserialize;
24+
use rand::RngExt;
25+
use serde::{Deserialize, Serialize};
2526
use serde_json::json;
2627
use sha2::{Digest, Sha256};
2728

@@ -818,6 +819,86 @@ async fn wait_for_start(
818819
}
819820
}
820821

822+
/// What `stop` left behind; the `state` of its response.
823+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
824+
#[serde(rename_all = "lowercase")]
825+
pub enum StopState {
826+
/// Suspended and draining; the reaper writes the tombstone.
827+
Ending,
828+
/// Already a tombstone.
829+
Ended,
830+
/// No Sandbox exists for this thread.
831+
Absent,
832+
}
833+
834+
/// Read-and-patch attempts before `stop` gives up on a changing Sandbox.
835+
const STOP_ATTEMPTS: u32 = 5;
836+
/// Base pause after a conflicting `stop` patch; grows per attempt.
837+
const STOP_BACKOFF_MS: u64 = 100;
838+
839+
/// End a (possibly wedged) session: one CAS merge patch sets
840+
/// `lifecycle=ending`, reason `stopped`, and `operatingMode: Suspended`. The
841+
/// controller then deletes the Pod (SIGTERM, so the harness still gets its
842+
/// checkpoint window). The normal stop remains `!shutdown`.
843+
pub async fn stop(
844+
client: Client,
845+
identity: &naming::AgentIdentity,
846+
namespace: &str,
847+
scope: &Options,
848+
owner: &str,
849+
) -> Result<(String, StopState), ProviderError> {
850+
let sandboxes = api(client, namespace);
851+
let name = scope.name(identity);
852+
for attempt in 1..=STOP_ATTEMPTS {
853+
let Some(sandbox) = sandboxes
854+
.get_opt(&name)
855+
.await
856+
.map_err(|e| format!("read Sandbox: {e}"))?
857+
else {
858+
return Ok((name, StopState::Absent));
859+
};
860+
verify_identity(&sandbox, identity, scope, owner)?;
861+
match Lifecycle::of(&sandbox) {
862+
Lifecycle::Ended => return Ok((name, StopState::Ended)),
863+
Lifecycle::Ending => return Ok((name, StopState::Ending)),
864+
// Never overwrite a record this version does not understand.
865+
Lifecycle::Unrecognized => {
866+
return Err(ProviderError::refused(
867+
ErrorCode::LifecycleUnknown,
868+
"Sandbox has a lifecycle this provider does not know; update Buzz",
869+
))
870+
}
871+
Lifecycle::Live => {}
872+
}
873+
let patch = json!({
874+
"metadata": {
875+
"resourceVersion": sandbox.metadata.resource_version,
876+
"annotations": {
877+
(LIFECYCLE): LIFECYCLE_ENDING,
878+
(ENDED_REASON): EndedReason::Stopped.as_str(),
879+
},
880+
},
881+
"spec": {"operatingMode": "Suspended"},
882+
});
883+
match sandboxes
884+
.patch(&name, &PatchParams::default(), &Patch::Merge(&patch))
885+
.await
886+
{
887+
Ok(_) => return Ok((name, StopState::Ending)),
888+
Err(e) if reason_is(&e, "Conflict") => {
889+
if attempt < STOP_ATTEMPTS {
890+
// Jittered, so concurrent writers stop colliding.
891+
let jitter = rand::rng().random_range(0..=STOP_BACKOFF_MS);
892+
let pause = STOP_BACKOFF_MS * u64::from(attempt) + jitter;
893+
tokio::time::sleep(Duration::from_millis(pause)).await;
894+
}
895+
}
896+
Err(e) => return Err(format!("stop Sandbox: {e}").into()),
897+
}
898+
}
899+
Err("Sandbox kept changing while stopping; retry".into())
900+
}
901+
821902
#[cfg(test)]
822903
#[path = "sandbox_tests.rs"]
823904
mod tests;

‎crates/buzz-backend-kubernetes/src/sandbox_fake.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ pub struct State {
3535
pub fail_secret_creates: usize,
3636
/// Store the Secret, then fail this many create responses with a 500.
3737
pub fail_secret_create_responses: usize,
38+
/// Answer this many Sandbox patches with a 409 Conflict.
39+
pub conflict_patches: usize,
40+
/// Every merge patch body received.
41+
pub patches: Vec<Value>,
3842
}
3943

4044
pub type Shared = Arc<Mutex<State>>;
@@ -55,6 +59,21 @@ fn ok(value: &Value) -> Response<Body> {
5559
.unwrap()
5660
}
5761

62+
fn merge(target: &mut Value, patch: &Value) {
63+
match (target, patch) {
64+
(Value::Object(target), Value::Object(patch)) => {
65+
for (key, value) in patch {
66+
if value.is_null() {
67+
target.remove(key);
68+
} else {
69+
merge(target.entry(key.clone()).or_insert(Value::Null), value);
70+
}
71+
}
72+
}
73+
(target, patch) => *target = patch.clone(),
74+
}
75+
}
76+
5877
impl State {
5978
fn next_rv(&mut self) -> String {
6079
self.rv += 1;
@@ -202,6 +221,27 @@ impl State {
202221
self.reconcile(name);
203222
ok(&sandbox)
204223
}
224+
(Method::PATCH, Some(name)) => {
225+
let patch: Value = serde_json::from_slice(body).unwrap();
226+
self.patches.push(patch.clone());
227+
if self.conflict_patches > 0 {
228+
self.conflict_patches -= 1;
229+
return status(409, "Conflict");
230+
}
231+
let rv = self.next_rv();
232+
let Some(current) = self.sandboxes.get_mut(name) else {
233+
return status(404, "NotFound");
234+
};
235+
let expected = &patch["metadata"]["resourceVersion"];
236+
if !expected.is_null() && *expected != current["metadata"]["resourceVersion"] {
237+
return status(409, "Conflict");
238+
}
239+
merge(current, &patch);
240+
current["metadata"]["resourceVersion"] = rv.into();
241+
let updated = current.clone();
242+
self.reconcile(name);
243+
ok(&updated)
244+
}
205245
_ => status(405, "MethodNotAllowed"),
206246
}
207247
}

0 commit comments

Comments
 (0)