Skip to content

Commit 575515e

Browse files
committed
feat(agents): add an explicit stop op for Sandbox sessions
Add a provider `stop` op for a wedged Agent Sandbox session. It takes the deploy payload, requires provider_config.sandbox and an approved identity_policy before any cluster contact, and sets lifecycle=ending, reason stopped, and operatingMode Suspended in one resourceVersion CAS merge patch (retried on conflict). The controller then deletes the Pod with SIGTERM, so the harness keeps its checkpoint window; the normal stop remains !shutdown. `info` now advertises `ops`, additive within protocol version 1, so the desktop can gate its session controls on `stop`. Adds wire tests, a refusal fixture, and fake-apiserver coverage for the CAS patch. Signed-off-by: Tee8z <[email protected]>
1 parent 16384d5 commit 575515e

10 files changed

Lines changed: 432 additions & 20 deletions

File tree

‎crates/buzz-backend-kubernetes/src/main.rs‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -99,22 +99,49 @@ fn respond(input: &str) -> Response {
9999

100100
match request {
101101
Request::Info => Response::info(),
102-
Request::Deploy(deploy) => {
103-
let runtime = match tokio::runtime::Builder::new_current_thread()
104-
.enable_all()
105-
.build()
106-
{
107-
Ok(runtime) => runtime,
108-
Err(e) => return Response::error(format!("could not start the runtime: {e}")),
109-
};
110-
match runtime.block_on(deploy_agent(&deploy)) {
102+
Request::Deploy(deploy) => match runtime() {
103+
Ok(runtime) => match runtime.block_on(deploy_agent(&deploy)) {
111104
Ok(agent_id) => Response::deployed(agent_id),
112105
Err(e) => Response::error(e),
113-
}
114-
}
106+
},
107+
Err(e) => Response::error(e),
108+
},
109+
Request::Stop(stop) => match runtime() {
110+
Ok(runtime) => match runtime.block_on(stop_session(&stop)) {
111+
Ok((agent_id, state)) => Response::stopped(agent_id, state.as_str()),
112+
Err(e) => Response::error(e),
113+
},
114+
Err(e) => Response::error(e),
115+
},
115116
}
116117
}
117118

119+
fn runtime() -> Result<tokio::runtime::Runtime, String> {
120+
tokio::runtime::Builder::new_current_thread()
121+
.enable_all()
122+
.build()
123+
.map_err(|e| format!("could not start the runtime: {e}"))
124+
}
125+
126+
/// End one Sandbox session. Refusals (identity, scope) precede any cluster
127+
/// contact, exactly as for `deploy`.
128+
async fn stop_session(
129+
request: &wire::DeployRequest,
130+
) -> Result<(String, sandbox::StopState), String> {
131+
let cfg = config::parse(&request.provider_config)?;
132+
let scope = sandbox::Options::parse(&request.provider_config)?
133+
.ok_or("stop requires provider_config.sandbox")?;
134+
let identity = identity::check(request, true)?;
135+
let owner = request
136+
.agent
137+
.launch
138+
.as_ref()
139+
.and_then(|launch| launch.owner_pubkey.as_deref())
140+
.ok_or("Sandbox requires a verified launch owner")?;
141+
let client = client::connect(cfg.context.as_deref()).await?;
142+
sandbox::stop(client, &identity, &cfg.namespace, &scope, owner).await
143+
}
144+
118145
/// Refuse a shared-compute agent, reading the **raw wire value**.
119146
///
120147
/// Trimmed before comparing: the desktop's own layers disagree about padding
@@ -226,6 +253,27 @@ mod tests {
226253
assert_eq!(json["ok"], true);
227254
assert_eq!(json["protocol_version"], wire::PROTOCOL_VERSION);
228255
assert!(json["config_schema"]["properties"]["namespace"].is_object());
256+
assert!(json.get("ops").is_none());
257+
}
258+
259+
/// `stop` is gated like a preflight: no approved identity policy, no
260+
/// cluster contact.
261+
#[test]
262+
fn stop_requires_sandbox_scope_and_identity_policy_before_cluster_contact() {
263+
use nostr::nips::nip19::ToBech32;
264+
let agent = nostr::Keys::generate();
265+
let mut request = serde_json::json!({
266+
"op": "stop",
267+
"agent": {"relay_url": "wss://example.invalid",
268+
"private_key_nsec": agent.secret_key().to_bech32().unwrap()},
269+
"provider_config": {"namespace": "test",
270+
"image": format!("example.invalid/pilot@sha256:{}", "a".repeat(64))}
271+
});
272+
assert!(error_of(&respond(&request.to_string())).contains("provider_config.sandbox"));
273+
request["provider_config"]["sandbox"] = serde_json::json!({
274+
"channel_id": uuid::Uuid::new_v4().to_string(), "thread_root": "a".repeat(64)
275+
});
276+
assert!(error_of(&respond(&request.to_string())).contains("identity_policy"));
229277
}
230278

231279
#[test]

‎crates/buzz-backend-kubernetes/src/sandbox.rs‎

Lines changed: 80 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,14 @@ use std::collections::BTreeMap;
1010
use std::time::Duration;
1111

1212
use buzz_backend_kubernetes::lifecycle::{
13-
annotation, owned_pod, sandbox_resource, Lifecycle, OperatingMode, CHANNEL, CHECKPOINTING,
14-
CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED, ENDED_REASON, GENERATION, INITIAL_POD, OWNER,
15-
RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION, SESSION, THREAD,
13+
annotation, owned_pod, sandbox_resource, EndedReason, Lifecycle, OperatingMode, CHANNEL,
14+
CHECKPOINTING, CHECKPOINTING_DISABLED, CHECKPOINTING_ENABLED, ENDED_REASON, GENERATION,
15+
INITIAL_POD, LIFECYCLE, LIFECYCLE_ENDING, OWNER, RESTORE_CHECKPOINT, SANDBOX_BINDING_VERSION,
16+
SESSION, THREAD,
1617
};
1718
use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, Pod, Secret};
1819
use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
19-
use kube::api::{Api, ListParams, PostParams};
20+
use kube::api::{Api, ListParams, Patch, PatchParams, PostParams};
2021
use kube::core::DynamicObject;
2122
use kube::{Client, ResourceExt};
2223
use serde::Deserialize;
@@ -712,6 +713,81 @@ async fn wait_for_start(
712713
}
713714
}
714715

716+
/// What `stop` left behind.
717+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
718+
pub enum StopState {
719+
/// Suspended and draining; the reaper writes the tombstone.
720+
Ending,
721+
/// Already a tombstone.
722+
Ended,
723+
/// No Sandbox exists for this thread.
724+
Absent,
725+
}
726+
727+
impl StopState {
728+
pub fn as_str(self) -> &'static str {
729+
match self {
730+
Self::Ending => "ending",
731+
Self::Ended => "ended",
732+
Self::Absent => "absent",
733+
}
734+
}
735+
}
736+
737+
/// End a (possibly wedged) session: one CAS merge patch sets
738+
/// `lifecycle=ending`, reason `stopped`, and `operatingMode: Suspended`. The
739+
/// controller then deletes the Pod (SIGTERM, so the harness still gets its
740+
/// checkpoint window). The normal stop remains `!shutdown`.
741+
pub async fn stop(
742+
client: Client,
743+
identity: &naming::AgentIdentity,
744+
namespace: &str,
745+
scope: &Options,
746+
owner: &str,
747+
) -> Result<(String, StopState), String> {
748+
let sandboxes = api(client, namespace);
749+
let name = scope.name(identity);
750+
for _ in 0..5 {
751+
let Some(sandbox) = sandboxes
752+
.get_opt(&name)
753+
.await
754+
.map_err(|e| format!("read Sandbox: {e}"))?
755+
else {
756+
return Ok((name, StopState::Absent));
757+
};
758+
verify_identity(&sandbox, identity, scope, owner)?;
759+
match Lifecycle::of(&sandbox) {
760+
Lifecycle::Ended => return Ok((name, StopState::Ended)),
761+
Lifecycle::Ending => return Ok((name, StopState::Ending)),
762+
Lifecycle::Unrecognized => {
763+
return Err(
764+
"Sandbox has a lifecycle this provider does not know; update Buzz".into(),
765+
)
766+
}
767+
Lifecycle::Live => {}
768+
}
769+
let patch = json!({
770+
"metadata": {
771+
"resourceVersion": sandbox.metadata.resource_version,
772+
"annotations": {
773+
(LIFECYCLE): LIFECYCLE_ENDING,
774+
(ENDED_REASON): EndedReason::Stopped.as_str(),
775+
},
776+
},
777+
"spec": {"operatingMode": "Suspended"},
778+
});
779+
match sandboxes
780+
.patch(&name, &PatchParams::default(), &Patch::Merge(&patch))
781+
.await
782+
{
783+
Ok(_) => return Ok((name, StopState::Ending)),
784+
Err(e) if reason_is(&e, "Conflict") => continue,
785+
Err(e) => return Err(format!("stop Sandbox: {e}")),
786+
}
787+
}
788+
Err("Sandbox kept changing while stopping; retry".into())
789+
}
790+
715791
#[cfg(test)]
716792
#[path = "sandbox_tests.rs"]
717793
mod tests;

‎crates/buzz-backend-kubernetes/src/sandbox_fake.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ pub struct State {
2828
pub fail_secret_creates: usize,
2929
/// Store the Secret, then fail this many create responses with a 500.
3030
pub fail_secret_create_responses: usize,
31+
/// Answer this many Sandbox patches with a 409 Conflict.
32+
pub conflict_patches: usize,
33+
/// Every merge patch body received.
34+
pub patches: Vec<Value>,
3135
}
3236

3337
pub type Shared = Arc<Mutex<State>>;
@@ -48,6 +52,21 @@ fn ok(value: &Value) -> Response<Body> {
4852
.unwrap()
4953
}
5054

55+
fn merge(target: &mut Value, patch: &Value) {
56+
match (target, patch) {
57+
(Value::Object(target), Value::Object(patch)) => {
58+
for (key, value) in patch {
59+
if value.is_null() {
60+
target.remove(key);
61+
} else {
62+
merge(target.entry(key.clone()).or_insert(Value::Null), value);
63+
}
64+
}
65+
}
66+
(target, patch) => *target = patch.clone(),
67+
}
68+
}
69+
5170
impl State {
5271
fn next_rv(&mut self) -> String {
5372
self.rv += 1;
@@ -185,6 +204,27 @@ impl State {
185204
self.reconcile(name);
186205
ok(&sandbox)
187206
}
207+
(Method::PATCH, Some(name)) => {
208+
let patch: Value = serde_json::from_slice(body).unwrap();
209+
self.patches.push(patch.clone());
210+
if self.conflict_patches > 0 {
211+
self.conflict_patches -= 1;
212+
return status(409, "Conflict");
213+
}
214+
let rv = self.next_rv();
215+
let Some(current) = self.sandboxes.get_mut(name) else {
216+
return status(404, "NotFound");
217+
};
218+
let expected = &patch["metadata"]["resourceVersion"];
219+
if !expected.is_null() && *expected != current["metadata"]["resourceVersion"] {
220+
return status(409, "Conflict");
221+
}
222+
merge(current, &patch);
223+
current["metadata"]["resourceVersion"] = rv.into();
224+
let updated = current.clone();
225+
self.reconcile(name);
226+
ok(&updated)
227+
}
188228
_ => status(405, "MethodNotAllowed"),
189229
}
190230
}

‎crates/buzz-backend-kubernetes/src/sandbox_tests.rs‎

Lines changed: 109 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
use super::*;
2+
use buzz_backend_kubernetes::lifecycle;
3+
use buzz_backend_kubernetes::lifecycle::LIFECYCLE_ENDED;
24
use nostr::nips::nip19::ToBech32;
35

46
#[path = "sandbox_fake.rs"]
@@ -239,18 +241,52 @@ async fn live_controller_reuses_scopes_and_refuses_lost_workspaces() {
239241
.await
240242
.unwrap_err();
241243
assert!(error.contains("replaced"), "{error}");
242-
let sandboxes = api(client, &namespace)
244+
let sandboxes = api(client.clone(), &namespace)
243245
.list(&ListParams::default())
244246
.await
245247
.unwrap();
246248
assert_eq!(sandboxes.items.len(), 2, "one Sandbox per canonical thread");
249+
250+
// Lifecycle: stop suspends the session; once tombstoned (the reaper's job,
251+
// simulated here) an ordinary deploy is refused.
252+
let (_, state) = stop(
253+
client.clone(),
254+
&identity,
255+
&namespace,
256+
&second_scope,
257+
"owner",
258+
)
259+
.await
260+
.unwrap();
261+
assert_eq!(state, StopState::Ending);
262+
let sandboxes = api(client.clone(), &namespace);
263+
let old = sandboxes.get(&second).await.unwrap();
264+
assert_eq!(old.data["spec"]["operatingMode"], "Suspended");
265+
let tombstone = json!({"metadata": {"annotations": {
266+
(LIFECYCLE): LIFECYCLE_ENDED, (lifecycle::CHECKPOINT): lifecycle::CHECKPOINT_MISSING,
267+
(lifecycle::ENDED_AT): "2026-01-01T00:00:00Z"}}});
268+
sandboxes
269+
.patch(&second, &PatchParams::default(), &Patch::Merge(&tombstone))
270+
.await
271+
.unwrap();
272+
let ended = deploy(
273+
client.clone(),
274+
&identity,
275+
&cfg,
276+
&second_scope,
277+
"owner",
278+
fixture_env(),
279+
)
280+
.await
281+
.unwrap_err();
282+
assert!(ended.contains("explicit recovery is required"), "{ended}");
247283
namespaces
248284
.delete(&namespace, &DeleteParams::default())
249285
.await
250286
.unwrap();
251287
}
252288

253-
// ── Fake-apiserver tests: the real deploy paths over kube::Client ───────────
289+
// ── Fake-apiserver tests: the real deploy/stop paths over kube::Client ──────
254290

255291
fn fake_identity() -> naming::AgentIdentity {
256292
naming::AgentIdentity::from_nsec(&nostr::Keys::generate().secret_key().to_bech32().unwrap())
@@ -509,3 +545,74 @@ async fn a_sandbox_from_before_the_restore_key_is_still_reused() {
509545
.unwrap_err()
510546
.contains("launch configuration differs"));
511547
}
548+
549+
#[tokio::test(start_paused = true)]
550+
async fn stop_suspends_with_a_cas_patch_and_retries_conflicts() {
551+
let state = fake::Shared::default();
552+
let (identity, cfg, scope) = (fake_identity(), fake_cfg(), fake_scope());
553+
assert_eq!(
554+
stop(fake::client(&state), &identity, "agents", &scope, "owner")
555+
.await
556+
.unwrap()
557+
.1,
558+
StopState::Absent
559+
);
560+
let name = deploy(
561+
fake::client(&state),
562+
&identity,
563+
&cfg,
564+
&scope,
565+
"owner",
566+
plain_env(),
567+
)
568+
.await
569+
.unwrap();
570+
assert!(stop(
571+
fake::client(&state),
572+
&identity,
573+
"agents",
574+
&scope,
575+
"other-owner"
576+
)
577+
.await
578+
.is_err());
579+
state.lock().unwrap().conflict_patches = 1;
580+
let (agent_id, stopped) = stop(fake::client(&state), &identity, "agents", &scope, "owner")
581+
.await
582+
.unwrap();
583+
assert_eq!(
584+
(agent_id.as_str(), stopped),
585+
(name.as_str(), StopState::Ending)
586+
);
587+
{
588+
let guard = state.lock().unwrap();
589+
let patch = guard.patches.last().unwrap();
590+
assert!(patch["metadata"]["resourceVersion"].is_string(), "{patch}");
591+
assert_eq!(patch["spec"]["operatingMode"], "Suspended");
592+
assert_eq!(
593+
patch["metadata"]["annotations"][LIFECYCLE],
594+
LIFECYCLE_ENDING
595+
);
596+
assert_eq!(patch["metadata"]["annotations"][ENDED_REASON], "stopped");
597+
assert_eq!(guard.patches.len(), 2, "one conflict, one retry");
598+
assert!(!guard.pods.contains_key(&name), "Suspended removes the Pod");
599+
}
600+
assert_eq!(
601+
stop(fake::client(&state), &identity, "agents", &scope, "owner")
602+
.await
603+
.unwrap()
604+
.1,
605+
StopState::Ending
606+
);
607+
let ending = deploy(
608+
fake::client(&state),
609+
&identity,
610+
&cfg,
611+
&scope,
612+
"owner",
613+
plain_env(),
614+
)
615+
.await
616+
.unwrap_err();
617+
assert!(ending.contains("ending"), "{ending}");
618+
}

0 commit comments

Comments
 (0)