Skip to content

Commit c3aa089

Browse files
committed
fix(microvm): handle multi-GB workload images on resume/restore paths
#287's micro-VM (kata + cloud-hypervisor) MVP builds the actor's virtio-blk rootfs synchronously with mkfs.ext4 -d, bound by the resume RPC deadline. For a small image that is sub-second; a real multi-GB image (OpenShell's ~3.2 GB helpdesk supervisor) takes minutes, so the context is cancelled and mkfs is SIGKILLed mid-write -- the actor never boots. - ateom-microvm RunWorkload (golden boot): build the rootfs on a deadline- detached context to a temp file + atomic rename; skip if it already exists. Idempotent + crash-safe so the controller retries converge. - ateom-microvm RunWorkload (restore reconstruct): detach the rebuild from the RPC deadline too (reset-each-restore, so no idempotency). - ateapi ResumeActor: hold the actor lock long enough to cover the build (30s -> 5m; suspend/delete stay short). - atenet router implicit-resume: 15s -> 5m, so a request that triggers a resume waits for the multi-GB rebuild + VM restore instead of cancelling it. Verified on CPU kind: the OpenShell helpdesk golden snapshot reaches Ready (supervisor boots inside the kata guest + is checkpointed) and a resumed user actor's agent serves /status, where before every path looped on 'mkfs.ext4 ... signal: killed'. Signed-off-by: Davanum Srinivas <[email protected]>
1 parent 89fd7b1 commit c3aa089

3 files changed

Lines changed: 37 additions & 6 deletions

File tree

‎cmd/ateapi/internal/controlapi/workflow.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,9 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, id string, boot bool) (
152152
state := &ResumeState{}
153153

154154
// Acquire lock and get the timeout context for the workflow
155-
// Lock TTL is 7 seconds, with 2 seconds padding for workflow timeout
156-
ctx, releaseLock, err := w.acquireActorLock(ctx, id, 30*time.Second, 2*time.Second)
155+
// Lock TTL 5m / 15s padding: a fresh golden BOOT builds the actor rootfs
156+
// (mkfs.ext4 -d), which takes minutes for large multi-GB workload images.
157+
ctx, releaseLock, err := w.acquireActorLock(ctx, id, 5*time.Minute, 15*time.Second)
157158
if err != nil {
158159
return nil, err
159160
}

‎cmd/atenet/internal/router/resumer.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@ func (r *ActorResumer) ResumeActor(ctx context.Context, actorID string) (*ateapi
4444
// We detach the context from the first caller using a fixed background timeout.
4545
// This guarantees that if Caller 1 disconnects or times out, the underlying
4646
// resume operation continues running for Caller 2 and Caller 3 without failing.
47-
bgCtx, bgCancel := context.WithTimeout(context.Background(), 15*time.Second)
47+
// Generous (5m): a micro-VM resume can reconstruct a multi-GB rootfs disk
48+
// (mkfs.ext4 -d), which takes minutes for large workload images.
49+
bgCtx, bgCancel := context.WithTimeout(context.Background(), 5*time.Minute)
4850
defer bgCancel()
4951

5052
backoff := wait.Backoff{

‎cmd/ateom-microvm/run.go‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,30 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload
215215
// Build the actor's writable rootfs as a raw ext4 virtio-blk disk from the
216216
// atelet-populated OCI bundle rootfs. This becomes /dev/vdb.
217217
diskPath := filepath.Join(actorDir, actorRootfsDiskName)
218-
if err := kata.BuildExt4Image(ctx, filepath.Join(bundle, "rootfs"), diskPath); err != nil {
219-
return nil, fmt.Errorf("while building actor rootfs disk: %w", err)
218+
// Building a multi-GB rootfs via mkfs.ext4 -d can take minutes and exceed the
219+
// caller's resume RPC deadline, which would SIGKILL mkfs mid-write and leave a
220+
// corrupt disk. Build on a deadline-detached context to a unique temp file,
221+
// then atomically rename it into place; skip entirely if a prior build already
222+
// produced the disk. Idempotent + crash-safe across the controller's retries.
223+
if _, statErr := os.Stat(diskPath); statErr != nil {
224+
buildCtx, buildCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute)
225+
tmp, tmpErr := os.CreateTemp(actorDir, "rootfs-*.ext4.building")
226+
if tmpErr != nil {
227+
buildCancel()
228+
return nil, fmt.Errorf("while creating temp rootfs disk: %w", tmpErr)
229+
}
230+
tmpPath := tmp.Name()
231+
_ = tmp.Close()
232+
if err := kata.BuildExt4Image(buildCtx, filepath.Join(bundle, "rootfs"), tmpPath); err != nil {
233+
buildCancel()
234+
_ = os.Remove(tmpPath)
235+
return nil, fmt.Errorf("while building actor rootfs disk: %w", err)
236+
}
237+
buildCancel()
238+
if err := os.Rename(tmpPath, diskPath); err != nil {
239+
_ = os.Remove(tmpPath)
240+
return nil, fmt.Errorf("while finalizing actor rootfs disk: %w", err)
241+
}
220242
}
221243

222244
// Sizing + agent params from the kata config.
@@ -866,7 +888,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
866888
slog.InfoContext(ctx, "Reset actor rootfs disk to golden (template)", slog.String("id", id))
867889
} else {
868890
bundleRootfs := filepath.Join(ateompath.OCIBundlePath(ns, name, id, containers[0].GetName()), "rootfs")
869-
if err := kata.BuildExt4Image(ctx, bundleRootfs, diskPath); err != nil {
891+
// Detach from the resume RPC deadline: reconstructing a multi-GB rootfs via
892+
// mkfs.ext4 -d can take minutes and would otherwise be SIGKILLed mid-write.
893+
// (Unlike the golden boot this is reset-each-restore, so it always rebuilds.)
894+
rbCtx, rbCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute)
895+
err := kata.BuildExt4Image(rbCtx, bundleRootfs, diskPath)
896+
rbCancel()
897+
if err != nil {
870898
return nil, fmt.Errorf("while reconstructing rootfs disk from image: %w", err)
871899
}
872900
slog.InfoContext(ctx, "Reconstructed actor rootfs disk from image", slog.String("id", id))

0 commit comments

Comments
 (0)