Skip to content

Commit a84e095

Browse files
committed
ateom-microvm: stage the rootfs upper inside the cold boot
The ROOTFS restore untarred the rootfs uppers before handing off to the cold boot, which only later cleared stale sandbox state. A previous activation's overlay mounts could therefore still sit on the upper dir when the untar removed it, the ordering teardown documents as unsafe. The untar also ran serially, so a large upper added its full extraction time to the resume, where the MEMORY path overlaps the same work with bundle preparation. Make the cold boot own the upper dir. It clears sandbox state first, moved ahead of bundle preparation, which touches only the atelet-owned bundle dirs, then stages the upper in the background and joins it right before the overlay mounts need it: pristine for a plain boot, re-materialized from the snapshot's tars when the boot parameters name a snapshot dir. A retry stages again from that dir, so nothing depends on what an earlier attempt left behind, and the ROOTFS restore arm shrinks to naming the dir. Spell the ladder out once: includesRootfs and includesMemory decide the checkpoint's capture set and whether a restore resumes the guest, and stageRootfsUpper is the one place a cold boot decides between a pristine and a restored upper. Both have table tests, so a regression in what each fidelity carries shows up without a KVM lane.
1 parent f3eee6c commit a84e095

7 files changed

Lines changed: 162 additions & 36 deletions

File tree

‎cmd/ateom-microvm/checkpoint.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
186186
// - Rootfs upper tars (ROOTFS and MEMORY): host-backed like the durable
187187
// volumes — the memory snapshot does not carry rootfs writes. Under
188188
// VOLUMES the workload cold-starts on restore, discarding rootfs state.
189-
shipsRootfs := scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS ||
190-
scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY
189+
shipsRootfs := includesRootfs(scope)
191190
var durableFiles []string
192191
g, gctx := errgroup.WithContext(ctx)
193192
if scope == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY {

‎cmd/ateom-microvm/fidelity.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//go:build linux
2+
3+
// Copyright 2026 Google LLC
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
package main
18+
19+
import "github.com/agent-substrate/substrate/internal/proto/ateompb"
20+
21+
// The fidelity ladder, lowest to highest, each level including the ones
22+
// below it: VOLUMES holds the durable volumes, ROOTFS adds each container's
23+
// rootfs upper, MEMORY adds the guest. These two predicates are the only
24+
// place the runtime spells out which levels carry which layer.
25+
26+
// includesRootfs reports whether snapshots at this fidelity carry the
27+
// containers' rootfs uppers, so a checkpoint tars them and a restore puts
28+
// them back.
29+
func includesRootfs(f ateompb.SnapshotFidelity) bool {
30+
return f == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS || f == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY
31+
}
32+
33+
// includesMemory reports whether snapshots at this fidelity carry the guest
34+
// itself, so a restore resumes it (and its CPU counters) rather than
35+
// cold-booting a fresh one.
36+
func includesMemory(f ateompb.SnapshotFidelity) bool {
37+
return f == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY
38+
}

‎cmd/ateom-microvm/fidelity_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,30 @@ import (
2626
"github.com/agent-substrate/substrate/internal/proto/ateompb"
2727
)
2828

29+
// TestFidelityLadder pins which layers each level carries: the checkpoint's
30+
// capture set and the restore's resume-versus-cold-boot decision both key off
31+
// these two predicates.
32+
func TestFidelityLadder(t *testing.T) {
33+
for _, tc := range []struct {
34+
fidelity ateompb.SnapshotFidelity
35+
wantRootfs, wantMemory bool
36+
}{
37+
{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED, false, false},
38+
{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, false, false},
39+
{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, true, false},
40+
{ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, true, true},
41+
} {
42+
t.Run(tc.fidelity.String(), func(t *testing.T) {
43+
if got := includesRootfs(tc.fidelity); got != tc.wantRootfs {
44+
t.Errorf("includesRootfs = %t, want %t", got, tc.wantRootfs)
45+
}
46+
if got := includesMemory(tc.fidelity); got != tc.wantMemory {
47+
t.Errorf("includesMemory = %t, want %t", got, tc.wantMemory)
48+
}
49+
})
50+
}
51+
}
52+
2953
// TestValidateFidelity pins what this runtime serves: every level of the
3054
// ladder, ROOTFS included, since the rootfs upper is host-backed and ships
3155
// with or without guest memory. Anything else is refused before the request

‎cmd/ateom-microvm/restore.go‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
157157
// attributed. Only a MEMORY restore resumes the guest and its CPU counters,
158158
// so only it counts from the first reading; the lower fidelities cold-boot
159159
// a fresh guest whose CPU counts from zero.
160-
resumesGuest := req.GetFidelity() == ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY
160+
resumesGuest := includesMemory(req.GetFidelity())
161161
if _, err := s.hostActor(ctx, attribution, resumesGuest); err != nil {
162162
return nil, err
163163
}
@@ -186,24 +186,18 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
186186
}
187187
case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS:
188188
// A ROOTFS snapshot holds no guest state either, but it carries each
189-
// container's rootfs upper: put those back before the overlays are
190-
// assembled, then cold-boot on top of them instead of a pristine upper.
191-
tUpper := time.Now()
192-
if err := untarRootfsUpper(rootfsUpperDir(p.actorDirs), restoreDir, containerNames(p.containers)); err != nil {
193-
return nil, err
194-
}
195-
dUpper := time.Since(tUpper)
196-
p.keepRootfsUpper = true
189+
// container's rootfs upper. The cold boot re-materializes the uppers
190+
// from the snapshot instead of starting from a pristine dir, overlapped
191+
// with its bundle preparation like the MEMORY path does.
192+
p.rootfsUpperSnapshotDir = restoreDir
197193
if err := s.coldBootActorRetrying(ctx, p); err != nil {
198194
return nil, err
199195
}
200196
dTotal := time.Since(tStart)
201197
slog.InfoContext(ctx, "Actor restored (rootfs upper + durable-dir volumes, cold boot)",
202-
slog.String("id", p.actorUID), slog.Duration("rootfs_upper", dUpper), slog.Duration("total", dTotal))
198+
slog.String("id", p.actorUID), slog.Duration("total", dTotal))
203199
ateomphaselog.LogSnapshotPhases(ctx, "Restore timing breakdown", attribution, scope,
204-
ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{
205-
{Name: phaseRootfsUpper, D: dUpper}, {Name: phaseTotal, D: dTotal},
206-
})
200+
ateomphaselog.RestoreDurationKey, nil, []ateomphaselog.Phase{{Name: phaseTotal, D: dTotal}})
207201
case ateompb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES:
208202
// A VOLUMES snapshot holds no guest state, so this is a cold boot that
209203
// happens to start with the volumes already populated. wakeup probe gating comes

‎cmd/ateom-microvm/rootfsupper.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,18 @@ func containerNames(containers []*ateompb.Container) []string {
8080
return names
8181
}
8282

83+
// stageRootfsUpper prepares the host upper dir for a cold boot: pristine when
84+
// snapshotDir is empty, otherwise re-materialized from the per-container tars
85+
// a ROOTFS or MEMORY snapshot left in snapshotDir. Either way any previous
86+
// activation's contents are gone first. Must run after CleanupSandboxState
87+
// has dropped the overlay mounts that used the dir.
88+
func stageRootfsUpper(actorDirs *ateompb.ActorDirs, snapshotDir string, containers []string) error {
89+
if snapshotDir == "" {
90+
return resetRootfsUpperDir(actorDirs)
91+
}
92+
return untarRootfsUpper(rootfsUpperDir(actorDirs), snapshotDir, containers)
93+
}
94+
8395
// resetRootfsUpperDir gives a cold boot a pristine upper directory: a cold
8496
// boot must start from the bare image, and atelet's actor-dir reset does not
8597
// know about this directory, so ateom wipes any previous activation's contents

‎cmd/ateom-microvm/rootfsupper_test.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import (
2323
"testing"
2424

2525
"github.com/agent-substrate/substrate/internal/tarutil"
26+
27+
"github.com/agent-substrate/substrate/internal/proto/ateompb"
2628
)
2729

2830
// upperDirWith returns a rootfs upper directory laid out the way the host
@@ -96,6 +98,51 @@ func TestRootfsUpperRoundTrip(t *testing.T) {
9698
}
9799
}
98100

101+
// TestStageRootfsUpper pins the one decision a cold boot makes about the
102+
// host upper dir: without a snapshot dir it starts pristine, wiping whatever
103+
// a previous activation left; with one it is re-materialized from the tars.
104+
func TestStageRootfsUpper(t *testing.T) {
105+
containers := []string{"app_ovl"}
106+
src := upperDirWith(t, map[string]string{"app_ovl/fs/home/agent/notes.txt": "rootfs write"})
107+
snapshotDir := t.TempDir()
108+
if err := tarRootfsUpper(t.Context(), src, snapshotDir, containers); err != nil {
109+
t.Fatalf("tarRootfsUpper: %v", err)
110+
}
111+
112+
t.Run("pristine without a snapshot", func(t *testing.T) {
113+
stale := upperDirWith(t, map[string]string{"app_ovl/fs/stale.txt": "stale"})
114+
dirs := &ateompb.ActorDirs{RootDir: filepath.Dir(stale)}
115+
if err := os.Rename(stale, rootfsUpperDir(dirs)); err != nil {
116+
t.Fatal(err)
117+
}
118+
if err := stageRootfsUpper(dirs, "", containers); err != nil {
119+
t.Fatalf("stageRootfsUpper: %v", err)
120+
}
121+
entries, err := os.ReadDir(rootfsUpperDir(dirs))
122+
if err != nil || len(entries) != 0 {
123+
t.Errorf("upper dir after pristine staging: ReadDir = %v, %v; want an empty directory", entries, err)
124+
}
125+
})
126+
127+
t.Run("re-materialized from a snapshot", func(t *testing.T) {
128+
stale := upperDirWith(t, map[string]string{"app_ovl/fs/stale.txt": "stale"})
129+
dirs := &ateompb.ActorDirs{RootDir: filepath.Dir(stale)}
130+
if err := os.Rename(stale, rootfsUpperDir(dirs)); err != nil {
131+
t.Fatal(err)
132+
}
133+
if err := stageRootfsUpper(dirs, snapshotDir, containers); err != nil {
134+
t.Fatalf("stageRootfsUpper: %v", err)
135+
}
136+
got, err := os.ReadFile(filepath.Join(rootfsUpperDir(dirs), "app_ovl/fs/home/agent/notes.txt"))
137+
if err != nil || string(got) != "rootfs write" {
138+
t.Errorf("restored upper content = %q, %v; want the snapshot's write", got, err)
139+
}
140+
if _, err := os.Stat(filepath.Join(rootfsUpperDir(dirs), "app_ovl/fs/stale.txt")); !os.IsNotExist(err) {
141+
t.Errorf("stale content survived staging from a snapshot (stat err = %v), want it wiped", err)
142+
}
143+
})
144+
}
145+
99146
// The snapshot is untrusted, so it must not decide any directory above the
100147
// upperdirs: a symlink there would point the overlay mount, or the workdir
101148
// wipe in kata.StageMergedRootfs, outside the actor's directory.

‎cmd/ateom-microvm/run.go‎

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -312,10 +312,11 @@ type actorBootParams struct {
312312
// memory); a container's own cgroup limit comes from its declared resources.
313313
// Zero fields keep the kata defaults.
314314
size sizing.SandboxSize
315-
// keepRootfsUpper boots on top of whatever the host rootfs upper dir holds
316-
// instead of wiping it first: a ROOTFS restore re-materializes the upper
317-
// from the snapshot before cold-booting.
318-
keepRootfsUpper bool
315+
// rootfsUpperSnapshotDir, when set, names a snapshot directory holding the
316+
// per-container rootfs upper tars; the cold boot re-materializes the uppers
317+
// from it instead of starting from a pristine dir. A ROOTFS restore sets it
318+
// to the restore dir.
319+
rootfsUpperSnapshotDir string
319320
}
320321

321322
func (p actorBootParams) attribution() resources.ActorAttribution {
@@ -420,6 +421,29 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re
420421
}
421422
}()
422423

424+
// Clean stale per-sandbox state before anything touches the host upper dir.
425+
// Also create the runtime dir for the sockets.
426+
427+
s.cleanupSandboxState(ctx, actorUID)
428+
if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil {
429+
return fmt.Errorf("while creating VM dir: %w", err)
430+
}
431+
432+
// Stage the host upper dir in the background (pristine, or from a ROOTFS
433+
// snapshot's tars) and join it right before the overlay mounts need it,
434+
// hiding it behind the bundle preparation below. Every return before the
435+
// join must drain the goroutine, or a retry's staging would race it.
436+
upperDone := make(chan error, 1)
437+
upperJoined := false
438+
go func() {
439+
upperDone <- stageRootfsUpper(p.actorDirs, p.rootfsUpperSnapshotDir, containerNames(containers))
440+
}()
441+
defer func() {
442+
if !upperJoined {
443+
<-upperDone
444+
}
445+
}()
446+
423447
// Guest sizing + agent kernel params.
424448
memMiB, vcpus, kparams := s.guestConfig()
425449

@@ -460,21 +484,10 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re
460484
return err
461485
}
462486

463-
// Clean stale per-sandbox state + create the runtime dir for the sockets.
464-
s.cleanupSandboxState(ctx, actorUID)
465-
if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil {
466-
return fmt.Errorf("while creating VM dir: %w", err)
467-
}
468-
469-
// A cold boot starts from the bare image: give it a pristine host upper dir
470-
// (atelet's actor-dir reset does not know this directory; see rootfsupper.go).
471-
// A ROOTFS restore has already re-materialized the upper and boots on it;
472-
// that holds across a retry too, since a guest that never reached its agent
473-
// wrote nothing and cleanupSandboxState only drops the overlay mounts.
474-
if !p.keepRootfsUpper {
475-
if err := resetRootfsUpperDir(p.actorDirs); err != nil {
476-
return err
477-
}
487+
// The overlay mounts below need the upper dir in its final state.
488+
upperJoined = true
489+
if err := <-upperDone; err != nil {
490+
return err
478491
}
479492

480493
// Assemble each container's merged rootfs on the host (overlay of image lower +
@@ -675,8 +688,7 @@ func (s *AteomService) buildActorContainers(actorUID string, actorDirs *ateompb.
675688
// CSI volumes, and system-info volumes (if any) under SharedDir(id)/durable,
676689
// SharedDir(id)/csi, and SharedDir(id)/system-info,
677690
// then starts the ONE virtiofsd that serves them all. Must run AFTER CleanupSandboxState (which
678-
// wipes SharedDir) and resetRootfsUpperDir/untarRootfsUpper (which own the
679-
// upper contents). The returned virtiofsd cmd outlives this call (CH
691+
// wipes SharedDir) and stageRootfsUpper (which owns the upper contents). The returned virtiofsd cmd outlives this call (CH
680692
// demand-pages from it); the caller owns it (tracked on runningActor, killed
681693
// in teardownActor).
682694
func (s *AteomService) stageMergedRootfs(ctx context.Context, rr resolvedRuntime, id string, actorDirs *ateompb.ActorDirs, ctrs []actorContainer, containers []*ateompb.Container, procAttr *syscall.SysProcAttr) (*exec.Cmd, error) {

0 commit comments

Comments
 (0)