Skip to content

Commit a6442f9

Browse files
authored
Resuming a microVM actor on the same node no longer reads its rootfs upper back from disk (#3485)
Why: In the microVM burn-in (GKE downstream testing), wake p50 went from ~350ms to ~1.1s (!), almost all of it in upper_join. Reported to me by @annapendleton though I'm not sure who originally found it. #3353 started writing out snapshot tars as they're written, so their dirty pages can't run a tight worker pod out of memory. It also dropped the written pages from the page cache, so a resume on the same node read the rootfs upper tar back from disk ... which is slow. It also made every suspend write the tars at disk speed. Fixes by commit order: 1. Pull summing memory.stat keys out of memoryUsed, no behavior change. 2. Clean pages are reclaimable, so dropping them is irrelevant for OOM safety. - The tars now stay cached. Write-out waits on the previous chunk instead of the one just written, so dirty pages stay at about two chunks without stalling on each one. 4. Start snapshot writeback after the OnDemand delta merge, which replaces memory-ranges, so it tracks the merged file instead of the unlinked delta. 5. Write snapshot files back in the background instead of with a blocking fdatasync. - Starting write-out can block for a long time on a busy disk (https://github.com/torvalds/linux/blob/v6.12/fs/sync.c#L315-L317), and holding deleted snapshot files open meanwhile leaked disk under load, so it starts write-out once, 64MiB at a time, and closes a file as soon as it's unlinked, which drops its dirty pages. 7. What snapshot writes can borrow leaves clean page cache out, so cached tars don't make a pod look "full". - A worker pod without a memory limit (e.g. the deploy_locust.sh default) has nothing to protect, so it lends all of it instead of capping every suspend as if the pod were full. 9. When the pod can lend memory for the whole archive, the tar skips the write-behind and goes to the page cache at full speed, and the loan comes back once it's on disk. Otherwise the write-behind still bounds its dirty pages. That said, with respect to workers without limits: We should probably consider recommending memory limits on worker pods. I will think about it overnight and propose docs updates tomorrow. Tested with locust burn-in tests (see comments in the PR). AI use: AI-assisted, I reviewed and tested it. - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Benjamin Elder <[email protected]>
1 parent dc039f1 commit a6442f9

18 files changed

Lines changed: 703 additions & 101 deletions

‎cmd/ateworker-microvm/checkpoint.go‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -215,15 +215,15 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke
215215
g.Go(func() error {
216216
t := time.Now()
217217
var err error
218-
durableFiles, err = tarDurableVolumes(gctx, actorDirs.GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec().GetContainers()))
218+
durableFiles, err = s.tarDurableVolumes(gctx, actorDirs.GetDurableDirVolumeMountsDir(), checkpointDir, durableVolumeNames(req.GetSpec().GetContainers()))
219219
dDurable = time.Since(t)
220220
return err
221221
})
222222
}
223223
if shipsRootfs {
224224
g.Go(func() error {
225225
t := time.Now()
226-
err := tarRootfsUpper(gctx, rootfsUpperDir(actorDirs), checkpointDir, containerNames(req.GetSpec().GetContainers()))
226+
err := s.tarRootfsUpper(gctx, rootfsUpperDir(actorDirs), checkpointDir, containerNames(req.GetSpec().GetContainers()))
227227
dUpper = time.Since(t)
228228
return err
229229
})
@@ -290,7 +290,8 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien
290290

291291
slog.InfoContext(ctx, "Snapshotting guest", slog.String("id", actorUID), slog.String("dir", checkpointDir))
292292
tSnapshot := time.Now()
293-
if err := s.writeSnapshot(ctx, client, ra, actorUID, checkpointDir); err != nil {
293+
borrowed, err := s.writeSnapshot(ctx, client, ra, actorUID, checkpointDir)
294+
if err != nil {
294295
return 0, err
295296
}
296297
dSnapshot := time.Since(tSnapshot)
@@ -310,12 +311,18 @@ func (s *AteWorkerService) snapshotVMState(ctx context.Context, client *ch.Clien
310311
} else if ra != nil && ra.restoreSourceDir != "" {
311312
tMerge := time.Now()
312313
if err := mergeOnDemandDelta(ctx, ra.restoreSourceDir, checkpointDir, ra.preserveRestoreSource); err != nil {
314+
s.writeBackSnapshot(ctx, actorUID, checkpointDir, borrowed)
313315
return 0, fmt.Errorf("while merging OnDemand delta into restore source: %w", err)
314316
}
315317
slog.InfoContext(ctx, "Merged OnDemand delta into base (complete snapshot)",
316318
slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge)))
317319
}
318320

321+
// Clean pages can be reclaimed, and dropped once a restore has read them.
322+
// Start writeback after mergeOnDemandDelta so it tracks the final merged
323+
// memory-ranges inode rather than the unlinked delta file.
324+
s.writeBackSnapshot(ctx, actorUID, checkpointDir, borrowed)
325+
319326
// The RO lower never ships (reconstructed from the OCI image at restore).
320327
// The disk-backed upper ships as its own tar from CheckpointWorkload; a
321328
return dSnapshot, nil
@@ -337,13 +344,17 @@ const (
337344
// the whole pod, and every actor on it, so the leaf may only grow into what the
338345
// actor booked and the guest left untouched, plus what it can borrow, and the
339346
// kernel throttles CH to the disk past that.
340-
func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) error {
347+
//
348+
// It returns what it borrowed, for the caller to Return once the snapshot is on
349+
// disk. On error the loan comes back once whatever CH wrote is on disk or
350+
// removed.
351+
func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client, ra *runningActor, actorUID, checkpointDir string) (int64, error) {
341352
if s.snapshotWrites != nil {
342353
select {
343354
case s.snapshotWrites <- struct{}{}:
344355
defer func() { <-s.snapshotWrites }()
345356
case <-ctx.Done():
346-
return fmt.Errorf("while waiting to snapshot: %w", ctx.Err())
357+
return 0, fmt.Errorf("while waiting to snapshot: %w", ctx.Err())
347358
}
348359
}
349360
// the page cache the write adds isn't the actor running away
@@ -380,11 +391,15 @@ func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client,
380391
}
381392
}
382393
if err := client.Snapshot(ctx, checkpointDir); err != nil {
383-
s.borrow.Return(borrowed)
384-
return fmt.Errorf("while snapshotting guest: %w", err)
394+
s.writeBackSnapshot(ctx, actorUID, checkpointDir, borrowed)
395+
return 0, fmt.Errorf("while snapshotting guest: %w", err)
385396
}
386-
// clean pages can be reclaimed, and dropped once a restore has read them.
387-
// the tars sync themselves
397+
return borrowed, nil
398+
}
399+
400+
// writeBackSnapshot returns borrowed once the snapshot files in checkpointDir
401+
// are on disk or removed, or right away if there are none.
402+
func (s *AteWorkerService) writeBackSnapshot(ctx context.Context, actorUID, checkpointDir string, borrowed int64) {
388403
var written []string
389404
for _, name := range []string{"config.json", "state.json", memoryRangesFile} {
390405
p := filepath.Join(checkpointDir, name)
@@ -396,7 +411,6 @@ func (s *AteWorkerService) writeSnapshot(ctx context.Context, client *ch.Client,
396411
s.borrow.Return(borrowed)
397412
slog.WarnContext(ctx, "Failed to start snapshot writeback", slog.String("id", actorUID), slog.Any("err", err))
398413
}
399-
return nil
400414
}
401415

402416
// mergeOnDemandDelta merges the OnDemand delta in checkpointDir with the base in

‎cmd/ateworker-microvm/durable.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,14 +108,14 @@ func (s *AteWorkerService) stageDurableVolumes(ctx context.Context, actorUID, sr
108108
//
109109
// Sockets the workload left behind are skipped rather than archived (tarutil
110110
// logs them); they hold no data and the workload recreates them on start.
111-
func tarDurableVolumes(ctx context.Context, dir, checkpointDir string, volumes []string) ([]string, error) {
111+
func (s *AteWorkerService) tarDurableVolumes(ctx context.Context, dir, checkpointDir string, volumes []string) ([]string, error) {
112112
var files []string
113113
for _, vol := range volumes {
114114
name, err := durableTarFile(vol)
115115
if err != nil {
116116
return nil, err
117117
}
118-
if err := tarutil.CreateWithRoot(ctx, filepath.Join(checkpointDir, name), filepath.Join(dir, vol)); err != nil {
118+
if err := s.archiveDir(ctx, filepath.Join(checkpointDir, name), filepath.Join(dir, vol)); err != nil {
119119
return nil, fmt.Errorf("while archiving durable-dir volume %q: %w", vol, err)
120120
}
121121
files = append(files, name)

‎cmd/ateworker-microvm/durable_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func TestDurableVolumesRoundTrip(t *testing.T) {
9797
t.Fatal(err)
9898
}
9999
checkpointDir := t.TempDir()
100-
files, err := tarDurableVolumes(t.Context(), src, checkpointDir, []string{"cache", "data"})
100+
files, err := (&AteWorkerService{}).tarDurableVolumes(t.Context(), src, checkpointDir, []string{"cache", "data"})
101101
if err != nil {
102102
t.Fatalf("tarDurableVolumes: %v", err)
103103
}

‎cmd/ateworker-microvm/rootfsupper.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,14 @@ func resetRootfsUpperDir(actorDirs *ateworkerpb.ActorDirs) error {
113113
// guest first: virtiofsd is write-through, so a completed guest write has
114114
// reached the host overlay's upper by then, but a running guest could still
115115
// add more after the walk.
116-
func tarRootfsUpper(ctx context.Context, dir, checkpointDir string, containers []string) error {
116+
func (s *AteWorkerService) tarRootfsUpper(ctx context.Context, dir, checkpointDir string, containers []string) error {
117117
for _, cid := range containers {
118118
name, err := rootfsUpperTarFile(cid)
119119
if err != nil {
120120
return err
121121
}
122122
upper, _ := kata.UpperWorkDirs(dir, cid)
123-
if err := tarutil.CreateWithRoot(ctx, filepath.Join(checkpointDir, name), upper); err != nil {
123+
if err := s.archiveDir(ctx, filepath.Join(checkpointDir, name), upper); err != nil {
124124
return fmt.Errorf("while archiving rootfs upper %q: %w", upper, err)
125125
}
126126
}

‎cmd/ateworker-microvm/rootfsupper_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func TestRootfsUpperRoundTrip(t *testing.T) {
6363
t.Fatal(err)
6464
}
6565
checkpointDir := t.TempDir()
66-
if err := tarRootfsUpper(t.Context(), src, checkpointDir, containers); err != nil {
66+
if err := (&AteWorkerService{}).tarRootfsUpper(t.Context(), src, checkpointDir, containers); err != nil {
6767
t.Fatalf("tarRootfsUpper: %v", err)
6868
}
6969
// Restore: onto a directory holding a stale previous activation's contents,
@@ -105,7 +105,7 @@ func TestStageRootfsUpper(t *testing.T) {
105105
containers := []string{"app_ovl"}
106106
src := upperDirWith(t, map[string]string{"app_ovl/fs/home/agent/notes.txt": "rootfs write"})
107107
snapshotDir := t.TempDir()
108-
if err := tarRootfsUpper(t.Context(), src, snapshotDir, containers); err != nil {
108+
if err := (&AteWorkerService{}).tarRootfsUpper(t.Context(), src, snapshotDir, containers); err != nil {
109109
t.Fatalf("tarRootfsUpper: %v", err)
110110
}
111111

‎cmd/ateworker-microvm/snapshotborrow.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,18 @@
1616

1717
package main
1818

19+
import (
20+
"context"
21+
"io/fs"
22+
"log/slog"
23+
"math"
24+
"os"
25+
"path/filepath"
26+
27+
"go.ate.dev/substrate/internal/snapshotcache"
28+
"go.ate.dev/substrate/internal/tarutil"
29+
)
30+
1931
// unbookedMemory is the worker's reported memory capacity less what its
2032
// actors booked, overhead included.
2133
func (s *AteWorkerService) unbookedMemory(podLimit int64) int64 {
@@ -35,3 +47,65 @@ func actorBooking(memoryBytes int64) int64 {
3547
perGiB := actorOverhead.GetMemoryBytesPerGib()
3648
return memoryBytes + actorOverhead.GetMemoryBytes() + memoryBytes/gib*perGiB + memoryBytes%gib*perGiB/gib
3749
}
50+
51+
// archiveDir archives srcDir to tarPath. When the pod can lend the whole
52+
// directory, the archive goes to the page cache without a write-behind, and
53+
// the loan comes back once it's on disk. Otherwise its dirty pages are bounded.
54+
// On error the partial archive is removed and nothing stays borrowed.
55+
func (s *AteWorkerService) archiveDir(ctx context.Context, tarPath, srcDir string) error {
56+
var size int64
57+
lent := false
58+
if s.borrow != nil {
59+
var err error
60+
if size, err = dirSize(ctx, srcDir); err == nil {
61+
lent = s.borrow.TakeAll(size)
62+
}
63+
}
64+
if !lent {
65+
return tarutil.CreateWithRoot(ctx, tarPath, srcDir)
66+
}
67+
if err := tarutil.CreateWithRoot(ctx, tarPath, srcDir, tarutil.WriteBehind(0)); err != nil {
68+
// create has closed it, so unlinking drops its dirty pages without
69+
// writing them
70+
_ = os.Remove(tarPath)
71+
s.borrow.Return(size)
72+
return err
73+
}
74+
if err := snapshotcache.StartWriteback([]string{tarPath}, func() { s.borrow.Return(size) }); err != nil {
75+
s.borrow.Return(size)
76+
slog.WarnContext(ctx, "Failed to start archive writeback", slog.String("path", tarPath), slog.Any("err", err))
77+
}
78+
return nil
79+
}
80+
81+
// dirSize estimates how big a tar of dir is: each regular file's data padded
82+
// to a 512 byte block, plus 2KiB per entry for its header and a PAX header
83+
// carrying its xattrs. Hardlinks count once per link, which leaves room for
84+
// the archive's end blocks.
85+
func dirSize(ctx context.Context, dir string) (int64, error) {
86+
var size int64
87+
err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
88+
if err != nil {
89+
return err
90+
}
91+
if err := ctx.Err(); err != nil {
92+
return err
93+
}
94+
n := int64(2 << 10)
95+
if d.Type().IsRegular() {
96+
info, err := d.Info()
97+
if err != nil {
98+
return err
99+
}
100+
n += (info.Size() + 511) &^ 511
101+
}
102+
// sparse files can claim far more than the disk holds
103+
if size > math.MaxInt64-n {
104+
size = math.MaxInt64
105+
return filepath.SkipAll
106+
}
107+
size += n
108+
return nil
109+
})
110+
return size, err
111+
}

‎cmd/ateworker-microvm/snapshotborrow_test.go‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@
1616

1717
package main
1818

19-
import "testing"
19+
import (
20+
"os"
21+
"path/filepath"
22+
"testing"
23+
24+
"go.ate.dev/substrate/internal/ateworker/memborrow"
25+
)
2026

2127
func TestUnbookedMemory(t *testing.T) {
2228
const limit = 4 << 30
@@ -30,3 +36,34 @@ func TestUnbookedMemory(t *testing.T) {
3036
t.Errorf("unbookedMemory() = %d, want %d", got, want)
3137
}
3238
}
39+
40+
func TestDirSize(t *testing.T) {
41+
dir := t.TempDir()
42+
if err := os.Mkdir(filepath.Join(dir, "sub"), 0o755); err != nil {
43+
t.Fatal(err)
44+
}
45+
if err := os.WriteFile(filepath.Join(dir, "sub", "f"), make([]byte, 1000), 0o644); err != nil {
46+
t.Fatal(err)
47+
}
48+
if err := os.Symlink("f", filepath.Join(dir, "sub", "l")); err != nil {
49+
t.Fatal(err)
50+
}
51+
// dir, sub, f and l are 4 entries, and f's 1000 bytes pad to 1024
52+
if got, err := dirSize(t.Context(), dir); err != nil || got != 4*2048+1024 {
53+
t.Errorf("dirSize() = %d, %v, want %d", got, err, 4*2048+1024)
54+
}
55+
}
56+
57+
func TestArchiveDirReturnsTheLoanOnError(t *testing.T) {
58+
l := &memborrow.Ledger{
59+
Unbooked: func(int64) int64 { return 1 << 30 },
60+
PodMemory: func() (int64, int64, error) { return 4 << 30, 0, nil },
61+
}
62+
s := &AteWorkerService{borrow: l}
63+
if err := s.archiveDir(t.Context(), filepath.Join(t.TempDir(), "missing", "upper.tar"), t.TempDir()); err == nil {
64+
t.Fatal("archiveDir() into a missing directory succeeded")
65+
}
66+
if got := l.Take(1 << 30); got != 1<<30 {
67+
t.Errorf("Take() after a failed archiveDir = %d, want all %d back", got, int64(1<<30))
68+
}
69+
}

‎cmd/ateworker-microvm/snapshotcache.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,14 @@ func dropStagedMemoryImage(ctx context.Context, restoreDir string) {
4949
}
5050

5151
// dropActorSnapshotCacheAsync pins the actor's snapshot files in restoreDir
52-
// while the caller holds the actor lock, then evicts them from the host page
53-
// cache in the background, once they are on disk. A snapshot taken moments ago
54-
// is still being written back.
52+
// while the caller holds the actor lock, then evicts their clean pages from the
53+
// host page cache in the background.
5554
func dropActorSnapshotCacheAsync(restoreDir string, includeMemory bool) {
5655
targets := snapshotcache.FindFiles(restoreDir, func(name string) bool {
5756
return isSnapshotDataFile(name, includeMemory)
5857
})
5958
if len(targets) == 0 {
6059
return
6160
}
62-
go snapshotcache.SyncAndEvict(targets)
61+
go snapshotcache.EvictWithRetries(targets)
6362
}

‎internal/ateworker/cgroup/actor.go‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"io/fs"
2424
"os"
2525
"path/filepath"
26+
"slices"
2627
"strconv"
2728
"strings"
2829
"syscall"
@@ -169,31 +170,40 @@ func actorLeafMemory(root, actorUID string) (int64, error) {
169170
return current, nil
170171
}
171172

172-
// PodMemory returns the worker pod's memory limit and usage. limit is 0 when
173-
// the pod has none.
174-
func PodMemory() (limit, current int64, err error) {
173+
// PodMemory returns the worker pod's memory limit and what it uses besides
174+
// clean page cache, which the kernel can reclaim. Both are 0 when the pod has no
175+
// limit.
176+
func PodMemory() (limit, unreclaimable int64, err error) {
175177
return podMemory(Root)
176178
}
177179

178-
func podMemory(root string) (limit, current int64, err error) {
180+
func podMemory(root string) (limit, unreclaimable int64, err error) {
179181
raw, err := os.ReadFile(filepath.Join(root, "memory.max"))
180182
if err != nil {
181183
return 0, 0, err
182184
}
183-
if v := strings.TrimSpace(string(raw)); v != "max" {
184-
if limit, err = parseBytes(raw); err != nil {
185-
return 0, 0, fmt.Errorf("while parsing memory.max %q: %w", raw, err)
186-
}
185+
if strings.TrimSpace(string(raw)) == "max" {
186+
return 0, 0, nil
187187
}
188-
if raw, err = os.ReadFile(filepath.Join(root, "memory.current")); err != nil {
189-
return 0, 0, err
188+
if limit, err = parseBytes(raw); err != nil {
189+
return 0, 0, fmt.Errorf("while parsing memory.max %q: %w", raw, err)
190190
}
191-
if current, err = parseBytes(raw); err != nil {
192-
return 0, 0, fmt.Errorf("while parsing memory.current %q: %w", raw, err)
191+
if unreclaimable, err = memoryUnreclaimable(root); err != nil {
192+
return 0, 0, err
193193
}
194-
return limit, current, nil
194+
return limit, unreclaimable, nil
195195
}
196196

197+
// memoryUnreclaimable is memoryUsed plus page cache that is dirty or under
198+
// writeback, which reclaim can't drop either.
199+
func memoryUnreclaimable(root string) (int64, error) {
200+
sum, err := sumMemoryStat(root, memoryUnreclaimableStats...)
201+
return int64(sum), err
202+
}
203+
204+
// memoryUnreclaimableStats are the memory.stat keys memoryUnreclaimable sums
205+
var memoryUnreclaimableStats = slices.Concat(memoryUsedStats, []string{"file_dirty", "file_writeback"})
206+
197207
func parseBytes(raw []byte) (int64, error) {
198208
return strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64)
199209
}

0 commit comments

Comments
 (0)