Skip to content

Commit 34cce1e

Browse files
authored
ateworker: reuse a completed checkpoint only on the same boot (#3213)
Follow-up to #3015. Part of #2836. A retried CheckpointWorkload reuses the worker's record of a completed checkpoint. The worker does not sync the snapshot files, so after a node reboot the record and the files can survive with torn contents. The retry then reused them, and atelet stamped the local snapshot with the new boot ID, so its pre-reboot check no longer catches them. - RecordCheckpoint records the node's boot ID, and ReuseCheckpoint skips a record from another boot. After a reboot, the retry fails instead of committing files that may be torn. - atelet's boot ID reader moves to internal/bootid so the workers can use it. TestRecordAndReuseCheckpoint covers a record from another boot. AI use: AI-assisted. I reviewed and tested it. - [x] Tests pass --------- Signed-off-by: Chenyi Wang <[email protected]>
1 parent 893a458 commit 34cce1e

11 files changed

Lines changed: 161 additions & 90 deletions

File tree

‎cmd/atelet/bootid.go‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,32 +15,9 @@
1515
package main
1616

1717
import (
18-
"fmt"
19-
"os"
20-
"strings"
21-
2218
"go.ate.dev/substrate/cmd/atelet/internal/ateletpath"
2319
)
2420

25-
// bootIDPath is where Linux exposes the boot ID: a random UUID generated once
26-
// per boot, so it changes on every reboot, clean or not, and on nothing else.
27-
// It is not namespaced, so atelet reads the node's.
28-
const bootIDPath = "/proc/sys/kernel/random/boot_id"
29-
30-
// readBootID returns the boot ID read from path. It is never empty, so a local
31-
// snapshot whose manifest records none never matches it.
32-
func readBootID(path string) (string, error) {
33-
b, err := os.ReadFile(path)
34-
if err != nil {
35-
return "", fmt.Errorf("while reading the boot ID: %w", err)
36-
}
37-
id := strings.TrimSpace(string(b))
38-
if id == "" {
39-
return "", fmt.Errorf("boot ID file %s is empty", path)
40-
}
41-
return id, nil
42-
}
43-
4421
// localSnapshotBootCheck returns the snapshot tracker's check of whether an
4522
// actor's local snapshot was written on the boot bootID: whether the
4623
// snapshot's manifest records bootID. A manifest that is missing or does not

‎cmd/atelet/bootid_test.go‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,36 +22,6 @@ import (
2222
"go.ate.dev/substrate/cmd/atelet/internal/ateletpath"
2323
)
2424

25-
func TestReadBootID(t *testing.T) {
26-
t.Run("reads and trims boot ID", func(t *testing.T) {
27-
p := filepath.Join(t.TempDir(), "boot_id")
28-
const want = "5f1c0d2e-3a4b-4c5d-8e9f-0a1b2c3d4e5f"
29-
if err := os.WriteFile(p, []byte(want+"\n"), 0o600); err != nil {
30-
t.Fatal(err)
31-
}
32-
got, err := readBootID(p)
33-
if err != nil || got != want {
34-
t.Errorf("readBootID = (%q, %v), want (%q, nil)", got, err, want)
35-
}
36-
})
37-
38-
t.Run("rejects empty file", func(t *testing.T) {
39-
p := filepath.Join(t.TempDir(), "boot_id")
40-
if err := os.WriteFile(p, []byte(" \n\t"), 0o600); err != nil {
41-
t.Fatal(err)
42-
}
43-
if _, err := readBootID(p); err == nil {
44-
t.Error("readBootID accepted an empty boot ID file")
45-
}
46-
})
47-
48-
t.Run("rejects missing file", func(t *testing.T) {
49-
if _, err := readBootID(filepath.Join(t.TempDir(), "missing")); err == nil {
50-
t.Error("readBootID accepted a missing file")
51-
}
52-
})
53-
}
54-
5525
// localSnapshotBootCheck matches a local snapshot only when its manifest
5626
// records the given boot ID.
5727
func TestLocalSnapshotBootCheck(t *testing.T) {

‎cmd/atelet/main.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ import (
5252
"go.ate.dev/substrate/internal/ateattr"
5353
"go.ate.dev/substrate/internal/ateinterceptors"
5454
"go.ate.dev/substrate/internal/atelet"
55+
"go.ate.dev/substrate/internal/bootid"
5556
"go.ate.dev/substrate/internal/clustertrustbundle"
5657
"go.ate.dev/substrate/internal/credbundle"
5758
"go.ate.dev/substrate/internal/imagecache"
@@ -384,7 +385,7 @@ func main() {
384385
defer ateapiConn.Close()
385386
workerService := ateapi.NewWorkerServiceClient(ateapiConn)
386387

387-
bootID, err := readBootID(bootIDPath)
388+
bootID, err := bootid.Read(bootid.Path)
388389
if err != nil {
389390
serverboot.Fatal(ctx, "Failed to read the node's boot ID", err)
390391
}
@@ -752,7 +753,7 @@ type Atelet struct {
752753
// failures, so a restore that can never succeed stops being retried (see
753754
// boundRetryableRestoreError).
754755
restoreFailures restoreFailureCounter
755-
// bootID is the node's boot ID (see readBootID), recorded in the manifest
756+
// bootID is the node's boot ID (see bootid.Read), recorded in the manifest
756757
// of each local snapshot. A local snapshot recording another, or none,
757758
// was written before the node last rebooted.
758759
bootID string

‎cmd/ateworker-gvisor/main.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import (
4949
"go.ate.dev/substrate/internal/ateworker/register"
5050
"go.ate.dev/substrate/internal/ateworkervalidation"
5151
"go.ate.dev/substrate/internal/atunnel"
52+
"go.ate.dev/substrate/internal/bootid"
5253
"go.ate.dev/substrate/internal/imagecache"
5354
"go.ate.dev/substrate/internal/nodepath"
5455
"go.ate.dev/substrate/internal/ocispec"
@@ -140,6 +141,10 @@ func do(ctx context.Context) error {
140141
if *usageSampleInterval < minUsageSampleInterval {
141142
return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval)
142143
}
144+
bootID, err := bootid.Read(bootid.Path)
145+
if err != nil {
146+
return err
147+
}
143148

144149
const serviceName = "ateworker-gvisor"
145150
// Export through atelet's node-local relay when it is there, so telemetry
@@ -239,7 +244,7 @@ func do(ctx context.Context) error {
239244
if err != nil {
240245
return err
241246
}
242-
ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags)
247+
ateworkerService := NewService(tunnel, actorLogger, *maxActors, flags, bootID)
243248
ateworkerService.throttleMemory = psiEnabled(defaultCgroupRoot)
244249
if !ateworkerService.throttleMemory {
245250
slog.WarnContext(ctx, "Kernel has no cgroup memory pressure (PSI), so sandboxes keep memory.max and an OOM kill in one can restart the worker")
@@ -345,6 +350,9 @@ type AteWorkerService struct {
345350
// crashReporter reports actors whose sandbox died. Nil if it could not be
346351
// set up.
347352
crashReporter crashReporter
353+
// bootID is the node's boot ID, recorded with each completed checkpoint so
354+
// a retry after a reboot does not reuse it.
355+
bootID string
348356

349357
// Guards actors and draining against concurrent stats reads.
350358
actorsMu sync.RWMutex
@@ -394,7 +402,7 @@ type AteWorkerService struct {
394402
var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil)
395403

396404
// NewService creates a new AteWorkerService.
397-
func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore) *AteWorkerService {
405+
func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxActors int, flags *flagStore, bootID string) *AteWorkerService {
398406
s := &AteWorkerService{
399407
locks: actorlock.New(),
400408
inFlight: actorlock.NewInFlight(),
@@ -405,6 +413,7 @@ func NewService(tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, maxAc
405413
cgroupRoot: defaultCgroupRoot,
406414
flags: flags,
407415
checkpointWrites: make(chan struct{}, checkpointWriteSlots),
416+
bootID: bootID,
408417
}
409418
s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory}
410419
return s
@@ -787,7 +796,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke
787796
defer s.locks.Unlock(req.GetActorUid())
788797

789798
if s.lookupActor(req.GetActorUid()) == nil {
790-
if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil {
799+
if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil {
791800
slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid()))
792801
return resp, nil
793802
}
@@ -980,7 +989,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke
980989
s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointed", attribution)
981990

982991
resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles}
983-
if err := ateworker.RecordCheckpoint(checkpointPath, fidelity, resp); err != nil {
992+
if err := ateworker.RecordCheckpoint(checkpointPath, s.bootID, fidelity, resp); err != nil {
984993
slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", req.GetActorUid()), slog.Any("err", err))
985994
}
986995
return resp, nil

‎cmd/ateworker-microvm/checkpoint.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke
8282
defer s.locks.Unlock(req.GetActorUid())
8383

8484
if s.lookupActor(req.GetActorUid()) == nil {
85-
if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), req.GetFidelity()); resp != nil {
85+
if resp := ateworker.ReuseCheckpoint(req.GetActorDirs().GetCheckpointDir(), s.bootID, req.GetFidelity()); resp != nil {
8686
slog.InfoContext(ctx, "Reusing completed checkpoint from previous attempt", slog.String("actorUID", req.GetActorUid()))
8787
return resp, nil
8888
}
@@ -263,7 +263,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke
263263
slog.Duration("teardown", dTeardown))
264264
s.recordFinalIfEnded(ctx, hosted)
265265
resp := &ateworkerpb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles, DataSnapshotFiles: durableFiles}
266-
if err := ateworker.RecordCheckpoint(checkpointDir, scope, resp); err != nil {
266+
if err := ateworker.RecordCheckpoint(checkpointDir, s.bootID, scope, resp); err != nil {
267267
slog.WarnContext(ctx, "Failed to record checkpoint completion marker", slog.String("actorUID", actorUID), slog.Any("err", err))
268268
}
269269
return resp, nil

‎cmd/ateworker-microvm/main.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import (
4949
"go.ate.dev/substrate/internal/ateworker/pid1"
5050
"go.ate.dev/substrate/internal/ateworker/register"
5151
"go.ate.dev/substrate/internal/atunnel"
52+
"go.ate.dev/substrate/internal/bootid"
5253
"go.ate.dev/substrate/internal/nodepath"
5354
"go.ate.dev/substrate/internal/otlprelay"
5455
"go.ate.dev/substrate/internal/preview"
@@ -124,6 +125,10 @@ func do(ctx context.Context) error {
124125
if *usageSampleInterval < minUsageSampleInterval {
125126
return fmt.Errorf("--usage-sample-interval must be at least %v, got %v", minUsageSampleInterval, *usageSampleInterval)
126127
}
128+
bootID, err := bootid.Read(bootid.Path)
129+
if err != nil {
130+
return err
131+
}
127132

128133
const serviceName = "ateworker-microvm"
129134
// Export through atelet's node-local relay when it is there, so telemetry
@@ -233,7 +238,7 @@ func do(ctx context.Context) error {
233238
if err != nil {
234239
return err
235240
}
236-
ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger)
241+
ateworkerService := NewService(*podUID, *chBinary, *guestDebug, *maxActors, tunnel, actorLogger, bootID)
237242
// The controller sets both from the downward API.
238243
pool := workerstats.Pool{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("WORKER_POOL_NAME")}
239244
if pool.Namespace == "" || pool.Name == "" {
@@ -357,6 +362,9 @@ type AteWorkerService struct {
357362
// crashReporter reports actors whose micro-VM died. Nil if it could not
358363
// be set up.
359364
crashReporter crashReporter
365+
// bootID is the node's boot ID, recorded with each completed checkpoint so
366+
// a retry after a reboot does not reuse it.
367+
bootID string
360368

361369
// shuttingDown is set once SIGTERM has been received. While true, new workload
362370
// RPCs are rejected with codes.Unavailable so the control plane reschedules.
@@ -402,7 +410,7 @@ type AteWorkerService struct {
402410
var _ ateworkerpb.WorkerServer = (*AteWorkerService)(nil)
403411

404412
// NewService creates a new AteWorkerService.
405-
func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteWorkerService {
413+
func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel *atunnel.Tunnel, actorLogger *actorlog.ActorLogger, bootID string) *AteWorkerService {
406414
s := &AteWorkerService{
407415
locks: actorlock.New(),
408416
inFlight: actorlock.NewInFlight(),
@@ -415,6 +423,7 @@ func NewService(podUID, chBinary string, guestDebug bool, maxActors int, tunnel
415423
snapshotWrites: make(chan struct{}, snapshotWriteSlots),
416424
tunnel: tunnel,
417425
actorLogger: actorLogger,
426+
bootID: bootID,
418427
}
419428
s.borrow = &memborrow.Ledger{Unbooked: s.unbookedMemory}
420429
return s

‎internal/ateworker/checkpoint.go‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,20 @@ func CheckpointContext(ctx context.Context) (context.Context, context.CancelFunc
4242
const checkpointDoneFile = ".checkpoint-done.json"
4343

4444
type checkpointRecord struct {
45+
// BootID is the node's boot ID when the checkpoint finished. The worker
46+
// does not sync the snapshot files, so after a reboot they may be torn.
47+
BootID string `json:"boot_id"`
4548
Fidelity ateworkerpb.SnapshotFidelity `json:"fidelity"`
4649
SnapshotFiles []string `json:"snapshot_files,omitempty"`
4750
DataSnapshotFiles []string `json:"data_snapshot_files,omitempty"`
4851
}
4952

5053
// RecordCheckpoint writes the completion marker into checkpointDir so a retry
51-
// after a caller timeout or atelet restart can reuse the finished
52-
// CheckpointWorkloadResponse.
53-
func RecordCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelity, resp *ateworkerpb.CheckpointWorkloadResponse) error {
54+
// after a caller timeout or atelet restart on the boot bootID can reuse the
55+
// finished CheckpointWorkloadResponse.
56+
func RecordCheckpoint(checkpointDir, bootID string, fidelity ateworkerpb.SnapshotFidelity, resp *ateworkerpb.CheckpointWorkloadResponse) error {
5457
b, err := json.Marshal(checkpointRecord{
58+
BootID: bootID,
5559
Fidelity: fidelity,
5660
SnapshotFiles: resp.GetSnapshotFiles(),
5761
DataSnapshotFiles: resp.GetDataSnapshotFiles(),
@@ -67,15 +71,16 @@ func RecordCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelit
6771
}
6872

6973
// ReuseCheckpoint returns the recorded CheckpointWorkloadResponse from
70-
// checkpointDir if the completion marker exists, its fidelity matches, and all
71-
// listed snapshot files are still present on disk.
72-
func ReuseCheckpoint(checkpointDir string, fidelity ateworkerpb.SnapshotFidelity) *ateworkerpb.CheckpointWorkloadResponse {
74+
// checkpointDir if the completion marker exists, was recorded on the boot
75+
// bootID, its fidelity matches, and all listed snapshot files are still present
76+
// on disk.
77+
func ReuseCheckpoint(checkpointDir, bootID string, fidelity ateworkerpb.SnapshotFidelity) *ateworkerpb.CheckpointWorkloadResponse {
7378
b, err := os.ReadFile(filepath.Join(checkpointDir, checkpointDoneFile))
7479
if err != nil {
7580
return nil
7681
}
7782
var rec checkpointRecord
78-
if err := json.Unmarshal(b, &rec); err != nil || rec.Fidelity != fidelity {
83+
if err := json.Unmarshal(b, &rec); err != nil || rec.BootID != bootID || rec.Fidelity != fidelity {
7984
return nil
8085
}
8186
// DataSnapshotFiles is the VOLUMES subset of SnapshotFiles: the same file

‎internal/ateworker/checkpoint_test.go‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ import (
2525
"go.ate.dev/substrate/internal/proto/ateworkerpb"
2626
)
2727

28+
// testBootID is the boot the tests record and reuse checkpoints on.
29+
const testBootID = "boot-1"
30+
2831
func TestRecordAndReuseCheckpoint(t *testing.T) {
2932
checkpointDir := t.TempDir()
3033
imgPath := filepath.Join(checkpointDir, "checkpoint.img")
@@ -34,33 +37,38 @@ func TestRecordAndReuseCheckpoint(t *testing.T) {
3437

3538
// Without a completion marker (e.g. partial files from an interrupted attempt),
3639
// ReuseCheckpoint returns nil.
37-
if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
40+
if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
3841
t.Fatalf("ReuseCheckpoint(before RecordCheckpoint) = %v, want nil", got)
3942
}
4043

4144
wantResp := &ateworkerpb.CheckpointWorkloadResponse{
4245
SnapshotFiles: []string{"checkpoint.img"},
4346
}
44-
if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantResp); err != nil {
47+
if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, wantResp); err != nil {
4548
t.Fatalf("RecordCheckpoint: %v", err)
4649
}
4750

4851
// Reusing with matching fidelity and intact file succeeds.
49-
got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY)
52+
got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY)
5053
if got == nil || !slices.Equal(got.GetSnapshotFiles(), wantResp.GetSnapshotFiles()) {
5154
t.Fatalf("ReuseCheckpoint() = %v, want %v", got, wantResp)
5255
}
5356

57+
// A marker recorded before the node rebooted is not reused.
58+
if got := ReuseCheckpoint(checkpointDir, "boot-2", ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
59+
t.Fatalf("ReuseCheckpoint(another boot) = %v, want nil", got)
60+
}
61+
5462
// Mismatched fidelity is not reused.
55-
if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil {
63+
if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil {
5664
t.Fatalf("ReuseCheckpoint(different fidelity) = %v, want nil", got)
5765
}
5866

59-
// Once atelet moves/removes the file from checkpoint-state/, the marker is not reused.
67+
// Once the file is gone from checkpoint-state/, the marker is not reused.
6068
if err := os.Remove(imgPath); err != nil {
6169
t.Fatal(err)
6270
}
63-
if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
71+
if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
6472
t.Fatalf("ReuseCheckpoint(after file removed) = %v, want nil", got)
6573
}
6674
}
@@ -82,10 +90,10 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) {
8290
SnapshotFiles: []string{"checkpoint.img", "fs/fscheckpoint.pb", "fs/multitar.img"},
8391
DataSnapshotFiles: []string{"fs/fscheckpoint.pb", "fs/multitar.img"},
8492
}
85-
if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, memResp); err != nil {
93+
if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, memResp); err != nil {
8694
t.Fatalf("RecordCheckpoint(MEMORY): %v", err)
8795
}
88-
got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY)
96+
got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY)
8997
if got == nil || !slices.Equal(got.GetSnapshotFiles(), memResp.GetSnapshotFiles()) || !slices.Equal(got.GetDataSnapshotFiles(), memResp.GetDataSnapshotFiles()) {
9098
t.Fatalf("ReuseCheckpoint(MEMORY) = %v, want %v", got, memResp)
9199
}
@@ -95,10 +103,10 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) {
95103
SnapshotFiles: []string{"checkpoint.img"},
96104
DataSnapshotFiles: []string{"fs/fscheckpoint.pb"},
97105
}
98-
if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, badSubset); err != nil {
106+
if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, badSubset); err != nil {
99107
t.Fatalf("RecordCheckpoint(badSubset): %v", err)
100108
}
101-
if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
109+
if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY); got != nil {
102110
t.Fatalf("ReuseCheckpoint(data file not in SnapshotFiles) = %v, want nil", got)
103111
}
104112

@@ -108,19 +116,19 @@ func TestReuseCheckpointDataSnapshotFilesByFidelity(t *testing.T) {
108116
SnapshotFiles: []string{"fs/multitar.img", "fs/fscheckpoint.pb"},
109117
DataSnapshotFiles: []string{"fs/fscheckpoint.pb", "fs/multitar.img"},
110118
}
111-
if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, volResp); err != nil {
119+
if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, volResp); err != nil {
112120
t.Fatalf("RecordCheckpoint(VOLUMES): %v", err)
113121
}
114-
got = ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES)
122+
got = ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES)
115123
if got == nil || !slices.Equal(got.GetSnapshotFiles(), volResp.GetSnapshotFiles()) || !slices.Equal(got.GetDataSnapshotFiles(), volResp.GetDataSnapshotFiles()) {
116124
t.Fatalf("ReuseCheckpoint(VOLUMES) = %v, want %v", got, volResp)
117125
}
118126

119127
// VOLUMES fidelity rejects non-data files in SnapshotFiles.
120-
if err := RecordCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, memResp); err != nil {
128+
if err := RecordCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, memResp); err != nil {
121129
t.Fatalf("RecordCheckpoint(VOLUMES with non-data files): %v", err)
122130
}
123-
if got := ReuseCheckpoint(checkpointDir, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil {
131+
if got := ReuseCheckpoint(checkpointDir, testBootID, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES); got != nil {
124132
t.Fatalf("ReuseCheckpoint(VOLUMES with non-data files) = %v, want nil", got)
125133
}
126134
}

0 commit comments

Comments
 (0)