Skip to content

Commit 9b9de47

Browse files
dmcgowandmcgowan
authored
Merge pull request #4824 from dcantah/dcantah/reuse-scratch
Add scratch space re-use functionality to LCOW snapshotter
2 parents 392b285 + 3e5acb9 commit 9b9de47

1 file changed

Lines changed: 95 additions & 34 deletions

File tree

‎snapshots/lcow/lcow.go‎

Lines changed: 95 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@ func init() {
5959
}
6060

6161
const (
62-
rootfsSizeLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb"
62+
rootfsSizeLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb"
63+
rootfsLocLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.location"
64+
reuseScratchLabel = "containerd.io/snapshot/io.microsoft.container.storage.reuse-scratch"
65+
reuseScratchOwnerKeyLabel = "containerd.io/snapshot/io.microsoft.owner.key"
6366
)
6467

6568
type snapshotter struct {
@@ -306,7 +309,7 @@ func (s *snapshotter) getSnapshotDir(id string) string {
306309
return filepath.Join(s.root, "snapshots", id)
307310
}
308311

309-
func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) ([]mount.Mount, error) {
312+
func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) {
310313
ctx, t, err := s.ms.TransactionContext(ctx, true)
311314
if err != nil {
312315
return nil, err
@@ -330,43 +333,65 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
330333
for _, o := range opts {
331334
o(&snapshotInfo)
332335
}
336+
337+
defer func() {
338+
if err != nil {
339+
os.RemoveAll(snDir)
340+
}
341+
}()
342+
333343
// IO/disk space optimization
334344
//
335345
// We only need one sandbox.vhd for the container. Skip making one for this
336346
// snapshot if this isn't the snapshot that just houses the final sandbox.vhd
337-
// that will be mounted as the containers scratch. Currently the key for a snapshot
338-
// where a layer.vhd will be extracted to it will have the string `extract-` in it.
347+
// that will be mounted as the containers scratch. The key for a snapshot
348+
// where a layer.vhd will be extracted to it will have the substring `extract-` in it.
339349
// If this is changed this will also need to be changed.
340350
//
341351
// We save about 17MB per layer (if the default scratch vhd size of 20GB is used) and of
342-
// course the time to copy the vhd per snapshot.
352+
// course the time to copy the vhdx per snapshot.
343353
if !strings.Contains(key, snapshots.UnpackKeyPrefix) {
344-
var sizeGB int
345-
if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeLabel]; ok {
346-
i32, err := strconv.ParseInt(sizeGBstr, 10, 32)
347-
if err != nil {
348-
return nil, errors.Wrapf(err, "failed to parse label %q=%q", rootfsSizeLabel, sizeGBstr)
354+
// This is the code path that handles re-using a scratch disk that has already been
355+
// made/mounted for an LCOW UVM. In the non sharing case, we create a new disk and mount this
356+
// into the LCOW UVM for every container but there are certain scenarios where we'd rather
357+
// just mount a single disk and then have every container share this one storage space instead of
358+
// every container having it's own xGB of space to play around with.
359+
//
360+
// This is accomplished by just making a symlink to the disk that we'd like to share and then
361+
// using ref counting later on down the stack in hcsshim if we see that we've already mounted this
362+
// disk.
363+
shareScratch := snapshotInfo.Labels[reuseScratchLabel]
364+
ownerKey := snapshotInfo.Labels[reuseScratchOwnerKeyLabel]
365+
if shareScratch == "true" && ownerKey != "" {
366+
if err = s.handleSharing(ctx, ownerKey, snDir); err != nil {
367+
return nil, err
368+
}
369+
} else {
370+
var sizeGB int
371+
if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeLabel]; ok {
372+
i64, _ := strconv.ParseInt(sizeGBstr, 10, 32)
373+
sizeGB = int(i64)
349374
}
350-
sizeGB = int(i32)
351-
}
352375

353-
scratchSource, err := s.openOrCreateScratch(ctx, sizeGB)
354-
if err != nil {
355-
return nil, err
356-
}
357-
defer scratchSource.Close()
376+
scratchLocation := snapshotInfo.Labels[rootfsLocLabel]
377+
scratchSource, err := s.openOrCreateScratch(ctx, sizeGB, scratchLocation)
378+
if err != nil {
379+
return nil, err
380+
}
381+
defer scratchSource.Close()
358382

359-
// Create the sandbox.vhdx for this snapshot from the cache.
360-
destPath := filepath.Join(snDir, "sandbox.vhdx")
361-
dest, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE, 0700)
362-
if err != nil {
363-
return nil, errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
364-
}
365-
defer dest.Close()
366-
if _, err := io.Copy(dest, scratchSource); err != nil {
367-
dest.Close()
368-
os.Remove(destPath)
369-
return nil, errors.Wrap(err, "failed to copy cached scratch.vhdx to sandbox.vhdx in snapshot")
383+
// Create the sandbox.vhdx for this snapshot from the cache
384+
destPath := filepath.Join(snDir, "sandbox.vhdx")
385+
dest, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE, 0700)
386+
if err != nil {
387+
return nil, errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
388+
}
389+
defer dest.Close()
390+
if _, err := io.Copy(dest, scratchSource); err != nil {
391+
dest.Close()
392+
os.Remove(destPath)
393+
return nil, errors.Wrap(err, "failed to copy cached scratch.vhdx to sandbox.vhdx in snapshot")
394+
}
370395
}
371396
}
372397
}
@@ -378,8 +403,38 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
378403
return s.mounts(newSnapshot), nil
379404
}
380405

381-
func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *os.File, err error) {
382-
// Create the scratch.vhdx cache file if it doesn't already exist.
406+
func (s *snapshotter) handleSharing(ctx context.Context, id, snDir string) error {
407+
var key string
408+
if err := s.Walk(ctx, func(ctx context.Context, info snapshots.Info) error {
409+
if strings.Contains(info.Name, id) {
410+
key = info.Name
411+
}
412+
return nil
413+
}); err != nil {
414+
return err
415+
}
416+
417+
mounts, err := s.Mounts(ctx, key)
418+
if err != nil {
419+
return errors.Wrap(err, "failed to get mounts for owner snapshot")
420+
}
421+
422+
sandboxPath := filepath.Join(mounts[0].Source, "sandbox.vhdx")
423+
linkPath := filepath.Join(snDir, "sandbox.vhdx")
424+
if _, err := os.Stat(sandboxPath); err != nil {
425+
return errors.Wrap(err, "failed to find sandbox.vhdx in snapshot directory")
426+
}
427+
428+
// We've found everything we need, now just make a symlink in our new snapshot to the
429+
// sandbox.vhdx in the scratch we're asking to share.
430+
if err := os.Symlink(sandboxPath, linkPath); err != nil {
431+
return errors.Wrap(err, "failed to create symlink for sandbox scratch space")
432+
}
433+
return nil
434+
}
435+
436+
func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int, scratchLoc string) (_ *os.File, err error) {
437+
// Create the scratch.vhdx cache file if it doesn't already exit.
383438
s.scratchLock.Lock()
384439
defer s.scratchLock.Unlock()
385440

@@ -389,13 +444,17 @@ func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *o
389444
}
390445

391446
scratchFinalPath := filepath.Join(s.root, vhdFileName)
447+
if scratchLoc != "" {
448+
scratchFinalPath = filepath.Join(scratchLoc, vhdFileName)
449+
}
450+
392451
scratchSource, err := os.OpenFile(scratchFinalPath, os.O_RDONLY, 0700)
393452
if err != nil {
394453
if !os.IsNotExist(err) {
395454
return nil, errors.Wrapf(err, "failed to open vhd %s for read", vhdFileName)
396455
}
397456

398-
log.G(ctx).Debugf("vhd %s not found, creating a new one", vhdFileName)
457+
log.G(ctx).Debugf("vhdx %s not found, creating a new one", vhdFileName)
399458

400459
// Golang logic for ioutil.TempFile without the file creation
401460
r := uint32(time.Now().UnixNano() + int64(os.Getpid()))
@@ -417,18 +476,20 @@ func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *o
417476
}
418477

419478
if err := rhcs.CreateScratchWithOpts(ctx, scratchTempPath, &opt); err != nil {
420-
_ = os.Remove(scratchTempPath)
479+
os.Remove(scratchTempPath)
421480
return nil, errors.Wrapf(err, "failed to create '%s' temp file", scratchTempName)
422481
}
423482
if err := os.Rename(scratchTempPath, scratchFinalPath); err != nil {
424-
_ = os.Remove(scratchTempPath)
483+
os.Remove(scratchTempPath)
425484
return nil, errors.Wrapf(err, "failed to rename '%s' temp file to 'scratch.vhdx'", scratchTempName)
426485
}
427486
scratchSource, err = os.OpenFile(scratchFinalPath, os.O_RDONLY, 0700)
428487
if err != nil {
429-
_ = os.Remove(scratchFinalPath)
488+
os.Remove(scratchFinalPath)
430489
return nil, errors.Wrap(err, "failed to open scratch.vhdx for read after creation")
431490
}
491+
} else {
492+
log.G(ctx).Debugf("scratch vhd %s was already present. Retrieved from cache", vhdFileName)
432493
}
433494
return scratchSource, nil
434495
}

0 commit comments

Comments
 (0)