Skip to content

Commit 3e5acb9

Browse files
dcantahdcantah
committed
Add scratch space re-use functionality to LCOW snapshotter
Currently we would create a new disk and mount this into the LCOW UVM for every container but there are certain scenarios where we'd rather just mount a single disk and then have every container share this one storage space instead of every container having it's own xGB of space to play around with. This is accomplished by just making a symlink to the disk that we'd like to share and then using ref counting later on down the stack in hcsshim if we see that we've already mounted this disk. Signed-off-by: Daniel Canter <[email protected]>
1 parent 7b0149a commit 3e5acb9

1 file changed

Lines changed: 95 additions & 34 deletions

File tree

‎snapshots/lcow/lcow.go‎

Lines changed: 95 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@ func init() {
5959
}
6060

6161
const (
62-
rootfsSizeLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb"
62+
rootfsSizeLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb"
63+
rootfsLocLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.location"
64+
reuseScratchLabel = "containerd.io/snapshot/io.microsoft.container.storage.reuse-scratch"
65+
reuseScratchOwnerKeyLabel = "containerd.io/snapshot/io.microsoft.owner.key"
6366
)
6467

6568
type snapshotter struct {
@@ -306,7 +309,7 @@ func (s *snapshotter) getSnapshotDir(id string) string {
306309
return filepath.Join(s.root, "snapshots", id)
307310
}
308311

309-
func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) ([]mount.Mount, error) {
312+
func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) {
310313
ctx, t, err := s.ms.TransactionContext(ctx, true)
311314
if err != nil {
312315
return nil, err
@@ -330,43 +333,65 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
330333
for _, o := range opts {
331334
o(&snapshotInfo)
332335
}
336+
337+
defer func() {
338+
if err != nil {
339+
os.RemoveAll(snDir)
340+
}
341+
}()
342+
333343
// IO/disk space optimization
334344
//
335345
// We only need one sandbox.vhd for the container. Skip making one for this
336346
// snapshot if this isn't the snapshot that just houses the final sandbox.vhd
337-
// that will be mounted as the containers scratch. Currently the key for a snapshot
338-
// where a layer.vhd will be extracted to it will have the string `extract-` in it.
347+
// that will be mounted as the containers scratch. The key for a snapshot
348+
// where a layer.vhd will be extracted to it will have the substring `extract-` in it.
339349
// If this is changed this will also need to be changed.
340350
//
341351
// We save about 17MB per layer (if the default scratch vhd size of 20GB is used) and of
342-
// course the time to copy the vhd per snapshot.
352+
// course the time to copy the vhdx per snapshot.
343353
if !strings.Contains(key, snapshots.UnpackKeyPrefix) {
344-
var sizeGB int
345-
if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeLabel]; ok {
346-
i32, err := strconv.ParseInt(sizeGBstr, 10, 32)
347-
if err != nil {
348-
return nil, errors.Wrapf(err, "failed to parse label %q=%q", rootfsSizeLabel, sizeGBstr)
354+
// This is the code path that handles re-using a scratch disk that has already been
355+
// made/mounted for an LCOW UVM. In the non sharing case, we create a new disk and mount this
356+
// into the LCOW UVM for every container but there are certain scenarios where we'd rather
357+
// just mount a single disk and then have every container share this one storage space instead of
358+
// every container having it's own xGB of space to play around with.
359+
//
360+
// This is accomplished by just making a symlink to the disk that we'd like to share and then
361+
// using ref counting later on down the stack in hcsshim if we see that we've already mounted this
362+
// disk.
363+
shareScratch := snapshotInfo.Labels[reuseScratchLabel]
364+
ownerKey := snapshotInfo.Labels[reuseScratchOwnerKeyLabel]
365+
if shareScratch == "true" && ownerKey != "" {
366+
if err = s.handleSharing(ctx, ownerKey, snDir); err != nil {
367+
return nil, err
368+
}
369+
} else {
370+
var sizeGB int
371+
if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeLabel]; ok {
372+
i64, _ := strconv.ParseInt(sizeGBstr, 10, 32)
373+
sizeGB = int(i64)
349374
}
350-
sizeGB = int(i32)
351-
}
352375

353-
scratchSource, err := s.openOrCreateScratch(ctx, sizeGB)
354-
if err != nil {
355-
return nil, err
356-
}
357-
defer scratchSource.Close()
376+
scratchLocation := snapshotInfo.Labels[rootfsLocLabel]
377+
scratchSource, err := s.openOrCreateScratch(ctx, sizeGB, scratchLocation)
378+
if err != nil {
379+
return nil, err
380+
}
381+
defer scratchSource.Close()
358382

359-
// Create the sandbox.vhdx for this snapshot from the cache.
360-
destPath := filepath.Join(snDir, "sandbox.vhdx")
361-
dest, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE, 0700)
362-
if err != nil {
363-
return nil, errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
364-
}
365-
defer dest.Close()
366-
if _, err := io.Copy(dest, scratchSource); err != nil {
367-
dest.Close()
368-
os.Remove(destPath)
369-
return nil, errors.Wrap(err, "failed to copy cached scratch.vhdx to sandbox.vhdx in snapshot")
383+
// Create the sandbox.vhdx for this snapshot from the cache
384+
destPath := filepath.Join(snDir, "sandbox.vhdx")
385+
dest, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE, 0700)
386+
if err != nil {
387+
return nil, errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
388+
}
389+
defer dest.Close()
390+
if _, err := io.Copy(dest, scratchSource); err != nil {
391+
dest.Close()
392+
os.Remove(destPath)
393+
return nil, errors.Wrap(err, "failed to copy cached scratch.vhdx to sandbox.vhdx in snapshot")
394+
}
370395
}
371396
}
372397
}
@@ -378,8 +403,38 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
378403
return s.mounts(newSnapshot), nil
379404
}
380405

381-
func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *os.File, err error) {
382-
// Create the scratch.vhdx cache file if it doesn't already exist.
406+
func (s *snapshotter) handleSharing(ctx context.Context, id, snDir string) error {
407+
var key string
408+
if err := s.Walk(ctx, func(ctx context.Context, info snapshots.Info) error {
409+
if strings.Contains(info.Name, id) {
410+
key = info.Name
411+
}
412+
return nil
413+
}); err != nil {
414+
return err
415+
}
416+
417+
mounts, err := s.Mounts(ctx, key)
418+
if err != nil {
419+
return errors.Wrap(err, "failed to get mounts for owner snapshot")
420+
}
421+
422+
sandboxPath := filepath.Join(mounts[0].Source, "sandbox.vhdx")
423+
linkPath := filepath.Join(snDir, "sandbox.vhdx")
424+
if _, err := os.Stat(sandboxPath); err != nil {
425+
return errors.Wrap(err, "failed to find sandbox.vhdx in snapshot directory")
426+
}
427+
428+
// We've found everything we need, now just make a symlink in our new snapshot to the
429+
// sandbox.vhdx in the scratch we're asking to share.
430+
if err := os.Symlink(sandboxPath, linkPath); err != nil {
431+
return errors.Wrap(err, "failed to create symlink for sandbox scratch space")
432+
}
433+
return nil
434+
}
435+
436+
func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int, scratchLoc string) (_ *os.File, err error) {
437+
// Create the scratch.vhdx cache file if it doesn't already exit.
383438
s.scratchLock.Lock()
384439
defer s.scratchLock.Unlock()
385440

@@ -389,13 +444,17 @@ func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *o
389444
}
390445

391446
scratchFinalPath := filepath.Join(s.root, vhdFileName)
447+
if scratchLoc != "" {
448+
scratchFinalPath = filepath.Join(scratchLoc, vhdFileName)
449+
}
450+
392451
scratchSource, err := os.OpenFile(scratchFinalPath, os.O_RDONLY, 0700)
393452
if err != nil {
394453
if !os.IsNotExist(err) {
395454
return nil, errors.Wrapf(err, "failed to open vhd %s for read", vhdFileName)
396455
}
397456

398-
log.G(ctx).Debugf("vhd %s not found, creating a new one", vhdFileName)
457+
log.G(ctx).Debugf("vhdx %s not found, creating a new one", vhdFileName)
399458

400459
// Golang logic for ioutil.TempFile without the file creation
401460
r := uint32(time.Now().UnixNano() + int64(os.Getpid()))
@@ -417,18 +476,20 @@ func (s *snapshotter) openOrCreateScratch(ctx context.Context, sizeGB int) (_ *o
417476
}
418477

419478
if err := rhcs.CreateScratchWithOpts(ctx, scratchTempPath, &opt); err != nil {
420-
_ = os.Remove(scratchTempPath)
479+
os.Remove(scratchTempPath)
421480
return nil, errors.Wrapf(err, "failed to create '%s' temp file", scratchTempName)
422481
}
423482
if err := os.Rename(scratchTempPath, scratchFinalPath); err != nil {
424-
_ = os.Remove(scratchTempPath)
483+
os.Remove(scratchTempPath)
425484
return nil, errors.Wrapf(err, "failed to rename '%s' temp file to 'scratch.vhdx'", scratchTempName)
426485
}
427486
scratchSource, err = os.OpenFile(scratchFinalPath, os.O_RDONLY, 0700)
428487
if err != nil {
429-
_ = os.Remove(scratchFinalPath)
488+
os.Remove(scratchFinalPath)
430489
return nil, errors.Wrap(err, "failed to open scratch.vhdx for read after creation")
431490
}
491+
} else {
492+
log.G(ctx).Debugf("scratch vhd %s was already present. Retrieved from cache", vhdFileName)
432493
}
433494
return scratchSource, nil
434495
}

0 commit comments

Comments
 (0)