Skip to content

Commit ff1451c

Browse files
dcantahdcantah
committed
Scratch size customization and UVM scratch creation for WCOW snapshotter
* Currently we rely on making the UVMs sandbox.vhdx in the shim itself instead of this being made by the snapshotter itself. This change adds a label that affects whether to create the UVMs scratch layer in the snapshotter itself. * Adds container scratch size customization. Before adding the computestorage calls (vendored in with #4859) there was no way to make a containers or UVMs scratch size less than the default (20 for containers and 10 for the UVM). Signed-off-by: Daniel Canter <[email protected]>
1 parent 18ad79d commit ff1451c

1 file changed

Lines changed: 115 additions & 13 deletions

File tree

‎snapshots/windows/windows.go‎

Lines changed: 115 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ package windows
2121
import (
2222
"context"
2323
"encoding/json"
24+
"fmt"
25+
"io"
2426
"os"
2527
"path/filepath"
2628
"strconv"
@@ -29,6 +31,7 @@ import (
2931
winfs "github.com/Microsoft/go-winio/pkg/fs"
3032
"github.com/Microsoft/go-winio/vhd"
3133
"github.com/Microsoft/hcsshim"
34+
"github.com/Microsoft/hcsshim/computestorage"
3235
"github.com/containerd/containerd/errdefs"
3336
"github.com/containerd/containerd/log"
3437
"github.com/containerd/containerd/mount"
@@ -53,6 +56,9 @@ func init() {
5356
}
5457

5558
const (
59+
// Label to specify that we should make a scratch space for a UtilityVM.
60+
uvmScratchLabel = "containerd.io/snapshot/io.microsoft.vm.storage.scratch"
61+
// Label to control a containers scratch space size (sandbox.vhdx).
5662
rootfsSizeLabel = "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb"
5763
)
5864

@@ -345,15 +351,6 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
345351
if !strings.Contains(key, snapshots.UnpackKeyPrefix) {
346352
parentLayerPaths := s.parentIDsToParentPaths(newSnapshot.ParentIDs)
347353

348-
var parentPath string
349-
if len(parentLayerPaths) != 0 {
350-
parentPath = parentLayerPaths[0]
351-
}
352-
353-
if err := hcsshim.CreateSandboxLayer(s.info, newSnapshot.ID, parentPath, parentLayerPaths); err != nil {
354-
return nil, errors.Wrap(err, "failed to create sandbox layer")
355-
}
356-
357354
var snapshotInfo snapshots.Info
358355
for _, o := range opts {
359356
o(&snapshotInfo)
@@ -368,12 +365,20 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
368365
sizeGB = int(i32)
369366
}
370367

371-
if sizeGB > 0 {
372-
const gbToByte = 1024 * 1024 * 1024
373-
if err := hcsshim.ExpandSandboxSize(s.info, newSnapshot.ID, uint64(gbToByte*sizeGB)); err != nil {
374-
return nil, errors.Wrapf(err, "failed to expand scratch size to %d GB", sizeGB)
368+
var makeUVMScratch bool
369+
if _, ok := snapshotInfo.Labels[uvmScratchLabel]; ok {
370+
makeUVMScratch = true
371+
}
372+
373+
// This has to be run first to avoid clashing with the containers sandbox.vhdx.
374+
if makeUVMScratch {
375+
if err := s.createUVMScratchLayer(ctx, snDir, parentLayerPaths); err != nil {
376+
return nil, errors.Wrap(err, "failed to make UVM's scratch layer")
375377
}
376378
}
379+
if err := s.createScratchLayer(ctx, snDir, parentLayerPaths, sizeGB); err != nil {
380+
return nil, errors.Wrap(err, "failed to create scratch layer")
381+
}
377382
}
378383
}
379384

@@ -391,3 +396,100 @@ func (s *snapshotter) parentIDsToParentPaths(parentIDs []string) []string {
391396
}
392397
return parentLayerPaths
393398
}
399+
400+
// This is essentially a recreation of what HCS' CreateSandboxLayer does with some extra bells and
401+
// whistles like expanding the volume if a size is specified. This will create a 1GB scratch
402+
// vhdx to be used if a different sized scratch that is not equal to the default of 20 is requested.
403+
func (s *snapshotter) createScratchLayer(ctx context.Context, snDir string, parentLayers []string, sizeGB int) error {
404+
parentLen := len(parentLayers)
405+
if parentLen == 0 {
406+
return errors.New("no parent layers present")
407+
}
408+
baseLayer := parentLayers[parentLen-1]
409+
410+
var (
411+
templateBase = filepath.Join(baseLayer, "blank-base.vhdx")
412+
templateDiffDisk = filepath.Join(baseLayer, "blank.vhdx")
413+
newDisks = sizeGB > 0 && sizeGB < 20
414+
expand = sizeGB > 0 && sizeGB != 20
415+
)
416+
417+
// If a size greater than 0 and less than 20 (the default size produced by hcs)
418+
// was specified we make a new set of disks to be used. We make it a 1GB disk and just
419+
// expand it to the size specified so for future container runs we don't need to remake a disk.
420+
if newDisks {
421+
templateBase = filepath.Join(baseLayer, "scratch.vhdx")
422+
templateDiffDisk = filepath.Join(baseLayer, "scratch-diff.vhdx")
423+
}
424+
425+
if _, err := os.Stat(templateDiffDisk); os.IsNotExist(err) {
426+
// Scratch disk not present so lets make it.
427+
if err := computestorage.SetupContainerBaseLayer(ctx, baseLayer, templateBase, templateDiffDisk, 1); err != nil {
428+
return errors.Wrapf(err, "failed to create scratch vhdx at %q", baseLayer)
429+
}
430+
}
431+
432+
dest := filepath.Join(snDir, "sandbox.vhdx")
433+
if err := copyScratchDisk(templateDiffDisk, dest); err != nil {
434+
return err
435+
}
436+
437+
if expand {
438+
gbToByte := 1024 * 1024 * 1024
439+
if err := hcsshim.ExpandSandboxSize(s.info, filepath.Base(snDir), uint64(gbToByte*sizeGB)); err != nil {
440+
return errors.Wrapf(err, "failed to expand sandbox vhdx size to %d GB", sizeGB)
441+
}
442+
}
443+
return nil
444+
}
445+
446+
// This handles creating the UVMs scratch layer.
447+
func (s *snapshotter) createUVMScratchLayer(ctx context.Context, snDir string, parentLayers []string) error {
448+
parentLen := len(parentLayers)
449+
if parentLen == 0 {
450+
return errors.New("no parent layers present")
451+
}
452+
baseLayer := parentLayers[parentLen-1]
453+
454+
// Make sure base layer has a UtilityVM folder.
455+
uvmPath := filepath.Join(baseLayer, "UtilityVM")
456+
if _, err := os.Stat(uvmPath); os.IsNotExist(err) {
457+
return errors.Wrapf(err, "failed to find UtilityVM directory in base layer %q", baseLayer)
458+
}
459+
460+
templateDiffDisk := filepath.Join(uvmPath, "SystemTemplate.vhdx")
461+
462+
// Check if SystemTemplate disk doesn't exist for some reason (this should be made during the unpacking
463+
// of the base layer).
464+
if _, err := os.Stat(templateDiffDisk); os.IsNotExist(err) {
465+
return fmt.Errorf("%q does not exist in Utility VM image", templateDiffDisk)
466+
}
467+
468+
// Move the sandbox.vhdx into a nested vm folder to avoid clashing with a containers sandbox.vhdx.
469+
vmScratchDir := filepath.Join(snDir, "vm")
470+
if err := os.MkdirAll(vmScratchDir, 0777); err != nil {
471+
return errors.Wrap(err, "failed to make `vm` directory for vm's scratch space")
472+
}
473+
474+
return copyScratchDisk(templateDiffDisk, filepath.Join(vmScratchDir, "sandbox.vhdx"))
475+
}
476+
477+
func copyScratchDisk(source, dest string) error {
478+
scratchSource, err := os.OpenFile(source, os.O_RDWR, 0700)
479+
if err != nil {
480+
return errors.Wrapf(err, "failed to open %s", source)
481+
}
482+
defer scratchSource.Close()
483+
484+
f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0700)
485+
if err != nil {
486+
return errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
487+
}
488+
defer f.Close()
489+
490+
if _, err := io.Copy(f, scratchSource); err != nil {
491+
os.Remove(dest)
492+
return errors.Wrapf(err, "failed to copy cached %q to %q in snapshot", source, dest)
493+
}
494+
return nil
495+
}

0 commit comments

Comments
 (0)