Skip to content

Commit 81386df

Browse files
Random-Liudmcgowan
Random-Liu
authored and
dmcgowan
committed
Add direct unpack support for overlay and aufs
Signed-off-by: Lantao Liu <[email protected]>
1 parent f06e605 commit 81386df

9 files changed

Lines changed: 333 additions & 91 deletions

File tree

‎archive/tar.go‎

Lines changed: 46 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@ package archive
1919
import (
2020
"archive/tar"
2121
"context"
22-
"fmt"
2322
"io"
24-
"io/ioutil"
2523
"os"
2624
"path/filepath"
2725
"runtime"
@@ -91,11 +89,6 @@ const (
9189
// archives.
9290
whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix
9391

94-
// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other
95-
// layers. Normally these should not go into exported archives and all changed
96-
// hardlinks should be copied to the top layer.
97-
whiteoutLinkDir = whiteoutMetaPrefix + "plnk"
98-
9992
// whiteoutOpaqueDir file means directory has been made opaque - meaning
10093
// readdir calls to this directory do not follow to lower layers.
10194
whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq"
@@ -130,10 +123,6 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
130123
// Used for handling opaque directory markers which
131124
// may occur out of order
132125
unpackedPaths = make(map[string]struct{})
133-
134-
// Used for aufs plink directory
135-
aufsTempdir = ""
136-
aufsHardlinks = make(map[string]*tar.Header)
137126
)
138127

139128
// Iterate through the files in the archive.
@@ -201,40 +190,15 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
201190
}
202191
}
203192

204-
// Skip AUFS metadata dirs
205-
if strings.HasPrefix(hdr.Name, whiteoutMetaPrefix) {
206-
// Regular files inside /.wh..wh.plnk can be used as hardlink targets
207-
// We don't want this directory, but we need the files in them so that
208-
// such hardlinks can be resolved.
209-
if strings.HasPrefix(hdr.Name, whiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
210-
basename := filepath.Base(hdr.Name)
211-
aufsHardlinks[basename] = hdr
212-
if aufsTempdir == "" {
213-
if aufsTempdir, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "dockerplnk"); err != nil {
214-
return 0, err
215-
}
216-
defer os.RemoveAll(aufsTempdir)
217-
}
218-
p, err := fs.RootPath(aufsTempdir, basename)
219-
if err != nil {
220-
return 0, err
221-
}
222-
if err := createTarFile(ctx, p, root, hdr, tr); err != nil {
223-
return 0, err
224-
}
225-
}
226-
227-
if hdr.Name != whiteoutOpaqueDir {
228-
continue
229-
}
230-
}
231-
232-
if strings.HasPrefix(base, whiteoutPrefix) {
193+
// Naive whiteout convert function which handles whiteout files by
194+
// removing the target files.
195+
convertWhiteout := func(hdr *tar.Header, path string) (bool, error) {
196+
base := filepath.Base(path)
233197
dir := filepath.Dir(path)
234198
if base == whiteoutOpaqueDir {
235199
_, err := os.Lstat(dir)
236200
if err != nil {
237-
return 0, err
201+
return false, err
238202
}
239203
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
240204
if err != nil {
@@ -252,26 +216,29 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
252216
}
253217
return nil
254218
})
255-
if err != nil {
256-
return 0, err
257-
}
258-
continue
219+
return false, err
259220
}
260221

261-
originalBase := base[len(whiteoutPrefix):]
262-
originalPath := filepath.Join(dir, originalBase)
222+
if strings.HasPrefix(base, whiteoutPrefix) {
223+
originalBase := base[len(whiteoutPrefix):]
224+
originalPath := filepath.Join(dir, originalBase)
263225

264-
// Ensure originalPath is under dir
265-
if dir[len(dir)-1] != filepath.Separator {
266-
dir += string(filepath.Separator)
267-
}
268-
if !strings.HasPrefix(originalPath, dir) {
269-
return 0, errors.Wrapf(errInvalidArchive, "invalid whiteout name: %v", base)
226+
return false, os.RemoveAll(originalPath)
270227
}
271228

272-
if err := os.RemoveAll(originalPath); err != nil {
273-
return 0, err
274-
}
229+
return true, nil
230+
}
231+
if options.ConvertWhiteout != nil {
232+
convertWhiteout = options.ConvertWhiteout
233+
}
234+
if err := validateWhiteout(path); err != nil {
235+
return 0, err
236+
}
237+
writeFile, err := convertWhiteout(hdr, path)
238+
if err != nil {
239+
return 0, errors.Wrapf(err, "failed to convert whiteout file %q", hdr.Name)
240+
}
241+
if !writeFile {
275242
continue
276243
}
277244
// If path exits we almost always just want to remove and replace it.
@@ -289,26 +256,6 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
289256
srcData := io.Reader(tr)
290257
srcHdr := hdr
291258

292-
// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
293-
// we manually retarget these into the temporary files we extracted them into
294-
if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), whiteoutLinkDir) {
295-
linkBasename := filepath.Base(hdr.Linkname)
296-
srcHdr = aufsHardlinks[linkBasename]
297-
if srcHdr == nil {
298-
return 0, fmt.Errorf("invalid aufs hardlink")
299-
}
300-
p, err := fs.RootPath(aufsTempdir, linkBasename)
301-
if err != nil {
302-
return 0, err
303-
}
304-
tmpFile, err := os.Open(p)
305-
if err != nil {
306-
return 0, err
307-
}
308-
defer tmpFile.Close()
309-
srcData = tmpFile
310-
}
311-
312259
if err := createTarFile(ctx, path, root, srcHdr, srcData); err != nil {
313260
return 0, err
314261
}
@@ -684,3 +631,26 @@ func hardlinkRootPath(root, linkname string) (string, error) {
684631
}
685632
return targetPath, nil
686633
}
634+
635+
func validateWhiteout(path string) error {
636+
base := filepath.Base(path)
637+
dir := filepath.Dir(path)
638+
639+
if base == whiteoutOpaqueDir {
640+
return nil
641+
}
642+
643+
if strings.HasPrefix(base, whiteoutPrefix) {
644+
originalBase := base[len(whiteoutPrefix):]
645+
originalPath := filepath.Join(dir, originalBase)
646+
647+
// Ensure originalPath is under dir
648+
if dir[len(dir)-1] != filepath.Separator {
649+
dir += string(filepath.Separator)
650+
}
651+
if !strings.HasPrefix(originalPath, dir) {
652+
return errors.Wrapf(errInvalidArchive, "invalid whiteout name: %v", base)
653+
}
654+
}
655+
return nil
656+
}

‎archive/tar_opts.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ type ApplyOpt func(options *ApplyOptions) error
2424
// Filter specific files from the archive
2525
type Filter func(*tar.Header) (bool, error)
2626

27+
// ConvertWhiteout converts whiteout files from the archive
28+
type ConvertWhiteout func(*tar.Header, string) (bool, error)
29+
2730
// all allows all files
2831
func all(_ *tar.Header) (bool, error) {
2932
return true, nil
@@ -36,3 +39,11 @@ func WithFilter(f Filter) ApplyOpt {
3639
return nil
3740
}
3841
}
42+
43+
// WithConvertWhiteout uses the convert function to convert the whiteout files.
44+
func WithConvertWhiteout(c ConvertWhiteout) ApplyOpt {
45+
return func(options *ApplyOptions) error {
46+
options.ConvertWhiteout = c
47+
return nil
48+
}
49+
}

‎archive/tar_opts_linux.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// +build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package archive
20+
21+
import (
22+
"archive/tar"
23+
"os"
24+
"path/filepath"
25+
"strings"
26+
27+
"golang.org/x/sys/unix"
28+
)
29+
30+
// ApplyOptions provides additional options for an Apply operation
31+
type ApplyOptions struct {
32+
Filter Filter // Filter tar headers
33+
ConvertWhiteout ConvertWhiteout // Convert whiteout files
34+
}
35+
36+
// AufsConvertWhiteout converts whiteout files for aufs.
37+
func AufsConvertWhiteout(_ *tar.Header, _ string) (bool, error) {
38+
return true, nil
39+
}
40+
41+
// OverlayConvertWhiteout converts whiteout files for overlay.
42+
func OverlayConvertWhiteout(hdr *tar.Header, path string) (bool, error) {
43+
base := filepath.Base(path)
44+
dir := filepath.Dir(path)
45+
46+
// if a directory is marked as opaque, we need to translate that to overlay
47+
if base == whiteoutOpaqueDir {
48+
// don't write the file itself
49+
return false, unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0)
50+
}
51+
52+
// if a file was deleted and we are using overlay, we need to create a character device
53+
if strings.HasPrefix(base, whiteoutPrefix) {
54+
originalBase := base[len(whiteoutPrefix):]
55+
originalPath := filepath.Join(dir, originalBase)
56+
57+
if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil {
58+
return false, err
59+
}
60+
// don't write the file itself
61+
return false, os.Chown(originalPath, hdr.Uid, hdr.Gid)
62+
}
63+
64+
return true, nil
65+
}

archive/tar_opts_unix.go renamed to archive/tar_opts_other.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// +build !windows
1+
// +build !linux,!windows
22

33
/*
44
Copyright The containerd Authors.
@@ -20,5 +20,6 @@ package archive
2020

2121
// ApplyOptions provides additional options for an Apply operation
2222
type ApplyOptions struct {
23-
Filter Filter // Filter tar headers
23+
Filter Filter // Filter tar headers
24+
ConvertWhiteout ConvertWhiteout // Convert whiteout files
2425
}

‎archive/tar_opts_windows.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,10 @@ package archive
2020

2121
// ApplyOptions provides additional options for an Apply operation
2222
type ApplyOptions struct {
23-
ParentLayerPaths []string // Parent layer paths used for Windows layer apply
24-
IsWindowsContainerLayer bool // True if the tar stream to be applied is a Windows Container Layer
25-
Filter Filter // Filter tar headers
23+
ParentLayerPaths []string // Parent layer paths used for Windows layer apply
24+
IsWindowsContainerLayer bool // True if the tar stream to be applied is a Windows Container Layer
25+
Filter Filter // Filter tar headers
26+
ConvertWhiteout ConvertWhiteout // Convert whiteout files
2627
}
2728

2829
// WithParentLayers adds parent layers to the apply process this is required

‎diff/apply/apply.go‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@ package apply
1919
import (
2020
"context"
2121
"io"
22-
"io/ioutil"
2322
"time"
2423

25-
"github.com/containerd/containerd/archive"
2624
"github.com/containerd/containerd/content"
2725
"github.com/containerd/containerd/diff"
2826
"github.com/containerd/containerd/log"
@@ -94,15 +92,8 @@ func (s *fsApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts [
9492
rc := &readCounter{
9593
r: io.TeeReader(processor, digester.Hash()),
9694
}
97-
if err := mount.WithTempMount(ctx, mounts, func(root string) error {
98-
if _, err := archive.Apply(ctx, root, rc); err != nil {
99-
return err
100-
}
10195

102-
// Read any trailing data
103-
_, err := io.Copy(ioutil.Discard, rc)
104-
return err
105-
}); err != nil {
96+
if err := apply(ctx, mounts, rc); err != nil {
10697
return emptyDesc, err
10798
}
10899

0 commit comments

Comments
 (0)