Skip to content

Commit 0ab7f03

Browse files
crosbymichaelcrosbymichael
authored
Merge pull request #3528 from dmcgowan/overlay-direct-unpack
Add direct unpack support for overlay
2 parents eac100a + bcc4a14 commit 0ab7f03

27 files changed

Lines changed: 957 additions & 242 deletions

‎archive/tar.go‎

Lines changed: 144 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@ package archive
1919
import (
2020
"archive/tar"
2121
"context"
22-
"fmt"
2322
"io"
24-
"io/ioutil"
2523
"os"
2624
"path/filepath"
2725
"runtime"
@@ -91,11 +89,6 @@ const (
9189
// archives.
9290
whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix
9391

94-
// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other
95-
// layers. Normally these should not go into exported archives and all changed
96-
// hardlinks should be copied to the top layer.
97-
whiteoutLinkDir = whiteoutMetaPrefix + "plnk"
98-
9992
// whiteoutOpaqueDir file means directory has been made opaque - meaning
10093
// readdir calls to this directory do not follow to lower layers.
10194
whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq"
@@ -117,11 +110,15 @@ func Apply(ctx context.Context, root string, r io.Reader, opts ...ApplyOpt) (int
117110
if options.Filter == nil {
118111
options.Filter = all
119112
}
113+
if options.applyFunc == nil {
114+
options.applyFunc = applyNaive
115+
}
120116

121-
return apply(ctx, root, tar.NewReader(r), options)
117+
return options.applyFunc(ctx, root, tar.NewReader(r), options)
122118
}
123119

124-
// applyNaive applies a tar stream of an OCI style diff tar.
120+
// applyNaive applies a tar stream of an OCI style diff tar to a directory
121+
// applying each file as either a whole file or whiteout.
125122
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
126123
func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
127124
var (
@@ -131,11 +128,49 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
131128
// may occur out of order
132129
unpackedPaths = make(map[string]struct{})
133130

134-
// Used for aufs plink directory
135-
aufsTempdir = ""
136-
aufsHardlinks = make(map[string]*tar.Header)
131+
convertWhiteout = options.ConvertWhiteout
137132
)
138133

134+
if convertWhiteout == nil {
135+
// handle whiteouts by removing the target files
136+
convertWhiteout = func(hdr *tar.Header, path string) (bool, error) {
137+
base := filepath.Base(path)
138+
dir := filepath.Dir(path)
139+
if base == whiteoutOpaqueDir {
140+
_, err := os.Lstat(dir)
141+
if err != nil {
142+
return false, err
143+
}
144+
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
145+
if err != nil {
146+
if os.IsNotExist(err) {
147+
err = nil // parent was deleted
148+
}
149+
return err
150+
}
151+
if path == dir {
152+
return nil
153+
}
154+
if _, exists := unpackedPaths[path]; !exists {
155+
err := os.RemoveAll(path)
156+
return err
157+
}
158+
return nil
159+
})
160+
return false, err
161+
}
162+
163+
if strings.HasPrefix(base, whiteoutPrefix) {
164+
originalBase := base[len(whiteoutPrefix):]
165+
originalPath := filepath.Join(dir, originalBase)
166+
167+
return false, os.RemoveAll(originalPath)
168+
}
169+
170+
return true, nil
171+
}
172+
}
173+
139174
// Iterate through the files in the archive.
140175
for {
141176
select {
@@ -193,85 +228,21 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
193228
if base == "" {
194229
parentPath = filepath.Dir(path)
195230
}
196-
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
197-
err = mkdirAll(parentPath, 0755)
198-
if err != nil {
199-
return 0, err
200-
}
231+
if err := mkparent(ctx, parentPath, root, options.Parents); err != nil {
232+
return 0, err
201233
}
202234
}
203235

204-
// Skip AUFS metadata dirs
205-
if strings.HasPrefix(hdr.Name, whiteoutMetaPrefix) {
206-
// Regular files inside /.wh..wh.plnk can be used as hardlink targets
207-
// We don't want this directory, but we need the files in them so that
208-
// such hardlinks can be resolved.
209-
if strings.HasPrefix(hdr.Name, whiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
210-
basename := filepath.Base(hdr.Name)
211-
aufsHardlinks[basename] = hdr
212-
if aufsTempdir == "" {
213-
if aufsTempdir, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "dockerplnk"); err != nil {
214-
return 0, err
215-
}
216-
defer os.RemoveAll(aufsTempdir)
217-
}
218-
p, err := fs.RootPath(aufsTempdir, basename)
219-
if err != nil {
220-
return 0, err
221-
}
222-
if err := createTarFile(ctx, p, root, hdr, tr); err != nil {
223-
return 0, err
224-
}
225-
}
226-
227-
if hdr.Name != whiteoutOpaqueDir {
228-
continue
229-
}
236+
// Naive whiteout convert function which handles whiteout files by
237+
// removing the target files.
238+
if err := validateWhiteout(path); err != nil {
239+
return 0, err
230240
}
231-
232-
if strings.HasPrefix(base, whiteoutPrefix) {
233-
dir := filepath.Dir(path)
234-
if base == whiteoutOpaqueDir {
235-
_, err := os.Lstat(dir)
236-
if err != nil {
237-
return 0, err
238-
}
239-
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
240-
if err != nil {
241-
if os.IsNotExist(err) {
242-
err = nil // parent was deleted
243-
}
244-
return err
245-
}
246-
if path == dir {
247-
return nil
248-
}
249-
if _, exists := unpackedPaths[path]; !exists {
250-
err := os.RemoveAll(path)
251-
return err
252-
}
253-
return nil
254-
})
255-
if err != nil {
256-
return 0, err
257-
}
258-
continue
259-
}
260-
261-
originalBase := base[len(whiteoutPrefix):]
262-
originalPath := filepath.Join(dir, originalBase)
263-
264-
// Ensure originalPath is under dir
265-
if dir[len(dir)-1] != filepath.Separator {
266-
dir += string(filepath.Separator)
267-
}
268-
if !strings.HasPrefix(originalPath, dir) {
269-
return 0, errors.Wrapf(errInvalidArchive, "invalid whiteout name: %v", base)
270-
}
271-
272-
if err := os.RemoveAll(originalPath); err != nil {
273-
return 0, err
274-
}
241+
writeFile, err := convertWhiteout(hdr, path)
242+
if err != nil {
243+
return 0, errors.Wrapf(err, "failed to convert whiteout file %q", hdr.Name)
244+
}
245+
if !writeFile {
275246
continue
276247
}
277248
// If path exits we almost always just want to remove and replace it.
@@ -289,26 +260,6 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
289260
srcData := io.Reader(tr)
290261
srcHdr := hdr
291262

292-
// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
293-
// we manually retarget these into the temporary files we extracted them into
294-
if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), whiteoutLinkDir) {
295-
linkBasename := filepath.Base(hdr.Linkname)
296-
srcHdr = aufsHardlinks[linkBasename]
297-
if srcHdr == nil {
298-
return 0, fmt.Errorf("invalid aufs hardlink")
299-
}
300-
p, err := fs.RootPath(aufsTempdir, linkBasename)
301-
if err != nil {
302-
return 0, err
303-
}
304-
tmpFile, err := os.Open(p)
305-
if err != nil {
306-
return 0, err
307-
}
308-
defer tmpFile.Close()
309-
srcData = tmpFile
310-
}
311-
312263
if err := createTarFile(ctx, path, root, srcHdr, srcData); err != nil {
313264
return 0, err
314265
}
@@ -428,6 +379,66 @@ func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header
428379
return chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime))
429380
}
430381

382+
func mkparent(ctx context.Context, path, root string, parents []string) error {
383+
if dir, err := os.Lstat(path); err == nil {
384+
if dir.IsDir() {
385+
return nil
386+
}
387+
return &os.PathError{
388+
Op: "mkparent",
389+
Path: path,
390+
Err: syscall.ENOTDIR,
391+
}
392+
} else if !os.IsNotExist(err) {
393+
return err
394+
}
395+
396+
i := len(path)
397+
for i > len(root) && !os.IsPathSeparator(path[i-1]) {
398+
i--
399+
}
400+
401+
if i > len(root)+1 {
402+
if err := mkparent(ctx, path[:i-1], root, parents); err != nil {
403+
return err
404+
}
405+
}
406+
407+
if err := mkdir(path, 0755); err != nil {
408+
// Check that still doesn't exist
409+
dir, err1 := os.Lstat(path)
410+
if err1 == nil && dir.IsDir() {
411+
return nil
412+
}
413+
return err
414+
}
415+
416+
for _, p := range parents {
417+
ppath, err := fs.RootPath(p, path[len(root):])
418+
if err != nil {
419+
return err
420+
}
421+
422+
dir, err := os.Lstat(ppath)
423+
if err == nil {
424+
if !dir.IsDir() {
425+
// Replaced, do not copy attributes
426+
break
427+
}
428+
if err := copyDirInfo(dir, path); err != nil {
429+
return err
430+
}
431+
return copyUpXAttrs(path, ppath)
432+
} else if !os.IsNotExist(err) {
433+
return err
434+
}
435+
}
436+
437+
log.G(ctx).Debugf("parent directory %q not found: default permissions(0755) used", path)
438+
439+
return nil
440+
}
441+
431442
type changeWriter struct {
432443
tw *tar.Writer
433444
source string
@@ -598,6 +609,9 @@ func (cw *changeWriter) Close() error {
598609
}
599610

600611
func (cw *changeWriter) includeParents(hdr *tar.Header) error {
612+
if cw.addedDirs == nil {
613+
return nil
614+
}
601615
name := strings.TrimRight(hdr.Name, "/")
602616
fname := filepath.Join(cw.source, name)
603617
parent := filepath.Dir(name)
@@ -684,3 +698,26 @@ func hardlinkRootPath(root, linkname string) (string, error) {
684698
}
685699
return targetPath, nil
686700
}
701+
702+
func validateWhiteout(path string) error {
703+
base := filepath.Base(path)
704+
dir := filepath.Dir(path)
705+
706+
if base == whiteoutOpaqueDir {
707+
return nil
708+
}
709+
710+
if strings.HasPrefix(base, whiteoutPrefix) {
711+
originalBase := base[len(whiteoutPrefix):]
712+
originalPath := filepath.Join(dir, originalBase)
713+
714+
// Ensure originalPath is under dir
715+
if dir[len(dir)-1] != filepath.Separator {
716+
dir += string(filepath.Separator)
717+
}
718+
if !strings.HasPrefix(originalPath, dir) {
719+
return errors.Wrapf(errInvalidArchive, "invalid whiteout name: %v", base)
720+
}
721+
}
722+
return nil
723+
}

0 commit comments

Comments
 (0)