Skip to content

Commit edc72bb

Browse files
crosbymichaelcrosbymichael
authored
Merge pull request #1980 from dmcgowan/cherry-pick-fix-1723
[release/1.0] archive: track and include parent directories
2 parents c59bc7e + 2e3f26f commit edc72bb

4 files changed

Lines changed: 235 additions & 140 deletions

File tree

‎archive/tar.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ type changeWriter struct {
291291
whiteoutT time.Time
292292
inodeSrc map[uint64]string
293293
inodeRefs map[uint64][]string
294+
addedDirs map[string]struct{}
294295
}
295296

296297
func newChangeWriter(w io.Writer, source string) *changeWriter {
@@ -300,6 +301,7 @@ func newChangeWriter(w io.Writer, source string) *changeWriter {
300301
whiteoutT: time.Now(),
301302
inodeSrc: map[uint64]string{},
302303
inodeRefs: map[uint64][]string{},
304+
addedDirs: map[string]struct{}{},
303305
}
304306
}
305307

@@ -312,6 +314,7 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
312314
whiteOutBase := filepath.Base(p)
313315
whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase)
314316
hdr := &tar.Header{
317+
Typeflag: tar.TypeReg,
315318
Name: whiteOut[1:],
316319
Size: 0,
317320
ModTime: cw.whiteoutT,
@@ -395,6 +398,9 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
395398
hdr.PAXRecords[paxSchilyXattr+"security.capability"] = string(capability)
396399
}
397400

401+
if err := cw.includeParents(hdr); err != nil {
402+
return err
403+
}
398404
if err := cw.tw.WriteHeader(hdr); err != nil {
399405
return errors.Wrap(err, "failed to write file header")
400406
}
@@ -424,6 +430,10 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
424430
hdr.Typeflag = tar.TypeLink
425431
hdr.Linkname = source
426432
hdr.Size = 0
433+
434+
if err := cw.includeParents(hdr); err != nil {
435+
return err
436+
}
427437
if err := cw.tw.WriteHeader(hdr); err != nil {
428438
return errors.Wrap(err, "failed to write file header")
429439
}
@@ -533,6 +543,29 @@ func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header
533543
return chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime))
534544
}
535545

546+
func (cw *changeWriter) includeParents(hdr *tar.Header) error {
547+
name := strings.TrimRight(hdr.Name, "/")
548+
fname := filepath.Join(cw.source, name)
549+
parent := filepath.Dir(name)
550+
pname := filepath.Join(cw.source, parent)
551+
552+
// Do not include root directory as parent
553+
if fname != cw.source && pname != cw.source {
554+
_, ok := cw.addedDirs[parent]
555+
if !ok {
556+
cw.addedDirs[parent] = struct{}{}
557+
fi, err := os.Stat(pname)
558+
if err != nil {
559+
return err
560+
}
561+
if err := cw.HandleChange(fs.ChangeKindModify, parent, fi, nil); err != nil {
562+
return err
563+
}
564+
}
565+
}
566+
return nil
567+
}
568+
536569
func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) {
537570
buf := bufferPool.Get().(*[]byte)
538571
defer bufferPool.Put(buf)

‎archive/tar_test.go‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -666,6 +666,201 @@ func makeWriterToTarTest(wt WriterToTar, a fstest.Applier, validate func(string)
666666
}
667667
}
668668

669+
func TestDiffTar(t *testing.T) {
670+
tests := []struct {
671+
name string
672+
validators []tarEntryValidator
673+
a fstest.Applier
674+
b fstest.Applier
675+
}{
676+
{
677+
name: "EmptyDiff",
678+
validators: []tarEntryValidator{},
679+
a: fstest.Apply(
680+
fstest.CreateDir("/etc/", 0755),
681+
),
682+
b: fstest.Apply(),
683+
},
684+
{
685+
name: "ParentInclusion",
686+
validators: []tarEntryValidator{
687+
dirEntry("d1/", 0755),
688+
dirEntry("d1/d/", 0700),
689+
dirEntry("d2/", 0770),
690+
fileEntry("d2/f", []byte("ok"), 0644),
691+
},
692+
a: fstest.Apply(
693+
fstest.CreateDir("/d1/", 0755),
694+
fstest.CreateDir("/d2/", 0770),
695+
),
696+
b: fstest.Apply(
697+
fstest.CreateDir("/d1/d", 0700),
698+
fstest.CreateFile("/d2/f", []byte("ok"), 0644),
699+
),
700+
},
701+
{
702+
name: "HardlinkParentInclusion",
703+
validators: []tarEntryValidator{
704+
dirEntry("d2/", 0755),
705+
fileEntry("d2/l1", []byte("link me"), 0644),
706+
// d1/f1 and its parent is included after the new link,
707+
// before the new link was included, these files would
708+
// not habe needed
709+
dirEntry("d1/", 0755),
710+
linkEntry("d1/f1", "d2/l1"),
711+
dirEntry("d3/", 0755),
712+
fileEntry("d3/l1", []byte("link me"), 0644),
713+
dirEntry("d4/", 0755),
714+
linkEntry("d4/f1", "d3/l1"),
715+
whiteoutEntry("d6/l1"),
716+
whiteoutEntry("d6/l2"),
717+
},
718+
a: fstest.Apply(
719+
fstest.CreateDir("/d1/", 0755),
720+
fstest.CreateFile("/d1/f1", []byte("link me"), 0644),
721+
fstest.CreateDir("/d2/", 0755),
722+
fstest.CreateFile("/d2/f1", []byte("link me"), 0644),
723+
fstest.CreateDir("/d3/", 0755),
724+
fstest.CreateDir("/d4/", 0755),
725+
fstest.CreateFile("/d4/f1", []byte("link me"), 0644),
726+
fstest.CreateDir("/d5/", 0755),
727+
fstest.CreateFile("/d5/f1", []byte("link me"), 0644),
728+
fstest.CreateDir("/d6/", 0755),
729+
fstest.Link("/d1/f1", "/d6/l1"),
730+
fstest.Link("/d5/f1", "/d6/l2"),
731+
),
732+
b: fstest.Apply(
733+
fstest.Link("/d1/f1", "/d2/l1"),
734+
fstest.Link("/d4/f1", "/d3/l1"),
735+
fstest.Remove("/d6/l1"),
736+
fstest.Remove("/d6/l2"),
737+
),
738+
},
739+
}
740+
741+
for _, at := range tests {
742+
t.Run(at.name, makeDiffTarTest(at.validators, at.a, at.b))
743+
}
744+
}
745+
746+
type tarEntryValidator func(*tar.Header, []byte) error
747+
748+
func dirEntry(name string, mode int) tarEntryValidator {
749+
return func(hdr *tar.Header, b []byte) error {
750+
if hdr.Typeflag != tar.TypeDir {
751+
return errors.New("not directory type")
752+
}
753+
if hdr.Name != name {
754+
return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
755+
}
756+
if hdr.Mode != int64(mode) {
757+
return errors.Errorf("wrong mode %o, expected %o", hdr.Mode, mode)
758+
}
759+
return nil
760+
}
761+
}
762+
763+
func fileEntry(name string, expected []byte, mode int) tarEntryValidator {
764+
return func(hdr *tar.Header, b []byte) error {
765+
if hdr.Typeflag != tar.TypeReg {
766+
return errors.New("not file type")
767+
}
768+
if hdr.Name != name {
769+
return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
770+
}
771+
if hdr.Mode != int64(mode) {
772+
return errors.Errorf("wrong mode %o, expected %o", hdr.Mode, mode)
773+
}
774+
if bytes.Compare(b, expected) != 0 {
775+
return errors.Errorf("different file content")
776+
}
777+
return nil
778+
}
779+
}
780+
781+
func linkEntry(name, link string) tarEntryValidator {
782+
return func(hdr *tar.Header, b []byte) error {
783+
if hdr.Typeflag != tar.TypeLink {
784+
return errors.New("not link type")
785+
}
786+
if hdr.Name != name {
787+
return errors.Errorf("wrong name %q, expected %q", hdr.Name, name)
788+
}
789+
if hdr.Linkname != link {
790+
return errors.Errorf("wrong link %q, expected %q", hdr.Linkname, link)
791+
}
792+
return nil
793+
}
794+
}
795+
796+
func whiteoutEntry(name string) tarEntryValidator {
797+
whiteOutDir := filepath.Dir(name)
798+
whiteOutBase := filepath.Base(name)
799+
whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase)
800+
801+
return func(hdr *tar.Header, b []byte) error {
802+
if hdr.Typeflag != tar.TypeReg {
803+
return errors.Errorf("not file type: %q", hdr.Typeflag)
804+
}
805+
if hdr.Name != whiteOut {
806+
return errors.Errorf("wrong name %q, expected whiteout %q", hdr.Name, name)
807+
}
808+
return nil
809+
}
810+
}
811+
812+
func makeDiffTarTest(validators []tarEntryValidator, a, b fstest.Applier) func(*testing.T) {
813+
return func(t *testing.T) {
814+
ad, err := ioutil.TempDir("", "test-make-diff-tar-")
815+
if err != nil {
816+
t.Fatalf("failed to create temp dir: %v", err)
817+
}
818+
defer os.RemoveAll(ad)
819+
if err := a.Apply(ad); err != nil {
820+
t.Fatalf("failed to apply a: %v", err)
821+
}
822+
823+
bd, err := ioutil.TempDir("", "test-make-diff-tar-")
824+
if err != nil {
825+
t.Fatalf("failed to create temp dir: %v", err)
826+
}
827+
defer os.RemoveAll(bd)
828+
if err := fs.CopyDir(bd, ad); err != nil {
829+
t.Fatalf("failed to copy dir: %v", err)
830+
}
831+
if err := b.Apply(bd); err != nil {
832+
t.Fatalf("failed to apply b: %v", err)
833+
}
834+
835+
rc := Diff(context.Background(), ad, bd)
836+
defer rc.Close()
837+
838+
tr := tar.NewReader(rc)
839+
for i := 0; ; i++ {
840+
hdr, err := tr.Next()
841+
if err != nil {
842+
if err == io.EOF {
843+
break
844+
}
845+
t.Fatalf("tar read error: %v", err)
846+
}
847+
var b []byte
848+
if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
849+
b, err = ioutil.ReadAll(tr)
850+
if err != nil {
851+
t.Fatalf("tar read file error: %v", err)
852+
}
853+
}
854+
if i >= len(validators) {
855+
t.Fatal("no validator for entry")
856+
}
857+
if err := validators[i](hdr, b); err != nil {
858+
t.Fatalf("tar entry[%d] validation fail: %#v", i, err)
859+
}
860+
}
861+
}
862+
}
863+
669864
type diffApplier struct{}
670865

671866
func (d diffApplier) TestContext(ctx context.Context) (context.Context, func(), error) {

‎fs/diff.go‎

Lines changed: 6 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,8 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
222222
c1 = make(chan *currentPath)
223223
c2 = make(chan *currentPath)
224224

225-
f1, f2 *currentPath
226-
rmdir string
227-
lastEmittedDir = string(filepath.Separator)
228-
parents []os.FileInfo
225+
f1, f2 *currentPath
226+
rmdir string
229227
)
230228
g.Go(func() error {
231229
defer close(c1)
@@ -260,10 +258,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
260258
continue
261259
}
262260

263-
var (
264-
f os.FileInfo
265-
emit = true
266-
)
261+
var f os.FileInfo
267262
k, p := pathChange(f1, f2)
268263
switch k {
269264
case ChangeKindAdd:
@@ -299,83 +294,17 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
299294
f2 = nil
300295
if same {
301296
if !isLinked(f) {
302-
emit = false
297+
continue
303298
}
304299
k = ChangeKindUnmodified
305300
}
306301
}
307-
if emit {
308-
emittedDir, emitParents := commonParents(lastEmittedDir, p, parents)
309-
for _, pf := range emitParents {
310-
p := filepath.Join(emittedDir, pf.Name())
311-
if err := changeFn(ChangeKindUnmodified, p, pf, nil); err != nil {
312-
return err
313-
}
314-
emittedDir = p
315-
}
316-
317-
if err := changeFn(k, p, f, nil); err != nil {
318-
return err
319-
}
320-
321-
if f != nil && f.IsDir() {
322-
lastEmittedDir = p
323-
} else {
324-
lastEmittedDir = emittedDir
325-
}
326-
327-
parents = parents[:0]
328-
} else if f.IsDir() {
329-
lastEmittedDir, parents = commonParents(lastEmittedDir, p, parents)
330-
parents = append(parents, f)
302+
if err := changeFn(k, p, f, nil); err != nil {
303+
return err
331304
}
332305
}
333306
return nil
334307
})
335308

336309
return g.Wait()
337310
}
338-
339-
func commonParents(base, updated string, dirs []os.FileInfo) (string, []os.FileInfo) {
340-
if basePrefix := makePrefix(base); strings.HasPrefix(updated, basePrefix) {
341-
var (
342-
parents []os.FileInfo
343-
last = base
344-
)
345-
for _, d := range dirs {
346-
next := filepath.Join(last, d.Name())
347-
if strings.HasPrefix(updated, makePrefix(last)) {
348-
parents = append(parents, d)
349-
last = next
350-
} else {
351-
break
352-
}
353-
}
354-
return base, parents
355-
}
356-
357-
baseS := strings.Split(base, string(filepath.Separator))
358-
updatedS := strings.Split(updated, string(filepath.Separator))
359-
commonS := []string{string(filepath.Separator)}
360-
361-
min := len(baseS)
362-
if len(updatedS) < min {
363-
min = len(updatedS)
364-
}
365-
for i := 0; i < min; i++ {
366-
if baseS[i] == updatedS[i] {
367-
commonS = append(commonS, baseS[i])
368-
} else {
369-
break
370-
}
371-
}
372-
373-
return filepath.Join(commonS...), []os.FileInfo{}
374-
}
375-
376-
func makePrefix(d string) string {
377-
if d == "" || d[len(d)-1] != filepath.Separator {
378-
return d + string(filepath.Separator)
379-
}
380-
return d
381-
}

0 commit comments

Comments
 (0)