Skip to content

Commit 05bd043

Browse files
jstarksjstarks
committed
Support layers from external URLs
This is used to support downloading Windows base images from Microsoft servers. Signed-off-by: John Starks <[email protected]>
1 parent 9c90236 commit 05bd043

16 files changed

Lines changed: 231 additions & 53 deletions

‎distribution/pull_v2.go‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ type v2LayerDescriptor struct {
133133
V2MetadataService *metadata.V2MetadataService
134134
tmpFile *os.File
135135
verifier digest.Verifier
136+
foreignSrc *distribution.Descriptor
136137
}
137138

138139
func (ld *v2LayerDescriptor) Key() string {
@@ -180,9 +181,8 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre
180181
}
181182

182183
tmpFile := ld.tmpFile
183-
blobs := ld.repo.Blobs(ctx)
184184

185-
layerDownload, err := blobs.Open(ctx, ld.digest)
185+
layerDownload, err := ld.open(ctx)
186186
if err != nil {
187187
logrus.Errorf("Error initiating layer download: %v", err)
188188
if err == distribution.ErrBlobUnknown {
@@ -501,6 +501,29 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
501501
return imageID, manifestDigest, nil
502502
}
503503

504+
var descriptors []xfer.DownloadDescriptor
505+
506+
// Note that the order of this loop is in the direction of bottom-most
507+
// to top-most, so that the downloads slice gets ordered correctly.
508+
for _, d := range mfst.Layers {
509+
layerDescriptor := &v2LayerDescriptor{
510+
digest: d.Digest,
511+
repo: p.repo,
512+
repoInfo: p.repoInfo,
513+
V2MetadataService: p.V2MetadataService,
514+
}
515+
516+
if d.MediaType == schema2.MediaTypeForeignLayer && len(d.URLs) > 0 {
517+
if !layer.ForeignSourceSupported() {
518+
return "", "", errors.New("foreign layers are not supported on this OS")
519+
}
520+
521+
layerDescriptor.foreignSrc = &d
522+
}
523+
524+
descriptors = append(descriptors, layerDescriptor)
525+
}
526+
504527
configChan := make(chan []byte, 1)
505528
errChan := make(chan error, 1)
506529
var cancel func()
@@ -517,21 +540,6 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
517540
configChan <- configJSON
518541
}()
519542

520-
var descriptors []xfer.DownloadDescriptor
521-
522-
// Note that the order of this loop is in the direction of bottom-most
523-
// to top-most, so that the downloads slice gets ordered correctly.
524-
for _, d := range mfst.References() {
525-
layerDescriptor := &v2LayerDescriptor{
526-
digest: d.Digest,
527-
repo: p.repo,
528-
repoInfo: p.repoInfo,
529-
V2MetadataService: p.V2MetadataService,
530-
}
531-
532-
descriptors = append(descriptors, layerDescriptor)
533-
}
534-
535543
var (
536544
configJSON []byte // raw serialized image config
537545
unmarshalledConfig image.Image // deserialized image config

‎distribution/pull_v2_unix.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,17 @@
33
package distribution
44

55
import (
6+
"github.com/docker/distribution"
7+
"github.com/docker/distribution/context"
68
"github.com/docker/distribution/manifest/schema1"
79
"github.com/docker/docker/image"
810
)
911

1012
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
1113
return nil
1214
}
15+
16+
func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) {
17+
blobs := ld.repo.Blobs(ctx)
18+
return blobs.Open(ctx, ld.digest)
19+
}

‎distribution/pull_v2_windows.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@ package distribution
55
import (
66
"encoding/json"
77
"fmt"
8+
"net/http"
9+
"os"
810

11+
"github.com/docker/distribution"
12+
"github.com/docker/distribution/context"
913
"github.com/docker/distribution/manifest/schema1"
14+
"github.com/docker/distribution/registry/client/transport"
1015
"github.com/docker/docker/image"
16+
"github.com/docker/docker/layer"
1117
)
1218

1319
func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS) error {
@@ -28,3 +34,33 @@ func detectBaseLayer(is image.Store, m *schema1.Manifest, rootFS *image.RootFS)
2834
}
2935
return fmt.Errorf("Invalid base layer %q", v1img.Parent)
3036
}
37+
38+
var _ layer.ForeignSourcer = &v2LayerDescriptor{}
39+
40+
func (ld *v2LayerDescriptor) ForeignSource() *distribution.Descriptor {
41+
return ld.foreignSrc
42+
}
43+
44+
func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) {
45+
if ld.foreignSrc == nil {
46+
blobs := ld.repo.Blobs(ctx)
47+
return blobs.Open(ctx, ld.digest)
48+
}
49+
50+
var (
51+
err error
52+
rsc distribution.ReadSeekCloser
53+
)
54+
55+
// Find the first URL that results in a 200 result code.
56+
for _, url := range ld.foreignSrc.URLs {
57+
rsc = transport.NewHTTPReadSeeker(http.DefaultClient, url, nil)
58+
_, err = rsc.Seek(0, os.SEEK_SET)
59+
if err == nil {
60+
break
61+
}
62+
rsc.Close()
63+
rsc = nil
64+
}
65+
return rsc, err
66+
}

‎distribution/push_v2.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,13 @@ func (pd *v2PushDescriptor) DiffID() layer.DiffID {
240240
}
241241

242242
func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
243+
if fs, ok := pd.layer.(layer.ForeignSourcer); ok {
244+
if d := fs.ForeignSource(); d != nil {
245+
progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
246+
return *d, nil
247+
}
248+
}
249+
243250
diffID := pd.DiffID()
244251

245252
pd.pushState.Lock()

‎distribution/xfer/download.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"time"
88

99
"github.com/Sirupsen/logrus"
10+
"github.com/docker/distribution"
1011
"github.com/docker/docker/image"
1112
"github.com/docker/docker/layer"
1213
"github.com/docker/docker/pkg/archive"
@@ -318,7 +319,11 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor,
318319
return
319320
}
320321

321-
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
322+
var src *distribution.Descriptor
323+
if fs, ok := descriptor.(layer.ForeignSourcer); ok {
324+
src = fs.ForeignSource()
325+
}
326+
d.layer, err = d.layerStore.RegisterForeign(inflatedLayerData, parentLayer, src)
322327
if err != nil {
323328
select {
324329
case <-d.Transfer.Context().Done():
@@ -409,7 +414,11 @@ func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor Downloa
409414
}
410415
defer layerReader.Close()
411416

412-
d.layer, err = d.layerStore.Register(layerReader, parentLayer)
417+
var src *distribution.Descriptor
418+
if fs, ok := l.(layer.ForeignSourcer); ok {
419+
src = fs.ForeignSource()
420+
}
421+
d.layer, err = d.layerStore.RegisterForeign(layerReader, parentLayer, src)
413422
if err != nil {
414423
d.err = fmt.Errorf("failed to register layer: %v", err)
415424
return

‎distribution/xfer/download_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111
"time"
1212

13+
"github.com/docker/distribution"
1314
"github.com/docker/distribution/digest"
1415
"github.com/docker/docker/image"
1516
"github.com/docker/docker/layer"
@@ -71,6 +72,10 @@ func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.
7172
}
7273

7374
func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
75+
return ls.RegisterForeign(reader, parentID, nil)
76+
}
77+
78+
func (ls *mockLayerStore) RegisterForeign(reader io.Reader, parentID layer.ChainID, _ *distribution.Descriptor) (layer.Layer, error) {
7479
var (
7580
parent layer.Layer
7681
err error

‎image/tarexport/load.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"reflect"
1111

1212
"github.com/Sirupsen/logrus"
13+
"github.com/docker/distribution"
1314
"github.com/docker/docker/image"
1415
"github.com/docker/docker/image/v1"
1516
"github.com/docker/docker/layer"
@@ -63,6 +64,10 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
6364
var parentLinks []parentLink
6465

6566
for _, m := range manifest {
67+
if m.LayerSources != nil && !layer.ForeignSourceSupported() {
68+
return fmt.Errorf("invalid manifest, foreign layers not supported on this operating system")
69+
}
70+
6671
configPath, err := safePath(tmpDir, m.Config)
6772
if err != nil {
6873
return err
@@ -92,7 +97,7 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
9297
r.Append(diffID)
9398
newLayer, err := l.ls.Get(r.ChainID())
9499
if err != nil {
95-
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput)
100+
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), m.LayerSources[diffID], progressOutput)
96101
if err != nil {
97102
return err
98103
}
@@ -151,7 +156,7 @@ func (l *tarexporter) setParentID(id, parentID image.ID) error {
151156
return l.is.SetParent(id, parentID)
152157
}
153158

154-
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressOutput progress.Output) (layer.Layer, error) {
159+
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc *distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
155160
rawTar, err := os.Open(filename)
156161
if err != nil {
157162
logrus.Debugf("Error reading embedded tar: %v", err)
@@ -174,9 +179,9 @@ func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string,
174179

175180
progressReader := progress.NewProgressReader(inflatedLayerData, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
176181

177-
return l.ls.Register(progressReader, rootFS.ChainID())
182+
return l.ls.RegisterForeign(progressReader, rootFS.ChainID(), foreignSrc)
178183
}
179-
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
184+
return l.ls.RegisterForeign(inflatedLayerData, rootFS.ChainID(), foreignSrc)
180185
}
181186

182187
func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error {
@@ -298,7 +303,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
298303
if err != nil {
299304
return err
300305
}
301-
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, progressOutput)
306+
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, nil, progressOutput)
302307
if err != nil {
303308
return err
304309
}

0 commit comments

Comments
 (0)