Skip to content

Commit fcc8209

Browse files
ctalledothaJeztah
ctalledo
authored and
thaJeztah
committed
Add support for multiple platforms in image export and loading.
Currently the image export and load APIs can be used to export or load all platforms for the image, or a single specified platform. This commit updates the API so that it accepts a list of platforms to export or load, thereby giving clients the ability to export only selected platforms of an image into a tar file, or load selected platforms from a tar file. Unit and integration tests were updated accordingly. As this requires a daemon API change, the API version was bumped. Signed-off-by: Cesar Talledo <[email protected]>
1 parent c55a163 commit fcc8209

12 files changed

Lines changed: 482 additions & 114 deletions

File tree

‎api/swagger.yaml‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10443,7 +10443,10 @@ paths:
1044310443
type: "string"
1044410444
required: true
1044510445
- name: "platform"
10446-
type: "string"
10446+
type: "array"
10447+
items:
10448+
type: "string"
10449+
collectionFormat: "multi"
1044710450
in: "query"
1044810451
description: |
1044910452
JSON encoded OCI platform describing a platform which will be used
@@ -10488,13 +10491,16 @@ paths:
1048810491
items:
1048910492
type: "string"
1049010493
- name: "platform"
10491-
type: "string"
10494+
type: "array"
10495+
items:
10496+
type: "string"
10497+
collectionFormat: "multi"
1049210498
in: "query"
1049310499
description: |
10494-
JSON encoded OCI platform describing a platform which will be used
10495-
to select a platform-specific image to be saved if the image is
10496-
multi-platform.
10497-
If not provided, the full multi-platform image will be saved.
10500+
JSON encoded OCI platform(s) which will be used to select the
10501+
platform-specific image(s) to be saved if the image is
10502+
multi-platform. If not provided, the full multi-platform image
10503+
will be saved.
1049810504
1049910505
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
1050010506
tags: ["Image"]
@@ -10530,13 +10536,16 @@ paths:
1053010536
type: "boolean"
1053110537
default: false
1053210538
- name: "platform"
10533-
type: "string"
10539+
type: "array"
10540+
items:
10541+
type: "string"
10542+
collectionFormat: "multi"
1053410543
in: "query"
1053510544
description: |
10536-
JSON encoded OCI platform describing a platform which will be used
10537-
to select a platform-specific image to be load if the image is
10538-
multi-platform.
10539-
If not provided, the full multi-platform image will be loaded.
10545+
JSON encoded OCI platform(s) which will be used to select the
10546+
platform-specific image(s) to load if the image is
10547+
multi-platform. If not provided, the full multi-platform image
10548+
will be loaded.
1054010549
1054110550
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
1054210551
tags: ["Image"]

‎daemon/containerd/image_exporter.go‎

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,15 @@ import (
3131
// outStream is the writer which the images are written to.
3232
//
3333
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
34-
func (i *ImageService) ExportImage(ctx context.Context, names []string, platform *ocispec.Platform, outStream io.Writer) error {
35-
pm := i.matchRequestedOrDefault(platforms.OnlyStrict, platform)
34+
func (i *ImageService) ExportImage(ctx context.Context, names []string, platformList []ocispec.Platform, outStream io.Writer) error {
35+
var pm platforms.MatchComparer
36+
37+
// Get the platform matcher for the requested platforms
38+
if len(platformList) == 0 {
39+
pm = matchAllWithPreference(i.hostPlatformMatcher())
40+
} else {
41+
pm = matchAnyWithPreference(i.hostPlatformMatcher(), platformList)
42+
}
3643

3744
opts := []archive.ExportOpt{
3845
archive.WithSkipNonDistributableBlobs(),
@@ -65,8 +72,12 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, platform
6572
exportImage := func(ctx context.Context, img c8dimages.Image, ref reference.Named) error {
6673
target := img.Target
6774

68-
if platform != nil {
69-
newTarget, err := i.getPushDescriptor(ctx, img, platform)
75+
// If a single platform is requested, export the manifest for the specific platform only
76+
// (single-level index). Otherwise export the full index (two-level, nested). Note that
77+
// since opts includes WithPlatform and WithSkipMissing, the index will contain the
78+
// requested platforms only, and only if they are available in the content store.
79+
if len(platformList) == 1 {
80+
newTarget, err := i.getPushDescriptor(ctx, img, &platformList[0])
7081
if err != nil {
7182
return errors.Wrap(err, "no suitable export target found")
7283
}
@@ -229,14 +240,23 @@ func (i *ImageService) leaseContent(ctx context.Context, store content.Store, de
229240
// LoadImage uploads a set of images into the repository. This is the
230241
// complement of ExportImage. The input stream is an uncompressed tar
231242
// ball containing images and metadata.
232-
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error {
243+
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platformList []ocispec.Platform, outStream io.Writer, quiet bool) error {
233244
decompressed, err := compression.DecompressStream(inTar)
234245
if err != nil {
235246
return errors.Wrap(err, "failed to decompress input tar archive")
236247
}
237248
defer decompressed.Close()
238249

239-
pm := i.matchRequestedOrDefault(platforms.OnlyStrict, platform)
250+
specificPlatforms := len(platformList) > 0
251+
252+
// Get the platform matcher for the requested platforms
253+
var pm platforms.MatchComparer
254+
if specificPlatforms {
255+
pm = platforms.Any(platformList...)
256+
} else {
257+
// All platforms
258+
pm = matchAllWithPreference(i.hostPlatformMatcher())
259+
}
240260

241261
opts := []containerd.ImportOpt{
242262
containerd.WithImportPlatform(pm),
@@ -266,16 +286,19 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
266286
}),
267287
}
268288

269-
if platform == nil {
289+
if !specificPlatforms {
270290
// Allow variants to be missing if no specific platform is requested.
271291
opts = append(opts, containerd.WithSkipMissing())
272292
}
273293

274294
imgs, err := i.client.Import(ctx, decompressed, opts...)
275295
if err != nil {
276-
if platform != nil {
277-
p := platforms.FormatAll(*platform)
278-
log.G(ctx).WithFields(log.Fields{"error": err, "platform": p}).Debug("failed to import image to containerd")
296+
if specificPlatforms {
297+
platformNames := make([]string, 0, len(platformList))
298+
for _, p := range platformList {
299+
platformNames = append(platformNames, platforms.FormatAll(p))
300+
}
301+
log.G(ctx).WithFields(log.Fields{"error": err, "platform(s)": platformNames}).Debug("failed to import image to containerd")
279302

280303
// Note: ErrEmptyWalk will not be returned in most cases as
281304
// index.json will contain a descriptor of the actual OCI index or
@@ -284,22 +307,26 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
284307
// doesn't have a platform set, so it won't be filtered out by the
285308
// FilterPlatform containerd handler.
286309
if errors.Is(err, c8dimages.ErrEmptyWalk) {
287-
return errdefs.NotFound(errors.Wrapf(err, "requested platform (%s) not found", p))
310+
return errdefs.NotFound(errors.Wrapf(err, "requested platform(s) (%v) not found", platformNames))
288311
}
289312
if cerrdefs.IsNotFound(err) {
290-
return errdefs.NotFound(errors.Wrapf(err, "requested platform (%s) found, but some content is missing", p))
313+
return errdefs.NotFound(errors.Wrapf(err, "requested platform(s) (%v) found, but some content is missing", platformNames))
291314
}
292315
}
293316
log.G(ctx).WithError(err).Debug("failed to import image to containerd")
294317
return errdefs.System(err)
295318
}
296319

297-
if platform != nil {
298-
// Verify that the requested platform is available for the loaded images.
320+
if specificPlatforms {
321+
// Verify that the requested platform(s) are available for the loaded images.
299322
// While the ideal behavior here would be to verify whether the input
300323
// archive actually supplied them, we're not able to determine that
301324
// as the imported index is not returned by the import operation.
302-
if err := i.verifyImagesProvidePlatform(ctx, imgs, *platform, pm); err != nil {
325+
platformNames := make([]string, 0, len(platformList))
326+
for _, p := range platformList {
327+
platformNames = append(platformNames, platforms.FormatAll(p))
328+
}
329+
if err := i.verifyImagesProvidePlatform(ctx, imgs, platformNames, pm); err != nil {
303330
return err
304331
}
305332
}
@@ -308,7 +335,7 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
308335
// Unpack only an image of the host platform
309336
unpackPm := i.hostPlatformMatcher()
310337
// If a load of specific platform is requested, unpack it
311-
if platform != nil {
338+
if specificPlatforms {
312339
unpackPm = pm
313340
}
314341

@@ -378,9 +405,9 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
378405

379406
// verifyImagesProvidePlatform checks if the requested platform is loaded.
380407
// If the requested platform is not loaded, it returns an error.
381-
func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c8dimages.Image, platform ocispec.Platform, pm platforms.Matcher) error {
408+
func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c8dimages.Image, platformNames []string, pm platforms.Matcher) error {
382409
if len(imgs) == 0 {
383-
return errdefs.NotFound(fmt.Errorf("no images providing the requested platform %s found", platforms.FormatAll(platform)))
410+
return errdefs.NotFound(fmt.Errorf("no images providing the requested platform(s) found: %v", platformNames))
384411
}
385412
var incompleteImgs []string
386413
for _, img := range imgs {
@@ -399,7 +426,7 @@ func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c
399426
}
400427
available, err := platformImg.CheckContentAvailable(ctx)
401428
if err != nil {
402-
return errors.Wrapf(err, "failed to determine image content availability for platform %s", platforms.FormatAll(platform))
429+
return errors.Wrapf(err, "failed to determine image content availability for platform(s) %s", platformNames)
403430
}
404431

405432
if available {
@@ -427,5 +454,5 @@ func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c
427454
msg = "images [%s] were loaded, but don't provide the requested platform (%s)"
428455
}
429456

430-
return errdefs.NotFound(fmt.Errorf(msg, strings.Join(incompleteImgs, ", "), platforms.FormatAll(platform)))
457+
return errdefs.NotFound(fmt.Errorf(msg, strings.Join(incompleteImgs, ", "), platformNames))
431458
}

‎daemon/containerd/image_load_test.go‎

Lines changed: 87 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package containerd
33
import (
44
"bytes"
55
"context"
6+
"fmt"
67
"math/rand"
78
"os"
89
"path/filepath"
@@ -16,15 +17,18 @@ import (
1617
"github.com/docker/docker/internal/testutils/labelstore"
1718
"github.com/docker/docker/internal/testutils/specialimage"
1819
"github.com/moby/go-archive"
20+
"github.com/moby/moby/api/types/backend"
21+
"github.com/moby/moby/api/types/image"
1922
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2023
"gotest.tools/v3/assert"
2124
is "gotest.tools/v3/assert/cmp"
2225
)
2326

24-
func TestImageLoadMissing(t *testing.T) {
27+
func TestImageLoad(t *testing.T) {
2528
linuxAmd64 := ocispec.Platform{OS: "linux", Architecture: "amd64"}
2629
linuxArm64 := ocispec.Platform{OS: "linux", Architecture: "arm64"}
2730
linuxArmv5 := ocispec.Platform{OS: "linux", Architecture: "arm", Variant: "v5"}
31+
linuxRiscv64 := ocispec.Platform{OS: "linux", Architecture: "riskv64"}
2832

2933
ctx := namespaces.WithNamespace(context.TODO(), "testing-"+t.Name())
3034

@@ -35,7 +39,7 @@ func TestImageLoadMissing(t *testing.T) {
3539
// Mock the daemon platform.
3640
imgSvc.defaultPlatformOverride = platforms.Only(linuxAmd64)
3741

38-
tryLoad := func(ctx context.Context, t *testing.T, dir string, platform ocispec.Platform) error {
42+
tryLoad := func(ctx context.Context, t *testing.T, dir string, platformList []ocispec.Platform) error {
3943
tarRc, err := archive.Tar(dir, archive.Uncompressed)
4044
assert.NilError(t, err)
4145
defer tarRc.Close()
@@ -46,10 +50,19 @@ func TestImageLoadMissing(t *testing.T) {
4650
t.Log(buf.String())
4751
}()
4852

49-
return imgSvc.LoadImage(ctx, tarRc, &platform, &buf, true)
53+
return imgSvc.LoadImage(ctx, tarRc, platformList, &buf, true)
5054
}
5155

52-
clearStore := func(ctx context.Context, t *testing.T) {
56+
cleanup := func(ctx context.Context, t *testing.T) {
57+
// Remove all existing images to start fresh
58+
images, err := imgSvc.Images(ctx, image.ListOptions{})
59+
assert.NilError(t, err)
60+
for _, img := range images {
61+
_, err := imgSvc.ImageDelete(ctx, img.ID, image.RemoveOptions{PruneChildren: true})
62+
assert.NilError(t, err)
63+
}
64+
65+
// Remove all content from the store
5366
assert.NilError(t, store.Walk(ctx, func(info content.Info) error {
5467
return store.Delete(ctx, info.Digest)
5568
}), "failed to delete all content")
@@ -60,39 +73,64 @@ func TestImageLoadMissing(t *testing.T) {
6073
_, err := specialimage.EmptyIndex(imgDataDir)
6174
assert.NilError(t, err)
6275

63-
err = tryLoad(ctx, t, imgDataDir, linuxAmd64)
64-
assert.Check(t, is.Error(err, "image emptyindex:latest was loaded, but doesn't provide the requested platform (linux/amd64)"))
76+
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxAmd64})
77+
assert.Check(t, is.Error(err, "image emptyindex:latest was loaded, but doesn't provide the requested platform ([linux/amd64])"))
6578
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
6679
})
67-
clearStore(ctx, t)
80+
cleanup(ctx, t)
6881

6982
t.Run("single platform", func(t *testing.T) {
7083
imgDataDir := t.TempDir()
7184
r := rand.NewSource(0x9127371238)
72-
_, err := specialimage.RandomSinglePlatform(imgDataDir, linuxAmd64, r)
85+
_, err = specialimage.RandomSinglePlatform(imgDataDir, linuxAmd64, r)
7386
assert.NilError(t, err)
7487

75-
err = tryLoad(ctx, t, imgDataDir, linuxArm64)
76-
assert.Check(t, is.ErrorContains(err, "doesn't provide the requested platform (linux/arm64)"))
88+
platforms := []ocispec.Platform{linuxAmd64}
89+
err = tryLoad(ctx, t, imgDataDir, platforms)
90+
assert.NilError(t, err)
91+
92+
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArm64})
93+
assert.Check(t, is.ErrorContains(err, "doesn't provide the requested platform ([linux/arm64])"))
7794
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
7895
})
96+
cleanup(ctx, t)
7997

80-
clearStore(ctx, t)
81-
82-
t.Run("2 platform image", func(t *testing.T) {
98+
t.Run("multi-platform image", func(t *testing.T) {
8399
imgDataDir := t.TempDir()
84-
_, mfstDescs, err := specialimage.MultiPlatform(imgDataDir, "multiplatform:latest", []ocispec.Platform{linuxAmd64, linuxArm64})
100+
imgRef := "multiplatform:latest"
101+
_, mfstDescs, err := specialimage.MultiPlatform(imgDataDir, imgRef, []ocispec.Platform{linuxAmd64, linuxArm64, linuxRiscv64})
85102
assert.NilError(t, err)
86103

104+
t.Run("one platform in index", func(t *testing.T) {
105+
platforms := []ocispec.Platform{linuxAmd64}
106+
err = tryLoad(ctx, t, imgDataDir, platforms)
107+
assert.NilError(t, err)
108+
109+
// verify that the loaded image has the correct platform
110+
err = verifyImagePlatforms(ctx, imgSvc, imgRef, platforms)
111+
assert.NilError(t, err)
112+
})
113+
cleanup(ctx, t)
114+
115+
t.Run("all platforms in index", func(t *testing.T) {
116+
platforms := []ocispec.Platform{linuxAmd64, linuxArm64, linuxRiscv64}
117+
err = tryLoad(ctx, t, imgDataDir, platforms)
118+
assert.NilError(t, err)
119+
120+
// verify that the loaded image has the correct platforms
121+
err = verifyImagePlatforms(ctx, imgSvc, imgRef, platforms)
122+
assert.NilError(t, err)
123+
})
124+
cleanup(ctx, t)
125+
87126
t.Run("platform not included in index", func(t *testing.T) {
88-
err = tryLoad(ctx, t, imgDataDir, linuxArmv5)
89-
assert.Check(t, is.Error(err, "image multiplatform:latest was loaded, but doesn't provide the requested platform (linux/arm/v5)"))
127+
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArmv5})
128+
assert.Check(t, is.Error(err, "image multiplatform:latest was loaded, but doesn't provide the requested platform ([linux/arm/v5])"))
90129
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
91130
})
131+
cleanup(ctx, t)
92132

93-
clearStore(ctx, t)
94-
95-
t.Run("platform blobs missing", func(t *testing.T) {
133+
t.Run("platform included but blobs missing", func(t *testing.T) {
96134
// Assumption: arm64 image is second in the index (implementation detail of specialimage.MultiPlatform)
97135
mfstDesc := mfstDescs[1]
98136
assert.Assert(t, mfstDesc.Platform.Architecture == linuxArm64.Architecture)
@@ -104,9 +142,37 @@ func TestImageLoadMissing(t *testing.T) {
104142
mfstPath := filepath.Join(imgDataDir, "blobs/sha256", mfstDesc.Digest.Encoded())
105143
assert.NilError(t, os.Remove(mfstPath))
106144

107-
err = tryLoad(ctx, t, imgDataDir, linuxArm64)
108-
assert.Check(t, is.ErrorContains(err, "requested platform (linux/arm64) found, but some content is missing"))
145+
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArm64})
146+
assert.Check(t, is.ErrorContains(err, "requested platform(s) ([linux/arm64]) found, but some content is missing"))
109147
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
110148
})
149+
cleanup(ctx, t)
111150
})
112151
}
152+
153+
func verifyImagePlatforms(ctx context.Context, imgSvc *ImageService, imgRef string, expectedPlatforms []ocispec.Platform) error {
154+
// get the manifest(s) for the image
155+
img, err := imgSvc.ImageInspect(ctx, imgRef, backend.ImageInspectOpts{Manifests: true})
156+
if err != nil {
157+
return err
158+
}
159+
// verify that the image manifest has the expected platforms
160+
for _, ep := range expectedPlatforms {
161+
want := platforms.FormatAll(ep)
162+
found := false
163+
for _, m := range img.Manifests {
164+
if m.Descriptor.Platform != nil {
165+
got := platforms.FormatAll(*m.Descriptor.Platform)
166+
if got == want {
167+
found = true
168+
break
169+
}
170+
}
171+
}
172+
if !found {
173+
return fmt.Errorf("expected platform %q not found in loaded images", want)
174+
}
175+
}
176+
177+
return nil
178+
}

0 commit comments

Comments
 (0)