Skip to content

Commit 7fea467

Browse files
vrothbergvrothberg
committed
support multi-image (docker) archives
Support loading and saving tarballs with more than one image. Add a new `/libpod/images/export` endpoint to the rest API to allow for exporting/saving multiple images into an archive. Note that a non-release version of containers/image is vendored. A release version must be vendored before cutting a new Podman release. We force the containers/image version via a replace in the go.mod file; this way go won't try to match the versions. Signed-off-by: Valentin Rothberg <[email protected]>
1 parent be7778d commit 7fea467

File tree

115 files changed

+3780
-1670
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

115 files changed

+3780
-1670
lines changed

‎cmd/podman/images/save.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ import (
1616
"golang.org/x/crypto/ssh/terminal"
1717
)
1818

19-
var validFormats = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
19+
var (
20+
validFormats = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
21+
containerConfig = registry.PodmanConfig()
22+
)
2023

2124
var (
2225
saveDescription = `Save an image to docker-archive or oci-archive on the local machine. Default is docker-archive.`
@@ -79,7 +82,7 @@ func saveFlags(flags *pflag.FlagSet) {
7982
flags.StringVar(&saveOpts.Format, "format", define.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
8083
flags.StringVarP(&saveOpts.Output, "output", "o", "", "Write to a specified file (default: stdout, which must be redirected)")
8184
flags.BoolVarP(&saveOpts.Quiet, "quiet", "q", false, "Suppress the output")
82-
85+
flags.BoolVarP(&saveOpts.MultiImageArchive, "multi-image-archive", "m", containerConfig.Engine.MultiImageArchive, "Interpret additional arguments as images not tags and create a multi-image-archive (only for docker-archive)")
8386
}
8487

8588
func save(cmd *cobra.Command, args []string) (finalErr error) {
@@ -118,6 +121,13 @@ func save(cmd *cobra.Command, args []string) (finalErr error) {
118121
if len(args) > 1 {
119122
tags = args[1:]
120123
}
124+
125+
// Decide whether c/image's progress bars should use stderr or stdout.
126+
// If the output is set of stdout, any log message there would corrupt
127+
// the tarfile.
128+
if saveOpts.Output == os.Stdout.Name() {
129+
saveOpts.Quiet = true
130+
}
121131
err := registry.ImageEngine().Save(context.Background(), args[0], tags, saveOpts)
122132
if err == nil {
123133
succeeded = true

‎docs/source/markdown/podman-save.1.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ Save image to **oci-archive, oci-dir** (directory with oci manifest type), or **
4040
--format docker-dir
4141
```
4242

43+
**--multi-image-archive**, **-m**
44+
45+
Allow for creating archives with more than one image. Additional names will be interpreted as images instead of tags. Only supported for **docker-archive**.
46+
4347
**--quiet**, **-q**
4448

4549
Suppress the output

‎go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ require (
1111
github.com/containernetworking/cni v0.8.0
1212
github.com/containernetworking/plugins v0.8.7
1313
github.com/containers/buildah v1.15.1-0.20200813183340-0a8dc1f8064c
14-
github.com/containers/common v0.20.3-0.20200827091701-a550d6a98aa3
14+
github.com/containers/common v0.21.0
1515
github.com/containers/conmon v2.0.20+incompatible
1616
github.com/containers/image/v5 v5.5.2
1717
github.com/containers/psgo v1.5.1
@@ -60,8 +60,10 @@ require (
6060
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
6161
golang.org/x/net v0.0.0-20200707034311-ab3426394381
6262
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
63-
golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
63+
golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed
6464
k8s.io/api v0.0.0-20190620084959-7cf5895f2711
6565
k8s.io/apimachinery v0.19.0
6666
k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
6767
)
68+
69+
replace github.com/containers/image/v5 => github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0

‎go.sum‎

Lines changed: 13 additions & 31 deletions
Large diffs are not rendered by default.

‎libpod/image/image.go‎

Lines changed: 171 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/containers/common/pkg/retry"
1818
cp "github.com/containers/image/v5/copy"
1919
"github.com/containers/image/v5/directory"
20+
"github.com/containers/image/v5/docker/archive"
2021
dockerarchive "github.com/containers/image/v5/docker/archive"
2122
"github.com/containers/image/v5/docker/reference"
2223
"github.com/containers/image/v5/image"
@@ -173,13 +174,182 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
173174
return newImage, nil
174175
}
175176

177+
// SaveImages stores one more images in a multi-image archive.
178+
// Note that only `docker-archive` supports storing multiple
179+
// image.
180+
func (ir *Runtime) SaveImages(ctx context.Context, namesOrIDs []string, format string, outputFile string, quiet bool) (finalErr error) {
181+
if format != DockerArchive {
182+
return errors.Errorf("multi-image archives are only supported in in the %q format", DockerArchive)
183+
}
184+
185+
sys := GetSystemContext("", "", false)
186+
187+
archWriter, err := archive.NewWriter(sys, outputFile)
188+
if err != nil {
189+
return err
190+
}
191+
defer func() {
192+
err := archWriter.Close()
193+
if err == nil {
194+
return
195+
}
196+
if finalErr == nil {
197+
finalErr = err
198+
return
199+
}
200+
finalErr = errors.Wrap(finalErr, err.Error())
201+
}()
202+
203+
// Decide whether c/image's progress bars should use stderr or stdout.
204+
// Use stderr in case we need to be quiet or if the output is set to
205+
// stdout. If the output is set of stdout, any log message there would
206+
// corrupt the tarfile.
207+
writer := os.Stdout
208+
if quiet {
209+
writer = os.Stderr
210+
}
211+
212+
// extend an image with additional tags
213+
type imageData struct {
214+
*Image
215+
tags []reference.NamedTagged
216+
}
217+
218+
// Look up the images (and their tags) in the local storage.
219+
imageMap := make(map[string]*imageData) // to group tags for an image
220+
imageQueue := []string{} // to preserve relative image order
221+
for _, nameOrID := range namesOrIDs {
222+
// Look up the name or ID in the local image storage.
223+
localImage, err := ir.NewFromLocal(nameOrID)
224+
if err != nil {
225+
return err
226+
}
227+
id := localImage.ID()
228+
229+
iData, exists := imageMap[id]
230+
if !exists {
231+
imageQueue = append(imageQueue, id)
232+
iData = &imageData{Image: localImage}
233+
imageMap[id] = iData
234+
}
235+
236+
// Unless we referred to an ID, add the input as a tag.
237+
if !strings.HasPrefix(id, nameOrID) {
238+
tag, err := NormalizedTag(nameOrID)
239+
if err != nil {
240+
return err
241+
}
242+
refTagged, isTagged := tag.(reference.NamedTagged)
243+
if isTagged {
244+
iData.tags = append(iData.tags, refTagged)
245+
}
246+
}
247+
}
248+
249+
policyContext, err := getPolicyContext(sys)
250+
if err != nil {
251+
return err
252+
}
253+
defer func() {
254+
if err := policyContext.Destroy(); err != nil {
255+
logrus.Errorf("failed to destroy policy context: %q", err)
256+
}
257+
}()
258+
259+
// Now copy the images one-by-one.
260+
for _, id := range imageQueue {
261+
dest, err := archWriter.NewReference(nil)
262+
if err != nil {
263+
return err
264+
}
265+
266+
img := imageMap[id]
267+
copyOptions := getCopyOptions(sys, writer, nil, nil, SigningOptions{}, "", img.tags)
268+
copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath()
269+
270+
// For copying, we need a source reference that we can create
271+
// from the image.
272+
src, err := is.Transport.NewStoreReference(img.imageruntime.store, nil, id)
273+
if err != nil {
274+
return errors.Wrapf(err, "error getting source imageReference for %q", img.InputName)
275+
}
276+
_, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
277+
if err != nil {
278+
return err
279+
}
280+
}
281+
282+
return nil
283+
}
284+
285+
// LoadAllImagesFromDockerArchive loads all images from the docker archive that
286+
// fileName points to.
287+
func (ir *Runtime) LoadAllImagesFromDockerArchive(ctx context.Context, fileName string, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
288+
if signaturePolicyPath == "" {
289+
signaturePolicyPath = ir.SignaturePolicyPath
290+
}
291+
292+
sc := GetSystemContext(signaturePolicyPath, "", false)
293+
reader, err := archive.NewReader(sc, fileName)
294+
if err != nil {
295+
return nil, err
296+
}
297+
298+
defer func() {
299+
if err := reader.Close(); err != nil {
300+
logrus.Errorf(err.Error())
301+
}
302+
}()
303+
304+
refLists, err := reader.List()
305+
if err != nil {
306+
return nil, err
307+
}
308+
309+
refPairs := []pullRefPair{}
310+
for _, refList := range refLists {
311+
for _, ref := range refList {
312+
pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, ref, sc)
313+
if err != nil {
314+
return nil, err
315+
}
316+
refPairs = append(refPairs, pairs...)
317+
}
318+
}
319+
320+
goal := pullGoal{
321+
pullAllPairs: true,
322+
usedSearchRegistries: false,
323+
refPairs: refPairs,
324+
searchedRegistries: nil,
325+
}
326+
327+
defer goal.cleanUp()
328+
imageNames, err := ir.doPullImage(ctx, sc, goal, writer, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{}, nil)
329+
if err != nil {
330+
return nil, err
331+
}
332+
333+
newImages := make([]*Image, 0, len(imageNames))
334+
for _, name := range imageNames {
335+
newImage, err := ir.NewFromLocal(name)
336+
if err != nil {
337+
return nil, errors.Wrapf(err, "error retrieving local image after pulling %s", name)
338+
}
339+
newImages = append(newImages, newImage)
340+
}
341+
ir.newImageEvent(events.LoadFromArchive, "")
342+
return newImages, nil
343+
}
344+
176345
// LoadFromArchiveReference creates a new image object for images pulled from a tar archive and the like (podman load)
177346
// This function is needed because it is possible for a tar archive to have multiple tags for one image
178347
func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
179348
if signaturePolicyPath == "" {
180349
signaturePolicyPath = ir.SignaturePolicyPath
181350
}
182-
imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
351+
352+
imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{})
183353
if err != nil {
184354
return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
185355
}

0 commit comments

Comments
 (0)