Skip to content

Commit 40c67fd

Browse files
dmcgowandmcgowan
authored
Merge pull request #1880 from AkihiroSuda/refactor-importer
importer: refactor and fix GC
2 parents 7e1bc9c + 6340197 commit 40c67fd

11 files changed

Lines changed: 394 additions & 292 deletions

File tree

‎client.go‎

Lines changed: 49 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
"github.com/containerd/containerd/namespaces"
3131
"github.com/containerd/containerd/platforms"
3232
"github.com/containerd/containerd/plugin"
33-
"github.com/containerd/containerd/reference"
3433
"github.com/containerd/containerd/remotes"
3534
"github.com/containerd/containerd/remotes/docker"
3635
"github.com/containerd/containerd/remotes/docker/schema1"
@@ -503,95 +502,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
503502
}, nil
504503
}
505504

506-
type imageFormat string
507-
508-
const (
509-
ociImageFormat imageFormat = "oci"
510-
)
511-
512505
type importOpts struct {
513-
format imageFormat
514-
refObject string
515-
labels map[string]string
516506
}
517507

518508
// ImportOpt allows the caller to specify import specific options
519509
type ImportOpt func(c *importOpts) error
520510

521-
// WithImportLabel sets a label to be associated with an imported image
522-
func WithImportLabel(key, value string) ImportOpt {
523-
return func(opts *importOpts) error {
524-
if opts.labels == nil {
525-
opts.labels = make(map[string]string)
526-
}
527-
528-
opts.labels[key] = value
529-
return nil
530-
}
531-
}
532-
533-
// WithImportLabels associates a set of labels to an imported image
534-
func WithImportLabels(labels map[string]string) ImportOpt {
535-
return func(opts *importOpts) error {
536-
if opts.labels == nil {
537-
opts.labels = make(map[string]string)
538-
}
539-
540-
for k, v := range labels {
541-
opts.labels[k] = v
542-
}
543-
return nil
544-
}
545-
}
546-
547-
// WithOCIImportFormat sets the import format for an OCI image format
548-
func WithOCIImportFormat() ImportOpt {
549-
return func(c *importOpts) error {
550-
if c.format != "" {
551-
return errors.New("format already set")
552-
}
553-
c.format = ociImageFormat
554-
return nil
555-
}
556-
}
557-
558-
// WithRefObject specifies the ref object to import.
559-
// If refObject is empty, it is copied from the ref argument of Import().
560-
func WithRefObject(refObject string) ImportOpt {
561-
return func(c *importOpts) error {
562-
c.refObject = refObject
563-
return nil
564-
}
565-
}
566-
567-
func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
511+
func resolveImportOpt(opts ...ImportOpt) (importOpts, error) {
568512
var iopts importOpts
569513
for _, o := range opts {
570514
if err := o(&iopts); err != nil {
571515
return iopts, err
572516
}
573517
}
574-
// use OCI as the default format
575-
if iopts.format == "" {
576-
iopts.format = ociImageFormat
577-
}
578-
// if refObject is not explicitly specified, use the one specified in ref
579-
if iopts.refObject == "" {
580-
refSpec, err := reference.Parse(ref)
581-
if err != nil {
582-
return iopts, err
583-
}
584-
iopts.refObject = refSpec.Object
585-
}
586518
return iopts, nil
587519
}
588520

589521
// Import imports an image from a Tar stream using reader.
590-
// OCI format is assumed by default.
591-
//
592-
// Note that unreferenced blobs are imported to the content store as well.
593-
func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
594-
iopts, err := resolveImportOpt(ref, opts...)
522+
// Caller needs to specify importer. Future version may use oci.v1 as the default.
523+
// Note that unreferrenced blobs may be imported to the content store as well.
524+
func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) {
525+
_, err := resolveImportOpt(opts...) // unused now
595526
if err != nil {
596527
return nil, err
597528
}
@@ -602,58 +533,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts
602533
}
603534
defer done()
604535

605-
switch iopts.format {
606-
case ociImageFormat:
607-
return c.importFromOCITar(ctx, ref, reader, iopts)
608-
default:
609-
return nil, errors.Errorf("unsupported format: %s", iopts.format)
536+
imgrecs, err := importer.Import(ctx, c.ContentStore(), reader)
537+
if err != nil {
538+
// is.Update() is not called on error
539+
return nil, err
540+
}
541+
542+
is := c.ImageService()
543+
var images []Image
544+
for _, imgrec := range imgrecs {
545+
if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
546+
if !errdefs.IsNotFound(err) {
547+
return nil, err
548+
}
549+
550+
created, err := is.Create(ctx, imgrec)
551+
if err != nil {
552+
return nil, err
553+
}
554+
555+
imgrec = created
556+
} else {
557+
imgrec = updated
558+
}
559+
560+
images = append(images, &image{
561+
client: c,
562+
i: imgrec,
563+
})
610564
}
565+
return images, nil
611566
}
612567

613568
type exportOpts struct {
614-
format imageFormat
615569
}
616570

617-
// ExportOpt allows callers to set export options
571+
// ExportOpt allows the caller to specify export-specific options
618572
type ExportOpt func(c *exportOpts) error
619573

620-
// WithOCIExportFormat sets the OCI image format as the export target
621-
func WithOCIExportFormat() ExportOpt {
622-
return func(c *exportOpts) error {
623-
if c.format != "" {
624-
return errors.New("format already set")
574+
func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) {
575+
var eopts exportOpts
576+
for _, o := range opts {
577+
if err := o(&eopts); err != nil {
578+
return eopts, err
625579
}
626-
c.format = ociImageFormat
627-
return nil
628580
}
581+
return eopts, nil
629582
}
630583

631-
// TODO: add WithMediaTypeTranslation that transforms media types according to the format.
632-
// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
633-
// -> application/vnd.oci.image.layer.v1.tar+gzip
634-
635584
// Export exports an image to a Tar stream.
636585
// OCI format is used by default.
637586
// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
638-
func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
639-
var eopts exportOpts
640-
for _, o := range opts {
641-
if err := o(&eopts); err != nil {
642-
return nil, err
643-
}
644-
}
645-
// use OCI as the default format
646-
if eopts.format == "" {
647-
eopts.format = ociImageFormat
587+
// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream.
588+
func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
589+
_, err := resolveExportOpt(opts...) // unused now
590+
if err != nil {
591+
return nil, err
648592
}
649593
pr, pw := io.Pipe()
650-
switch eopts.format {
651-
case ociImageFormat:
652-
go func() {
653-
pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts))
654-
}()
655-
default:
656-
return nil, errors.Errorf("unsupported format: %s", eopts.format)
657-
}
594+
go func() {
595+
pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw))
596+
}()
658597
return pr, nil
659598
}

‎cmd/ctr/commands/images/export.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"os"
66

77
"github.com/containerd/containerd/cmd/ctr/commands"
8+
oci "github.com/containerd/containerd/images/oci"
89
"github.com/containerd/containerd/reference"
910
digest "github.com/opencontainers/go-digest"
1011
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -13,11 +14,14 @@ import (
1314
)
1415

1516
var exportCommand = cli.Command{
16-
Name: "export",
17-
Usage: "export an image",
18-
ArgsUsage: "[flags] <out> <image>",
19-
Description: "export an image to a tar stream",
17+
Name: "export",
18+
Usage: "export an image",
19+
ArgsUsage: "[flags] <out> <image>",
20+
Description: `Export an image to a tar stream.
21+
Currently, only OCI format is supported.
22+
`,
2023
Flags: []cli.Flag{
24+
// TODO(AkihiroSuda): make this map[string]string as in moby/moby#33355?
2125
cli.StringFlag{
2226
Name: "oci-ref-name",
2327
Value: "",
@@ -78,7 +82,7 @@ var exportCommand = cli.Command{
7882
return nil
7983
}
8084
}
81-
r, err := client.Export(ctx, desc)
85+
r, err := client.Export(ctx, &oci.V1Exporter{}, desc)
8286
if err != nil {
8387
return err
8488
}

‎cmd/ctr/commands/images/import.go‎

Lines changed: 56 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,37 +5,66 @@ import (
55
"io"
66
"os"
77

8-
"github.com/containerd/containerd"
98
"github.com/containerd/containerd/cmd/ctr/commands"
9+
"github.com/containerd/containerd/images"
10+
oci "github.com/containerd/containerd/images/oci"
1011
"github.com/containerd/containerd/log"
1112
"github.com/urfave/cli"
1213
)
1314

1415
var importCommand = cli.Command{
15-
Name: "import",
16-
Usage: "import an image",
17-
ArgsUsage: "[flags] <ref> <in>",
18-
Description: "import an image from a tar stream",
19-
Flags: []cli.Flag{
16+
Name: "import",
17+
Usage: "import images",
18+
ArgsUsage: "[flags] <in>",
19+
Description: `Import images from a tar stream.
20+
Implemented formats:
21+
- oci.v1 (default)
22+
23+
24+
For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags)
25+
but does not contain the base image name.
26+
27+
e.g.
28+
$ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar
29+
30+
If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadbeef", the command will create
31+
"foo/bar:latest" and "foo/bar@sha256:deadbeef" images in the containerd store.
32+
`,
33+
Flags: append([]cli.Flag{
2034
cli.StringFlag{
21-
Name: "ref-object",
22-
Value: "",
23-
Usage: "reference object e.g. tag@digest (default: use the object specified in ref)",
35+
Name: "format",
36+
Value: "oci.v1",
37+
Usage: "image format. See DESCRIPTION.",
2438
},
25-
commands.LabelFlag,
26-
},
39+
cli.StringFlag{
40+
Name: "oci-name",
41+
Value: "unknown/unknown",
42+
Usage: "prefix added to either oci.v1 ref annotation or digest",
43+
},
44+
// TODO(AkihiroSuda): support commands.LabelFlag (for all children objects)
45+
}, commands.SnapshotterFlags...),
46+
2747
Action: func(context *cli.Context) error {
2848
var (
29-
ref = context.Args().First()
30-
in = context.Args().Get(1)
31-
refObject = context.String("ref-object")
32-
labels = commands.LabelArgs(context.StringSlice("label"))
49+
in = context.Args().First()
50+
imageImporter images.Importer
3351
)
52+
53+
switch format := context.String("format"); format {
54+
case "oci.v1":
55+
imageImporter = &oci.V1Importer{
56+
ImageName: context.String("oci-name"),
57+
}
58+
default:
59+
return fmt.Errorf("unknown format %s", format)
60+
}
61+
3462
client, ctx, cancel, err := commands.NewClient(context)
3563
if err != nil {
3664
return err
3765
}
3866
defer cancel()
67+
3968
var r io.ReadCloser
4069
if in == "-" {
4170
r = os.Stdin
@@ -45,25 +74,25 @@ var importCommand = cli.Command{
4574
return err
4675
}
4776
}
48-
img, err := client.Import(ctx,
49-
ref,
50-
r,
51-
containerd.WithRefObject(refObject),
52-
containerd.WithImportLabels(labels),
53-
)
77+
imgs, err := client.Import(ctx, imageImporter, r)
5478
if err != nil {
5579
return err
5680
}
5781
if err = r.Close(); err != nil {
5882
return err
5983
}
6084

61-
log.G(ctx).WithField("image", ref).Debug("unpacking")
85+
log.G(ctx).Debugf("unpacking %d images", len(imgs))
6286

63-
// TODO: Show unpack status
64-
fmt.Printf("unpacking %s...", img.Target().Digest)
65-
err = img.Unpack(ctx, context.String("snapshotter"))
66-
fmt.Println("done")
67-
return err
87+
for _, img := range imgs {
88+
// TODO: Show unpack status
89+
fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest)
90+
err = img.Unpack(ctx, context.String("snapshotter"))
91+
if err != nil {
92+
return err
93+
}
94+
fmt.Println("done")
95+
}
96+
return nil
6897
},
6998
}

‎export_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@ import (
55
"io"
66
"runtime"
77
"testing"
8+
9+
"github.com/containerd/containerd/images/oci"
810
)
911

10-
// TestExport exports testImage as a tar stream
11-
func TestExport(t *testing.T) {
12+
// TestOCIExport exports testImage as a tar stream
13+
func TestOCIExport(t *testing.T) {
1214
// TODO: support windows
1315
if testing.Short() || runtime.GOOS == "windows" {
1416
t.Skip()
@@ -26,8 +28,7 @@ func TestExport(t *testing.T) {
2628
if err != nil {
2729
t.Fatal(err)
2830
}
29-
30-
exportedStream, err := client.Export(ctx, pulled.Target())
31+
exportedStream, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target())
3132
if err != nil {
3233
t.Fatal(err)
3334
}

0 commit comments

Comments
 (0)