Add option to pull all metadata

dmcgowan · dmcgowan · commit a40c3830df92 · 2019-08-26T11:05:21.000-07:00
Add flags to pull and fetch to grab all metadata.
Add fetch option to pull only metadata.

Signed-off-by: Derek McGowan &lt;derek@mcgstyle.net&gt;
diff --git a/client.go b/client.go
@@ -333,9 +333,8 @@ type RemoteContext struct {
 	// MaxConcurrentDownloads is the max concurrent content downloads for each pull.
 	MaxConcurrentDownloads int
 
-	// AppendDistributionSourceLabel allows fetcher to add distribute source
-	// label for each blob content, which doesn't work for legacy schema1.
-	AppendDistributionSourceLabel bool
+	// AllMetadata downloads all manifests and known-configuration files
+	AllMetadata bool
 }
 
 func defaultRemoteContext() *RemoteContext {
diff --git a/client_opts.go b/client_opts.go
@@ -195,11 +195,10 @@ func WithMaxConcurrentDownloads(max int) RemoteOpt {
 	}
 }
 
-// WithAppendDistributionSourceLabel allows fetcher to add distribute source
-// label for each blob content, which doesn't work for legacy schema1.
-func WithAppendDistributionSourceLabel() RemoteOpt {
+// WithAllMetadata downloads all manifests and known-configuration files
+func WithAllMetadata() RemoteOpt {
 	return func(_ *Client, c *RemoteContext) error {
-		c.AppendDistributionSourceLabel = true
+		c.AllMetadata = true
 		return nil
 	}
 }
diff --git a/cmd/ctr/commands/content/fetch.go b/cmd/ctr/commands/content/fetch.go
@@ -67,8 +67,12 @@ Most of this is experimental and there are few leaps to make this work.`,
 			Usage: "pull content from all platforms",
 		},
 		cli.BoolFlag{
-			Name:  "all-manifests",
-			Usage: "Pull manifests from all platforms and layers for a specific platform",
+			Name:  "all-metadata",
+			Usage: "Pull metadata for all platforms",
+		},
+		cli.BoolFlag{
+			Name:  "metadata-only",
+			Usage: "Pull all metadata including manifests and configs",
 		},
 	),
 	Action: func(clicontext *cli.Context) error {
@@ -84,6 +88,7 @@ Most of this is experimental and there are few leaps to make this work.`,
 		if err != nil {
 			return err
 		}
+
 		_, err = Fetch(ctx, client, ref, config)
 		return err
 	},
@@ -97,10 +102,12 @@ type FetchConfig struct {
 	ProgressOutput io.Writer
 	// Labels to set on the content
 	Labels []string
+	// PlatformMatcher matches platforms, supersedes Platforms
+	PlatformMatcher platforms.MatchComparer
 	// Platforms to fetch
 	Platforms []string
-	// Whether or not download all manifests
-	IsAllManifests bool
+	// Whether or not download all metadata
+	AllMetadata bool
 }
 
 // NewFetchConfig returns the default FetchConfig from cli flags
@@ -124,7 +131,13 @@ func NewFetchConfig(ctx context.Context, clicontext *cli.Context) (*FetchConfig,
 		config.Platforms = p
 	}
 
-	config.IsAllManifests = clicontext.Bool("all-manifests")
+	if clicontext.Bool("metadata-only") {
+		config.AllMetadata = true
+		// Any with an empty set is None
+		config.PlatformMatcher = platforms.Any()
+	} else if clicontext.Bool("all-metadata") {
+		config.AllMetadata = true
+	}
 
 	return config, nil
 }
@@ -160,12 +173,16 @@ func Fetch(ctx context.Context, client *containerd.Client, ref string, config *F
 		containerd.WithSchema1Conversion,
 	}
 
-	if config.IsAllManifests {
-		opts = append(opts, containerd.WithAppendDistributionSourceLabel())
+	if config.AllMetadata {
+		opts = append(opts, containerd.WithAllMetadata())
 	}
 
-	for _, platform := range config.Platforms {
-		opts = append(opts, containerd.WithPlatform(platform))
+	if config.PlatformMatcher != nil {
+		opts = append(opts, containerd.WithPlatformMatcher(config.PlatformMatcher))
+	} else {
+		for _, platform := range config.Platforms {
+			opts = append(opts, containerd.WithPlatform(platform))
+		}
 	}
 
 	img, err := client.Fetch(pctx, ref, opts...)
diff --git a/cmd/ctr/commands/images/pull.go b/cmd/ctr/commands/images/pull.go
@@ -51,11 +51,11 @@ command. As part of this process, we do the following:
 		},
 		cli.BoolFlag{
 			Name:  "all-platforms",
-			Usage: "pull content from all platforms",
+			Usage: "pull content and metadata from all platforms",
 		},
 		cli.BoolFlag{
-			Name:  "all-manifests",
-			Usage: "Pull manifests from all platforms and layers for a specific platform",
+			Name:  "all-metadata",
+			Usage: "Pull metadata for all platforms",
 		},
 	),
 	Action: func(context *cli.Context) error {
diff --git a/image_test.go b/image_test.go
@@ -99,9 +99,7 @@ func TestImagePullWithDistSourceLabel(t *testing.T) {
 	pMatcher := platforms.Default()
 
 	// pull content without unpack and add distribution source label
-	image, err := client.Pull(ctx, imageName,
-		WithPlatformMatcher(pMatcher),
-		WithAppendDistributionSourceLabel())
+	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(pMatcher))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +181,7 @@ func TestImageUsage(t *testing.T) {
 	imageName = imageName + "@" + image.Target().Digest.String()
 
 	// Fetch single platforms, but all manifests pulled
-	if _, err := client.Fetch(ctx, imageName, WithPlatformMatcher(testPlatform)); err != nil {
+	if _, err := client.Fetch(ctx, imageName, WithPlatformMatcher(testPlatform), WithAllMetadata()); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/pull.go b/pull.go
@@ -140,7 +140,7 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
 		childrenHandler := images.ChildrenHandler(store)
 		// Set any children labels for that content
 		childrenHandler = images.SetChildrenLabels(store, childrenHandler)
-		if rCtx.AppendDistributionSourceLabel {
+		if rCtx.AllMetadata {
 			// Filter manifests by platforms but allow to handle manifest
 			// and configuration for not-target platforms
 			childrenHandler = remotes.FilterManifestByPlatformHandler(childrenHandler, rCtx.PlatformMatcher)
@@ -164,22 +164,18 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
 			},
 		)
 
+		appendDistSrcLabelHandler, err := docker.AppendDistributionSourceLabel(store, ref)
+		if err != nil {
+			return images.Image{}, err
+		}
+
 		handlers := append(rCtx.BaseHandlers,
 			remotes.FetchHandler(store, fetcher),
 			convertibleHandler,
 			childrenHandler,
+			appendDistSrcLabelHandler,
 		)
 
-		// append distribution source label to blob data
-		if rCtx.AppendDistributionSourceLabel {
-			appendDistSrcLabelHandler, err := docker.AppendDistributionSourceLabel(store, ref)
-			if err != nil {
-				return images.Image{}, err
-			}
-
-			handlers = append(handlers, appendDistSrcLabelHandler)
-		}
-
 		handler = images.Handlers(handlers...)
 
 		converterFunc = func(ctx context.Context, desc ocispec.Descriptor) (ocispec.Descriptor, error) {

Original file line number	Diff line number	Diff line change
`@@ -195,11 +195,10 @@ func WithMaxConcurrentDownloads(max int) RemoteOpt {`
`195`	`195`	`}`
`196`	`196`	`}`
`197`	`197`
`198`		`-// WithAppendDistributionSourceLabel allows fetcher to add distribute source`
`199`		`-// label for each blob content, which doesn't work for legacy schema1.`
`200`		`-func WithAppendDistributionSourceLabel() RemoteOpt {`
	`198`	`+// WithAllMetadata downloads all manifests and known-configuration files`
	`199`	`+func WithAllMetadata() RemoteOpt {`
`201`	`200`	`return func(_ Client, c RemoteContext) error {`
`202`		`- c.AppendDistributionSourceLabel = true`
	`201`	`+ c.AllMetadata = true`
`203`	`202`	`return nil`
`204`	`203`	`}`
`205`	`204`	`}`