Skip to content

Add configuration options to local transfer service #8062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fuweid merged 1 commit into from
Feb 25, 2023
Merged

Add configuration options to local transfer service #8062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions cmd/ctr/commands/images/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ import (
"os"
"time"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/urfave/cli"

"github.com/containerd/containerd"
"github.com/containerd/containerd/cmd/ctr/commands"
"github.com/containerd/containerd/images/archive"
Expand All @@ -30,7 +33,6 @@ import (
tarchive "github.com/containerd/containerd/pkg/transfer/archive"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/platforms"
"github.com/urfave/cli"
)

var importCommand = cli.Command{
Expand Down Expand Up @@ -127,9 +129,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
opts = append(opts, image.WithNamedPrefix(prefix, overwrite))
}

// TODO: Add platform options
var platSpec ocispec.Platform
//Only when all-platforms not specified, we will check platform value
Comment thread
fangn2 marked this conversation as resolved.
//Implicitly if the platforms is empty, it means all-platforms
if !context.Bool("all-platforms") {
//If platform specified, use that one, if not use default
if platform := context.String("platform"); platform != "" {
platSpec, err = platforms.Parse(platform)
if err != nil {
return err
}
} else {
platSpec = platforms.DefaultSpec()
}
opts = append(opts, image.WithPlatforms(platSpec))
}

// TODO: Add unpack options
if !context.Bool("no-unpack") {
snapshotter := context.String("snapshotter")
//If OS field is not empty, it means platSpec was updated in the above block
//i.e all-platforms was not specified
if platSpec.OS != "" {
opts = append(opts, image.WithUnpack(platSpec, snapshotter))
} else {
//empty spec means all platforms
var emptySpec ocispec.Platform
opts = append(opts, image.WithUnpack(emptySpec, snapshotter))
}
}

is := image.NewStore(context.String("index-name"), opts...)

Expand Down
8 changes: 7 additions & 1 deletion cmd/ctr/commands/images/pull.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/containerd/containerd/pkg/progress"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/platforms"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -112,6 +113,11 @@ command. As part of this process, we do the following:
p = append(p, platforms.DefaultSpec())
}
sopts = append(sopts, image.WithPlatforms(p...))

//set unpack configuration
for _, platform := range p {
sopts = append(sopts, image.WithUnpack(platform, context.String("snapshotter")))
}
}
// TODO: Support unpack for all platforms..?
// Pass in a *?
Expand All @@ -125,7 +131,7 @@ command. As part of this process, we do the following:
sopts = append(sopts, image.WithAllMetadata)
}

reg := image.NewOCIRegistry(ref, nil, ch)
reg := registry.NewOCIRegistry(ref, nil, ch)
is := image.NewStore(ref, sopts...)

pf, done := ProgressHandler(ctx, os.Stdout)
Expand Down
3 changes: 2 additions & 1 deletion cmd/ctr/commands/images/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/containerd/containerd/pkg/progress"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
Expand Down Expand Up @@ -103,7 +104,7 @@ var pushCommand = cli.Command{
if local == "" {
local = ref
}
reg := image.NewOCIRegistry(ref, nil, ch)
reg := registry.NewOCIRegistry(ref, nil, ch)
is := image.NewStore(local)

pf, done := ProgressHandler(ctx, os.Stdout)
Expand Down
10 changes: 5 additions & 5 deletions cmd/ctr/commands/resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (

"github.com/containerd/console"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/transfer/image"
"github.com/containerd/containerd/pkg/transfer/registry"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/config"
Expand Down Expand Up @@ -218,7 +218,7 @@ type staticCredentials struct {
}

// NewStaticCredentials gets credentials from passing in cli context
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (image.CredentialHelper, error) {
func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (registry.CredentialHelper, error) {
username := clicontext.String("user")
var secret string
if i := strings.IndexByte(username, ':'); i > 0 {
Expand Down Expand Up @@ -248,12 +248,12 @@ func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref st
}, nil
}

func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (image.Credentials, error) {
func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (registry.Credentials, error) {
if ref == sc.ref {
return image.Credentials{
return registry.Credentials{
Username: sc.username,
Secret: sc.secret,
}, nil
}
return image.Credentials{}, nil
return registry.Credentials{}, nil
}
28 changes: 10 additions & 18 deletions pkg/transfer/image/imagestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/images/archive"
"github.com/containerd/containerd/pkg/streaming"
"github.com/containerd/containerd/pkg/transfer"
"github.com/containerd/containerd/pkg/transfer/plugins"
"github.com/containerd/containerd/pkg/unpack"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/typeurl/v2"
Expand All @@ -51,7 +51,7 @@ type Store struct {
// extraReferences are used to store or lookup multiple references
extraReferences []Reference

unpacks []UnpackConfiguration
unpacks []transfer.UnpackConfiguration
}

// Reference is used to create or find a reference for an image
Expand Down Expand Up @@ -84,14 +84,6 @@ type Reference struct {
SkipNamedDigest bool
}

// UnpackConfiguration specifies the platform and snapshotter to use for resolving
// the unpack Platform, if snapshotter is not specified the platform default will
// be used.
type UnpackConfiguration struct {
Platform ocispec.Platform
Snapshotter string
}

// StoreOpt defines options when configuring an image store source or destination
type StoreOpt func(*Store)

Expand Down Expand Up @@ -171,7 +163,7 @@ func WithExtraReference(name string) StoreOpt {
// WithUnpack specifies a platform to unpack for and an optional snapshotter to use
func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt {
return func(s *Store) {
s.unpacks = append(s.unpacks, UnpackConfiguration{
s.unpacks = append(s.unpacks, transfer.UnpackConfiguration{
Platform: p,
Snapshotter: snapshotter,
})
Expand Down Expand Up @@ -333,11 +325,11 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err
return store.Get(ctx, is.imageName)
}

func (is *Store) UnpackPlatforms() []unpack.Platform {
unpacks := make([]unpack.Platform, len(is.unpacks))
func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration {
unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks))
for i, uc := range is.unpacks {
unpacks[i].SnapshotterKey = uc.Snapshotter
unpacks[i].Platform = platforms.Only(uc.Platform)
unpacks[i].Snapshotter = uc.Snapshotter
unpacks[i].Platform = uc.Platform
}
return unpacks
}
Expand Down Expand Up @@ -424,7 +416,7 @@ func referencesFromProto(references []*transfertypes.ImageReference) []Reference
}
return or
}
func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration {
func unpackToProto(uc []transfer.UnpackConfiguration) []*transfertypes.UnpackConfiguration {
auc := make([]*transfertypes.UnpackConfiguration, len(uc))
for i := range uc {
p := types.Platform{
Expand All @@ -440,8 +432,8 @@ func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguratio
return auc
}

func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfiguration {
uc := make([]UnpackConfiguration, len(auc))
func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []transfer.UnpackConfiguration {
uc := make([]transfer.UnpackConfiguration, len(auc))
for i := range auc {
uc[i].Snapshotter = auc[i].Snapshotter
if auc[i].Platform != nil {
Expand Down
41 changes: 37 additions & 4 deletions pkg/transfer/local/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@ package local
import (
"context"
"encoding/json"
"fmt"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/transfer"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/pkg/unpack"
)

func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error {
Expand All @@ -46,12 +49,16 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
return err
}

var descriptors []ocispec.Descriptor
var (
descriptors []ocispec.Descriptor
handler images.Handler
unpacker *unpack.Unpacker
)

// If save index, add index
descriptors = append(descriptors, index)

var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var handlerFunc images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// Only save images at top level
if desc.Digest != index.Digest {
return images.Children(ctx, ts.content, desc)
Expand All @@ -76,7 +83,33 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
}

if f, ok := is.(transfer.ImageFilterer); ok {
handler = f.ImageFilter(handler, ts.content)
handlerFunc = f.ImageFilter(handlerFunc, ts.content)
}

handler = images.Handlers(handlerFunc)

// First find suitable platforms to unpack into
//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
if iu, ok := is.(transfer.ImageUnpacker); ok {
unpacks := iu.UnpackPlatforms()
if len(unpacks) > 0 {
uopts := []unpack.UnpackerOpt{}
for _, u := range unpacks {
matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
if matched {
uopts = append(uopts, unpack.WithUnpackPlatform(mu))
}
}

if ts.config.DuplicationSuppressor != nil {
uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
}
unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
if err != nil {
return fmt.Errorf("unable to initialize unpacker: %w", err)
}
handler = unpacker.Unpack(handler)
}
}

if err := images.WalkNotEmpty(ctx, handler, index); err != nil {
Expand Down
Loading