Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 76 additions & 3 deletions cmd/ctr/commands/images/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ package images
import (
"errors"
"fmt"
"strings"

"github.com/containerd/containerd/v2/cmd/ctr/commands"
"github.com/containerd/containerd/v2/core/images/converter"
"github.com/containerd/containerd/v2/core/images/converter/erofs"
"github.com/containerd/containerd/v2/core/images/converter/uncompress"
"github.com/containerd/containerd/v2/internal/erofsutils/seekable"
"github.com/containerd/platforms"
"github.com/urfave/cli/v2"
)
Expand All @@ -33,7 +36,11 @@ var convertCommand = &cli.Command{
ArgsUsage: "[flags] <source_ref> <target_ref>",
Description: `Convert an image format.

e.g., 'ctr convert --uncompress --oci example.com/foo:orig example.com/foo:converted'
e.g., 'ctr images convert --uncompress --oci example.com/foo:orig example.com/foo:converted'
'ctr images convert --erofs example.com/foo:orig example.com/foo:erofs'
'ctr images convert --erofs --erofs-compression='lzma:lz4hc,12' example.com/foo:orig example.com/foo:erofs'
'ctr images convert --erofs-seekable example.com/foo:orig example.com/foo:seekable-erofs'
'ctr images convert --erofs-seekable --erofs-dm-verity example.com/foo:orig example.com/foo:seekable-erofs-verity'

Use '--platform' to define the output platform.
When '--all-platforms' is given all images in a manifest list must be available.
Expand All @@ -48,6 +55,38 @@ When '--all-platforms' is given all images in a manifest list must be available.
Name: "oci",
Usage: "Convert Docker media types to OCI media types",
},
// erofs flags
&cli.BoolFlag{
Name: "erofs",
Usage: "Convert layers to EROFS format",
},
&cli.StringFlag{
Name: "erofs-compression",
Usage: "Compression algorithms for EROFS, separated by colon (e.g., 'lz4:lz4hc,12')",
},
&cli.StringFlag{
Name: "erofs-mkfs-opts",
Usage: "Extra options for mkfs.erofs (e.g., '-zlz4')",
},
// seekable erofs flags
&cli.BoolFlag{
Name: "erofs-seekable",
Usage: "Wrap EROFS in seekable zstd frames with custom chunk table",
},
&cli.IntFlag{
Name: "erofs-chunk-size",
Usage: "Uncompressed bytes per zstd frame (random access granularity)",
Value: seekable.DefaultChunkSize,
},
&cli.BoolFlag{
Name: "erofs-dm-verity",
Usage: "Include dm-verity payload at EOF",
},
&cli.IntFlag{
Name: "erofs-dm-verity-block-size",
Usage: "dm-verity block size in bytes",
Value: seekable.DefaultDMVerityBlockSize,
},
// platform flags
&cli.StringSliceFlag{
Name: "platform",
Expand All @@ -68,8 +107,8 @@ When '--all-platforms' is given all images in a manifest list must be available.
}

if !cliContext.Bool("all-platforms") {
if pss := cliContext.StringSlice("platform"); len(pss) > 0 {
all, err := platforms.ParseAll(pss)
if platformStrs := cliContext.StringSlice("platform"); len(platformStrs) > 0 {
all, err := platforms.ParseAll(platformStrs)
if err != nil {
return err
}
Expand All @@ -83,6 +122,28 @@ When '--all-platforms' is given all images in a manifest list must be available.
convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
}

if cliContext.Bool("erofs-seekable") {
// Seekable EROFS: wrap raw EROFS in zstd frames with chunk table.
var seekableOpts []erofs.SeekableConvertOpt
if cliContext.IsSet("erofs-chunk-size") {
seekableOpts = append(seekableOpts, erofs.WithSeekableChunkSize(cliContext.Int("erofs-chunk-size")))
}
seekableOpts = append(seekableOpts, erofs.WithSeekableDMVerity(cliContext.Bool("erofs-dm-verity")))
if cliContext.IsSet("erofs-dm-verity-block-size") {
seekableOpts = append(seekableOpts, erofs.WithSeekableDMVerityBlockSize(cliContext.Int("erofs-dm-verity-block-size")))
}
// Pass through raw EROFS options (compression, mkfs-opts).
rawOpts := buildErofsOpts(cliContext)
if len(rawOpts) > 0 {
seekableOpts = append(seekableOpts, erofs.WithSeekableRawErofsOpts(rawOpts...))
}
convertOpts = append(convertOpts, converter.WithLayerConvertFunc(erofs.SeekableLayerConvertFunc(seekableOpts...)))
} else if cliContext.Bool("erofs") {
// Raw EROFS (non-seekable).
erofsOpts := buildErofsOpts(cliContext)
convertOpts = append(convertOpts, converter.WithLayerConvertFunc(erofs.LayerConvertFunc(erofsOpts...)))
}

if cliContext.Bool("oci") {
convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
}
Expand All @@ -101,3 +162,15 @@ When '--all-platforms' is given all images in a manifest list must be available.
return nil
},
}

func buildErofsOpts(cliContext *cli.Context) []erofs.ConvertOpt {
var opts []erofs.ConvertOpt
if compressors := cliContext.String("erofs-compression"); compressors != "" {
opts = append(opts, erofs.WithCompressors(compressors))
}
if mkfsOptsStr := cliContext.String("erofs-mkfs-opts"); mkfsOptsStr != "" {
mkfsOpts := strings.Fields(mkfsOptsStr)
opts = append(opts, erofs.WithMkfsOptions(mkfsOpts))
}
return opts
}
172 changes: 172 additions & 0 deletions core/images/converter/erofs/erofs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
/*
Copyright The containerd Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package erofs

import (
"context"
"fmt"
"io"
"os"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"

"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/core/images"
"github.com/containerd/containerd/v2/core/images/converter"
"github.com/containerd/containerd/v2/core/images/converter/uncompress"
"github.com/containerd/containerd/v2/internal/erofsutils"
"github.com/containerd/containerd/v2/pkg/labels"
"github.com/containerd/containerd/v2/plugins/diff/erofs"
"github.com/containerd/errdefs"
"github.com/containerd/log"
)

type ConvertOpt func(*convertOptions)

type convertOptions struct {
compressors string
mkfsExtraOpts []string
}

func WithCompressors(compressors string) ConvertOpt {
return func(opts *convertOptions) {
opts.compressors = compressors
}
}

func WithMkfsOptions(extraOpts []string) ConvertOpt {
return func(opts *convertOptions) {
opts.mkfsExtraOpts = extraOpts
}
}

// LayerConvertFunc converts tar.gz layers into EROFS layers with optional configuration.
func LayerConvertFunc(opts ...ConvertOpt) converter.ConvertFunc {
return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
var convertOpts convertOptions
for _, opt := range opts {
opt(&convertOpts)
}

if !images.IsLayerType(desc.MediaType) || erofs.IsErofsMediaType(desc.MediaType) {
return nil, nil
}

uncompressedDesc := &desc
if !uncompress.IsUncompressedType(desc.MediaType) {
var err error
uncompressedDesc, err = uncompress.LayerConvertFunc(ctx, cs, desc)
if err != nil {
return nil, err
}
if uncompressedDesc == nil {
return nil, fmt.Errorf("unexpectedly got the same blob after compression (%s, %q)", desc.Digest, desc.MediaType)
}
log.G(ctx).Debugf("uncompressed %s into %s", desc.Digest, uncompressedDesc.Digest)
}

info, err := cs.Info(ctx, desc.Digest)
if err != nil {
return nil, fmt.Errorf("failed to get content info: %w", err)
}

labelz := info.Labels
if labelz == nil {
labelz = make(map[string]string)
}

ra, err := cs.ReaderAt(ctx, *uncompressedDesc)
if err != nil {
return nil, fmt.Errorf("failed to get reader: %w", err)
}
defer ra.Close()

sr := io.NewSectionReader(ra, 0, uncompressedDesc.Size)

blob, err := os.CreateTemp("", "erofs-layer-*.img")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
blobPath := blob.Name()
blob.Close()

defer func() {
if err := os.Remove(blobPath); err != nil && !os.IsNotExist(err) {
log.G(ctx).WithError(err).Warnf("failed to remove temp file %s", blobPath)
}
}()

var mkfsArgs []string
if convertOpts.compressors != "" {
compressionArg := "-z" + convertOpts.compressors
mkfsArgs = append(mkfsArgs, compressionArg)
mkfsArgs = append(mkfsArgs, []string{"-C", "65536"}...)
}
mkfsArgs = append(mkfsArgs, convertOpts.mkfsExtraOpts...)

if err := erofsutils.ConvertTarErofs(ctx, sr, blobPath, "", mkfsArgs); err != nil {
return nil, fmt.Errorf("failed to convert to EROFS: %w", err)
}
log.G(ctx).Debugf("converted %s to EROFS", desc.Digest)

erofsFile, err := os.Open(blobPath)
if err != nil {
return nil, fmt.Errorf("failed to open converted file: %w", err)
}
defer erofsFile.Close()

stat, err := erofsFile.Stat()
if err != nil {
return nil, fmt.Errorf("failed to stat converted file: %w", err)
}

ref := fmt.Sprintf("convert-erofs-from-%s", desc.Digest)
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
if err != nil {
return nil, fmt.Errorf("failed to open content writer: %w", err)
}
defer w.Close()

if err := w.Truncate(0); err != nil {
return nil, fmt.Errorf("failed to truncate writer: %w", err)
}

n, err := io.Copy(w, erofsFile)
if err != nil {
return nil, fmt.Errorf("failed to copy to content store: %w", err)
}

if n != stat.Size() {
return nil, fmt.Errorf("size mismatch: copied %d bytes, expected %d bytes", n, stat.Size())
}

labelz[labels.LabelUncompressed] = w.Digest().String()
if err = w.Commit(ctx, n, "", content.WithLabels(labelz)); err != nil && !errdefs.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to commit: %w", err)
}

if err := w.Close(); err != nil {
return nil, fmt.Errorf("failed to close writer: %w", err)
}

newDesc := desc
newDesc.MediaType = images.MediaTypeErofsLayer
newDesc.Digest = w.Digest()
newDesc.Size = n
return &newDesc, nil
}
}
Loading
Loading