Skip to content

Commit 129942c

Browse files
estespestesp
authored
Merge pull request #3135 from dmcgowan/archive-importer-docker-types
Compress import blobs in Docker compatibility code
2 parents 36e4c8e + 8d1ae23 commit 129942c

2 files changed

Lines changed: 109 additions & 16 deletions

File tree

‎images/archive/importer.go‎

Lines changed: 98 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ import (
2222
"bytes"
2323
"context"
2424
"encoding/json"
25+
"fmt"
2526
"io"
2627
"io/ioutil"
2728
"path"
2829

2930
"github.com/containerd/containerd/archive/compression"
3031
"github.com/containerd/containerd/content"
32+
"github.com/containerd/containerd/errdefs"
3133
"github.com/containerd/containerd/images"
3234
"github.com/containerd/containerd/log"
3335
digest "github.com/opencontainers/go-digest"
@@ -137,19 +139,23 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (oc
137139
if !ok {
138140
return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
139141
}
140-
config.MediaType = ocispec.MediaTypeImageConfig
142+
config.MediaType = images.MediaTypeDockerSchema2Config
141143

142144
layers, err := resolveLayers(ctx, store, mfst.Layers, blobs)
143145
if err != nil {
144146
return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
145147
}
146148

147-
manifest := ocispec.Manifest{
148-
Versioned: specs.Versioned{
149-
SchemaVersion: 2,
150-
},
151-
Config: config,
152-
Layers: layers,
149+
manifest := struct {
150+
SchemaVersion int `json:"schemaVersion"`
151+
MediaType string `json:"mediaType"`
152+
Config ocispec.Descriptor `json:"config"`
153+
Layers []ocispec.Descriptor `json:"layers"`
154+
}{
155+
SchemaVersion: 2,
156+
MediaType: images.MediaTypeDockerSchema2Manifest,
157+
Config: config,
158+
Layers: layers,
153159
}
154160

155161
desc, err := writeManifest(ctx, store, manifest, ocispec.MediaTypeImageManifest)
@@ -212,35 +218,111 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size
212218
}
213219

214220
func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) {
215-
var layers []ocispec.Descriptor
216-
for _, f := range layerFiles {
221+
layers := make([]ocispec.Descriptor, len(layerFiles))
222+
descs := map[digest.Digest]*ocispec.Descriptor{}
223+
filters := []string{}
224+
for i, f := range layerFiles {
217225
desc, ok := blobs[f]
218226
if !ok {
219227
return nil, errors.Errorf("layer %q not found", f)
220228
}
229+
layers[i] = desc
230+
descs[desc.Digest] = &layers[i]
231+
filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
232+
}
233+
234+
err := store.Walk(ctx, func(info content.Info) error {
235+
dgst, ok := info.Labels["containerd.io/uncompressed"]
236+
if ok {
237+
desc := descs[digest.Digest(dgst)]
238+
if desc != nil {
239+
desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
240+
desc.Digest = info.Digest
241+
desc.Size = info.Size
242+
}
243+
}
244+
return nil
245+
}, filters...)
246+
if err != nil {
247+
return nil, errors.Wrap(err, "failure checking for compressed blobs")
248+
}
221249

250+
for i, desc := range layers {
251+
if desc.MediaType != "" {
252+
continue
253+
}
222254
// Open blob, resolve media type
223255
ra, err := store.ReaderAt(ctx, desc)
224256
if err != nil {
225-
return nil, errors.Wrapf(err, "failed to open %q (%s)", f, desc.Digest)
257+
return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
226258
}
227259
s, err := compression.DecompressStream(content.NewReader(ra))
228260
if err != nil {
229-
return nil, errors.Wrapf(err, "failed to detect compression for %q", f)
261+
return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
230262
}
231263
if s.GetCompression() == compression.Uncompressed {
232-
// TODO: Support compressing and writing back to content store
233-
desc.MediaType = ocispec.MediaTypeImageLayer
234-
} else {
235-
desc.MediaType = ocispec.MediaTypeImageLayerGzip
264+
ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
265+
labels := map[string]string{
266+
"containerd.io/uncompressed": desc.Digest.String(),
267+
}
268+
layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
269+
if err != nil {
270+
s.Close()
271+
return nil, err
272+
}
236273
}
274+
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
237275
s.Close()
238276

239-
layers = append(layers, desc)
240277
}
241278
return layers, nil
242279
}
243280

281+
func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
282+
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
283+
if err != nil {
284+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
285+
}
286+
287+
defer func() {
288+
w.Close()
289+
if err != nil {
290+
cs.Abort(ctx, ref)
291+
}
292+
}()
293+
if err := w.Truncate(0); err != nil {
294+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
295+
}
296+
297+
cw, err := compression.CompressStream(w, compression.Gzip)
298+
if err != nil {
299+
return ocispec.Descriptor{}, err
300+
}
301+
302+
if _, err := io.Copy(cw, r); err != nil {
303+
return ocispec.Descriptor{}, err
304+
}
305+
if err := cw.Close(); err != nil {
306+
return ocispec.Descriptor{}, err
307+
}
308+
309+
cst, err := w.Status()
310+
if err != nil {
311+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
312+
}
313+
314+
desc.Digest = w.Digest()
315+
desc.Size = cst.Offset
316+
317+
if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
318+
if !errdefs.IsAlreadyExists(err) {
319+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
320+
}
321+
}
322+
323+
return desc, nil
324+
}
325+
244326
func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
245327
manifestBytes, err := json.Marshal(manifest)
246328
if err != nil {

‎import_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"runtime"
2929
"testing"
3030

31+
"github.com/containerd/containerd/archive/compression"
3132
"github.com/containerd/containerd/archive/tartest"
3233
"github.com/containerd/containerd/images"
3334
"github.com/containerd/containerd/images/archive"
@@ -289,6 +290,16 @@ func createContent(size int64, seed int64) ([]byte, digest.Digest) {
289290
if err != nil {
290291
panic(err)
291292
}
293+
wb := bytes.NewBuffer(nil)
294+
cw, err := compression.CompressStream(wb, compression.Gzip)
295+
if err != nil {
296+
panic(err)
297+
}
298+
299+
if _, err := cw.Write(b); err != nil {
300+
panic(err)
301+
}
302+
b = wb.Bytes()
292303
return b, digest.FromBytes(b)
293304
}
294305

0 commit comments

Comments
 (0)