Skip to content

Commit 8d1ae23

Browse files
dmcgowandmcgowan
committed
Always add compressed blobs to Docker import manifest
Ensure the manifest which gets created using the Docker compatibility code compresses the blob before creating the manifest. This ensures consistency with manifests used by Docker. Signed-off-by: Derek McGowan <[email protected]>
1 parent b90eead commit 8d1ae23

2 files changed

Lines changed: 110 additions & 16 deletions

File tree

‎images/archive/importer.go‎

Lines changed: 98 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ import (
2222
"bytes"
2323
"context"
2424
"encoding/json"
25+
"fmt"
2526
"io"
2627
"io/ioutil"
2728
"path"
2829

2930
"github.com/containerd/containerd/archive/compression"
3031
"github.com/containerd/containerd/content"
32+
"github.com/containerd/containerd/errdefs"
3133
"github.com/containerd/containerd/images"
3234
"github.com/containerd/containerd/log"
3335
digest "github.com/opencontainers/go-digest"
@@ -137,19 +139,23 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (oc
137139
if !ok {
138140
return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
139141
}
140-
config.MediaType = ocispec.MediaTypeImageConfig
142+
config.MediaType = images.MediaTypeDockerSchema2Config
141143

142144
layers, err := resolveLayers(ctx, store, mfst.Layers, blobs)
143145
if err != nil {
144146
return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
145147
}
146148

147-
manifest := ocispec.Manifest{
148-
Versioned: specs.Versioned{
149-
SchemaVersion: 2,
150-
},
151-
Config: config,
152-
Layers: layers,
149+
manifest := struct {
150+
SchemaVersion int `json:"schemaVersion"`
151+
MediaType string `json:"mediaType"`
152+
Config ocispec.Descriptor `json:"config"`
153+
Layers []ocispec.Descriptor `json:"layers"`
154+
}{
155+
SchemaVersion: 2,
156+
MediaType: images.MediaTypeDockerSchema2Manifest,
157+
Config: config,
158+
Layers: layers,
153159
}
154160

155161
desc, err := writeManifest(ctx, store, manifest, ocispec.MediaTypeImageManifest)
@@ -214,35 +220,111 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size
214220
}
215221

216222
func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) {
217-
var layers []ocispec.Descriptor
218-
for _, f := range layerFiles {
223+
layers := make([]ocispec.Descriptor, len(layerFiles))
224+
descs := map[digest.Digest]*ocispec.Descriptor{}
225+
filters := []string{}
226+
for i, f := range layerFiles {
219227
desc, ok := blobs[f]
220228
if !ok {
221229
return nil, errors.Errorf("layer %q not found", f)
222230
}
231+
layers[i] = desc
232+
descs[desc.Digest] = &layers[i]
233+
filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
234+
}
235+
236+
err := store.Walk(ctx, func(info content.Info) error {
237+
dgst, ok := info.Labels["containerd.io/uncompressed"]
238+
if ok {
239+
desc := descs[digest.Digest(dgst)]
240+
if desc != nil {
241+
desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
242+
desc.Digest = info.Digest
243+
desc.Size = info.Size
244+
}
245+
}
246+
return nil
247+
}, filters...)
248+
if err != nil {
249+
return nil, errors.Wrap(err, "failure checking for compressed blobs")
250+
}
223251

252+
for i, desc := range layers {
253+
if desc.MediaType != "" {
254+
continue
255+
}
224256
// Open blob, resolve media type
225257
ra, err := store.ReaderAt(ctx, desc)
226258
if err != nil {
227-
return nil, errors.Wrapf(err, "failed to open %q (%s)", f, desc.Digest)
259+
return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
228260
}
229261
s, err := compression.DecompressStream(content.NewReader(ra))
230262
if err != nil {
231-
return nil, errors.Wrapf(err, "failed to detect compression for %q", f)
263+
return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
232264
}
233265
if s.GetCompression() == compression.Uncompressed {
234-
// TODO: Support compressing and writing back to content store
235-
desc.MediaType = ocispec.MediaTypeImageLayer
236-
} else {
237-
desc.MediaType = ocispec.MediaTypeImageLayerGzip
266+
ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
267+
labels := map[string]string{
268+
"containerd.io/uncompressed": desc.Digest.String(),
269+
}
270+
layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
271+
if err != nil {
272+
s.Close()
273+
return nil, err
274+
}
238275
}
276+
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
239277
s.Close()
240278

241-
layers = append(layers, desc)
242279
}
243280
return layers, nil
244281
}
245282

283+
func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
284+
w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
285+
if err != nil {
286+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
287+
}
288+
289+
defer func() {
290+
w.Close()
291+
if err != nil {
292+
cs.Abort(ctx, ref)
293+
}
294+
}()
295+
if err := w.Truncate(0); err != nil {
296+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
297+
}
298+
299+
cw, err := compression.CompressStream(w, compression.Gzip)
300+
if err != nil {
301+
return ocispec.Descriptor{}, err
302+
}
303+
304+
if _, err := io.Copy(cw, r); err != nil {
305+
return ocispec.Descriptor{}, err
306+
}
307+
if err := cw.Close(); err != nil {
308+
return ocispec.Descriptor{}, err
309+
}
310+
311+
cst, err := w.Status()
312+
if err != nil {
313+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
314+
}
315+
316+
desc.Digest = w.Digest()
317+
desc.Size = cst.Offset
318+
319+
if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
320+
if !errdefs.IsAlreadyExists(err) {
321+
return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
322+
}
323+
}
324+
325+
return desc, nil
326+
}
327+
246328
func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
247329
manifestBytes, err := json.Marshal(manifest)
248330
if err != nil {

‎import_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package containerd
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"encoding/json"
2223
"io"
@@ -26,6 +27,7 @@ import (
2627
"runtime"
2728
"testing"
2829

30+
"github.com/containerd/containerd/archive/compression"
2931
"github.com/containerd/containerd/archive/tartest"
3032
"github.com/containerd/containerd/images"
3133
"github.com/containerd/containerd/images/archive"
@@ -274,6 +276,16 @@ func createContent(size int64, seed int64) ([]byte, digest.Digest) {
274276
if err != nil {
275277
panic(err)
276278
}
279+
wb := bytes.NewBuffer(nil)
280+
cw, err := compression.CompressStream(wb, compression.Gzip)
281+
if err != nil {
282+
panic(err)
283+
}
284+
285+
if _, err := cw.Write(b); err != nil {
286+
panic(err)
287+
}
288+
b = wb.Bytes()
277289
return b, digest.FromBytes(b)
278290
}
279291

0 commit comments

Comments
 (0)