Skip to content

Commit 30802fa

Browse files
giuseppegiuseppe
committed
compression: add support for the zstd algorithm
zstd is a compression algorithm that has a very fast decoder, while providing also good compression ratios. The fast decoder makes it suitable for container images, as decompressing the tarballs is a very expensive operation. opencontainers/image-spec#788 added support for zstd to the OCI image specs. Signed-off-by: Giuseppe Scrivano <[email protected]>
1 parent 9c398e1 commit 30802fa

71 files changed

Lines changed: 17101 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎archive/compression/compression.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"sync"
3030

3131
"github.com/containerd/containerd/log"
32+
"github.com/klauspost/compress/zstd"
3233
)
3334

3435
type (
@@ -41,6 +42,8 @@ const (
4142
Uncompressed Compression = iota
4243
// Gzip is gzip compression algorithm.
4344
Gzip
45+
// Zstd is zstd compression algorithm.
46+
Zstd
4447
)
4548

4649
const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ"
@@ -126,6 +129,7 @@ func (r *bufferedReader) Peek(n int) ([]byte, error) {
126129
func DetectCompression(source []byte) Compression {
127130
for compression, m := range map[Compression][]byte{
128131
Gzip: {0x1F, 0x8B, 0x08},
132+
Zstd: {0x28, 0xb5, 0x2f, 0xfd},
129133
} {
130134
if len(source) < len(m) {
131135
// Len too short
@@ -174,6 +178,19 @@ func DecompressStream(archive io.Reader) (DecompressReadCloser, error) {
174178
return gzReader.Close()
175179
},
176180
}, nil
181+
case Zstd:
182+
zstdReader, err := zstd.NewReader(buf)
183+
if err != nil {
184+
return nil, err
185+
}
186+
return &readCloserWrapper{
187+
Reader: zstdReader,
188+
compression: compression,
189+
closer: func() error {
190+
zstdReader.Close()
191+
return nil
192+
},
193+
}, nil
177194

178195
default:
179196
return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension())
@@ -187,6 +204,8 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
187204
return &writeCloserWrapper{dest, nil}, nil
188205
case Gzip:
189206
return gzip.NewWriter(dest), nil
207+
case Zstd:
208+
return zstd.NewWriter(dest)
190209
default:
191210
return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension())
192211
}
@@ -197,6 +216,8 @@ func (compression *Compression) Extension() string {
197216
switch *compression {
198217
case Gzip:
199218
return "gz"
219+
case Zstd:
220+
return "zst"
200221
}
201222
return ""
202223
}

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ require (
3636
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
3737
github.com/hashicorp/go-multierror v1.0.0
3838
github.com/imdario/mergo v0.3.10
39+
github.com/klauspost/compress v1.11.3
3940
github.com/moby/sys/symlink v0.1.0
4041
github.com/opencontainers/go-digest v1.0.0
4142
github.com/opencontainers/image-spec v1.0.1

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
315315
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
316316
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
317317
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
318+
github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
319+
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
318320
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
319321
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
320322
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

‎images/mediatypes.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ func DiffCompression(ctx context.Context, mediaType string) (string, error) {
7878
switch ext[len(ext)-1] {
7979
case "gzip":
8080
return "gzip", nil
81+
case "zstd":
82+
return "zstd", nil
8183
}
8284
}
8385
return "", nil

‎vendor/github.com/klauspost/compress/LICENSE‎

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/klauspost/compress/fse/README.md‎

Lines changed: 79 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/klauspost/compress/fse/bitreader.go‎

Lines changed: 122 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)