Skip to content

Commit 53110e5

Browse files
committed
Support remote snapshotter to speed up image pulling
Among container's lifecycle, pulling image is one of the biggest performance bottleneck on container startup processes. One research shows that time for pulling accounts for 76% of container startup time[FAST '16](https://www.usenix.org/node/194431). Remote snapshotter is one of the solutions of the issue. And discussed on some issue threads(containerd#2943, etc.). This implementation partially based on the discussion but slightly different to make it work with metadata snapshotter which binds each snapshots to namespaces. Signed-off-by: Kohei Tokunaga <[email protected]>
1 parent be6bead commit 53110e5

6 files changed

Lines changed: 245 additions & 0 deletions

File tree

client.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,9 @@ type RemoteContext struct {
342342

343343
// AllMetadata downloads all manifests and known-configuration files
344344
AllMetadata bool
345+
346+
// SkipDownloadForExistingSnapshot skips downloading layers existing as remote snapshots
347+
SkipDownloadForExistingSnapshot bool
345348
}
346349

347350
func defaultRemoteContext() *RemoteContext {

client_opts.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,11 @@ func WithAllMetadata() RemoteOpt {
211211
return nil
212212
}
213213
}
214+
215+
// WithSkipDownloadForExistingSnapshot skip downloading layers existing as remote snapshots
216+
func WithSkipDownloadForExistingSnapshot() RemoteOpt {
217+
return func(_ *Client, c *RemoteContext) error {
218+
c.SkipDownloadForExistingSnapshot = true
219+
return nil
220+
}
221+
}

cmd/ctr/commands/content/fetch.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,10 @@ type FetchConfig struct {
108108
Platforms []string
109109
// Whether or not download all metadata
110110
AllMetadata bool
111+
// Snapshotter
112+
Snapshotter string
113+
// Whether or not skip downloading layers existing as remote snapshots.
114+
SkipDownloadForExistingSnapshot bool
111115
}
112116

113117
// NewFetchConfig returns the default FetchConfig from cli flags
@@ -138,6 +142,10 @@ func NewFetchConfig(ctx context.Context, clicontext *cli.Context) (*FetchConfig,
138142
} else if clicontext.Bool("all-metadata") {
139143
config.AllMetadata = true
140144
}
145+
if clicontext.Bool("remote-snapshot") {
146+
config.SkipDownloadForExistingSnapshot = true
147+
config.Snapshotter = clicontext.String("snapshotter")
148+
}
141149

142150
return config, nil
143151
}
@@ -177,6 +185,13 @@ func Fetch(ctx context.Context, client *containerd.Client, ref string, config *F
177185
opts = append(opts, containerd.WithAllMetadata())
178186
}
179187

188+
if config.SkipDownloadForExistingSnapshot {
189+
opts = append(opts,
190+
containerd.WithSkipDownloadForExistingSnapshot(),
191+
containerd.WithPullSnapshotter(config.Snapshotter),
192+
)
193+
}
194+
180195
if config.PlatformMatcher != nil {
181196
opts = append(opts, containerd.WithPlatformMatcher(config.PlatformMatcher))
182197
} else {

cmd/ctr/commands/images/pull.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ command. As part of this process, we do the following:
5757
Name: "all-metadata",
5858
Usage: "Pull metadata for all platforms",
5959
},
60+
cli.BoolFlag{
61+
Name: "remote-snapshot",
62+
Usage: "Skip downloading layers existing as remote snapshots",
63+
},
6064
),
6165
Action: func(context *cli.Context) error {
6266
var (

pull.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/containerd/containerd/remotes"
2626
"github.com/containerd/containerd/remotes/docker"
2727
"github.com/containerd/containerd/remotes/docker/schema1"
28+
"github.com/containerd/containerd/snapshots"
2829
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2930
"github.com/pkg/errors"
3031
"golang.org/x/sync/semaphore"
@@ -148,6 +149,11 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
148149
// Filter children by platforms if specified.
149150
childrenHandler = images.FilterPlatforms(childrenHandler, rCtx.PlatformMatcher)
150151
}
152+
// Skip downloading layers existing as remote snapshots.
153+
if rCtx.SkipDownloadForExistingSnapshot {
154+
childrenHandler = snapshots.FilterLayerBySnapshotter(childrenHandler, c.SnapshotService(rCtx.Snapshotter), store, fetcher, ref)
155+
}
156+
151157
// Sort and limit manifests if a finite number is needed
152158
if limit > 0 {
153159
childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit)

snapshots/remote.go

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package snapshots
18+
19+
import (
20+
"context"
21+
"encoding/base64"
22+
"encoding/json"
23+
"fmt"
24+
"math/rand"
25+
"time"
26+
27+
"github.com/containerd/containerd/content"
28+
"github.com/containerd/containerd/images"
29+
"github.com/containerd/containerd/remotes"
30+
"github.com/opencontainers/go-digest"
31+
"github.com/opencontainers/image-spec/identity"
32+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
33+
)
34+
35+
const (
36+
RemoteSnapshotLabel string = "containerd.io/snapshot/remote_snapshot"
37+
RemoteRefLabel string = "containerd.io/snapshot/remote_snapshot/ref"
38+
RemoteDigestLabel string = "containerd.io/snapshot/remote_snapshot/digest"
39+
)
40+
41+
// FilterLayerBySnapshotter filters out layers from download candidates if we
42+
// can make a snapshot without downloading the actual contents of the layer.
43+
func FilterLayerBySnapshotter(f images.HandlerFunc, sn Snapshotter, store content.Store, fetcher remotes.Fetcher, ref string) images.HandlerFunc {
44+
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
45+
children, err := f(ctx, desc)
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
if desc.MediaType == ocispec.MediaTypeImageManifest ||
51+
desc.MediaType == images.MediaTypeDockerSchema2Manifest {
52+
p, err := content.ReadBlob(ctx, store, desc)
53+
if err != nil {
54+
return nil, err
55+
}
56+
var manifest ocispec.Manifest
57+
if err := json.Unmarshal(p, &manifest); err != nil {
58+
return nil, err
59+
}
60+
61+
configDesc := manifest.Config
62+
if _, err := remotes.FetchHandler(store, fetcher)(ctx, configDesc); err != nil {
63+
return nil, err
64+
}
65+
configLayers, err := images.RootFS(ctx, store, configDesc)
66+
if err != nil {
67+
return nil, err
68+
}
69+
70+
necessary, _, _ := createRemoteChain(ctx, manifest.Layers, configLayers, sn, ref)
71+
unnecessary := exclude(manifest.Layers, necessary)
72+
children = exclude(children, unnecessary)
73+
}
74+
75+
return children, nil
76+
}
77+
}
78+
79+
// createRemoteChain checks if each layer and the lower layers are "remote
80+
// layer" with which the remote snapshotter can make a snapshot without
81+
// downloading the actual layer contents.
82+
// If so, it filters out the layer from download candidates and make the
83+
// snapshot(we call "remote snapshot" here) NOW to avoid unpacking the
84+
// layer contents.
85+
func createRemoteChain(ctx context.Context, layers []ocispec.Descriptor, diffIDs []digest.Digest, sn Snapshotter, ref string) ([]ocispec.Descriptor, string, bool) {
86+
if len(layers) <= 0 {
87+
return nil, "", true
88+
}
89+
chainID := identity.ChainID(diffIDs).String()
90+
91+
// Make sure that all lower chains are remote snapshots.
92+
necessary, parentID, ok := createRemoteChain(ctx, layers[:len(layers)-1], diffIDs[:len(diffIDs)-1], sn, ref)
93+
if !ok {
94+
95+
// Some of lower chains aren't remote snapshots.
96+
// We need to fetch all layers above.
97+
return append(necessary, layers[len(layers)-1]), chainID, false
98+
}
99+
100+
if info, err := sn.Stat(ctx, chainID); err == nil {
101+
102+
// The snapshot is applied a special label "RemoteSnapshotLabel".
103+
// This label is automatically applied by the remote snapshotter
104+
// if the snapshot is a remote snapshot.
105+
if _, ok := info.Labels[RemoteSnapshotLabel]; ok {
106+
107+
// Snapshotter is remote snapshotter and the remote
108+
// snapshot already exists. We avoid to download it.
109+
return necessary, chainID, true
110+
}
111+
112+
// Snapshotter is not a remote snapshotter or the snapshot
113+
// isn't remote snapshot. We need to fetch all layers above.
114+
return append(necessary, layers[len(layers)-1]), chainID, false
115+
}
116+
117+
// We got error during Stat(), so the snapshot hasn't been made yet.
118+
//
119+
// Following cases are possible:
120+
// A. Snapshotter is a remote snapshotter and the layer is a remote
121+
// layer.
122+
// B. Snapshotter is a remote snapshotter and the layer isn't a remote
123+
// layer.
124+
// C. Snapshotter is a normal snapshotter.
125+
//
126+
// Only in the case of A, we want the remote snapshotter to make the
127+
// remote snapshot NOW and skip downloading the layer by filter out the
128+
// layer. To achive that, we need to:
129+
// 1. know that the underlyeing snapshotter is a remote snapshotter, and
130+
// 2. make the remote snapshot NOW if the layer is a remote layer.
131+
//
132+
// We acheve that by using Prepare(), Stat() and Commit() with special
133+
// labels.
134+
// The reason why we manually invoke Prepare() and Commit() is we want
135+
// containerd to recognise proper metadata which is binded to the
136+
// current namespace. It can't be achived with automatic snapshot
137+
// generation in the remote snapshotter internally.
138+
139+
// 1. Prepare()ing a snapshot with passing basic information about this
140+
// layer (ref and layer digest) as labels. Remote snapshotters MUST
141+
// recognise these labels and MUST check if the layer is a remote
142+
// layer. If the remote snapshot exists, remote snapshotter MUST
143+
// prepare the active snapshot WITH automatically applying a label
144+
// "RemoteSnapshotLabel".
145+
remoteOpt := WithLabels(map[string]string{
146+
RemoteRefLabel: ref,
147+
RemoteDigestLabel: layers[len(layers)-1].Digest.String(),
148+
})
149+
key := fmt.Sprintf("remote-%s %s", uniquePart(), chainID)
150+
if _, err := sn.Prepare(ctx, key, parentID, remoteOpt); err == nil {
151+
152+
// 2. Then we Stat() the prepared active snapshot. If the active
153+
// snapshot has a RemoteSnapshotLabel, it means we are in the case of
154+
// A(mentioned above). So we can safely Commit() the remote snapshot
155+
// without any opration on the active snapshot and skip downloading
156+
// this layer.
157+
// Through these steps, we don't explicitly apply RemoteSnapshotLabels
158+
// to any snapshots. This label is applied only in the remote
159+
// snapshotter fully automatically. So we can use this label to know
160+
// that the underlying snapshotters is a remote snapshotters or not.
161+
if info, err := sn.Stat(ctx, key); err == nil {
162+
if _, ok := info.Labels[RemoteSnapshotLabel]; ok {
163+
164+
// 3. The remote snapshot has a label RemoteSnapshotLabel which
165+
// we haven't applied above, it means the snapshotter is a remote
166+
// snapshotter and this layer is a remote layer. So we don't do
167+
// any operation on the active snapshot and simply Commit() it.
168+
// When Commit()-ing a remote snapshot, remote snapshotter MUST
169+
// recognise RemoteSnapshotLabel applied to the corresponding active
170+
// snapshot and MUST apply the RemoteSnapshotLabel to the
171+
// corresponding commiting snapshot automatically.
172+
if err := sn.Commit(ctx, chainID, key); err == nil {
173+
174+
// We succeeded to Commit() the remote snapshot.
175+
// Now, we can safely skip to download the layer.
176+
return necessary, chainID, true
177+
}
178+
}
179+
}
180+
}
181+
182+
// We failed to make the remote snapshotter, so we treat this layer as a
183+
// normal way.
184+
sn.Remove(ctx, key)
185+
return append(necessary, layers[len(layers)-1]), chainID, false
186+
}
187+
188+
func uniquePart() string {
189+
t := time.Now()
190+
var b [3]byte
191+
// Ignore read failures, just decreases uniqueness
192+
rand.Read(b[:])
193+
return fmt.Sprintf("%d-%s", t.Nanosecond(), base64.URLEncoding.EncodeToString(b[:]))
194+
}
195+
196+
func exclude(a []ocispec.Descriptor, b []ocispec.Descriptor) []ocispec.Descriptor {
197+
amap := map[string]ocispec.Descriptor{}
198+
for _, va := range a {
199+
amap[va.Digest.String()] = va
200+
}
201+
for _, vb := range b {
202+
delete(amap, vb.Digest.String())
203+
}
204+
var res []ocispec.Descriptor
205+
for _, va := range amap {
206+
res = append(res, va)
207+
}
208+
return res
209+
}

0 commit comments

Comments
 (0)