|
| 1 | +/* |
| 2 | + Copyright The containerd Authors. |
| 3 | +
|
| 4 | + Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + you may not use this file except in compliance with the License. |
| 6 | + You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | + Unless required by applicable law or agreed to in writing, software |
| 11 | + distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + See the License for the specific language governing permissions and |
| 14 | + limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package snapshots |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "encoding/base64" |
| 22 | + "encoding/json" |
| 23 | + "fmt" |
| 24 | + "math/rand" |
| 25 | + "time" |
| 26 | + |
| 27 | + "github.com/containerd/containerd/content" |
| 28 | + "github.com/containerd/containerd/images" |
| 29 | + "github.com/containerd/containerd/remotes" |
| 30 | + "github.com/opencontainers/go-digest" |
| 31 | + "github.com/opencontainers/image-spec/identity" |
| 32 | + ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 33 | +) |
| 34 | + |
| 35 | +const ( |
| 36 | + RemoteSnapshotLabel string = "containerd.io/snapshot/remote_snapshot" |
| 37 | + RemoteRefLabel string = "containerd.io/snapshot/remote_snapshot/ref" |
| 38 | + RemoteDigestLabel string = "containerd.io/snapshot/remote_snapshot/digest" |
| 39 | +) |
| 40 | + |
| 41 | +// FilterLayerBySnapshotter filters out layers from download candidates if we |
| 42 | +// can make a snapshot without downloading the actual contents of the layer. |
| 43 | +func FilterLayerBySnapshotter(f images.HandlerFunc, sn Snapshotter, store content.Store, fetcher remotes.Fetcher, ref string) images.HandlerFunc { |
| 44 | + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { |
| 45 | + children, err := f(ctx, desc) |
| 46 | + if err != nil { |
| 47 | + return nil, err |
| 48 | + } |
| 49 | + |
| 50 | + if desc.MediaType == ocispec.MediaTypeImageManifest || |
| 51 | + desc.MediaType == images.MediaTypeDockerSchema2Manifest { |
| 52 | + p, err := content.ReadBlob(ctx, store, desc) |
| 53 | + if err != nil { |
| 54 | + return nil, err |
| 55 | + } |
| 56 | + var manifest ocispec.Manifest |
| 57 | + if err := json.Unmarshal(p, &manifest); err != nil { |
| 58 | + return nil, err |
| 59 | + } |
| 60 | + |
| 61 | + configDesc := manifest.Config |
| 62 | + if _, err := remotes.FetchHandler(store, fetcher)(ctx, configDesc); err != nil { |
| 63 | + return nil, err |
| 64 | + } |
| 65 | + configLayers, err := images.RootFS(ctx, store, configDesc) |
| 66 | + if err != nil { |
| 67 | + return nil, err |
| 68 | + } |
| 69 | + |
| 70 | + necessary, _, _ := createRemoteChain(ctx, manifest.Layers, configLayers, sn, ref) |
| 71 | + unnecessary := exclude(manifest.Layers, necessary) |
| 72 | + children = exclude(children, unnecessary) |
| 73 | + } |
| 74 | + |
| 75 | + return children, nil |
| 76 | + } |
| 77 | +} |
| 78 | + |
| 79 | +// createRemoteChain checks if each layer and the lower layers are "remote |
| 80 | +// layer" with which the remote snapshotter can make a snapshot without |
| 81 | +// downloading the actual layer contents. |
| 82 | +// If so, it filters out the layer from download candidates and make the |
| 83 | +// snapshot(we call "remote snapshot" here) NOW to avoid unpacking the |
| 84 | +// layer contents. |
| 85 | +func createRemoteChain(ctx context.Context, layers []ocispec.Descriptor, diffIDs []digest.Digest, sn Snapshotter, ref string) ([]ocispec.Descriptor, string, bool) { |
| 86 | + if len(layers) <= 0 { |
| 87 | + return nil, "", true |
| 88 | + } |
| 89 | + chainID := identity.ChainID(diffIDs).String() |
| 90 | + |
| 91 | + // Make sure that all lower chains are remote snapshots. |
| 92 | + necessary, parentID, ok := createRemoteChain(ctx, layers[:len(layers)-1], diffIDs[:len(diffIDs)-1], sn, ref) |
| 93 | + if !ok { |
| 94 | + |
| 95 | + // Some of lower chains aren't remote snapshots. |
| 96 | + // We need to fetch all layers above. |
| 97 | + return append(necessary, layers[len(layers)-1]), chainID, false |
| 98 | + } |
| 99 | + |
| 100 | + if info, err := sn.Stat(ctx, chainID); err == nil { |
| 101 | + |
| 102 | + // The snapshot is applied a special label "RemoteSnapshotLabel". |
| 103 | + // This label is automatically applied by the remote snapshotter |
| 104 | + // if the snapshot is a remote snapshot. |
| 105 | + if _, ok := info.Labels[RemoteSnapshotLabel]; ok { |
| 106 | + |
| 107 | + // Snapshotter is remote snapshotter and the remote |
| 108 | + // snapshot already exists. We avoid to download it. |
| 109 | + return necessary, chainID, true |
| 110 | + } |
| 111 | + |
| 112 | + // Snapshotter is not a remote snapshotter or the snapshot |
| 113 | + // isn't remote snapshot. We need to fetch all layers above. |
| 114 | + return append(necessary, layers[len(layers)-1]), chainID, false |
| 115 | + } |
| 116 | + |
| 117 | + // We got error during Stat(), so the snapshot hasn't been made yet. |
| 118 | + // |
| 119 | + // Following cases are possible: |
| 120 | + // A. Snapshotter is a remote snapshotter and the layer is a remote |
| 121 | + // layer. |
| 122 | + // B. Snapshotter is a remote snapshotter and the layer isn't a remote |
| 123 | + // layer. |
| 124 | + // C. Snapshotter is a normal snapshotter. |
| 125 | + // |
| 126 | + // Only in the case of A, we want the remote snapshotter to make the |
| 127 | + // remote snapshot NOW and skip downloading the layer by filter out the |
| 128 | + // layer. To achive that, we need to: |
| 129 | + // 1. know that the underlyeing snapshotter is a remote snapshotter, and |
| 130 | + // 2. make the remote snapshot NOW if the layer is a remote layer. |
| 131 | + // |
| 132 | + // We acheve that by using Prepare(), Stat() and Commit() with special |
| 133 | + // labels. |
| 134 | + // The reason why we manually invoke Prepare() and Commit() is we want |
| 135 | + // containerd to recognise proper metadata which is binded to the |
| 136 | + // current namespace. It can't be achived with automatic snapshot |
| 137 | + // generation in the remote snapshotter internally. |
| 138 | + |
| 139 | + // 1. Prepare()ing a snapshot with passing basic information about this |
| 140 | + // layer (ref and layer digest) as labels. Remote snapshotters MUST |
| 141 | + // recognise these labels and MUST check if the layer is a remote |
| 142 | + // layer. If the remote snapshot exists, remote snapshotter MUST |
| 143 | + // prepare the active snapshot WITH automatically applying a label |
| 144 | + // "RemoteSnapshotLabel". |
| 145 | + remoteOpt := WithLabels(map[string]string{ |
| 146 | + RemoteRefLabel: ref, |
| 147 | + RemoteDigestLabel: layers[len(layers)-1].Digest.String(), |
| 148 | + }) |
| 149 | + key := fmt.Sprintf("remote-%s %s", uniquePart(), chainID) |
| 150 | + if _, err := sn.Prepare(ctx, key, parentID, remoteOpt); err == nil { |
| 151 | + |
| 152 | + // 2. Then we Stat() the prepared active snapshot. If the active |
| 153 | + // snapshot has a RemoteSnapshotLabel, it means we are in the case of |
| 154 | + // A(mentioned above). So we can safely Commit() the remote snapshot |
| 155 | + // without any opration on the active snapshot and skip downloading |
| 156 | + // this layer. |
| 157 | + // Through these steps, we don't explicitly apply RemoteSnapshotLabels |
| 158 | + // to any snapshots. This label is applied only in the remote |
| 159 | + // snapshotter fully automatically. So we can use this label to know |
| 160 | + // that the underlying snapshotters is a remote snapshotters or not. |
| 161 | + if info, err := sn.Stat(ctx, key); err == nil { |
| 162 | + if _, ok := info.Labels[RemoteSnapshotLabel]; ok { |
| 163 | + |
| 164 | + // 3. The remote snapshot has a label RemoteSnapshotLabel which |
| 165 | + // we haven't applied above, it means the snapshotter is a remote |
| 166 | + // snapshotter and this layer is a remote layer. So we don't do |
| 167 | + // any operation on the active snapshot and simply Commit() it. |
| 168 | + // When Commit()-ing a remote snapshot, remote snapshotter MUST |
| 169 | + // recognise RemoteSnapshotLabel applied to the corresponding active |
| 170 | + // snapshot and MUST apply the RemoteSnapshotLabel to the |
| 171 | + // corresponding commiting snapshot automatically. |
| 172 | + if err := sn.Commit(ctx, chainID, key); err == nil { |
| 173 | + |
| 174 | + // We succeeded to Commit() the remote snapshot. |
| 175 | + // Now, we can safely skip to download the layer. |
| 176 | + return necessary, chainID, true |
| 177 | + } |
| 178 | + } |
| 179 | + } |
| 180 | + } |
| 181 | + |
| 182 | + // We failed to make the remote snapshotter, so we treat this layer as a |
| 183 | + // normal way. |
| 184 | + sn.Remove(ctx, key) |
| 185 | + return append(necessary, layers[len(layers)-1]), chainID, false |
| 186 | +} |
| 187 | + |
| 188 | +func uniquePart() string { |
| 189 | + t := time.Now() |
| 190 | + var b [3]byte |
| 191 | + // Ignore read failures, just decreases uniqueness |
| 192 | + rand.Read(b[:]) |
| 193 | + return fmt.Sprintf("%d-%s", t.Nanosecond(), base64.URLEncoding.EncodeToString(b[:])) |
| 194 | +} |
| 195 | + |
| 196 | +func exclude(a []ocispec.Descriptor, b []ocispec.Descriptor) []ocispec.Descriptor { |
| 197 | + amap := map[string]ocispec.Descriptor{} |
| 198 | + for _, va := range a { |
| 199 | + amap[va.Digest.String()] = va |
| 200 | + } |
| 201 | + for _, vb := range b { |
| 202 | + delete(amap, vb.Digest.String()) |
| 203 | + } |
| 204 | + var res []ocispec.Descriptor |
| 205 | + for _, va := range amap { |
| 206 | + res = append(res, va) |
| 207 | + } |
| 208 | + return res |
| 209 | +} |
0 commit comments