Skip to content

Commit 8f63d2a

Browse files
estespestesp
authored
Merge pull request #3067 from fuweid/me-fetch-platforms
remotes: add distribution labels to blob data
2 parents 09da2d8 + 506b815 commit 8f63d2a

9 files changed

Lines changed: 315 additions & 6 deletions

File tree

‎client.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,10 @@ type RemoteContext struct {
300300

301301
// MaxConcurrentDownloads is the max concurrent content downloads for each pull.
302302
MaxConcurrentDownloads int
303+
304+
// AppendDistributionSourceLabel allows fetcher to add distribute source
305+
// label for each blob content, which doesn't work for legacy schema1.
306+
AppendDistributionSourceLabel bool
303307
}
304308

305309
func defaultRemoteContext() *RemoteContext {

‎client_opts.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,3 +194,12 @@ func WithMaxConcurrentDownloads(max int) RemoteOpt {
194194
return nil
195195
}
196196
}
197+
198+
// WithAppendDistributionSourceLabel allows fetcher to add distribute source
199+
// label for each blob content, which doesn't work for legacy schema1.
200+
func WithAppendDistributionSourceLabel() RemoteOpt {
201+
return func(_ *Client, c *RemoteContext) error {
202+
c.AppendDistributionSourceLabel = true
203+
return nil
204+
}
205+
}

‎client_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,10 @@ func TestImagePullSomePlatforms(t *testing.T) {
281281
count := 0
282282
for _, manifest := range manifests {
283283
children, err := images.Children(ctx, cs, manifest)
284+
if err != nil {
285+
t.Fatal(err)
286+
}
287+
284288
found := false
285289
for _, matcher := range m {
286290
if matcher.Match(*manifest.Platform) {
@@ -302,8 +306,6 @@ func TestImagePullSomePlatforms(t *testing.T) {
302306
}
303307
ra.Close()
304308
}
305-
} else if !found && err == nil {
306-
t.Fatal("manifest should not have pulled children content")
307309
}
308310
}
309311

‎cmd/ctr/commands/content/fetch.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,10 +149,13 @@ func Fetch(ctx context.Context, client *containerd.Client, ref string, config *F
149149
containerd.WithResolver(config.Resolver),
150150
containerd.WithImageHandler(h),
151151
containerd.WithSchema1Conversion,
152+
containerd.WithAppendDistributionSourceLabel(),
152153
}
154+
153155
for _, platform := range config.Platforms {
154156
opts = append(opts, containerd.WithPlatform(platform))
155157
}
158+
156159
img, err := client.Fetch(pctx, ref, opts...)
157160
stopProgress()
158161
if err != nil {

‎image_test.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,16 @@
1717
package containerd
1818

1919
import (
20+
"context"
21+
"fmt"
2022
"runtime"
23+
"strings"
2124
"testing"
2225

2326
"github.com/containerd/containerd/errdefs"
27+
"github.com/containerd/containerd/images"
28+
"github.com/containerd/containerd/platforms"
29+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2430
)
2531

2632
func TestImageIsUnpacked(t *testing.T) {
@@ -72,3 +78,60 @@ func TestImageIsUnpacked(t *testing.T) {
7278
t.Fatalf("image should be unpacked")
7379
}
7480
}
81+
82+
func TestImagePullWithDistSourceLabel(t *testing.T) {
83+
var (
84+
source = "docker.io"
85+
repoName = "library/busybox"
86+
tag = "latest"
87+
)
88+
89+
ctx, cancel := testContext()
90+
defer cancel()
91+
92+
client, err := newClient(t, address)
93+
if err != nil {
94+
t.Fatal(err)
95+
}
96+
defer client.Close()
97+
98+
imageName := fmt.Sprintf("%s/%s:%s", source, repoName, tag)
99+
pMatcher := platforms.Default()
100+
101+
// pull content without unpack and add distribution source label
102+
image, err := client.Pull(ctx, imageName,
103+
WithPlatformMatcher(pMatcher),
104+
WithAppendDistributionSourceLabel())
105+
if err != nil {
106+
t.Fatal(err)
107+
}
108+
defer client.ImageService().Delete(ctx, imageName)
109+
110+
cs := client.ContentStore()
111+
key := fmt.Sprintf("containerd.io/distribution.source.%s", source)
112+
113+
// only check the target platform
114+
childrenHandler := images.FilterPlatforms(images.ChildrenHandler(cs), pMatcher)
115+
116+
checkLabelHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
117+
children, err := childrenHandler(ctx, desc)
118+
if err != nil {
119+
return nil, err
120+
}
121+
122+
info, err := cs.Info(ctx, desc.Digest)
123+
if err != nil {
124+
return nil, err
125+
}
126+
127+
// check the label
128+
if got := info.Labels[key]; !strings.Contains(got, repoName) {
129+
return nil, fmt.Errorf("expected to have %s repo name in label, but got %s", repoName, got)
130+
}
131+
return children, nil
132+
}
133+
134+
if err := images.Dispatch(ctx, images.HandlerFunc(checkLabelHandler), nil, image.Target()); err != nil {
135+
t.Fatal(err)
136+
}
137+
}

‎pull.go‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,9 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
112112
childrenHandler := images.ChildrenHandler(store)
113113
// Set any children labels for that content
114114
childrenHandler = images.SetChildrenLabels(store, childrenHandler)
115-
// Filter children by platforms
116-
childrenHandler = images.FilterPlatforms(childrenHandler, rCtx.PlatformMatcher)
115+
// Filter manifests by platforms but allow to handle manifest
116+
// and configuration for not-target platforms
117+
childrenHandler = remotes.FilterManifestByPlatformHandler(childrenHandler, rCtx.PlatformMatcher)
117118
// Sort and limit manifests if a finite number is needed
118119
if limit > 0 {
119120
childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit)
@@ -130,11 +131,23 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
130131
},
131132
)
132133

133-
handler = images.Handlers(append(rCtx.BaseHandlers,
134+
handlers := append(rCtx.BaseHandlers,
134135
remotes.FetchHandler(store, fetcher),
135136
convertibleHandler,
136137
childrenHandler,
137-
)...)
138+
)
139+
140+
// append distribution source label to blob data
141+
if rCtx.AppendDistributionSourceLabel {
142+
appendDistSrcLabelHandler, err := docker.AppendDistributionSourceLabel(store, ref)
143+
if err != nil {
144+
return images.Image{}, err
145+
}
146+
147+
handlers = append(handlers, appendDistSrcLabelHandler)
148+
}
149+
150+
handler = images.Handlers(handlers...)
138151

139152
converterFunc = func(ctx context.Context, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
140153
return docker.ConvertManifest(ctx, store, desc)
@@ -148,6 +161,7 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
148161
if rCtx.MaxConcurrentDownloads > 0 {
149162
limiter = semaphore.NewWeighted(int64(rCtx.MaxConcurrentDownloads))
150163
}
164+
151165
if err := images.Dispatch(ctx, handler, limiter, desc); err != nil {
152166
return images.Image{}, err
153167
}

‎remotes/docker/handler.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package docker
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"net/url"
23+
"strings"
24+
25+
"github.com/containerd/containerd/content"
26+
"github.com/containerd/containerd/images"
27+
"github.com/containerd/containerd/labels"
28+
"github.com/containerd/containerd/log"
29+
"github.com/containerd/containerd/reference"
30+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
31+
)
32+
33+
var (
34+
// labelDistributionSource describes the source blob comes from.
35+
labelDistributionSource = "containerd.io/distribution.source"
36+
)
37+
38+
// AppendDistributionSourceLabel updates the label of blob with distribution source.
39+
func AppendDistributionSourceLabel(manager content.Manager, ref string) (images.HandlerFunc, error) {
40+
refspec, err := reference.Parse(ref)
41+
if err != nil {
42+
return nil, err
43+
}
44+
45+
u, err := url.Parse("dummy://" + refspec.Locator)
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/")
51+
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
52+
info, err := manager.Info(ctx, desc.Digest)
53+
if err != nil {
54+
return nil, err
55+
}
56+
57+
key := distributionSourceLabelKey(source)
58+
59+
originLabel := ""
60+
if info.Labels != nil {
61+
originLabel = info.Labels[key]
62+
}
63+
value := appendDistributionSourceLabel(originLabel, repo)
64+
65+
// The repo name has been limited under 256 and the distribution
66+
// label might hit the limitation of label size, when blob data
67+
// is used as the very, very common layer.
68+
if err := labels.Validate(key, value); err != nil {
69+
log.G(ctx).Warnf("skip to append distribution label: %s", err)
70+
return nil, nil
71+
}
72+
73+
info = content.Info{
74+
Digest: desc.Digest,
75+
Labels: map[string]string{
76+
key: value,
77+
},
78+
}
79+
_, err = manager.Update(ctx, info, fmt.Sprintf("labels.%s", key))
80+
return nil, err
81+
}, nil
82+
}
83+
84+
func appendDistributionSourceLabel(originLabel, repo string) string {
85+
repos := []string{}
86+
if originLabel != "" {
87+
repos = strings.Split(originLabel, ",")
88+
}
89+
repos = append(repos, repo)
90+
91+
// use emtpy string to present duplicate items
92+
for i := 1; i < len(repos); i++ {
93+
tmp, j := repos[i], i-1
94+
for ; j >= 0 && repos[j] >= tmp; j-- {
95+
if repos[j] == tmp {
96+
tmp = ""
97+
}
98+
repos[j+1] = repos[j]
99+
}
100+
repos[j+1] = tmp
101+
}
102+
103+
i := 0
104+
for ; i < len(repos) && repos[i] == ""; i++ {
105+
}
106+
107+
return strings.Join(repos[i:], ",")
108+
}
109+
110+
func distributionSourceLabelKey(source string) string {
111+
return fmt.Sprintf("%s.%s", labelDistributionSource, source)
112+
}

‎remotes/docker/handler_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package docker
18+
19+
import (
20+
"reflect"
21+
"testing"
22+
)
23+
24+
func TestAppendDistributionLabel(t *testing.T) {
25+
for _, tc := range []struct {
26+
originLabel string
27+
repo string
28+
expected string
29+
}{
30+
{
31+
originLabel: "",
32+
repo: "",
33+
expected: "",
34+
},
35+
{
36+
originLabel: "",
37+
repo: "library/busybox",
38+
expected: "library/busybox",
39+
},
40+
{
41+
originLabel: "library/busybox",
42+
repo: "library/busybox",
43+
expected: "library/busybox",
44+
},
45+
// remove the duplicate one in origin
46+
{
47+
originLabel: "library/busybox,library/redis,library/busybox",
48+
repo: "library/alpine",
49+
expected: "library/alpine,library/busybox,library/redis",
50+
},
51+
// remove the empty repo
52+
{
53+
originLabel: "library/busybox,library/redis,library/busybox",
54+
repo: "",
55+
expected: "library/busybox,library/redis",
56+
},
57+
{
58+
originLabel: "library/busybox,library/redis,library/busybox",
59+
repo: "library/redis",
60+
expected: "library/busybox,library/redis",
61+
},
62+
} {
63+
if got := appendDistributionSourceLabel(tc.originLabel, tc.repo); !reflect.DeepEqual(got, tc.expected) {
64+
t.Fatalf("expected %v, but got %v", tc.expected, got)
65+
}
66+
}
67+
}

0 commit comments

Comments
 (0)