Skip to content

Commit 8b085dd

Browse files
mxpvmxpv
authored
Merge pull request #12936 from fidencio/release-2.2/backport-12835
[release/2.2] cri: unpack images with per-layer labels for runtime-specific snapshotters
2 parents 7022bea + a5f83d8 commit 8b085dd

8 files changed

Lines changed: 212 additions & 6 deletions

File tree

‎integration/runtime_handler_unpack_labels_linux_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package integration
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"syscall"
25+
"testing"
26+
"time"
27+
28+
"github.com/containerd/errdefs"
29+
"github.com/opencontainers/image-spec/identity"
30+
"github.com/stretchr/testify/assert"
31+
"github.com/stretchr/testify/require"
32+
criruntime "k8s.io/cri-api/pkg/apis/runtime/v1"
33+
34+
containerd "github.com/containerd/containerd/v2/client"
35+
"github.com/containerd/containerd/v2/integration/images"
36+
snpkg "github.com/containerd/containerd/v2/pkg/snapshotters"
37+
"github.com/containerd/containerd/v2/plugins"
38+
)
39+
40+
func TestRuntimeHandlerUnpackWithSnapshotLabels(t *testing.T) {
41+
workDir := t.TempDir()
42+
cfgPath := filepath.Join(workDir, "config.toml")
43+
cfg := `
44+
version = 3
45+
46+
[plugins.'io.containerd.cri.v1.images']
47+
snapshotter = "overlayfs"
48+
disable_snapshot_annotations = false
49+
50+
[plugins.'io.containerd.cri.v1.runtime'.containerd]
51+
default_runtime_name = "runc"
52+
53+
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
54+
runtime_type = "io.containerd.runc.v2"
55+
snapshotter = "overlayfs"
56+
57+
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.erofs]
58+
runtime_type = "io.containerd.runc.v2"
59+
snapshotter = "erofs"
60+
`
61+
require.NoError(t, os.WriteFile(cfgPath, []byte(cfg), 0o600))
62+
63+
ctrd := newCtrdProc(t, *containerdBin, workDir, nil)
64+
require.NoError(t, ctrd.isReady())
65+
66+
rSvc := ctrd.criRuntimeService(t)
67+
iSvc := ctrd.criImageService(t)
68+
69+
ctrdClient, err := containerd.New(ctrd.grpcAddress(), containerd.WithDefaultNamespace(k8sNamespace))
70+
require.NoError(t, err)
71+
72+
t.Cleanup(func() {
73+
if t.Failed() {
74+
t.Log("Dumping containerd config and logs due to test failure")
75+
dumpFileContent(t, ctrd.configPath())
76+
dumpFileContent(t, ctrd.logPath())
77+
}
78+
assert.NoError(t, ctrdClient.Close())
79+
cleanupPods(t, rSvc)
80+
assert.NoError(t, ctrd.kill(syscall.SIGTERM))
81+
assert.NoError(t, ctrd.wait(5*time.Minute))
82+
})
83+
84+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
85+
defer cancel()
86+
resp, err := ctrdClient.IntrospectionService().Plugins(ctx, fmt.Sprintf("type==%s,id==%s", plugins.SnapshotPlugin, "erofs"))
87+
require.NoError(t, err)
88+
if len(resp.Plugins) == 0 {
89+
t.Skip("erofs snapshotter plugin is not registered")
90+
}
91+
if initErr := resp.Plugins[0].InitErr; initErr != nil {
92+
t.Skipf("erofs snapshotter plugin is not ready: %s", initErr.Message)
93+
}
94+
95+
nginxImage := images.Get(images.Nginx)
96+
pullImagesByCRI(t, iSvc, nginxImage)
97+
98+
img, err := ctrdClient.GetImage(context.Background(), nginxImage)
99+
require.NoError(t, err)
100+
diffIDs, err := img.RootFS(context.Background())
101+
require.NoError(t, err)
102+
chainIDs := identity.ChainIDs(diffIDs)
103+
104+
// First pod uses default runtime handler (overlayfs). No containers created.
105+
sb1Cfg := PodSandboxConfig("overlay-pod", "runtime-handler-unpack")
106+
_, err = rSvc.RunPodSandbox(sb1Cfg, "")
107+
require.NoError(t, err)
108+
109+
// Image is pulled with overlayfs; nginx snapshots should not exist on erofs yet.
110+
erofsSn := ctrdClient.SnapshotService("erofs")
111+
for _, chainID := range chainIDs {
112+
_, err := erofsSn.Stat(context.Background(), chainID.String())
113+
assert.Truef(t, errdefs.IsNotFound(err), "expected no erofs snapshot for chainID %s before erofs container creation, got err=%v", chainID, err)
114+
}
115+
116+
// Second pod uses erofs runtime handler. Creating nginx container should trigger
117+
// automatic unpack for erofs with snapshot labels.
118+
sb2Cfg := PodSandboxConfig("erofs-pod", "runtime-handler-unpack")
119+
sb2ID, err := rSvc.RunPodSandbox(sb2Cfg, "erofs")
120+
require.NoError(t, err)
121+
cn2Cfg := ContainerConfig("erofs-container", nginxImage, WithCommand("sleep", "1d"))
122+
cn2ID, err := rSvc.CreateContainer(sb2ID, cn2Cfg, sb2Cfg)
123+
require.NoError(t, err)
124+
125+
for _, chainID := range chainIDs {
126+
snInfo, err := erofsSn.Stat(context.Background(), chainID.String())
127+
require.NoErrorf(t, err, "failed to stat erofs snapshot for chainID %s", chainID)
128+
require.NotNil(t, snInfo.Labels)
129+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetRefLabel], "missing %s on chainID %s", snpkg.TargetRefLabel, chainID)
130+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetManifestDigestLabel], "missing %s on chainID %s", snpkg.TargetManifestDigestLabel, chainID)
131+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetLayerDigestLabel], "missing %s on chainID %s", snpkg.TargetLayerDigestLabel, chainID)
132+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetImageLayersLabel], "missing %s on chainID %s", snpkg.TargetImageLayersLabel, chainID)
133+
}
134+
135+
// Make sure the second pod really used the erofs runtime handler path.
136+
sb2Status, err := rSvc.PodSandboxStatus(sb2ID)
137+
require.NoError(t, err)
138+
assert.Equal(t, "erofs", sb2Status.RuntimeHandler)
139+
140+
// Container should be created.
141+
cn2Status, err := rSvc.ContainerStatus(cn2ID)
142+
require.NoError(t, err)
143+
assert.Equal(t, criruntime.ContainerState_CONTAINER_CREATED, cn2Status.State)
144+
}

‎internal/cri/opts/container.go‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,32 @@ import (
2424
"strings"
2525

2626
"github.com/containerd/continuity/fs"
27+
"github.com/containerd/errdefs"
28+
"github.com/containerd/log"
29+
"github.com/containerd/platforms"
2730
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
31+
"golang.org/x/sync/semaphore"
2832

2933
containerd "github.com/containerd/containerd/v2/client"
3034
"github.com/containerd/containerd/v2/core/containers"
35+
"github.com/containerd/containerd/v2/core/images"
3136
"github.com/containerd/containerd/v2/core/mount"
3237
"github.com/containerd/containerd/v2/core/snapshots"
33-
"github.com/containerd/errdefs"
34-
"github.com/containerd/log"
38+
"github.com/containerd/containerd/v2/core/unpack"
39+
snpkg "github.com/containerd/containerd/v2/pkg/snapshotters"
3540
)
3641

3742
// WithNewSnapshot wraps `containerd.WithNewSnapshot` so that if creating the
3843
// snapshot fails we make sure the image is actually unpacked and retry.
39-
func WithNewSnapshot(id string, i containerd.Image, opts ...snapshots.Opt) containerd.NewContainerOpts {
44+
func WithNewSnapshot(id string, i containerd.Image, appendSnapshotLabels bool, opts ...snapshots.Opt) containerd.NewContainerOpts {
4045
f := containerd.WithNewSnapshot(id, i, opts...)
4146
return func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
4247
if err := f(ctx, client, c); err != nil {
4348
if !errdefs.IsNotFound(err) {
4449
return err
4550
}
4651

47-
if err := i.Unpack(ctx, c.Snapshotter); err != nil {
52+
if err := unpackImage(ctx, client, i, c.Snapshotter, appendSnapshotLabels); err != nil {
4853
return fmt.Errorf("error unpacking image: %w", err)
4954
}
5055
return f(ctx, client, c)
@@ -53,6 +58,55 @@ func WithNewSnapshot(id string, i containerd.Image, opts ...snapshots.Opt) conta
5358
}
5459
}
5560

61+
func unpackImage(ctx context.Context, client *containerd.Client, i containerd.Image, snapshotter string, appendSnapshotLabels bool) error {
62+
ctx, done, err := client.WithLease(ctx)
63+
if err != nil {
64+
return err
65+
}
66+
defer done(ctx)
67+
68+
matcher := platforms.Default()
69+
70+
capabilities, err := client.GetSnapshotterCapabilities(ctx, snapshotter)
71+
if err != nil {
72+
return err
73+
}
74+
75+
u, err := unpack.NewUnpacker(
76+
ctx,
77+
i.ContentStore(),
78+
unpack.WithUnpackPlatform(unpack.Platform{
79+
Platform: matcher,
80+
SnapshotterKey: snapshotter,
81+
Snapshotter: client.SnapshotService(snapshotter),
82+
Applier: client.DiffService(),
83+
SnapshotterCapabilities: capabilities,
84+
}),
85+
unpack.WithUnpackLimiter(semaphore.NewWeighted(3)),
86+
)
87+
if err != nil {
88+
return fmt.Errorf("unable to initialize unpacker: %w", err)
89+
}
90+
91+
childrenHandler := images.ChildrenHandler(i.ContentStore())
92+
childrenHandler = images.FilterPlatforms(childrenHandler, matcher)
93+
childrenHandler = images.LimitManifests(childrenHandler, matcher, 1)
94+
95+
var h images.Handler = childrenHandler
96+
if appendSnapshotLabels {
97+
h = snpkg.AppendInfoHandlerWrapper(i.Name())(h)
98+
}
99+
100+
if err := images.Dispatch(ctx, u.Unpack(h), nil, i.Target()); err != nil {
101+
_, _ = u.Wait()
102+
return err
103+
}
104+
if _, err := u.Wait(); err != nil {
105+
return err
106+
}
107+
return nil
108+
}
109+
56110
// WithVolumes copies ownership of volume in rootfs to its corresponding host path.
57111
// It doesn't update runtime spec.
58112
// The passed in map is a host path to container path map for all volumes.

‎internal/cri/server/container_create.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr
357357
// the runtime (runc) a chance to modify (e.g. to create mount
358358
// points corresponding to spec.Mounts) before making the
359359
// rootfs readonly (requested by spec.Root.Readonly).
360-
customopts.WithNewSnapshot(r.containerID, *r.containerdImage, sOpts...),
360+
customopts.WithNewSnapshot(r.containerID, *r.containerdImage, !c.ImageService.DisableSnapshotAnnotations(), sOpts...),
361361
}
362362
if len(volumeMounts) > 0 {
363363
mountMap := make(map[string]string)

‎internal/cri/server/container_status_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,8 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error
300300

301301
func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) }
302302

303+
func (s *fakeImageService) DisableSnapshotAnnotations() bool { return false }
304+
303305
func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string) (string, error) {
304306
return "", errors.New("not implemented")
305307
}

‎internal/cri/server/images/service.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ func (c *CRIImageService) PinnedImage(name string) string {
205205
return c.config.PinnedImages[name]
206206
}
207207

208+
func (c *CRIImageService) DisableSnapshotAnnotations() bool {
209+
return c.config.DisableSnapshotAnnotations
210+
}
211+
208212
// GRPCService returns a new CRI Image Service grpc server.
209213
func (c *CRIImageService) GRPCService() runtime.ImageServiceServer {
210214
return &GRPCCRIImageService{c}

‎internal/cri/server/podsandbox/controller.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ type ImageService interface {
124124
PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
125125
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
126126
PinnedImage(string) string
127+
DisableSnapshotAnnotations() bool
127128
}
128129

129130
type Controller struct {

‎internal/cri/server/podsandbox/sandbox_run.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
190190

191191
opts := []containerd.NewContainerOpts{
192192
containerd.WithSnapshotter(c.imageService.RuntimeSnapshotter(ctx, ociRuntime)),
193-
customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt...),
193+
customopts.WithNewSnapshot(id, containerdImage, !c.imageService.DisableSnapshotAnnotations(), snapshotterOpt...),
194194
containerd.WithSpec(spec, specOpts...),
195195
containerd.WithContainerLabels(sandboxLabels),
196196
containerd.WithContainerExtension(crilabels.SandboxMetadataExtension, &metadata),

‎internal/cri/server/service.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ type ImageService interface {
107107
LocalResolve(refOrID string) (imagestore.Image, error)
108108

109109
ImageFSPaths() map[string]string
110+
DisableSnapshotAnnotations() bool
110111
}
111112

112113
// criService implements CRIService.

0 commit comments

Comments
 (0)