Skip to content

Commit a5f83d8

Browse files
fidenciofidencio
committed
cri: unpack images with per-layer labels for runtime-specific snapshotters
Remote/proxy snapshotters like nydus need per-layer annotations on each snapshot (cri.image-ref, cri.layer-digest, cri.manifest-digest, cri.image-layers) so they can lazily fetch content inside the guest VM. During a normal PullImage, these annotations are set by AppendInfoHandlerWrapper and flow through the core/unpack.Unpacker to each layer's Prepare/Commit call. However, when an image is already present for one snapshotter (e.g., overlayfs) and needs to be used with a different one (e.g., nydus for Kata), no pull occurs. The image must be unpacked into the target snapshotter with the correct per-layer labels. Replace the image.Unpack() fallback in customopts.WithNewSnapshot with unpackImage, which leverages the existing core/unpack.Unpacker and wraps the image handler with AppendInfoHandlerWrapper when snapshot annotations are enabled (!DisableSnapshotAnnotations). This reuses the same unpack machinery as PullImage, including retry handling, parallel layer support, and deduplication. Note: this is a manual backport of PR #12835 (targeting main). On release/2.2, DisableSnapshotAnnotations lives on ImageConfig which is not embedded in criconfig.Config, so we expose it via a new DisableSnapshotAnnotations() method on the ImageService interface instead of accessing the config field directly. Signed-off-by: Fabiano Fidêncio <[email protected]>
1 parent 842cbd0 commit a5f83d8

8 files changed

Lines changed: 212 additions & 6 deletions

File tree

‎integration/runtime_handler_unpack_labels_linux_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package integration
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"syscall"
25+
"testing"
26+
"time"
27+
28+
"github.com/containerd/errdefs"
29+
"github.com/opencontainers/image-spec/identity"
30+
"github.com/stretchr/testify/assert"
31+
"github.com/stretchr/testify/require"
32+
criruntime "k8s.io/cri-api/pkg/apis/runtime/v1"
33+
34+
containerd "github.com/containerd/containerd/v2/client"
35+
"github.com/containerd/containerd/v2/integration/images"
36+
snpkg "github.com/containerd/containerd/v2/pkg/snapshotters"
37+
"github.com/containerd/containerd/v2/plugins"
38+
)
39+
40+
func TestRuntimeHandlerUnpackWithSnapshotLabels(t *testing.T) {
41+
workDir := t.TempDir()
42+
cfgPath := filepath.Join(workDir, "config.toml")
43+
cfg := `
44+
version = 3
45+
46+
[plugins.'io.containerd.cri.v1.images']
47+
snapshotter = "overlayfs"
48+
disable_snapshot_annotations = false
49+
50+
[plugins.'io.containerd.cri.v1.runtime'.containerd]
51+
default_runtime_name = "runc"
52+
53+
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
54+
runtime_type = "io.containerd.runc.v2"
55+
snapshotter = "overlayfs"
56+
57+
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.erofs]
58+
runtime_type = "io.containerd.runc.v2"
59+
snapshotter = "erofs"
60+
`
61+
require.NoError(t, os.WriteFile(cfgPath, []byte(cfg), 0o600))
62+
63+
ctrd := newCtrdProc(t, *containerdBin, workDir, nil)
64+
require.NoError(t, ctrd.isReady())
65+
66+
rSvc := ctrd.criRuntimeService(t)
67+
iSvc := ctrd.criImageService(t)
68+
69+
ctrdClient, err := containerd.New(ctrd.grpcAddress(), containerd.WithDefaultNamespace(k8sNamespace))
70+
require.NoError(t, err)
71+
72+
t.Cleanup(func() {
73+
if t.Failed() {
74+
t.Log("Dumping containerd config and logs due to test failure")
75+
dumpFileContent(t, ctrd.configPath())
76+
dumpFileContent(t, ctrd.logPath())
77+
}
78+
assert.NoError(t, ctrdClient.Close())
79+
cleanupPods(t, rSvc)
80+
assert.NoError(t, ctrd.kill(syscall.SIGTERM))
81+
assert.NoError(t, ctrd.wait(5*time.Minute))
82+
})
83+
84+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
85+
defer cancel()
86+
resp, err := ctrdClient.IntrospectionService().Plugins(ctx, fmt.Sprintf("type==%s,id==%s", plugins.SnapshotPlugin, "erofs"))
87+
require.NoError(t, err)
88+
if len(resp.Plugins) == 0 {
89+
t.Skip("erofs snapshotter plugin is not registered")
90+
}
91+
if initErr := resp.Plugins[0].InitErr; initErr != nil {
92+
t.Skipf("erofs snapshotter plugin is not ready: %s", initErr.Message)
93+
}
94+
95+
nginxImage := images.Get(images.Nginx)
96+
pullImagesByCRI(t, iSvc, nginxImage)
97+
98+
img, err := ctrdClient.GetImage(context.Background(), nginxImage)
99+
require.NoError(t, err)
100+
diffIDs, err := img.RootFS(context.Background())
101+
require.NoError(t, err)
102+
chainIDs := identity.ChainIDs(diffIDs)
103+
104+
// First pod uses default runtime handler (overlayfs). No containers created.
105+
sb1Cfg := PodSandboxConfig("overlay-pod", "runtime-handler-unpack")
106+
_, err = rSvc.RunPodSandbox(sb1Cfg, "")
107+
require.NoError(t, err)
108+
109+
// Image is pulled with overlayfs; nginx snapshots should not exist on erofs yet.
110+
erofsSn := ctrdClient.SnapshotService("erofs")
111+
for _, chainID := range chainIDs {
112+
_, err := erofsSn.Stat(context.Background(), chainID.String())
113+
assert.Truef(t, errdefs.IsNotFound(err), "expected no erofs snapshot for chainID %s before erofs container creation, got err=%v", chainID, err)
114+
}
115+
116+
// Second pod uses erofs runtime handler. Creating nginx container should trigger
117+
// automatic unpack for erofs with snapshot labels.
118+
sb2Cfg := PodSandboxConfig("erofs-pod", "runtime-handler-unpack")
119+
sb2ID, err := rSvc.RunPodSandbox(sb2Cfg, "erofs")
120+
require.NoError(t, err)
121+
cn2Cfg := ContainerConfig("erofs-container", nginxImage, WithCommand("sleep", "1d"))
122+
cn2ID, err := rSvc.CreateContainer(sb2ID, cn2Cfg, sb2Cfg)
123+
require.NoError(t, err)
124+
125+
for _, chainID := range chainIDs {
126+
snInfo, err := erofsSn.Stat(context.Background(), chainID.String())
127+
require.NoErrorf(t, err, "failed to stat erofs snapshot for chainID %s", chainID)
128+
require.NotNil(t, snInfo.Labels)
129+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetRefLabel], "missing %s on chainID %s", snpkg.TargetRefLabel, chainID)
130+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetManifestDigestLabel], "missing %s on chainID %s", snpkg.TargetManifestDigestLabel, chainID)
131+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetLayerDigestLabel], "missing %s on chainID %s", snpkg.TargetLayerDigestLabel, chainID)
132+
assert.NotEmpty(t, snInfo.Labels[snpkg.TargetImageLayersLabel], "missing %s on chainID %s", snpkg.TargetImageLayersLabel, chainID)
133+
}
134+
135+
// Make sure the second pod really used the erofs runtime handler path.
136+
sb2Status, err := rSvc.PodSandboxStatus(sb2ID)
137+
require.NoError(t, err)
138+
assert.Equal(t, "erofs", sb2Status.RuntimeHandler)
139+
140+
// Container should be created.
141+
cn2Status, err := rSvc.ContainerStatus(cn2ID)
142+
require.NoError(t, err)
143+
assert.Equal(t, criruntime.ContainerState_CONTAINER_CREATED, cn2Status.State)
144+
}

‎internal/cri/opts/container.go‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,32 @@ import (
2424
"strings"
2525

2626
"github.com/containerd/continuity/fs"
27+
"github.com/containerd/errdefs"
28+
"github.com/containerd/log"
29+
"github.com/containerd/platforms"
2730
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
31+
"golang.org/x/sync/semaphore"
2832

2933
containerd "github.com/containerd/containerd/v2/client"
3034
"github.com/containerd/containerd/v2/core/containers"
35+
"github.com/containerd/containerd/v2/core/images"
3136
"github.com/containerd/containerd/v2/core/mount"
3237
"github.com/containerd/containerd/v2/core/snapshots"
33-
"github.com/containerd/errdefs"
34-
"github.com/containerd/log"
38+
"github.com/containerd/containerd/v2/core/unpack"
39+
snpkg "github.com/containerd/containerd/v2/pkg/snapshotters"
3540
)
3641

3742
// WithNewSnapshot wraps `containerd.WithNewSnapshot` so that if creating the
3843
// snapshot fails we make sure the image is actually unpacked and retry.
39-
func WithNewSnapshot(id string, i containerd.Image, opts ...snapshots.Opt) containerd.NewContainerOpts {
44+
func WithNewSnapshot(id string, i containerd.Image, appendSnapshotLabels bool, opts ...snapshots.Opt) containerd.NewContainerOpts {
4045
f := containerd.WithNewSnapshot(id, i, opts...)
4146
return func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
4247
if err := f(ctx, client, c); err != nil {
4348
if !errdefs.IsNotFound(err) {
4449
return err
4550
}
4651

47-
if err := i.Unpack(ctx, c.Snapshotter); err != nil {
52+
if err := unpackImage(ctx, client, i, c.Snapshotter, appendSnapshotLabels); err != nil {
4853
return fmt.Errorf("error unpacking image: %w", err)
4954
}
5055
return f(ctx, client, c)
@@ -53,6 +58,55 @@ func WithNewSnapshot(id string, i containerd.Image, opts ...snapshots.Opt) conta
5358
}
5459
}
5560

61+
func unpackImage(ctx context.Context, client *containerd.Client, i containerd.Image, snapshotter string, appendSnapshotLabels bool) error {
62+
ctx, done, err := client.WithLease(ctx)
63+
if err != nil {
64+
return err
65+
}
66+
defer done(ctx)
67+
68+
matcher := platforms.Default()
69+
70+
capabilities, err := client.GetSnapshotterCapabilities(ctx, snapshotter)
71+
if err != nil {
72+
return err
73+
}
74+
75+
u, err := unpack.NewUnpacker(
76+
ctx,
77+
i.ContentStore(),
78+
unpack.WithUnpackPlatform(unpack.Platform{
79+
Platform: matcher,
80+
SnapshotterKey: snapshotter,
81+
Snapshotter: client.SnapshotService(snapshotter),
82+
Applier: client.DiffService(),
83+
SnapshotterCapabilities: capabilities,
84+
}),
85+
unpack.WithUnpackLimiter(semaphore.NewWeighted(3)),
86+
)
87+
if err != nil {
88+
return fmt.Errorf("unable to initialize unpacker: %w", err)
89+
}
90+
91+
childrenHandler := images.ChildrenHandler(i.ContentStore())
92+
childrenHandler = images.FilterPlatforms(childrenHandler, matcher)
93+
childrenHandler = images.LimitManifests(childrenHandler, matcher, 1)
94+
95+
var h images.Handler = childrenHandler
96+
if appendSnapshotLabels {
97+
h = snpkg.AppendInfoHandlerWrapper(i.Name())(h)
98+
}
99+
100+
if err := images.Dispatch(ctx, u.Unpack(h), nil, i.Target()); err != nil {
101+
_, _ = u.Wait()
102+
return err
103+
}
104+
if _, err := u.Wait(); err != nil {
105+
return err
106+
}
107+
return nil
108+
}
109+
56110
// WithVolumes copies ownership of volume in rootfs to its corresponding host path.
57111
// It doesn't update runtime spec.
58112
// The passed in map is a host path to container path map for all volumes.

‎internal/cri/server/container_create.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr
357357
// the runtime (runc) a chance to modify (e.g. to create mount
358358
// points corresponding to spec.Mounts) before making the
359359
// rootfs readonly (requested by spec.Root.Readonly).
360-
customopts.WithNewSnapshot(r.containerID, *r.containerdImage, sOpts...),
360+
customopts.WithNewSnapshot(r.containerID, *r.containerdImage, !c.ImageService.DisableSnapshotAnnotations(), sOpts...),
361361
}
362362
if len(volumeMounts) > 0 {
363363
mountMap := make(map[string]string)

‎internal/cri/server/container_status_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,8 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error
300300

301301
func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) }
302302

303+
func (s *fakeImageService) DisableSnapshotAnnotations() bool { return false }
304+
303305
func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string) (string, error) {
304306
return "", errors.New("not implemented")
305307
}

‎internal/cri/server/images/service.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ func (c *CRIImageService) PinnedImage(name string) string {
205205
return c.config.PinnedImages[name]
206206
}
207207

208+
func (c *CRIImageService) DisableSnapshotAnnotations() bool {
209+
return c.config.DisableSnapshotAnnotations
210+
}
211+
208212
// GRPCService returns a new CRI Image Service grpc server.
209213
func (c *CRIImageService) GRPCService() runtime.ImageServiceServer {
210214
return &GRPCCRIImageService{c}

‎internal/cri/server/podsandbox/controller.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ type ImageService interface {
124124
PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
125125
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
126126
PinnedImage(string) string
127+
DisableSnapshotAnnotations() bool
127128
}
128129

129130
type Controller struct {

‎internal/cri/server/podsandbox/sandbox_run.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
190190

191191
opts := []containerd.NewContainerOpts{
192192
containerd.WithSnapshotter(c.imageService.RuntimeSnapshotter(ctx, ociRuntime)),
193-
customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt...),
193+
customopts.WithNewSnapshot(id, containerdImage, !c.imageService.DisableSnapshotAnnotations(), snapshotterOpt...),
194194
containerd.WithSpec(spec, specOpts...),
195195
containerd.WithContainerLabels(sandboxLabels),
196196
containerd.WithContainerExtension(crilabels.SandboxMetadataExtension, &metadata),

‎internal/cri/server/service.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ type ImageService interface {
107107
LocalResolve(refOrID string) (imagestore.Image, error)
108108

109109
ImageFSPaths() map[string]string
110+
DisableSnapshotAnnotations() bool
110111
}
111112

112113
// criService implements CRIService.

0 commit comments

Comments
 (0)