Skip to content

Commit 3f6ab6e

Browse files
fuweidfuweid
committed
internal/cri: simplify netns setup with pinned userns
Motivation: For pod-level user namespaces, it's impossible to force the container runtime to join an existing network namespace after creating a new user namespace. According to the capabilities section in [user_namespaces(7)][1], a network namespace created by containerd is owned by the root user namespace. When the container runtime (like runc or crun) creates a new user namespace, it becomes a child of the root user namespace. Processes within this child user namespace are not permitted to access resources owned by the parent user namespace. If the network namespace is not owned by the new user namespace, the container runtime will fail to mount /sys due to the [sysfs: Restrict mounting sysfs][2] patch. Referencing the [cap_capable][3] function in Linux, a process can access a resource if: * The resource is owned by the process's user namespace, and the process has the required capability. * The resource is owned by a child of the process's user namespace, and the owner's user namespace was created by the process's UID. In the context of pod-level user namespaces, the CRI plugin delegates the creation of the network namespace to the container runtime when running the pause container. After the pause container is initialized, the CRI plugin pins the pause container's network namespace into `/run/netns` and then executes the `CNI_ADD` command over it. However, if the pause container is terminated during the pinning process, the CRI plugin might encounter a PID cycle, leading to the `CNI_ADD` command operating on an incorrect network namespace. Moreover, rolling back the `RunPodSandbox` API is complex due to the delegation of network namespace creation. As highlighted in issue #10363, the CRI plugin can lose IP information after a containerd restart, making it challenging to maintain robustness in the RunPodSandbox API. Solution: Allow containerd to create a new user namespace and then create the network namespace within that user namespace. This way, the CRI plugin can force the container runtime to join both the user namespace and the network namespace. Since the network namespace is owned by the newly created user namespace, the container runtime will have the necessary permissions to mount `/sys` on the container's root filesystem. As a result, delegation of network namespace creation is no longer needed. NOTE: * The CRI plugin does not need to pin the newly created user namespace as it does with the network namespace, because the kernel allows retrieving a user namespace reference via [ioctl_ns(2)][4]. As a result, the podsandbox implementation can obtain the user namespace using the `netnsPath` parameter. [1]: <https://man7.org/linux/man-pages/man7/user_namespaces.7.html> [2]: <torvalds/linux@7dc5dbc> [3]: <https://github.com/torvalds/linux/blob/2c85ebc57b3e1817b6ce1a6b703928e113a90442/security/commoncap.c#L65> [4]: <https://man7.org/linux/man-pages/man2/ioctl_ns.2.html> Signed-off-by: Wei Fu <[email protected]>
1 parent 74bcff8 commit 3f6ab6e

15 files changed

Lines changed: 270 additions & 151 deletions

‎internal/cri/opts/spec_opts.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,23 @@ func WithoutNamespace(t runtimespec.LinuxNamespaceType) oci.SpecOpts {
310310
}
311311
}
312312

313+
// WithNamespacePath updates namespace with existing path.
314+
func WithNamespacePath(t runtimespec.LinuxNamespaceType, nsPath string) oci.SpecOpts {
315+
return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
316+
if s.Linux == nil {
317+
return fmt.Errorf("Linux spec is required")
318+
}
319+
320+
for i, ns := range s.Linux.Namespaces {
321+
if ns.Type == t {
322+
s.Linux.Namespaces[i].Path = nsPath
323+
return nil
324+
}
325+
}
326+
return fmt.Errorf("no such namespace %s", t)
327+
}
328+
}
329+
313330
// WithPodNamespaces sets the pod namespaces for the container
314331
func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, sandboxPid uint32, targetPid uint32, uids, gids []runtimespec.LinuxIDMapping) oci.SpecOpts {
315332
namespaces := config.GetNamespaceOptions()

‎internal/cri/server/podsandbox/controller.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,20 @@ func init() {
7979
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
8080
}
8181

82+
usernsInode, err := getCurrentUserNamespaceInode()
83+
if err != nil {
84+
return nil, fmt.Errorf("unable to cache user namespace inode: %w", err)
85+
}
86+
log.G(context.Background()).Infof("user namespace inode %d", usernsInode)
87+
8288
c := Controller{
8389
client: client,
8490
config: runtimeService.Config(),
8591
os: osinterface.RealOS{},
8692
runtimeService: runtimeService,
8793
imageService: criImagePlugin.(ImageService),
8894
store: NewStore(),
95+
usernsInode: usernsInode,
8996
}
9097

9198
eventMonitor := events.NewEventMonitor(&podSandboxEventHandler{
@@ -131,6 +138,10 @@ type Controller struct {
131138
eventMonitor *events.EventMonitor
132139

133140
store *Store
141+
142+
// usernsInode is the inode of current user namespace. Only available
143+
// in the linux platform.
144+
usernsInode uint64
134145
}
135146

136147
var _ sandbox.Controller = (*Controller)(nil)

‎internal/cri/server/podsandbox/helpers_linux.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"github.com/containerd/containerd/v2/core/snapshots"
4141
"github.com/containerd/containerd/v2/internal/cri/seutil"
4242
"github.com/containerd/containerd/v2/pkg/seccomp"
43+
"github.com/containerd/containerd/v2/pkg/sys"
4344
)
4445

4546
const (
@@ -68,6 +69,16 @@ func getCgroupsPath(cgroupsParent, id string) string {
6869
return filepath.Join(cgroupsParent, id)
6970
}
7071

72+
// getCurrentUserNamespaceInode returns inode of this process's user namespace.
73+
func getCurrentUserNamespaceInode() (uint64, error) {
74+
upath := fmt.Sprintf("/proc/%d/ns/user", os.Getpid())
75+
fi, err := os.Stat(upath)
76+
if err != nil {
77+
return 0, fmt.Errorf("failed to stat %s: %w", upath, err)
78+
}
79+
return fi.Sys().(*syscall.Stat_t).Ino, nil
80+
}
81+
7182
// getSandboxHostname returns the hostname file path inside the sandbox root directory.
7283
func (c *Controller) getSandboxHostname(id string) string {
7384
return filepath.Join(c.getSandboxRootDir(id), "hostname")
@@ -88,6 +99,50 @@ func (c *Controller) getSandboxDevShm(id string) string {
8899
return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
89100
}
90101

102+
// getSandboxPinnedNamespaces returns the pinned namespaces directory inside the
103+
// sandbox state directory.
104+
func (c *Controller) getSandboxPinnedNamespaces(id string) string {
105+
return filepath.Join(c.getVolatileSandboxRootDir(id), "pinned-namespaces")
106+
}
107+
108+
// getSandboxPinnedUserNamespace returns the pinned user namespace file.
109+
func (c *Controller) getSandboxPinnedUserNamespace(id string) string {
110+
return filepath.Join(c.getSandboxPinnedNamespaces(id), "user")
111+
}
112+
113+
// pinUserNamespace persists user namespace in namespace filesystem.
114+
func (c *Controller) pinUserNamespace(sandboxID string, netnsPath string) error {
115+
nsPath := c.getSandboxPinnedUserNamespace(sandboxID)
116+
117+
baseDir := filepath.Dir(nsPath)
118+
if err := os.MkdirAll(baseDir, 0755); err != nil {
119+
return fmt.Errorf("failed to init pinned-namespaces directory %s: %w", baseDir, err)
120+
}
121+
122+
emptyFd, err := os.OpenFile(nsPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
123+
if err != nil {
124+
return fmt.Errorf("failed to create empty file %s: %w", nsPath, err)
125+
}
126+
emptyFd.Close()
127+
128+
netnsFd, err := os.Open(netnsPath)
129+
if err != nil {
130+
return fmt.Errorf("failed to open netns(%s): %w", netnsPath, err)
131+
}
132+
defer netnsFd.Close()
133+
134+
usernsFd, err := sys.GetUsernsForNamespace(netnsFd.Fd())
135+
if err != nil {
136+
return fmt.Errorf("failed to get user namespace for netns(%s): %w", netnsPath, err)
137+
}
138+
defer usernsFd.Close()
139+
140+
if err = unix.Mount(usernsFd.Name(), nsPath, "none", unix.MS_BIND, ""); err != nil {
141+
return fmt.Errorf("failed to bind mount ns src: %v at %s: %w", usernsFd.Name(), nsPath, err)
142+
}
143+
return nil
144+
}
145+
91146
func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
92147
var labels []string
93148

‎internal/cri/server/podsandbox/helpers_other.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,7 @@ func ensureRemoveAll(ctx context.Context, dir string) error {
3636
func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
3737
return nil
3838
}
39+
40+
func getCurrentUserNamespaceInode() (uint64, error) {
41+
return 0, nil
42+
}

‎internal/cri/server/podsandbox/helpers_windows.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,7 @@ func ensureRemoveAll(_ context.Context, dir string) error {
3131
func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
3232
return nil
3333
}
34+
35+
func getCurrentUserNamespaceInode() (uint64, error) {
36+
return 0, nil
37+
}

‎internal/cri/server/podsandbox/sandbox_run.go‎

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,39 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
9595

9696
labels["oci_runtime_type"] = ociRuntime.Type
9797

98+
// Create sandbox container root directories.
99+
sandboxRootDir := c.getSandboxRootDir(id)
100+
if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
101+
return cin, fmt.Errorf("failed to create sandbox root directory %q: %w",
102+
sandboxRootDir, err)
103+
}
104+
defer func() {
105+
if retErr != nil && cleanupErr == nil {
106+
// Cleanup the sandbox root directory.
107+
if cleanupErr = c.os.RemoveAll(sandboxRootDir); cleanupErr != nil {
108+
log.G(ctx).WithError(cleanupErr).Errorf("Failed to remove sandbox root directory %q",
109+
sandboxRootDir)
110+
}
111+
}
112+
}()
113+
114+
volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
115+
if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
116+
return cin, fmt.Errorf("failed to create volatile sandbox root directory %q: %w",
117+
volatileSandboxRootDir, err)
118+
}
119+
defer func() {
120+
if retErr != nil && cleanupErr == nil {
121+
deferCtx, deferCancel := ctrdutil.DeferContext()
122+
defer deferCancel()
123+
// Cleanup the volatile sandbox root directory.
124+
if cleanupErr = ensureRemoveAll(deferCtx, volatileSandboxRootDir); cleanupErr != nil {
125+
log.G(ctx).WithError(cleanupErr).Errorf("Failed to remove volatile sandbox root directory %q",
126+
volatileSandboxRootDir)
127+
}
128+
}
129+
}()
130+
98131
// Create sandbox container.
99132
// NOTE: sandboxContainerSpec SHOULD NOT have side
100133
// effect, e.g. accessing/creating files, so that we can test
@@ -164,37 +197,6 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
164197
}
165198
}()
166199

167-
// Create sandbox container root directories.
168-
sandboxRootDir := c.getSandboxRootDir(id)
169-
if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
170-
return cin, fmt.Errorf("failed to create sandbox root directory %q: %w",
171-
sandboxRootDir, err)
172-
}
173-
defer func() {
174-
if retErr != nil && cleanupErr == nil {
175-
// Cleanup the sandbox root directory.
176-
if cleanupErr = c.os.RemoveAll(sandboxRootDir); cleanupErr != nil {
177-
log.G(ctx).WithError(cleanupErr).Errorf("Failed to remove sandbox root directory %q",
178-
sandboxRootDir)
179-
}
180-
}
181-
}()
182-
183-
volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
184-
if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
185-
return cin, fmt.Errorf("failed to create volatile sandbox root directory %q: %w",
186-
volatileSandboxRootDir, err)
187-
}
188-
defer func() {
189-
if retErr != nil && cleanupErr == nil {
190-
// Cleanup the volatile sandbox root directory.
191-
if cleanupErr = c.os.RemoveAll(volatileSandboxRootDir); cleanupErr != nil {
192-
log.G(ctx).WithError(cleanupErr).Errorf("Failed to remove volatile sandbox root directory %q",
193-
volatileSandboxRootDir)
194-
}
195-
}
196-
}()
197-
198200
// Setup files required for the sandbox.
199201
if err = c.setupSandboxFiles(id, config); err != nil {
200202
return cin, fmt.Errorf("failed to setup sandbox files: %w", err)

‎internal/cri/server/podsandbox/sandbox_run_linux.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,11 @@ func (c *Controller) sandboxContainerSpec(id string, config *runtime.PodSandboxC
107107
case runtime.NamespaceMode_POD:
108108
specOpts = append(specOpts, oci.WithUserNamespace(uids, gids))
109109
usernsEnabled = true
110+
111+
if err := c.pinUserNamespace(id, nsPath); err != nil {
112+
return nil, fmt.Errorf("failed to pin user namespace: %w", err)
113+
}
114+
specOpts = append(specOpts, customopts.WithNamespacePath(runtimespec.UserNamespace, c.getSandboxPinnedUserNamespace(id)))
110115
default:
111116
return nil, fmt.Errorf("unsupported user namespace mode: %q", mode)
112117
}

0 commit comments

Comments
 (0)