Skip to content

Commit 735238a

Browse files
committed
fix(kubernetes): force the kubevirt-csi temp NFS unmount after a timeout
The NFS publish path temp-mounts the export root and unmounts it in a defer with a plain umount, which has no deadline. If the NFS server is gone by then, that umount can block indefinitely: the publish hangs and the temp mount stays behind for the life of the node plugin container. The Linux mounter implements MounterForceUnmounter, whose UnmountWithForce runs umount under a timeout and retries with -f once the timeout fires, so use it when the mounter offers it. The timeout is kept well inside the 2m deadline kubelet puts on NodePublishVolume. A mounter without the interface keeps the plain unmount. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent d6e5c1b commit 735238a

2 files changed

Lines changed: 130 additions & 1 deletion

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"os"
77
"path/filepath"
8+
"time"
89

910
csi "github.com/container-storage-interface/spec/lib/go/csi"
1011
"google.golang.org/grpc/codes"
@@ -89,7 +90,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
8990
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
9091
}
9192
defer func() {
92-
if err := w.mounter.Unmount(tmpMount); err != nil {
93+
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
9394
klog.Warningf("Failed to unmount temp dir %s: %v", tmpMount, err)
9495
}
9596
}()
@@ -134,6 +135,26 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
134135
return &csi.NodePublishVolumeResponse{}, nil
135136
}
136137

138+
const (
139+
// csiTimeout in kubernetes pkg/volume/csi/csi_plugin.go: the deadline
140+
// kubelet puts on NodePublishVolume.
141+
kubeletCSIOperationTimeout = 2 * time.Minute
142+
// Leaves room inside kubeletCSIOperationTimeout for the mounts and the
143+
// migration that precede the unmount.
144+
tempUnmountTimeout = 30 * time.Second
145+
)
146+
147+
// unmountTempMount bounds the unmount where the mounter can: a plain umount
148+
// of an NFS mount whose server is gone blocks forever, which hangs the publish
149+
// and leaks the mount. UnmountWithForce retries with -f once the timeout
150+
// passes.
151+
func unmountTempMount(m mount.Interface, dir string) error {
152+
if fm, ok := m.(mount.MounterForceUnmounter); ok {
153+
return fm.UnmountWithForce(dir, tempUnmountTimeout)
154+
}
155+
return m.Unmount(dir)
156+
}
157+
137158
// NodeExpandVolume for NFS volumes is a no-op (LINSTOR handles NFS resize automatically).
138159
// This should not normally be called for NFS since ControllerExpandVolume returns
139160
// NodeExpansionRequired=false, but we handle it gracefully as a safety net.
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"path/filepath"
7+
"runtime"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
csi "github.com/container-storage-interface/spec/lib/go/csi"
13+
mount "k8s.io/mount-utils"
14+
)
15+
16+
// A path that does not exist, so the fake mounter's symlink resolution keeps
17+
// it as is and matches it against its mount table.
18+
const tempMountPath = "/nonexistent/nfs-init-pvc-1-123"
19+
20+
// forceFakeMounter adds mount.MounterForceUnmounter to the fake, as the real
21+
// Linux mounter implements it.
22+
type forceFakeMounter struct {
23+
*mount.FakeMounter
24+
forceErr error
25+
forcedTargets []string
26+
forcedTimeout time.Duration
27+
}
28+
29+
func (f *forceFakeMounter) UnmountWithForce(target string, timeout time.Duration) error {
30+
f.forcedTargets = append(f.forcedTargets, target)
31+
f.forcedTimeout = timeout
32+
return f.forceErr
33+
}
34+
35+
func mountedFake(unmountErr error) *mount.FakeMounter {
36+
m := mount.NewFakeMounter([]mount.MountPoint{{Path: tempMountPath, Type: "nfs"}})
37+
if unmountErr != nil {
38+
m.UnmountFunc = func(string) error { return unmountErr }
39+
}
40+
return m
41+
}
42+
43+
func TestUnmountTempMountUsesForceUnmounter(t *testing.T) {
44+
m := &forceFakeMounter{FakeMounter: mountedFake(errors.New("plain Unmount called"))}
45+
46+
if err := unmountTempMount(m, tempMountPath); err != nil {
47+
t.Fatalf("unmountTempMount: %v", err)
48+
}
49+
if len(m.forcedTargets) != 1 || m.forcedTargets[0] != tempMountPath {
50+
t.Fatalf("UnmountWithForce targets = %v, want [%s]", m.forcedTargets, tempMountPath)
51+
}
52+
if m.forcedTimeout <= 0 || m.forcedTimeout >= kubeletCSIOperationTimeout {
53+
t.Errorf("UnmountWithForce timeout = %v, want within (0, %v)", m.forcedTimeout, kubeletCSIOperationTimeout)
54+
}
55+
}
56+
57+
// main wires mount.New(""); without this the force path above is never taken
58+
// in production and the fallback is silent.
59+
func TestLinuxMounterIsForceUnmounter(t *testing.T) {
60+
if runtime.GOOS != "linux" {
61+
t.Skip("mount.New returns the unsupported mounter off Linux")
62+
}
63+
if _, ok := mount.New("").(mount.MounterForceUnmounter); !ok {
64+
t.Fatal("mount.New(\"\") does not implement mount.MounterForceUnmounter")
65+
}
66+
}
67+
68+
func TestUnmountTempMountReportsForceFailure(t *testing.T) {
69+
m := &forceFakeMounter{FakeMounter: mountedFake(nil), forceErr: errors.New("device busy")}
70+
71+
if err := unmountTempMount(m, tempMountPath); err == nil {
72+
t.Fatal("unmountTempMount succeeded although UnmountWithForce failed")
73+
}
74+
}
75+
76+
func TestUnmountTempMountFallsBackToPlainUnmount(t *testing.T) {
77+
m := mountedFake(nil)
78+
if err := unmountTempMount(m, tempMountPath); err != nil {
79+
t.Fatalf("unmountTempMount: %v", err)
80+
}
81+
if mps, _ := m.List(); len(mps) != 0 {
82+
t.Errorf("mount points after unmount = %v, want none", mps)
83+
}
84+
85+
if err := unmountTempMount(mountedFake(errors.New("device busy")), tempMountPath); err == nil {
86+
t.Error("unmountTempMount succeeded although Unmount failed")
87+
}
88+
}
89+
90+
func TestNodePublishVolumeNFSUnmountsTempMountWithForce(t *testing.T) {
91+
tmpRoot := t.TempDir()
92+
t.Setenv("TMPDIR", tmpRoot)
93+
94+
m := &forceFakeMounter{FakeMounter: mount.NewFakeMounter(nil)}
95+
w := &WrappedNodeService{mounter: m}
96+
req := &csi.NodePublishVolumeRequest{
97+
VolumeId: "pvc-1",
98+
TargetPath: filepath.Join(t.TempDir(), "target"),
99+
PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"},
100+
}
101+
if _, err := w.NodePublishVolume(context.Background(), req); err != nil {
102+
t.Fatalf("NodePublishVolume: %v", err)
103+
}
104+
105+
if len(m.forcedTargets) != 1 || !strings.HasPrefix(filepath.Base(m.forcedTargets[0]), "nfs-init-pvc-1-") {
106+
t.Fatalf("UnmountWithForce targets = %v, want the one nfs-init-pvc-1-* temp mount", m.forcedTargets)
107+
}
108+
}

0 commit comments

Comments
 (0)