Skip to content

Commit eb07d90

Browse files
committed
fix(kubernetes): force the kubevirt-csi temp NFS unmount after a timeout
The NFS publish path temp-mounts the export root and unmounts it in a defer with a plain umount, which has no deadline. If the NFS server is gone by then, that umount can block indefinitely: the publish hangs and the temp mount stays behind for the life of the node plugin container. The Linux mounter implements MounterForceUnmounter, whose UnmountWithForce runs umount under a timeout and retries with -f once the timeout fires, so use it when the mounter offers it. A mounter without the interface keeps the plain unmount. UnmountWithForce does not bound the whole operation, though: its -f retry runs with no deadline, and the first attempt is bounded only while the umount process can be killed. The cleanup therefore waits at most one fixed budget, well inside the 2m deadline kubelet puts on NodePublishVolume, and leaves a stalled umount running in the background instead of holding the publish. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent d6e5c1b commit eb07d90

2 files changed

Lines changed: 174 additions & 1 deletion

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node.go‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"os"
77
"path/filepath"
8+
"time"
89

910
csi "github.com/container-storage-interface/spec/lib/go/csi"
1011
"google.golang.org/grpc/codes"
@@ -89,7 +90,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
8990
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
9091
}
9192
defer func() {
92-
if err := w.mounter.Unmount(tmpMount); err != nil {
93+
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
9394
klog.Warningf("Failed to unmount temp dir %s: %v", tmpMount, err)
9495
}
9596
}()
@@ -134,6 +135,42 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
134135
return &csi.NodePublishVolumeResponse{}, nil
135136
}
136137

138+
const (
139+
// csiTimeout in kubernetes pkg/volume/csi/csi_plugin.go: the deadline
140+
// kubelet puts on NodePublishVolume.
141+
kubeletCSIOperationTimeout = 2 * time.Minute
142+
// Leaves room inside kubeletCSIOperationTimeout for the mounts and the
143+
// migration that precede the unmount.
144+
tempUnmountTimeout = 30 * time.Second
145+
)
146+
147+
// unmountTempMount bounds the unmount: a plain umount of an NFS mount whose
148+
// server is gone blocks forever, which hangs the publish and leaks the mount.
149+
func unmountTempMount(m mount.Interface, dir string) error {
150+
return unmountWithin(m, dir, tempUnmountTimeout)
151+
}
152+
153+
// unmountWithin gives the whole unmount one deadline. UnmountWithForce bounds
154+
// only its first umount, and only while that process can be killed; the -f
155+
// retry it falls back to runs with no deadline at all (mount-utils v0.36.0,
156+
// forceUmount). On timeout the umount keeps running in the background.
157+
func unmountWithin(m mount.Interface, dir string, budget time.Duration) error {
158+
done := make(chan error, 1)
159+
go func() {
160+
if fm, ok := m.(mount.MounterForceUnmounter); ok {
161+
done <- fm.UnmountWithForce(dir, budget/2)
162+
return
163+
}
164+
done <- m.Unmount(dir)
165+
}()
166+
select {
167+
case err := <-done:
168+
return err
169+
case <-time.After(budget):
170+
return fmt.Errorf("unmount of %s still running after %v", dir, budget)
171+
}
172+
}
173+
137174
// NodeExpandVolume for NFS volumes is a no-op (LINSTOR handles NFS resize automatically).
138175
// This should not normally be called for NFS since ControllerExpandVolume returns
139176
// NodeExpansionRequired=false, but we handle it gracefully as a safety net.
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"path/filepath"
7+
"runtime"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
csi "github.com/container-storage-interface/spec/lib/go/csi"
13+
mount "k8s.io/mount-utils"
14+
)
15+
16+
// A path that does not exist, so the fake mounter's symlink resolution keeps
17+
// it as is and matches it against its mount table.
18+
const tempMountPath = "/nonexistent/nfs-init-pvc-1-123"
19+
20+
// forceFakeMounter adds mount.MounterForceUnmounter to the fake, as the real
21+
// Linux mounter implements it.
22+
type forceFakeMounter struct {
23+
*mount.FakeMounter
24+
forceErr error
25+
forcedTargets []string
26+
forcedTimeout time.Duration
27+
}
28+
29+
func (f *forceFakeMounter) UnmountWithForce(target string, timeout time.Duration) error {
30+
f.forcedTargets = append(f.forcedTargets, target)
31+
f.forcedTimeout = timeout
32+
return f.forceErr
33+
}
34+
35+
func mountedFake(unmountErr error) *mount.FakeMounter {
36+
m := mount.NewFakeMounter([]mount.MountPoint{{Path: tempMountPath, Type: "nfs"}})
37+
if unmountErr != nil {
38+
m.UnmountFunc = func(string) error { return unmountErr }
39+
}
40+
return m
41+
}
42+
43+
func TestUnmountTempMountUsesForceUnmounter(t *testing.T) {
44+
m := &forceFakeMounter{FakeMounter: mountedFake(errors.New("plain Unmount called"))}
45+
46+
if err := unmountTempMount(m, tempMountPath); err != nil {
47+
t.Fatalf("unmountTempMount: %v", err)
48+
}
49+
if len(m.forcedTargets) != 1 || m.forcedTargets[0] != tempMountPath {
50+
t.Fatalf("UnmountWithForce targets = %v, want [%s]", m.forcedTargets, tempMountPath)
51+
}
52+
if m.forcedTimeout <= 0 || m.forcedTimeout >= kubeletCSIOperationTimeout {
53+
t.Errorf("UnmountWithForce timeout = %v, want within (0, %v)", m.forcedTimeout, kubeletCSIOperationTimeout)
54+
}
55+
}
56+
57+
// main wires mount.New(""); without this the force path above is never taken
58+
// in production and the fallback is silent.
59+
func TestLinuxMounterIsForceUnmounter(t *testing.T) {
60+
if runtime.GOOS != "linux" {
61+
t.Skip("mount.New returns the unsupported mounter off Linux")
62+
}
63+
if _, ok := mount.New("").(mount.MounterForceUnmounter); !ok {
64+
t.Fatal("mount.New(\"\") does not implement mount.MounterForceUnmounter")
65+
}
66+
}
67+
68+
// blockingForceMounter stands for an umount -f that never returns:
69+
// mount-utils runs the forced retry with no deadline of its own.
70+
type blockingForceMounter struct {
71+
*mount.FakeMounter
72+
release chan struct{}
73+
}
74+
75+
func (b *blockingForceMounter) UnmountWithForce(string, time.Duration) error {
76+
<-b.release
77+
return nil
78+
}
79+
80+
func TestUnmountTempMountGivesUpOnAStalledForceRetry(t *testing.T) {
81+
m := &blockingForceMounter{FakeMounter: mountedFake(nil), release: make(chan struct{})}
82+
defer close(m.release)
83+
84+
done := make(chan error, 1)
85+
go func() { done <- unmountWithin(m, tempMountPath, 50*time.Millisecond) }()
86+
select {
87+
case err := <-done:
88+
if err == nil {
89+
t.Fatal("unmountWithin reported success for an unmount still running")
90+
}
91+
case <-time.After(5 * time.Second):
92+
t.Fatal("unmountWithin did not return within its budget while the forced retry stalled")
93+
}
94+
}
95+
96+
func TestUnmountTempMountReportsForceFailure(t *testing.T) {
97+
m := &forceFakeMounter{FakeMounter: mountedFake(nil), forceErr: errors.New("device busy")}
98+
99+
if err := unmountTempMount(m, tempMountPath); err == nil {
100+
t.Fatal("unmountTempMount succeeded although UnmountWithForce failed")
101+
}
102+
}
103+
104+
func TestUnmountTempMountFallsBackToPlainUnmount(t *testing.T) {
105+
m := mountedFake(nil)
106+
if err := unmountTempMount(m, tempMountPath); err != nil {
107+
t.Fatalf("unmountTempMount: %v", err)
108+
}
109+
if mps, _ := m.List(); len(mps) != 0 {
110+
t.Errorf("mount points after unmount = %v, want none", mps)
111+
}
112+
113+
if err := unmountTempMount(mountedFake(errors.New("device busy")), tempMountPath); err == nil {
114+
t.Error("unmountTempMount succeeded although Unmount failed")
115+
}
116+
}
117+
118+
func TestNodePublishVolumeNFSUnmountsTempMountWithForce(t *testing.T) {
119+
tmpRoot := t.TempDir()
120+
t.Setenv("TMPDIR", tmpRoot)
121+
122+
m := &forceFakeMounter{FakeMounter: mount.NewFakeMounter(nil)}
123+
w := &WrappedNodeService{mounter: m}
124+
req := &csi.NodePublishVolumeRequest{
125+
VolumeId: "pvc-1",
126+
TargetPath: filepath.Join(t.TempDir(), "target"),
127+
PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"},
128+
}
129+
if _, err := w.NodePublishVolume(context.Background(), req); err != nil {
130+
t.Fatalf("NodePublishVolume: %v", err)
131+
}
132+
133+
if len(m.forcedTargets) != 1 || !strings.HasPrefix(filepath.Base(m.forcedTargets[0]), "nfs-init-pvc-1-") {
134+
t.Fatalf("UnmountWithForce targets = %v, want the one nfs-init-pvc-1-* temp mount", m.forcedTargets)
135+
}
136+
}

0 commit comments

Comments
 (0)