Skip to content

Commit b77235c

Browse files
committed
fix(kubernetes): force the kubevirt-csi temp NFS unmount after a timeout
The NFS publish path temp-mounts the export root and unmounts it in a defer with a plain umount, which has no deadline. If the NFS server is gone by then, that umount can block indefinitely: the publish hangs and the temp mount stays behind for the life of the node plugin container. mount-utils' UnmountWithForce cannot fix this. It retries with -f only after its first umount returns, and that umount forks the umount.nfs helper: killed on the timeout, umount leaves the helper stuck on the vanished server with the output pipe still open, and CombinedOutput waits for the pipe, so -f never runs. The plugin therefore runs umount itself with WaitDelay set, which closes the pipe shortly after the kill, and falls back to umount -f under a timeout of its own. As in mount-utils, "not mounted" counts as done: umount2 detaches the mount before it blocks tearing the superblock down, so the -f after a killed umount can find nothing left. A mounter that is not the exec-based one keeps its plain Unmount. An umount in uninterruptible sleep outlives any kill, so the cleanup also waits at most one fixed budget, well inside the 2m deadline kubelet puts on NodePublishVolume, and leaves a stalled unmount running in the background, where it logs its result once it ends. The temp dir is removed by that background unmount once it succeeds, not by the publish: removed at the deadline it is still a mount point, and nothing would retry. A forced unmount runs nfs_umount_begin, which kills every RPC in flight on the superblock, and NFS shares one superblock between mounts of the same export with the same options. The temp mount therefore gets nosharecache, so -f cannot fail the I/O of pods on the node that mount the same volume. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent ec4d26d commit b77235c

2 files changed

Lines changed: 445 additions & 9 deletions

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node.go‎

Lines changed: 107 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,13 @@ package main
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"os"
8+
"os/exec"
79
"path/filepath"
10+
"strings"
11+
"time"
812

913
csi "github.com/container-storage-interface/spec/lib/go/csi"
1014
"google.golang.org/grpc/codes"
@@ -81,21 +85,23 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
8185
if err != nil {
8286
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir: %v", err)
8387
}
84-
// Logged, not returned: a leftover temp dir does not make the volume
85-
// unusable, and failing an otherwise good publish only makes kubelet retry.
86-
defer func() {
87-
if err := os.Remove(tmpMount); err != nil {
88-
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, err)
89-
}
90-
}()
9188

9289
rootSource := fmt.Sprintf("%s:%s", host, path)
93-
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port)}
90+
// nosharecache gives the temp mount a superblock of its own: the forced
91+
// unmount below kills every RPC in flight on its superblock, which would
92+
// otherwise be the one the pods on this node use for the same export.
93+
// The price is a cache of its own too: pods already holding /data may not
94+
// see the entries the migration moves in until their attribute cache
95+
// expires (acdirmax).
96+
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"}
9497
if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil {
98+
if rmErr := os.Remove(tmpMount); rmErr != nil {
99+
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr)
100+
}
95101
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
96102
}
97103
defer func() {
98-
if err := w.mounter.Unmount(tmpMount); err != nil {
104+
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
99105
klog.Warningf("Failed to unmount temp dir %s: %v", tmpMount, err)
100106
}
101107
}()
@@ -140,6 +146,98 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
140146
return &csi.NodePublishVolumeResponse{}, nil
141147
}
142148

149+
const (
150+
// Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in
151+
// kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the
152+
// temp mount is a hard mount, so the mount and the migration before it
153+
// still block for as long as the server is gone.
154+
tempUnmountTimeout = 30 * time.Second
155+
// How long umount's output pipe may stay open once umount has exited or
156+
// been killed; the umount.nfs helper it forked can hold it.
157+
umountPipeGrace = time.Second
158+
)
159+
160+
// unmountTempMount bounds the unmount: a plain umount of an NFS mount whose
161+
// server is gone blocks forever, which hangs the publish and leaks the mount.
162+
func unmountTempMount(m mount.Interface, dir string) error {
163+
return unmountWithin(m, dir, tempUnmountTimeout)
164+
}
165+
166+
// unmountWithin gives the whole unmount one deadline. On timeout the unmount
167+
// keeps running in the background and logs its result when it ends; an umount
168+
// stuck in uninterruptible sleep cannot be killed, so no deadline on the
169+
// process itself can be relied on.
170+
//
171+
// The dir is removed by the worker once the unmount succeeds, so an unmount
172+
// that finishes after the deadline still cleans up; removed by the caller it
173+
// would still be a mount point at that moment. A failed unmount keeps the dir.
174+
// Plain Remove rather than RemoveAll: if the unmount did not take, the dir's
175+
// contents are the export's.
176+
func unmountWithin(m mount.Interface, dir string, budget time.Duration) error {
177+
done := make(chan error)
178+
abandoned := make(chan struct{})
179+
go func() {
180+
var err error
181+
if _, ok := m.(*mount.Mounter); ok {
182+
err = umountForcedAfter(dir, budget/3)
183+
} else {
184+
err = m.Unmount(dir)
185+
}
186+
if err == nil {
187+
if rmErr := os.Remove(dir); rmErr != nil && !os.IsNotExist(rmErr) {
188+
klog.Warningf("Failed to remove temp dir %s: %v", dir, rmErr)
189+
}
190+
}
191+
select {
192+
case done <- err:
193+
case <-abandoned:
194+
if err != nil {
195+
klog.Warningf("unmount of %s failed after the deadline: %v", dir, err)
196+
} else {
197+
klog.Infof("unmount of %s finished after the deadline", dir)
198+
}
199+
}
200+
}()
201+
select {
202+
case err := <-done:
203+
return err
204+
case <-time.After(budget):
205+
close(abandoned)
206+
return fmt.Errorf("unmount of %s still running after %v", dir, budget)
207+
}
208+
}
209+
210+
// umountForcedAfter replaces mount-utils' UnmountWithForce, which retries with
211+
// -f only once its first umount has returned. That umount forks umount.nfs;
212+
// killed on the deadline, it leaves the helper stuck on the vanished server
213+
// holding the output pipe, and CombinedOutput waits for that pipe, so the -f
214+
// never runs. WaitDelay closes the pipe instead.
215+
func umountForcedAfter(dir string, timeout time.Duration) error {
216+
err := runUmount(timeout, dir)
217+
if !errors.Is(err, context.DeadlineExceeded) {
218+
return err
219+
}
220+
klog.V(2).Infof("Timed out waiting for unmount of %s, trying with -f", dir)
221+
return runUmount(timeout, "-f", dir)
222+
}
223+
224+
func runUmount(timeout time.Duration, args ...string) error {
225+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
226+
defer cancel()
227+
cmd := exec.CommandContext(ctx, "umount", args...)
228+
cmd.WaitDelay = umountPipeGrace
229+
out, err := cmd.CombinedOutput()
230+
if ctx.Err() != nil {
231+
return fmt.Errorf("umount %s: %w", strings.Join(args, " "), ctx.Err())
232+
}
233+
// umount2 detaches the mount before it blocks tearing the superblock down,
234+
// so the -f after a killed umount can find nothing left to unmount.
235+
if err != nil && !strings.Contains(string(out), "not mounted") {
236+
return fmt.Errorf("umount %s: %v: %s", strings.Join(args, " "), err, out)
237+
}
238+
return nil
239+
}
240+
143241
// NodeExpandVolume for NFS volumes is a no-op (LINSTOR handles NFS resize automatically).
144242
// This should not normally be called for NFS since ControllerExpandVolume returns
145243
// NodeExpansionRequired=false, but we handle it gracefully as a safety net.

0 commit comments

Comments
 (0)