Skip to content

Commit fa88bd1

Browse files
lexfreigithub-actions[bot]
authored andcommitted
fix(kubernetes): keep NFS publish from hanging on a dead server
The kubevirt-csi node plugin mounts the NFS export root into a temp dir to create /data and migrate old files. That mount was hard, so with the server gone the mkdir, readdir and rename calls on it never returned. kubelet gives up after 2m and retries, and every retry blocked the same way, piling up goroutines and nfs-init-* dirs. Mount the temp root soft, so its RPCs fail once the connection to the server is declared dead instead of blocking forever; it only does metadata work that a retried publish redoes. retry=0 keeps mount.nfs from retrying a failed temp mount for 2m. timeo and retrans keep their defaults, because the first mount of a server sets them for the connection the pods' mounts of it share. The volume mount the pod gets stays hard, since a soft mount can report a failed write the server applied. Over NFSv4 that soft failure takes minutes, and a stat of a target that is already hard-mounted can block for good, so a publish for a target path that already has one running now returns Aborted instead of starting another. A soft stat can fail with EIO on a file /data already holds, and a rename issued once the server is back would replace it, so the migration now fails the publish on any stat error other than a missing destination. It checks the destination with lstat, so a user's symlink in /data that loops or dangles counts as present instead of failing every publish or being replaced by the rename. The skip it makes for a destination already in /data leaves the root entry where the pod cannot see it, so it is now logged as a warning. /data is created with a plain mkdir, since with the server gone MkdirAll fails its stat on the mount point and then reports the local temp dir as already existing instead of the EIO. The temp mount errors now name the volume, like the migration errors do. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]> (cherry picked from commit f76dedf)
1 parent 37c82f3 commit fa88bd1

2 files changed

Lines changed: 318 additions & 10 deletions

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node.go‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"io/fs"
78
"os"
89
"os/exec"
910
"path/filepath"
1011
"strings"
12+
"sync"
1113
"time"
1214

1315
csi "github.com/container-storage-interface/spec/lib/go/csi"
@@ -25,6 +27,16 @@ var _ csi.NodeServer = &WrappedNodeService{}
2527
type WrappedNodeService struct {
2628
*service.NodeService
2729
mounter mount.Interface
30+
// Target paths with an NFS publish running. kubelet retries a publish
31+
// that outlives its 2m deadline while the first call still runs, and on
32+
// a dead server every retry would block the same way: the soft temp mount
33+
// gives up only after minutes, and the stat of a target that is already
34+
// hard-mounted does not give up at all.
35+
publishing sync.Map
36+
// Replaced in tests, which cannot get the soft temp mount's EIO locally;
37+
// nil means the os function.
38+
lstat func(string) (os.FileInfo, error)
39+
mkdir func(string, os.FileMode) error
2840
}
2941

3042
// NodeStageVolume for NFS volumes is a no-op (NFS doesn't need staging).
@@ -53,7 +65,19 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
5365
return nil, status.Errorf(codes.Internal, "failed to parse NFS export: %v", err)
5466
}
5567

68+
lstat, mkdir := w.lstat, w.mkdir
69+
if lstat == nil {
70+
lstat = os.Lstat
71+
}
72+
if mkdir == nil {
73+
mkdir = os.Mkdir
74+
}
75+
5676
targetPath := req.GetTargetPath()
77+
if _, busy := w.publishing.LoadOrStore(targetPath, struct{}{}); busy {
78+
return nil, status.Errorf(codes.Aborted, "publish of NFS volume %s at %s already in progress", req.GetVolumeId(), targetPath)
79+
}
80+
defer w.publishing.Delete(targetPath)
5781

5882
// Check if already mounted
5983
notMnt, err := w.mounter.IsLikelyNotMountPoint(targetPath)
@@ -83,7 +107,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
83107
// user files that were written before this fix (backward compatibility).
84108
tmpMount, err := os.MkdirTemp("", fmt.Sprintf("nfs-init-%s-", req.GetVolumeId()))
85109
if err != nil {
86-
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir: %v", err)
110+
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir for volume %s: %v", req.GetVolumeId(), err)
87111
}
88112

89113
rootSource := fmt.Sprintf("%s:%s", host, path)
@@ -93,12 +117,21 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
93117
// The price is a cache of its own too: pods already holding /data may not
94118
// see the entries the migration moves in until their attribute cache
95119
// expires (acdirmax).
96-
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"}
120+
//
121+
// soft lets an RPC fail with EIO instead of blocking forever. That is
122+
// safe here because this mount only does metadata work that a retried
123+
// publish redoes, and it is why the volume mount below stays hard. Over
124+
// NFSv4 a soft RPC fails only once the TCP connection is declared dead,
125+
// which takes minutes with the default timeo and retrans. Those are not
126+
// lowered: the first mount of a server sets them for the connection that
127+
// every later mount of it on this node shares, the pods' hard mounts
128+
// included. retry=0 stops mount.nfs from retrying a failed mount for 2m.
129+
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache", "soft", "retry=0"}
97130
if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil {
98131
if rmErr := os.Remove(tmpMount); rmErr != nil {
99132
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr)
100133
}
101-
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
134+
return nil, status.Errorf(codes.Internal, "NFS temp mount for volume %s failed: %v", req.GetVolumeId(), err)
102135
}
103136
defer func() {
104137
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
@@ -107,8 +140,10 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
107140
}()
108141

109142
dataDir := filepath.Join(tmpMount, "data")
110-
if err := os.MkdirAll(dataDir, 0777); err != nil {
111-
return nil, status.Errorf(codes.Internal, "failed to create /data subdir: %v", err)
143+
// Not MkdirAll: with the server gone its stat fails on the mount point,
144+
// and it then reports the local temp dir as existing instead of the EIO.
145+
if err := mkdir(dataDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
146+
return nil, status.Errorf(codes.Internal, "failed to create /data subdir for volume %s: %v", req.GetVolumeId(), err)
112147
}
113148

114149
// Auto-migrate: move user files from root into /data (skip internal artifacts).
@@ -124,8 +159,14 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
124159
}
125160
src := filepath.Join(tmpMount, name)
126161
dst := filepath.Join(dataDir, name)
127-
if _, err := os.Stat(dst); err == nil {
128-
continue // already exists in /data, skip
162+
// Lstat: a user's symlink in /data counts as present whatever it
163+
// points at, and is not replaced.
164+
if _, err := lstat(dst); err == nil {
165+
klog.Warningf("Not migrating %s for volume %s: /data/%s already exists, so %s stays in the export root", name, req.GetVolumeId(), name, name)
166+
continue
167+
} else if !os.IsNotExist(err) {
168+
// The rename would replace whatever the failed stat could not see.
169+
return nil, status.Errorf(codes.Internal, "failed to check /data/%s for volume %s: %v", name, req.GetVolumeId(), err)
129170
}
130171
klog.Infof("Migrating %s to /data/%s for volume %s", name, name, req.GetVolumeId())
131172
if err := os.Rename(src, dst); err != nil {
@@ -148,9 +189,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
148189

149190
const (
150191
// Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in
151-
// kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the
152-
// temp mount is a hard mount, so the mount and the migration before it
153-
// still block for as long as the server is gone.
192+
// kubernetes pkg/volume/csi/csi_plugin.go).
154193
tempUnmountTimeout = 30 * time.Second
155194
// How long umount's output pipe may stay open once umount has exited or
156195
// been killed; the umount.nfs helper it forked can hold it.

0 commit comments

Comments
 (0)