Skip to content

Commit 155d9c5

Browse files
committed
fix(kubernetes): keep NFS publish from hanging on a dead server
The kubevirt-csi node plugin mounts the NFS export root into a temp dir to create /data and migrate old files. That mount was hard, so with the server gone the mkdir, readdir and rename calls on it never returned. kubelet gives up after 2m and retries, and every retry blocked the same way, piling up goroutines and nfs-init-* dirs. Mount the temp root soft, so its RPCs fail once the connection to the server is declared dead instead of blocking forever; it only does metadata work that a retried publish redoes. retry=0 keeps mount.nfs from retrying a failed temp mount for 2m. timeo and retrans keep their defaults, because the first mount of a server sets them for the connection the pods' mounts of it share. The volume mount the pod gets stays hard, since a soft mount can report a failed write the server applied. Over NFSv4 that soft failure takes minutes, and a stat of a target that is already hard-mounted can block for good, so a publish for a target path that already has one running now returns Aborted instead of starting another. A soft stat can fail with EIO on a file /data already holds, and a rename issued once the server is back would replace it, so the migration now fails the publish on any stat error other than a missing destination. It checks the destination with lstat, so a user's symlink in /data that loops or dangles counts as present instead of failing every publish or being replaced by the rename. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent d7f3f0d commit 155d9c5

2 files changed

Lines changed: 235 additions & 5 deletions

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node.go‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os/exec"
99
"path/filepath"
1010
"strings"
11+
"sync"
1112
"time"
1213

1314
csi "github.com/container-storage-interface/spec/lib/go/csi"
@@ -21,10 +22,19 @@ import (
2122

2223
var _ csi.NodeServer = &WrappedNodeService{}
2324

25+
// lstat is replaced in tests: the soft temp mount's EIO cannot be produced locally.
26+
var lstat = os.Lstat
27+
2428
// WrappedNodeService embeds the upstream NodeService and adds NFS mount support.
2529
type WrappedNodeService struct {
2630
*service.NodeService
2731
mounter mount.Interface
32+
// Target paths with an NFS publish running. kubelet retries a publish
33+
// that outlives its 2m deadline while the first call still runs, and on
34+
// a dead server every retry would block the same way: the soft temp mount
35+
// gives up only after minutes, and the stat of a target that is already
36+
// hard-mounted does not give up at all.
37+
publishing sync.Map
2838
}
2939

3040
// NodeStageVolume for NFS volumes is a no-op (NFS doesn't need staging).
@@ -54,6 +64,10 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
5464
}
5565

5666
targetPath := req.GetTargetPath()
67+
if _, busy := w.publishing.LoadOrStore(targetPath, struct{}{}); busy {
68+
return nil, status.Errorf(codes.Aborted, "publish of NFS volume %s at %s already in progress", req.GetVolumeId(), targetPath)
69+
}
70+
defer w.publishing.Delete(targetPath)
5771

5872
// Check if already mounted
5973
notMnt, err := w.mounter.IsLikelyNotMountPoint(targetPath)
@@ -93,7 +107,16 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
93107
// The price is a cache of its own too: pods already holding /data may not
94108
// see the entries the migration moves in until their attribute cache
95109
// expires (acdirmax).
96-
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"}
110+
//
111+
// soft lets an RPC fail with EIO instead of blocking forever. That is
112+
// safe here because this mount only does metadata work that a retried
113+
// publish redoes, and it is why the volume mount below stays hard. Over
114+
// NFSv4 a soft RPC fails only once the TCP connection is declared dead,
115+
// which takes minutes with the default timeo and retrans. Those are not
116+
// lowered: the first mount of a server sets them for the connection that
117+
// every later mount of it on this node shares, the pods' hard mounts
118+
// included. retry=0 stops mount.nfs from retrying a failed mount for 2m.
119+
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache", "soft", "retry=0"}
97120
if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil {
98121
if rmErr := os.Remove(tmpMount); rmErr != nil {
99122
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr)
@@ -124,8 +147,13 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
124147
}
125148
src := filepath.Join(tmpMount, name)
126149
dst := filepath.Join(dataDir, name)
127-
if _, err := os.Stat(dst); err == nil {
150+
// Lstat: a user's symlink in /data counts as present whatever it
151+
// points at, and is not replaced.
152+
if _, err := lstat(dst); err == nil {
128153
continue // already exists in /data, skip
154+
} else if !os.IsNotExist(err) {
155+
// The rename would replace whatever the failed stat could not see.
156+
return nil, status.Errorf(codes.Internal, "failed to check /data/%s for volume %s: %v", name, req.GetVolumeId(), err)
129157
}
130158
klog.Infof("Migrating %s to /data/%s for volume %s", name, name, req.GetVolumeId())
131159
if err := os.Rename(src, dst); err != nil {
@@ -148,9 +176,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
148176

149177
const (
150178
// Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in
151-
// kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the
152-
// temp mount is a hard mount, so the mount and the migration before it
153-
// still block for as long as the server is gone.
179+
// kubernetes pkg/volume/csi/csi_plugin.go).
154180
tempUnmountTimeout = 30 * time.Second
155181
// How long umount's output pipe may stay open once umount has exited or
156182
// been killed; the umount.nfs helper it forked can hold it.

‎packages/apps/kubernetes/images/kubevirt-csi-driver/node_test.go‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,15 @@ import (
55
"context"
66
"os"
77
"path/filepath"
8+
"slices"
89
"strings"
10+
"sync/atomic"
11+
"syscall"
912
"testing"
1013

1114
csi "github.com/container-storage-interface/spec/lib/go/csi"
15+
"google.golang.org/grpc/codes"
16+
"google.golang.org/grpc/status"
1217
"k8s.io/klog/v2"
1318
mount "k8s.io/mount-utils"
1419
)
@@ -69,3 +74,202 @@ func TestNodePublishVolumeNFSLogsTempDirRemovalFailure(t *testing.T) {
6974
}
7075
t.Errorf("removal failure of %s was not logged as a warning; log output:\n%s", leftovers[0], logs.String())
7176
}
77+
78+
func nfsPublishRequest(target string) *csi.NodePublishVolumeRequest {
79+
return &csi.NodePublishVolumeRequest{
80+
VolumeId: "pvc-1",
81+
TargetPath: target,
82+
PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"},
83+
}
84+
}
85+
86+
func hasRetryTuning(opts []string) bool {
87+
for _, o := range opts {
88+
if o == "soft" || strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") || strings.HasPrefix(o, "retry=") {
89+
return true
90+
}
91+
}
92+
return false
93+
}
94+
95+
// A soft mount can report a failed write the server did apply, harmless for
96+
// the metadata-only temp mount and not for the pods' data.
97+
func TestNodePublishVolumeNFSMountsOnlyTheTempRootSoft(t *testing.T) {
98+
t.Setenv("TMPDIR", t.TempDir())
99+
100+
// The fake drops the options of every mount an unmount leaves in place, so
101+
// both sets are read while the temp mount is being unmounted, after the
102+
// volume mount.
103+
var tempOpts, finalOpts []string
104+
m := mount.NewFakeMounter(nil)
105+
m.UnmountFunc = func(path string) error {
106+
for _, mp := range m.MountPoints {
107+
switch {
108+
case mp.Path == path:
109+
tempOpts = mp.Opts
110+
case mp.Device == "192.0.2.10:/export/data":
111+
finalOpts = mp.Opts
112+
}
113+
}
114+
return nil
115+
}
116+
w := &WrappedNodeService{mounter: m}
117+
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
118+
t.Fatalf("NodePublishVolume: %v", err)
119+
}
120+
121+
for _, want := range []string{"soft", "retry=0"} {
122+
if !slices.Contains(tempOpts, want) {
123+
t.Errorf("temp mount options = %v, want %s", tempOpts, want)
124+
}
125+
}
126+
// The first mount of a server sets the TCP timeouts of the connection the
127+
// pods' mounts of it share.
128+
for _, o := range tempOpts {
129+
if strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") {
130+
t.Errorf("temp mount options = %v, want the default timeo and retrans", tempOpts)
131+
}
132+
}
133+
if finalOpts == nil {
134+
t.Fatal("the /data subdir was not mounted before the temp mount was unmounted")
135+
}
136+
if hasRetryTuning(finalOpts) {
137+
t.Errorf("volume mount options = %v, want a hard mount with default retry", finalOpts)
138+
}
139+
}
140+
141+
// stalledMounter holds the first mount it gets until release is closed.
142+
type stalledMounter struct {
143+
*mount.FakeMounter
144+
held atomic.Bool
145+
entered chan struct{}
146+
release chan struct{}
147+
}
148+
149+
func (b *stalledMounter) Mount(source, target, fstype string, opts []string) error {
150+
if b.held.CompareAndSwap(false, true) {
151+
close(b.entered)
152+
<-b.release
153+
}
154+
return b.FakeMounter.Mount(source, target, fstype, opts)
155+
}
156+
157+
// kubelet retries a publish that outlives its deadline while the first call
158+
// still runs; on a dead server each retry would block the same way.
159+
func TestNodePublishVolumeNFSRefusesAPublishAlreadyInProgress(t *testing.T) {
160+
t.Setenv("TMPDIR", t.TempDir())
161+
m := &stalledMounter{
162+
FakeMounter: mount.NewFakeMounter(nil),
163+
entered: make(chan struct{}),
164+
release: make(chan struct{}),
165+
}
166+
w := &WrappedNodeService{mounter: m}
167+
target := filepath.Join(t.TempDir(), "target")
168+
169+
first := make(chan error, 1)
170+
go func() {
171+
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
172+
first <- err
173+
}()
174+
<-m.entered
175+
176+
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
177+
if status.Code(err) != codes.Aborted {
178+
t.Errorf("publish while one is in progress: err = %v, want code Aborted", err)
179+
}
180+
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "other"))); err != nil {
181+
t.Errorf("publish of the same volume at another target: %v", err)
182+
}
183+
184+
close(m.release)
185+
if err := <-first; err != nil {
186+
t.Fatalf("first publish: %v", err)
187+
}
188+
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)); err != nil {
189+
t.Errorf("publish after the first one finished: %v", err)
190+
}
191+
}
192+
193+
func TestNodePublishVolumeNFSReleasesTheTargetAfterAFailedPublish(t *testing.T) {
194+
t.Setenv("TMPDIR", t.TempDir())
195+
w := &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}}
196+
req := nfsPublishRequest(filepath.Join(t.TempDir(), "target"))
197+
for i := range 2 {
198+
_, err := w.NodePublishVolume(context.Background(), req)
199+
if code := status.Code(err); code != codes.Internal {
200+
t.Fatalf("publish %d: err = %v, want code Internal from the failed mount", i+1, err)
201+
}
202+
}
203+
}
204+
205+
// seedingMounter fills the first mount's target with what seed creates there.
206+
type seedingMounter struct {
207+
*mount.FakeMounter
208+
seed func(target string) error
209+
}
210+
211+
func (s *seedingMounter) Mount(source, target, fstype string, opts []string) error {
212+
if seed := s.seed; seed != nil {
213+
s.seed = nil
214+
if err := seed(target); err != nil {
215+
return err
216+
}
217+
}
218+
return s.FakeMounter.Mount(source, target, fstype, opts)
219+
}
220+
221+
func seedRootAndData(root *string, link string) func(string) error {
222+
return func(target string) error {
223+
*root = target
224+
if err := os.WriteFile(filepath.Join(target, "f"), nil, 0o600); err != nil {
225+
return err
226+
}
227+
if err := os.Mkdir(filepath.Join(target, "data"), 0o750); err != nil {
228+
return err
229+
}
230+
return os.Symlink(link, filepath.Join(target, "data", "f"))
231+
}
232+
}
233+
234+
func assertNotMigrated(t *testing.T, root string) {
235+
t.Helper()
236+
if _, err := os.Stat(filepath.Join(root, "f")); err != nil {
237+
t.Errorf("source was moved: %v", err)
238+
}
239+
if fi, err := os.Lstat(filepath.Join(root, "data", "f")); err != nil || fi.Mode()&os.ModeSymlink == 0 {
240+
t.Errorf("destination was replaced: %v, %v", fi, err)
241+
}
242+
}
243+
244+
// A soft temp mount answers a stat with EIO while the server is unreachable;
245+
// a rename issued once it is back would replace the file the stat missed.
246+
func TestNodePublishVolumeNFSDoesNotMigrateOverAnUncheckedDestination(t *testing.T) {
247+
t.Setenv("TMPDIR", t.TempDir())
248+
orig := lstat
249+
lstat = func(string) (os.FileInfo, error) { return nil, syscall.EIO }
250+
t.Cleanup(func() { lstat = orig })
251+
var root string
252+
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")}}
253+
254+
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
255+
if status.Code(err) != codes.Internal {
256+
t.Fatalf("err = %v, want code Internal from the failed stat", err)
257+
}
258+
assertNotMigrated(t, root)
259+
}
260+
261+
// The symlinks are the user's: one that loops or dangles is still an entry
262+
// in /data, neither a reason to fail the publish nor one to replace it.
263+
func TestNodePublishVolumeNFSLeavesASymlinkInDataAlone(t *testing.T) {
264+
for name, link := range map[string]string{"loop": "f", "dangling": "missing"} {
265+
t.Run(name, func(t *testing.T) {
266+
t.Setenv("TMPDIR", t.TempDir())
267+
var root string
268+
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, link)}}
269+
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
270+
t.Fatalf("NodePublishVolume: %v", err)
271+
}
272+
assertNotMigrated(t, root)
273+
})
274+
}
275+
}

0 commit comments

Comments
 (0)