Skip to content

Commit 38e9ef6

Browse files
authored
atelet: register the device plugin on a new socket each time (agent-substrate#3486)
Follow-up to agent-substrate#3323, as @SergeyKanzhelev suggested there. kubelet expects a device plugin to switch to a new socket when it re-registers. Since 1.37 it rejects a Register for a socket it still has connected, and two overlapping Registers for one socket can wedge the resource as "already connected" until kubelet restarts: kubernetes/kubernetes#139338 - the device plugin listens on a new `ate.dev-kvm-<random>.sock` for each registration, and removes sockets left over from earlier atelets - the device plugin stays up until atelet exits again. agent-substrate#3323 stopped it on SIGTERM to avoid overlapping with the replacement pod, but the DaemonSet only starts the replacement once the old pod is gone, so that just left `ate.dev/kvm` unhealthy for the whole drain <!-- e.g. "AI-assisted, I reviewed and tested it." No AI co-author trailers. --> AI use: I used Claude to draft this and reviewed it. <!-- Breaking change? Describe what users must do on upgrade and ask for the `breaking-change` label. --> - [x] Tests pass - [x] Docs updated --------- Signed-off-by: Benjamin Elder <[email protected]>
1 parent ac8c621 commit 38e9ef6

3 files changed

Lines changed: 99 additions & 38 deletions

File tree

‎cmd/atelet/main.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -235,10 +235,7 @@ func main() {
235235
}()
236236
}
237237

238-
// stopped on SIGTERM because the replacement pod starts while we drain and
239-
// registers the same socket, and kubelet can wedge the resource when two
240-
// registrations for one socket overlap
241-
startDevicePlugins(shutdownCtx)
238+
startDevicePlugins(ctx)
242239

243240
workerDialer := newWorkerDialer(256)
244241

‎internal/deviceplugin/deviceplugin.go‎

Lines changed: 58 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ package deviceplugin
2424

2525
import (
2626
"context"
27+
"crypto/rand"
2728
"errors"
2829
"fmt"
2930
"log/slog"
@@ -108,9 +109,10 @@ func Available(devs []HostDevice, devRoot string) []HostDevice {
108109
type Plugin struct {
109110
pluginapi.UnimplementedDevicePluginServer
110111

111-
dev HostDevice
112-
socket string
113-
devices []*pluginapi.Device
112+
dev HostDevice
113+
// each socket is socketPrefix plus a random suffix
114+
socketPrefix string
115+
devices []*pluginapi.Device
114116
}
115117

116118
var _ pluginapi.DevicePluginServer = (*Plugin)(nil)
@@ -126,17 +128,44 @@ func New(dev HostDevice) *Plugin {
126128
}
127129
return &Plugin{
128130
dev: dev,
129-
// One socket per resource, in the directory kubelet watches. The name is
130-
// derived from the resource so two plugins never collide.
131-
socket: filepath.Join(pluginapi.DevicePluginPath, socketName(dev.ResourceName)),
132-
devices: devices,
131+
// in the directory kubelet watches, named after the resource so two
132+
// plugins never collide
133+
socketPrefix: filepath.Join(pluginapi.DevicePluginPath, socketPrefix(dev.ResourceName)),
134+
devices: devices,
133135
}
134136
}
135137

136-
// socketName maps a resource name to a socket filename ("ate.dev/kvm" ->
137-
// "ate.dev-kvm.sock").
138-
func socketName(resourceName string) string {
139-
return filepath.Base(filepath.Dir(resourceName)) + "-" + filepath.Base(resourceName) + ".sock"
138+
// socketPrefix maps a resource name to a socket filename prefix
139+
// ("ate.dev/kvm" -> "ate.dev-kvm-").
140+
func socketPrefix(resourceName string) string {
141+
return filepath.Base(filepath.Dir(resourceName)) + "-" + filepath.Base(resourceName) + "-"
142+
}
143+
144+
// newSocket returns a fresh socket path. Since 1.37 kubelet rejects a Register
145+
// for a socket it still has connected, and can wedge the resource as "already
146+
// connected" if two Registers for one socket overlap.
147+
// https://github.com/kubernetes/kubernetes/blob/v1.37.0/pkg/kubelet/cm/devicemanager/plugin/v1beta1/handler.go#L82-L86
148+
func (p *Plugin) newSocket() string {
149+
return p.socketPrefix + rand.Text() + ".sock"
150+
}
151+
152+
// removeSockets removes the sockets newSocket made for prefix. The random part
153+
// is base32 and has no "-", which tells them apart from the sockets of a
154+
// resource named like ours plus a "-" suffix, e.g. "ate.dev/kvm-gpu".
155+
func removeSockets(prefix string) error {
156+
matches, err := filepath.Glob(prefix + "*.sock")
157+
if err != nil {
158+
return fmt.Errorf("while listing sockets: %w", err)
159+
}
160+
for _, path := range matches {
161+
if strings.Contains(strings.TrimPrefix(path, prefix), "-") {
162+
continue
163+
}
164+
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
165+
return fmt.Errorf("while removing socket %q: %w", path, err)
166+
}
167+
}
168+
return nil
140169
}
141170

142171
// Run serves the plugin and keeps it registered until ctx is cancelled. Kubelet
@@ -152,6 +181,11 @@ func (p *Plugin) Run(ctx context.Context) error {
152181
return fmt.Errorf("while watching %q: %w", pluginapi.DevicePluginPath, err)
153182
}
154183

184+
// earlier atelets leave their socket behind, as the server runs until exit
185+
if err := removeSockets(p.socketPrefix); err != nil {
186+
return err
187+
}
188+
155189
for ctx.Err() == nil {
156190
srv, err := p.serveAndRegister(ctx)
157191
if err != nil {
@@ -174,16 +208,13 @@ func (p *Plugin) Run(ctx context.Context) error {
174208
return ctx.Err()
175209
}
176210

177-
// serveAndRegister starts the gRPC server on the plugin socket and registers it
178-
// with kubelet. The returned server is stopped by the caller.
211+
// serveAndRegister starts the gRPC server on a new plugin socket and registers
212+
// it with kubelet. The returned server is stopped by the caller.
179213
func (p *Plugin) serveAndRegister(ctx context.Context) (*grpc.Server, error) {
180-
// A leftover socket from a previous run would make Listen fail with EADDRINUSE.
181-
if err := os.Remove(p.socket); err != nil && !errors.Is(err, os.ErrNotExist) {
182-
return nil, fmt.Errorf("while removing stale socket %q: %w", p.socket, err)
183-
}
184-
lis, err := net.Listen("unix", p.socket)
214+
socket := p.newSocket()
215+
lis, err := net.Listen("unix", socket)
185216
if err != nil {
186-
return nil, fmt.Errorf("while listening on %q: %w", p.socket, err)
217+
return nil, fmt.Errorf("while listening on %q: %w", socket, err)
187218
}
188219

189220
srv := grpc.NewServer()
@@ -195,15 +226,15 @@ func (p *Plugin) serveAndRegister(ctx context.Context) (*grpc.Server, error) {
195226
}
196227
}()
197228

198-
if err := p.register(ctx); err != nil {
229+
if err := p.register(ctx, socket); err != nil {
199230
srv.Stop()
200231
return nil, err
201232
}
202233
return srv, nil
203234
}
204235

205-
// register tells kubelet which resource this plugin's socket serves.
206-
func (p *Plugin) register(ctx context.Context) error {
236+
// register tells kubelet which resource socket serves.
237+
func (p *Plugin) register(ctx context.Context, socket string) error {
207238
ctx, cancel := context.WithTimeout(ctx, registerTimeout)
208239
defer cancel()
209240

@@ -216,7 +247,7 @@ func (p *Plugin) register(ctx context.Context) error {
216247

217248
_, err = pluginapi.NewRegistrationClient(conn).Register(ctx, &pluginapi.RegisterRequest{
218249
Version: pluginapi.Version,
219-
Endpoint: filepath.Base(p.socket),
250+
Endpoint: filepath.Base(socket),
220251
ResourceName: p.dev.ResourceName,
221252
})
222253
if err != nil {
@@ -253,9 +284,10 @@ func (p *Plugin) waitForKubeletRestart(ctx context.Context, watcher *fsnotify.Wa
253284
}
254285

255286
// kubeletRestarted reports whether event is kubelet recreating socket since we
256-
// stat'd it as registered. Re-registering on a stale event can wedge the
257-
// resource in kubelet as "already connected", if the second Register for our
258-
// socket arrives while kubelet is still closing the first stream.
287+
// stat'd it as registered. We don't re-register on a stale event, because
288+
// kubelet before 1.37 disconnects the new registration if it handles our old
289+
// stream closing after the new Register.
290+
// https://github.com/kubernetes/kubernetes/blob/v1.36.0/pkg/kubelet/cm/devicemanager/plugin/v1beta1/handler.go#L122-L134
259291
func kubeletRestarted(event fsnotify.Event, socket string, registered os.FileInfo) bool {
260292
if event.Name != socket || !event.Has(fsnotify.Create) {
261293
return false

‎internal/deviceplugin/deviceplugin_test.go‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ package deviceplugin
1616

1717
import (
1818
"context"
19+
"crypto/rand"
20+
"errors"
1921
"os"
2022
"path/filepath"
2123
"testing"
@@ -26,21 +28,51 @@ import (
2628
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
2729
)
2830

29-
// Each resource gets its own socket in the directory kubelet watches, and two
30-
// resources must never collide on one filename.
31+
// Each resource gets its own sockets in the directory kubelet watches, and two
32+
// resources never collide on one filename.
3133
func TestSocketPathsAreDistinctAndInKubeletDir(t *testing.T) {
3234
kvm := New(HostDevice{ResourceName: "ate.dev/kvm", Path: "/dev/kvm"})
3335
other := New(HostDevice{ResourceName: "ate.dev/other", Path: "/dev/other"})
3436

35-
if kvm.socket == other.socket {
36-
t.Errorf("sockets collide: %q", kvm.socket)
37+
if kvm.socketPrefix == other.socketPrefix {
38+
t.Errorf("socket prefixes collide: %q", kvm.socketPrefix)
3739
}
3840
for _, p := range []*Plugin{kvm, other} {
39-
if got := filepath.Dir(p.socket) + "/"; got != pluginapi.DevicePluginPath {
40-
t.Errorf("socket %q is not in %q", p.socket, pluginapi.DevicePluginPath)
41+
socket := p.newSocket()
42+
if got := filepath.Dir(socket) + "/"; got != pluginapi.DevicePluginPath {
43+
t.Errorf("socket %q is not in %q", socket, pluginapi.DevicePluginPath)
4144
}
42-
if filepath.Ext(p.socket) != ".sock" {
43-
t.Errorf("socket %q should end in .sock", p.socket)
45+
if filepath.Ext(socket) != ".sock" {
46+
t.Errorf("socket %q should end in .sock", socket)
47+
}
48+
}
49+
}
50+
51+
// removeSockets removes only our own sockets, not those of a resource whose
52+
// name extends ours.
53+
func TestRemoveSocketsKeepsOtherResources(t *testing.T) {
54+
dir := t.TempDir()
55+
prefix := filepath.Join(dir, socketPrefix("ate.dev/kvm"))
56+
ours := prefix + rand.Text() + ".sock"
57+
others := []string{
58+
filepath.Join(dir, socketPrefix("ate.dev/kvm-gpu")+rand.Text()+".sock"),
59+
filepath.Join(dir, "kubelet.sock"),
60+
}
61+
for _, path := range append(others, ours) {
62+
if err := os.WriteFile(path, nil, 0o600); err != nil {
63+
t.Fatal(err)
64+
}
65+
}
66+
67+
if err := removeSockets(prefix); err != nil {
68+
t.Fatalf("removeSockets() = %v", err)
69+
}
70+
if _, err := os.Stat(ours); !errors.Is(err, os.ErrNotExist) {
71+
t.Errorf("%q still exists, err = %v", ours, err)
72+
}
73+
for _, path := range others {
74+
if _, err := os.Stat(path); err != nil {
75+
t.Errorf("%q was removed: %v", path, err)
4476
}
4577
}
4678
}

0 commit comments

Comments
 (0)