@@ -24,6 +24,7 @@ package deviceplugin
2424
2525import (
2626 "context"
27+ "crypto/rand"
2728 "errors"
2829 "fmt"
2930 "log/slog"
@@ -108,9 +109,10 @@ func Available(devs []HostDevice, devRoot string) []HostDevice {
108109type Plugin struct {
109110 pluginapi.UnimplementedDevicePluginServer
110111
111- dev HostDevice
112- socket string
113- devices []* pluginapi.Device
112+ dev HostDevice
113+ // each socket is socketPrefix plus a random suffix
114+ socketPrefix string
115+ devices []* pluginapi.Device
114116}
115117
116118var _ pluginapi.DevicePluginServer = (* Plugin )(nil )
@@ -126,17 +128,44 @@ func New(dev HostDevice) *Plugin {
126128 }
127129 return & Plugin {
128130 dev : dev ,
129- // One socket per resource, in the directory kubelet watches. The name is
130- // derived from the resource so two plugins never collide.
131- socket : filepath .Join (pluginapi .DevicePluginPath , socketName (dev .ResourceName )),
132- devices : devices ,
131+ // in the directory kubelet watches, named after the resource so two
132+ // plugins never collide
133+ socketPrefix : filepath .Join (pluginapi .DevicePluginPath , socketPrefix (dev .ResourceName )),
134+ devices : devices ,
133135 }
134136}
135137
136- // socketName maps a resource name to a socket filename ("ate.dev/kvm" ->
137- // "ate.dev-kvm.sock").
138- func socketName (resourceName string ) string {
139- return filepath .Base (filepath .Dir (resourceName )) + "-" + filepath .Base (resourceName ) + ".sock"
138+ // socketPrefix maps a resource name to a socket filename prefix
139+ // ("ate.dev/kvm" -> "ate.dev-kvm-").
140+ func socketPrefix (resourceName string ) string {
141+ return filepath .Base (filepath .Dir (resourceName )) + "-" + filepath .Base (resourceName ) + "-"
142+ }
143+
144+ // newSocket returns a fresh socket path. Since 1.37 kubelet rejects a Register
145+ // for a socket it still has connected, and can wedge the resource as "already
146+ // connected" if two Registers for one socket overlap.
147+ // https://github.com/kubernetes/kubernetes/blob/v1.37.0/pkg/kubelet/cm/devicemanager/plugin/v1beta1/handler.go#L82-L86
148+ func (p * Plugin ) newSocket () string {
149+ return p .socketPrefix + rand .Text () + ".sock"
150+ }
151+
152+ // removeSockets removes the sockets newSocket made for prefix. The random part
153+ // is base32 and has no "-", which tells them apart from the sockets of a
154+ // resource named like ours plus a "-" suffix, e.g. "ate.dev/kvm-gpu".
155+ func removeSockets (prefix string ) error {
156+ matches , err := filepath .Glob (prefix + "*.sock" )
157+ if err != nil {
158+ return fmt .Errorf ("while listing sockets: %w" , err )
159+ }
160+ for _ , path := range matches {
161+ if strings .Contains (strings .TrimPrefix (path , prefix ), "-" ) {
162+ continue
163+ }
164+ if err := os .Remove (path ); err != nil && ! errors .Is (err , os .ErrNotExist ) {
165+ return fmt .Errorf ("while removing socket %q: %w" , path , err )
166+ }
167+ }
168+ return nil
140169}
141170
142171// Run serves the plugin and keeps it registered until ctx is cancelled. Kubelet
@@ -152,6 +181,11 @@ func (p *Plugin) Run(ctx context.Context) error {
152181 return fmt .Errorf ("while watching %q: %w" , pluginapi .DevicePluginPath , err )
153182 }
154183
184+ // earlier atelets leave their socket behind, as the server runs until exit
185+ if err := removeSockets (p .socketPrefix ); err != nil {
186+ return err
187+ }
188+
155189 for ctx .Err () == nil {
156190 srv , err := p .serveAndRegister (ctx )
157191 if err != nil {
@@ -174,16 +208,13 @@ func (p *Plugin) Run(ctx context.Context) error {
174208 return ctx .Err ()
175209}
176210
177- // serveAndRegister starts the gRPC server on the plugin socket and registers it
178- // with kubelet. The returned server is stopped by the caller.
211+ // serveAndRegister starts the gRPC server on a new plugin socket and registers
212+ // it with kubelet. The returned server is stopped by the caller.
179213func (p * Plugin ) serveAndRegister (ctx context.Context ) (* grpc.Server , error ) {
180- // A leftover socket from a previous run would make Listen fail with EADDRINUSE.
181- if err := os .Remove (p .socket ); err != nil && ! errors .Is (err , os .ErrNotExist ) {
182- return nil , fmt .Errorf ("while removing stale socket %q: %w" , p .socket , err )
183- }
184- lis , err := net .Listen ("unix" , p .socket )
214+ socket := p .newSocket ()
215+ lis , err := net .Listen ("unix" , socket )
185216 if err != nil {
186- return nil , fmt .Errorf ("while listening on %q: %w" , p . socket , err )
217+ return nil , fmt .Errorf ("while listening on %q: %w" , socket , err )
187218 }
188219
189220 srv := grpc .NewServer ()
@@ -195,15 +226,15 @@ func (p *Plugin) serveAndRegister(ctx context.Context) (*grpc.Server, error) {
195226 }
196227 }()
197228
198- if err := p .register (ctx ); err != nil {
229+ if err := p .register (ctx , socket ); err != nil {
199230 srv .Stop ()
200231 return nil , err
201232 }
202233 return srv , nil
203234}
204235
205- // register tells kubelet which resource this plugin's socket serves.
206- func (p * Plugin ) register (ctx context.Context ) error {
236+ // register tells kubelet which resource socket serves.
237+ func (p * Plugin ) register (ctx context.Context , socket string ) error {
207238 ctx , cancel := context .WithTimeout (ctx , registerTimeout )
208239 defer cancel ()
209240
@@ -216,7 +247,7 @@ func (p *Plugin) register(ctx context.Context) error {
216247
217248 _ , err = pluginapi .NewRegistrationClient (conn ).Register (ctx , & pluginapi.RegisterRequest {
218249 Version : pluginapi .Version ,
219- Endpoint : filepath .Base (p . socket ),
250+ Endpoint : filepath .Base (socket ),
220251 ResourceName : p .dev .ResourceName ,
221252 })
222253 if err != nil {
@@ -253,9 +284,10 @@ func (p *Plugin) waitForKubeletRestart(ctx context.Context, watcher *fsnotify.Wa
253284}
254285
255286// kubeletRestarted reports whether event is kubelet recreating socket since we
256- // stat'd it as registered. Re-registering on a stale event can wedge the
257- // resource in kubelet as "already connected", if the second Register for our
258- // socket arrives while kubelet is still closing the first stream.
287+ // stat'd it as registered. We don't re-register on a stale event, because
288+ // kubelet before 1.37 disconnects the new registration if it handles our old
289+ // stream closing after the new Register.
290+ // https://github.com/kubernetes/kubernetes/blob/v1.36.0/pkg/kubelet/cm/devicemanager/plugin/v1beta1/handler.go#L122-L134
259291func kubeletRestarted (event fsnotify.Event , socket string , registered os.FileInfo ) bool {
260292 if event .Name != socket || ! event .Has (fsnotify .Create ) {
261293 return false
0 commit comments