Skip to content

Commit 2fecf5b

Browse files
cpuguy83cpuguy83
committed
Make sure exit signals trigger an exit during init
Some cases can cause the server initialization to block (namely running a 2nd containerd instance by accident against the same root dir). In this case there is no way to quit the daemon except with `kill -9`. This changes context things so that server init is done in a goroutine and we wait on a channel for it to be ready while we also wait for a ctx.Done(), which will be cancelled if there is a termination signal. Signed-off-by: Brian Goff <[email protected]>
1 parent d081457 commit 2fecf5b

3 files changed

Lines changed: 62 additions & 23 deletions

File tree

‎cmd/containerd/command/main.go‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,15 @@ can be used and modified as necessary as a custom configuration.`
111111
}
112112
app.Action = func(context *cli.Context) error {
113113
var (
114-
start = time.Now()
115-
signals = make(chan os.Signal, 2048)
116-
serverC = make(chan *server.Server, 1)
117-
ctx = gocontext.Background()
118-
config = defaultConfig()
114+
start = time.Now()
115+
signals = make(chan os.Signal, 2048)
116+
serverC = make(chan *server.Server, 1)
117+
ctx, cancel = gocontext.WithCancel(gocontext.Background())
118+
config = defaultConfig()
119119
)
120120

121+
defer cancel()
122+
121123
// Only try to load the config if it either exists, or the user explicitly
122124
// told us to load this path.
123125
configPath := context.GlobalString("config")
@@ -161,7 +163,7 @@ can be used and modified as necessary as a custom configuration.`
161163
return nil
162164
}
163165

164-
done := handleSignals(ctx, signals, serverC)
166+
done := handleSignals(ctx, signals, serverC, cancel)
165167
// start the signal handler as soon as we can to make sure that
166168
// we don't miss any signals during boot
167169
signal.Notify(signals, handledSignals...)
@@ -193,17 +195,55 @@ can be used and modified as necessary as a custom configuration.`
193195
"revision": version.Revision,
194196
}).Info("starting containerd")
195197

196-
server, err := server.New(ctx, config)
197-
if err != nil {
198-
return err
198+
type srvResp struct {
199+
s *server.Server
200+
err error
199201
}
202+
// run server initialization in a goroutine so we don't end up blocking important things like SIGTERM handling
203+
// while the server is initializing.
204+
// As an example opening the bolt database will block forever if another containerd is already running and containerd
205+
// will have to be be `kill -9`'ed to recover.
206+
chsrv := make(chan srvResp)
207+
go func() {
208+
defer close(chsrv)
209+
210+
server, err := server.New(ctx, config)
211+
if err != nil {
212+
select {
213+
case chsrv <- srvResp{err: err}:
214+
case <-ctx.Done():
215+
}
216+
return
217+
}
200218

201-
// Launch as a Windows Service if necessary
202-
if err := launchService(server, done); err != nil {
203-
logrus.Fatal(err)
219+
// Launch as a Windows Service if necessary
220+
if err := launchService(server, done); err != nil {
221+
logrus.Fatal(err)
222+
}
223+
select {
224+
case <-ctx.Done():
225+
server.Stop()
226+
case chsrv <- srvResp{s: server}:
227+
}
228+
}()
229+
230+
var server *server.Server
231+
select {
232+
case <-ctx.Done():
233+
return ctx.Err()
234+
case r := <-chsrv:
235+
if r.err != nil {
236+
return err
237+
}
238+
server = r.s
204239
}
205240

206-
serverC <- server
241+
// We don't send the server down serverC directly in the goroutine above because we need it lower down.
242+
select {
243+
case <-ctx.Done():
244+
return ctx.Err()
245+
case serverC <- server:
246+
}
207247

208248
if config.Debug.Address != "" {
209249
var l net.Listener

‎cmd/containerd/command/main_unix.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ var handledSignals = []os.Signal{
3636
unix.SIGPIPE,
3737
}
3838

39-
func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} {
39+
func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} {
4040
done := make(chan struct{}, 1)
4141
go func() {
4242
var server *server.Server
@@ -61,11 +61,10 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se
6161
log.G(ctx).WithError(err).Error("notify stopping failed")
6262
}
6363

64-
if server == nil {
65-
close(done)
66-
return
64+
cancel()
65+
if server != nil {
66+
server.Stop()
6767
}
68-
server.Stop()
6968
close(done)
7069
return
7170
}

‎cmd/containerd/command/main_windows.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ var (
3939
}
4040
)
4141

42-
func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} {
42+
func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} {
4343
done := make(chan struct{})
4444
go func() {
4545
var server *server.Server
@@ -54,12 +54,12 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se
5454
log.G(ctx).WithError(err).Error("notify stopping failed")
5555
}
5656

57-
if server == nil {
58-
close(done)
59-
return
57+
cancel()
58+
if server != nil {
59+
server.Stop()
6060
}
61-
server.Stop()
6261
close(done)
62+
return
6363
}
6464
}
6565
}()

0 commit comments

Comments
 (0)