Skip to content

Commit dd2e19e

Browse files
dmcgowandmcgowan
committed
libcontainerd: split client and supervisor
Adds a supervisor package for starting and monitoring containerd. Separates grpc connection allowing access from daemon. Signed-off-by: Derek McGowan <[email protected]>
1 parent 724c5f3 commit dd2e19e

19 files changed

Lines changed: 338 additions & 474 deletions

‎cmd/dockerd/daemon.go‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import (
3636
"github.com/docker/docker/daemon/config"
3737
"github.com/docker/docker/daemon/listeners"
3838
"github.com/docker/docker/dockerversion"
39-
"github.com/docker/docker/libcontainerd"
39+
"github.com/docker/docker/libcontainerd/supervisor"
4040
dopts "github.com/docker/docker/opts"
4141
"github.com/docker/docker/pkg/authorization"
4242
"github.com/docker/docker/pkg/jsonmessage"
@@ -45,7 +45,6 @@ import (
4545
"github.com/docker/docker/pkg/signal"
4646
"github.com/docker/docker/pkg/system"
4747
"github.com/docker/docker/plugin"
48-
"github.com/docker/docker/registry"
4948
"github.com/docker/docker/runconfig"
5049
"github.com/docker/go-connections/tlsconfig"
5150
swarmapi "github.com/docker/swarmkit/api"
@@ -112,6 +111,10 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
112111
return err
113112
}
114113

114+
if err := system.MkdirAll(cli.Config.ExecRoot, 0700, ""); err != nil {
115+
return err
116+
}
117+
115118
if cli.Pidfile != "" {
116119
pf, err := pidfile.New(cli.Pidfile)
117120
if err != nil {
@@ -135,19 +138,27 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
135138
return fmt.Errorf("Failed to load listeners: %v", err)
136139
}
137140

138-
registryService, err := registry.NewService(cli.Config.ServiceOptions)
139-
if err != nil {
140-
return err
141-
}
141+
ctx, cancel := context.WithCancel(context.Background())
142+
if cli.Config.ContainerdAddr == "" && runtime.GOOS != "windows" {
143+
opts, err := cli.getContainerdDaemonOpts()
144+
if err != nil {
145+
cancel()
146+
return fmt.Errorf("Failed to generate containerd options: %v", err)
147+
}
142148

143-
rOpts, err := cli.getRemoteOptions()
144-
if err != nil {
145-
return fmt.Errorf("Failed to generate containerd options: %v", err)
146-
}
147-
containerdRemote, err := libcontainerd.New(filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), rOpts...)
148-
if err != nil {
149-
return err
149+
r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
150+
if err != nil {
151+
cancel()
152+
return fmt.Errorf("Failed to start containerd: %v", err)
153+
}
154+
155+
cli.Config.ContainerdAddr = r.Address()
156+
157+
// Try to wait for containerd to shutdown
158+
defer r.WaitTimeout(10 * time.Second)
150159
}
160+
defer cancel()
161+
151162
signal.Trap(func() {
152163
cli.stop()
153164
<-stopc // wait for daemonCli.start() to return
@@ -162,7 +173,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
162173
logrus.Fatalf("Error creating middlewares: %v", err)
163174
}
164175

165-
d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
176+
d, err := daemon.NewDaemon(ctx, cli.Config, pluginStore)
166177
if err != nil {
167178
return fmt.Errorf("Error starting daemon: %v", err)
168179
}
@@ -207,10 +218,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
207218

208219
initRouter(routerOptions)
209220

210-
// process cluster change notifications
211-
watchCtx, cancel := context.WithCancel(context.Background())
212-
defer cancel()
213-
go d.ProcessClusterNotifications(watchCtx, c.GetWatchStream())
221+
go d.ProcessClusterNotifications(ctx, c.GetWatchStream())
214222

215223
cli.setupConfigReloadTrap()
216224

@@ -227,8 +235,12 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
227235
// Wait for serve API to complete
228236
errAPI := <-serveAPIWait
229237
c.Cleanup()
238+
230239
shutdownDaemon(d)
231-
containerdRemote.Cleanup()
240+
241+
// Stop notification processing and any background processes
242+
cancel()
243+
232244
if errAPI != nil {
233245
return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
234246
}
@@ -511,14 +523,22 @@ func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config
511523
return nil
512524
}
513525

514-
func (cli *DaemonCli) getRemoteOptions() ([]libcontainerd.RemoteOption, error) {
515-
opts := []libcontainerd.RemoteOption{}
516-
517-
pOpts, err := cli.getPlatformRemoteOptions()
526+
func (cli *DaemonCli) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
527+
opts, err := cli.getPlatformContainerdDaemonOpts()
518528
if err != nil {
519529
return nil, err
520530
}
521-
opts = append(opts, pOpts...)
531+
532+
if cli.Config.Debug {
533+
opts = append(opts, supervisor.WithLogLevel("debug"))
534+
} else if cli.Config.LogLevel != "" {
535+
opts = append(opts, supervisor.WithLogLevel(cli.Config.LogLevel))
536+
}
537+
538+
if !cli.Config.CriContainerd {
539+
opts = append(opts, supervisor.WithPlugin("cri", nil))
540+
}
541+
522542
return opts, nil
523543
}
524544

‎cmd/dockerd/daemon_unix.go‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/containerd/containerd/runtime/linux"
1414
"github.com/docker/docker/cmd/dockerd/hack"
1515
"github.com/docker/docker/daemon"
16-
"github.com/docker/docker/libcontainerd"
16+
"github.com/docker/docker/libcontainerd/supervisor"
1717
"github.com/docker/libnetwork/portallocator"
1818
"golang.org/x/sys/unix"
1919
)
@@ -36,29 +36,16 @@ func getDaemonConfDir(_ string) string {
3636
return "/etc/docker"
3737
}
3838

39-
func (cli *DaemonCli) getPlatformRemoteOptions() ([]libcontainerd.RemoteOption, error) {
40-
opts := []libcontainerd.RemoteOption{
41-
libcontainerd.WithOOMScore(cli.Config.OOMScoreAdjust),
42-
libcontainerd.WithPlugin("linux", &linux.Config{
39+
func (cli *DaemonCli) getPlatformContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
40+
opts := []supervisor.DaemonOpt{
41+
supervisor.WithOOMScore(cli.Config.OOMScoreAdjust),
42+
supervisor.WithPlugin("linux", &linux.Config{
4343
Shim: daemon.DefaultShimBinary,
4444
Runtime: daemon.DefaultRuntimeBinary,
4545
RuntimeRoot: filepath.Join(cli.Config.Root, "runc"),
4646
ShimDebug: cli.Config.Debug,
4747
}),
4848
}
49-
if cli.Config.Debug {
50-
opts = append(opts, libcontainerd.WithLogLevel("debug"))
51-
} else if cli.Config.LogLevel != "" {
52-
opts = append(opts, libcontainerd.WithLogLevel(cli.Config.LogLevel))
53-
}
54-
if cli.Config.ContainerdAddr != "" {
55-
opts = append(opts, libcontainerd.WithRemoteAddr(cli.Config.ContainerdAddr))
56-
} else {
57-
opts = append(opts, libcontainerd.WithStartDaemon(true))
58-
}
59-
if !cli.Config.CriContainerd {
60-
opts = append(opts, libcontainerd.WithPlugin("cri", nil))
61-
}
6249

6350
return opts, nil
6451
}

‎cmd/dockerd/daemon_windows.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"os"
77
"path/filepath"
88

9-
"github.com/docker/docker/libcontainerd"
9+
"github.com/docker/docker/libcontainerd/supervisor"
1010
"github.com/sirupsen/logrus"
1111
"golang.org/x/sys/windows"
1212
)
@@ -48,7 +48,7 @@ func notifyShutdown(err error) {
4848
}
4949
}
5050

51-
func (cli *DaemonCli) getPlatformRemoteOptions() ([]libcontainerd.RemoteOption, error) {
51+
func (cli *DaemonCli) getPlatformContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
5252
return nil, nil
5353
}
5454

‎daemon/daemon.go‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ import (
1818
"sync"
1919
"time"
2020

21+
"google.golang.org/grpc"
22+
23+
"github.com/containerd/containerd"
24+
"github.com/containerd/containerd/defaults"
25+
"github.com/containerd/containerd/pkg/dialer"
2126
"github.com/docker/docker/api/types"
2227
containertypes "github.com/docker/docker/api/types/container"
2328
"github.com/docker/docker/api/types/swarm"
@@ -94,6 +99,7 @@ type Daemon struct {
9499
PluginStore *plugin.Store // todo: remove
95100
pluginManager *plugin.Manager
96101
linkIndex *linkIndex
102+
containerdCli *containerd.Client
97103
containerd libcontainerd.Client
98104
defaultIsolation containertypes.Isolation // Default isolation mode on Windows
99105
clusterProvider cluster.Provider
@@ -565,9 +571,14 @@ func (daemon *Daemon) IsSwarmCompatible() error {
565571

566572
// NewDaemon sets up everything for the daemon to be able to service
567573
// requests from the webserver.
568-
func NewDaemon(config *config.Config, registryService registry.Service, containerdRemote libcontainerd.Remote, pluginStore *plugin.Store) (daemon *Daemon, err error) {
574+
func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.Store) (daemon *Daemon, err error) {
569575
setDefaultMtu(config)
570576

577+
registryService, err := registry.NewService(config.ServiceOptions)
578+
if err != nil {
579+
return nil, err
580+
}
581+
571582
// Ensure that we have a correct root key limit for launching containers.
572583
if err := ModifyRootKeyLimit(); err != nil {
573584
logrus.Warnf("unable to modify root key limit, number of containers could be limited by this quota: %v", err)
@@ -720,8 +731,35 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
720731
}
721732
registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
722733

734+
gopts := []grpc.DialOption{
735+
grpc.WithInsecure(),
736+
grpc.WithBackoffMaxDelay(3 * time.Second),
737+
grpc.WithDialer(dialer.Dialer),
738+
739+
// TODO(stevvooe): We may need to allow configuration of this on the client.
740+
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
741+
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
742+
}
743+
if config.ContainerdAddr != "" {
744+
d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts))
745+
if err != nil {
746+
return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
747+
}
748+
}
749+
723750
createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {
724-
return pluginexec.New(getPluginExecRoot(config.Root), containerdRemote, m)
751+
var pluginCli *containerd.Client
752+
753+
// Windows is not currently using containerd, keep the
754+
// client as nil
755+
if config.ContainerdAddr != "" {
756+
pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts))
757+
if err != nil {
758+
return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
759+
}
760+
}
761+
762+
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, m)
725763
}
726764

727765
// Plugin system initialization should happen before restore. Do not change order.
@@ -880,7 +918,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
880918

881919
go d.execCommandGC()
882920

883-
d.containerd, err = containerdRemote.NewClient(ContainersNamespace, d)
921+
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), ContainersNamespace, d)
884922
if err != nil {
885923
return nil, err
886924
}
@@ -1037,6 +1075,10 @@ func (daemon *Daemon) Shutdown() error {
10371075
daemon.netController.Stop()
10381076
}
10391077

1078+
if daemon.containerdCli != nil {
1079+
daemon.containerdCli.Close()
1080+
}
1081+
10401082
return daemon.cleanupMounts()
10411083
}
10421084

0 commit comments

Comments
 (0)