Skip to content

Commit f1e0746

Browse files
robmryrobmry
committed
Tell RootlessKit about docker-proxy port mappings
Before this change, when running rootless, instead of running docker-proxy the daemon would run rootlesskit-docker-proxy. The job of rootlesskit-docker-proxy was to tell RootlessKit about mapped host ports before starting docker-proxy, and then to remove the mapping when it was stopped. So, rootlesskit-docker-proxy would need to be kept in-step with changes to docker-proxy (particuarly the upcoming change to bind TCP/UDP ports in the daemon and pass them to the proxy, but also possible-future changes like running proxy per-container rather than per-port-mapping). This change runs the docker-proxy in rootless mode, instead of rootlesskit-docker-proxy, and the daemon itself tells RootlessKit about changes in host port mappings. Signed-off-by: Rob Murray <[email protected]>
1 parent 384ca56 commit f1e0746

6 files changed

Lines changed: 368 additions & 39 deletions

File tree

‎daemon/config/config_linux.go‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ const (
3434
StockRuntimeName = "runc"
3535

3636
// userlandProxyBinary is the name of the userland-proxy binary.
37-
// In rootless-mode, [rootless.RootlessKitDockerProxyBinary] is used instead.
3837
userlandProxyBinary = "docker-proxy"
3938
)
4039

@@ -234,16 +233,25 @@ func setPlatformDefaults(cfg *Config) error {
234233
cfg.CgroupNamespaceMode = string(DefaultCgroupNamespaceMode)
235234
}
236235

236+
var err error
237+
cfg.BridgeConfig.UserlandProxyPath, err = lookupBinPath(userlandProxyBinary)
238+
if err != nil {
239+
// Log, but don't error here. This allows running a daemon with
240+
// userland-proxy disabled (which does not require the binary
241+
// to be present).
242+
//
243+
// An error is still produced by [Config.ValidatePlatformConfig] if
244+
// userland-proxy is enabled in the configuration.
245+
//
246+
// We log this at "debug" level, as this code is also executed
247+
// when running "--version", and we don't want to print logs in
248+
// that case..
249+
log.G(context.TODO()).WithError(err).Debug("failed to lookup default userland-proxy binary")
250+
}
251+
237252
if rootless.RunningWithRootlessKit() {
238253
cfg.Rootless = true
239254

240-
var err error
241-
// use rootlesskit-docker-proxy for exposing the ports in RootlessKit netns to the initial namespace.
242-
cfg.BridgeConfig.UserlandProxyPath, err = lookupBinPath(rootless.RootlessKitDockerProxyBinary)
243-
if err != nil {
244-
return errors.Wrapf(err, "running with RootlessKit, but %s not installed", rootless.RootlessKitDockerProxyBinary)
245-
}
246-
247255
dataHome, err := homedir.GetDataHome()
248256
if err != nil {
249257
return err
@@ -257,21 +265,6 @@ func setPlatformDefaults(cfg *Config) error {
257265
cfg.ExecRoot = filepath.Join(runtimeDir, "docker")
258266
cfg.Pidfile = filepath.Join(runtimeDir, "docker.pid")
259267
} else {
260-
var err error
261-
cfg.BridgeConfig.UserlandProxyPath, err = lookupBinPath(userlandProxyBinary)
262-
if err != nil {
263-
// Log, but don't error here. This allows running a daemon with
264-
// userland-proxy disabled (which does not require the binary
265-
// to be present).
266-
//
267-
// An error is still produced by [Config.ValidatePlatformConfig] if
268-
// userland-proxy is enabled in the configuration.
269-
//
270-
// We log this at "debug" level, as this code is also executed
271-
// when running "--version", and we don't want to print logs in
272-
// that case..
273-
log.G(context.TODO()).WithError(err).Debug("failed to lookup default userland-proxy binary")
274-
}
275268
cfg.Root = "/var/lib/docker"
276269
cfg.ExecRoot = "/var/run/docker"
277270
cfg.Pidfile = "/var/run/docker.pid"

‎daemon/daemon_unix.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,7 @@ func driverOptions(config *config.Config) nwconfig.Option {
915915
"EnableIP6Tables": config.BridgeConfig.EnableIP6Tables,
916916
"EnableUserlandProxy": config.BridgeConfig.EnableUserlandProxy,
917917
"UserlandProxyPath": config.BridgeConfig.UserlandProxyPath,
918+
"Rootless": config.Rootless,
918919
},
919920
})
920921
}

‎libnetwork/drivers/bridge/bridge_linux.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/docker/docker/errdefs"
1515
"github.com/docker/docker/libnetwork/datastore"
1616
"github.com/docker/docker/libnetwork/driverapi"
17+
"github.com/docker/docker/libnetwork/drivers/bridge/rlkclient"
1718
"github.com/docker/docker/libnetwork/internal/netiputil"
1819
"github.com/docker/docker/libnetwork/iptables"
1920
"github.com/docker/docker/libnetwork/netlabel"
@@ -56,6 +57,7 @@ type configuration struct {
5657
EnableIP6Tables bool
5758
EnableUserlandProxy bool
5859
UserlandProxyPath string
60+
Rootless bool
5961
}
6062

6163
// networkConfiguration for network specific configuration
@@ -131,6 +133,14 @@ type bridgeNetwork struct {
131133
sync.Mutex
132134
}
133135

136+
type portDriverClient interface {
137+
ChildHostIP(hostIP netip.Addr) netip.Addr
138+
AddPort(ctx context.Context, proto string, hostIP, childIP netip.Addr, hostPort int) (func() error, error)
139+
}
140+
141+
// Allow unit tests to supply a dummy RootlessKit port driver client.
142+
var newPortDriverClient = func() (portDriverClient, error) { return rlkclient.NewPortDriverClient() }
143+
134144
type driver struct {
135145
config configuration
136146
natChain *iptables.ChainInfo
@@ -144,6 +154,7 @@ type driver struct {
144154
networks map[string]*bridgeNetwork
145155
store *datastore.Store
146156
nlh *netlink.Handle
157+
portDriverClient portDriverClient
147158
configNetwork sync.Mutex
148159
sync.Mutex
149160
}
@@ -414,6 +425,15 @@ func (n *bridgeNetwork) userlandProxyPath() string {
414425
return n.driver.userlandProxyPath()
415426
}
416427

428+
func (n *bridgeNetwork) getPortDriverClient() portDriverClient {
429+
n.Lock()
430+
defer n.Unlock()
431+
if n.driver == nil {
432+
return nil
433+
}
434+
return n.driver.getPortDriverClient()
435+
}
436+
417437
func (n *bridgeNetwork) getEndpoint(eid string) (*bridgeEndpoint, error) {
418438
if eid == "" {
419439
return nil, InvalidEndpointIDError(eid)
@@ -465,6 +485,7 @@ func (d *driver) configure(option map[string]interface{}) error {
465485
filterChainV6 *iptables.ChainInfo
466486
isolationChain1V6 *iptables.ChainInfo
467487
isolationChain2V6 *iptables.ChainInfo
488+
pdc portDriverClient
468489
)
469490

470491
switch opt := option[netlabel.GenericData].(type) {
@@ -537,6 +558,14 @@ func (d *driver) configure(option map[string]interface{}) error {
537558
}
538559
}
539560

561+
if config.EnableUserlandProxy && config.Rootless {
562+
var err error
563+
pdc, err = newPortDriverClient()
564+
if err != nil {
565+
return err
566+
}
567+
}
568+
540569
d.Lock()
541570
d.natChain = natChain
542571
d.filterChain = filterChain
@@ -546,6 +575,7 @@ func (d *driver) configure(option map[string]interface{}) error {
546575
d.filterChainV6 = filterChainV6
547576
d.isolationChain1V6 = isolationChain1V6
548577
d.isolationChain2V6 = isolationChain2V6
578+
d.portDriverClient = pdc
549579
d.config = config
550580
d.Unlock()
551581

@@ -577,6 +607,12 @@ func (d *driver) userlandProxyPath() string {
577607
return ""
578608
}
579609

610+
func (d *driver) getPortDriverClient() portDriverClient {
611+
d.Lock()
612+
defer d.Unlock()
613+
return d.portDriverClient
614+
}
615+
580616
func parseNetworkGenericOptions(data interface{}) (*networkConfiguration, error) {
581617
var (
582618
err error

‎libnetwork/drivers/bridge/port_mapping_linux.go‎

Lines changed: 61 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,28 @@ import (
2323

2424
type portBinding struct {
2525
types.PortBinding
26+
// childHostIP is the host IP address, as seen from the daemon. This
27+
// is normally the same as PortBinding.HostIP but, in rootless mode, it
28+
// will be an address in the rootless network namespace. RootlessKit
29+
// binds the port on the real (parent) host address and maps it to the
30+
// same port number on the address dockerd sees in the child namespace.
31+
// So, for example, docker-proxy and DNAT rules need to use the child
32+
// namespace's host address. (PortBinding.HostIP isn't replaced by the
33+
// child address, because it's stored as user-config and the child
34+
// address may change if RootlessKit is configured differently.)
35+
childHostIP net.IP
36+
// portDriverRemove is a function that will inform the RootlessKit
37+
// port driver about removal of a port binding, or nil.
38+
portDriverRemove func() error
39+
// stopProxy is a function to stop the userland proxy for this binding,
40+
// if a proxy has been started - else nil.
2641
stopProxy func() error
2742
}
2843

2944
type portBindingReq struct {
3045
types.PortBinding
31-
disableNAT bool
46+
childHostIP net.IP
47+
disableNAT bool
3248
}
3349

3450
// addPortMappings takes cfg, the configuration for port mappings, selects host
@@ -79,6 +95,7 @@ func (n *bridgeNetwork) addPortMappings(
7995
sortAndNormPBs(sortedCfg)
8096

8197
proxyPath := n.userlandProxyPath()
98+
pdc := n.getPortDriverClient()
8299

83100
// toBind accumulates port bindings that should be allocated the same host port
84101
// (if required by NAT config). If the host address is unspecified, and defHostIP
@@ -91,7 +108,7 @@ func (n *bridgeNetwork) addPortMappings(
91108
// bindings to collect, they're applied and toBind is reset.
92109
var toBind []portBindingReq
93110
for i, c := range sortedCfg {
94-
if bindingIPv4, ok := configurePortBindingIPv4(disableNAT4, c, containerIPv4, defHostIP); ok {
111+
if bindingIPv4, ok := configurePortBindingIPv4(pdc, disableNAT4, c, containerIPv4, defHostIP); ok {
95112
toBind = append(toBind, bindingIPv4)
96113
}
97114

@@ -107,7 +124,7 @@ func (n *bridgeNetwork) addPortMappings(
107124
if proxyPath != "" && (containerIPv6 == nil) {
108125
containerIP = containerIPv4
109126
}
110-
if bindingIPv6, ok := configurePortBindingIPv6(disableNAT6, c, containerIP, defHostIP); ok {
127+
if bindingIPv6, ok := configurePortBindingIPv6(pdc, disableNAT6, c, containerIP, defHostIP); ok {
111128
toBind = append(toBind, bindingIPv6)
112129
}
113130

@@ -129,8 +146,24 @@ func (n *bridgeNetwork) addPortMappings(
129146
toBind = toBind[:0]
130147
}
131148

132-
for _, b := range bindings {
133-
if err := n.setPerPortIptables(b, true); err != nil {
149+
for i := range bindings {
150+
if pdc != nil && bindings[i].HostPort != 0 {
151+
var err error
152+
b := &bindings[i]
153+
hip, ok := netip.AddrFromSlice(b.HostIP)
154+
if !ok {
155+
return nil, fmt.Errorf("invalid host IP address in %s", b)
156+
}
157+
chip, ok := netip.AddrFromSlice(b.childHostIP)
158+
if !ok {
159+
return nil, fmt.Errorf("invalid child host IP address %s in %s", b.childHostIP, b)
160+
}
161+
b.portDriverRemove, err = pdc.AddPort(context.TODO(), b.Proto.String(), hip, chip, int(b.HostPort))
162+
if err != nil {
163+
return nil, err
164+
}
165+
}
166+
if err := n.setPerPortIptables(bindings[i], true); err != nil {
134167
return nil, err
135168
}
136169
}
@@ -263,7 +296,7 @@ func needSamePort(a, b types.PortBinding) bool {
263296

264297
// configurePortBindingIPv4 returns a new port binding with the HostIP field populated
265298
// if a binding is required, else nil.
266-
func configurePortBindingIPv4(disableNAT bool, bnd types.PortBinding, containerIPv4, defHostIP net.IP) (portBindingReq, bool) {
299+
func configurePortBindingIPv4(pdc portDriverClient, disableNAT bool, bnd types.PortBinding, containerIPv4, defHostIP net.IP) (portBindingReq, bool) {
267300
if len(containerIPv4) == 0 {
268301
return portBindingReq{}, false
269302
}
@@ -282,15 +315,15 @@ func configurePortBindingIPv4(disableNAT bool, bnd types.PortBinding, containerI
282315
// Unmap the addresses if they're IPv4-mapped IPv6.
283316
bnd.HostIP = bnd.HostIP.To4()
284317
bnd.IP = containerIPv4.To4()
285-
return portBindingReq{
318+
return setChildHostIP(pdc, portBindingReq{
286319
PortBinding: bnd,
287320
disableNAT: disableNAT,
288-
}, true
321+
}), true
289322
}
290323

291324
// configurePortBindingIPv6 returns a new port binding with the HostIP field populated
292325
// if a binding is required, else nil.
293-
func configurePortBindingIPv6(disableNAT bool, bnd types.PortBinding, containerIP, defHostIP net.IP) (portBindingReq, bool) {
326+
func configurePortBindingIPv6(pdc portDriverClient, disableNAT bool, bnd types.PortBinding, containerIP, defHostIP net.IP) (portBindingReq, bool) {
294327
if containerIP == nil {
295328
return portBindingReq{}, false
296329
}
@@ -317,10 +350,20 @@ func configurePortBindingIPv6(disableNAT bool, bnd types.PortBinding, containerI
317350
}
318351
}
319352
bnd.IP = containerIP
320-
return portBindingReq{
353+
return setChildHostIP(pdc, portBindingReq{
321354
PortBinding: bnd,
322355
disableNAT: disableNAT,
323-
}, true
356+
}), true
357+
}
358+
359+
func setChildHostIP(pdc portDriverClient, req portBindingReq) portBindingReq {
360+
if pdc == nil {
361+
req.childHostIP = req.HostIP
362+
return req
363+
}
364+
hip, _ := netip.AddrFromSlice(req.HostIP)
365+
req.childHostIP = pdc.ChildHostIP(hip).AsSlice()
366+
return req
324367
}
325368

326369
// bindHostPorts allocates ports and starts docker-proxy for the given cfg. The
@@ -410,7 +453,7 @@ func attemptBindHostPorts(
410453
if c.disableNAT {
411454
pb.HostPort = 0
412455
} else {
413-
pb.stopProxy, err = startProxy(c.Proto.String(), c.HostIP, port, c.IP, int(c.Port), proxyPath)
456+
pb.stopProxy, err = startProxy(c.Proto.String(), c.childHostIP, port, c.IP, int(c.Port), proxyPath)
414457
if err != nil {
415458
return nil, fmt.Errorf("failed to bind port %s:%d/%s: %w", c.HostIP, port, c.Proto, err)
416459
}
@@ -424,6 +467,7 @@ func attemptBindHostPorts(
424467
pb.HostPort = uint16(port)
425468
}
426469
pb.HostPortEnd = pb.HostPort
470+
pb.childHostIP = c.childHostIP
427471
res = append(res, pb)
428472
}
429473
return res, nil
@@ -442,7 +486,10 @@ func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint) error {
442486
func (n *bridgeNetwork) releasePortBindings(pbs []portBinding) error {
443487
var errs []error
444488
for _, pb := range pbs {
445-
var errP error
489+
var errPD, errP error
490+
if pb.portDriverRemove != nil {
491+
errPD = pb.portDriverRemove()
492+
}
446493
if pb.stopProxy != nil {
447494
errP = pb.stopProxy()
448495
if errP != nil {
@@ -456,7 +503,7 @@ func (n *bridgeNetwork) releasePortBindings(pbs []portBinding) error {
456503
if pb.HostPort > 0 {
457504
portallocator.Get().ReleasePort(pb.HostIP, pb.Proto.String(), int(pb.HostPort))
458505
}
459-
errs = append(errs, errP, errN)
506+
errs = append(errs, errPD, errP, errN)
460507
}
461508
return errors.Join(errs...)
462509
}

0 commit comments

Comments
 (0)