Skip to content

Commit 4740820

Browse files
robmryrobmry
committed
Use ioctl to try to trigger kernel module loads
An ioctl() call to get the "interface index" for a kernel module triggers the kernel to try to load the module, if the process is running with CAP_SYS_MODULE. This tends to be more reliable than "modprobe" for docker-in-docker. If the ioctl() method fails, fall back to trying "modprobe". Signed-off-by: Rob Murray <[email protected]>
1 parent efa041a commit 4740820

3 files changed

Lines changed: 120 additions & 30 deletions

File tree

‎internal/modprobe/modprobe_linux.go‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Package modprobe attempts to load kernel modules. It may have more success
2+
// than simply running "modprobe", particularly for docker-in-docker.
3+
package modprobe
4+
5+
import (
6+
"context"
7+
"errors"
8+
"fmt"
9+
"os/exec"
10+
"strings"
11+
12+
"github.com/containerd/log"
13+
"golang.org/x/sys/unix"
14+
)
15+
16+
// LoadModules attempts to load kernel modules, if necessary.
17+
//
18+
// isLoaded must be a function that checks whether the modules are loaded. It may
19+
// be called multiple times. isLoaded must return an error to indicate that the
20+
// modules still need to be loaded, otherwise nil.
21+
//
22+
// For each method of loading modules, LoadModules will attempt the load for each
23+
// of modNames, then it will call isLoaded to check the result - moving on to try
24+
// the next method if needed, and there is one.
25+
//
26+
// The returned error is the result of the final call to isLoaded.
27+
func LoadModules(ctx context.Context, isLoaded func() error, modNames ...string) error {
28+
if isLoaded() == nil {
29+
log.G(ctx).WithFields(log.Fields{
30+
"modules": modNames,
31+
}).Debug("Modules already loaded")
32+
return nil
33+
}
34+
35+
if err := tryLoad(ctx, isLoaded, modNames, ioctlLoader{}); err != nil {
36+
return tryLoad(ctx, isLoaded, modNames, modprobeLoader{})
37+
}
38+
return nil
39+
}
40+
41+
type loader interface {
42+
name() string
43+
load(modName string) error
44+
}
45+
46+
func tryLoad(ctx context.Context, isLoaded func() error, modNames []string, loader loader) error {
47+
var loadErrs []error
48+
for _, modName := range modNames {
49+
if err := loader.load(modName); err != nil {
50+
loadErrs = append(loadErrs, err)
51+
}
52+
}
53+
54+
if checkResult := isLoaded(); checkResult != nil {
55+
log.G(ctx).WithFields(log.Fields{
56+
"loader": loader.name(),
57+
"modules": modNames,
58+
"loadErrors": errors.Join(loadErrs...),
59+
"checkResult": checkResult,
60+
}).Debug("Modules not loaded")
61+
return checkResult
62+
}
63+
64+
log.G(ctx).WithFields(log.Fields{
65+
"loader": loader.name(),
66+
"modules": modNames,
67+
"loadErrors": errors.Join(loadErrs...),
68+
}).Debug("Modules loaded")
69+
return nil
70+
}
71+
72+
// ioctlLoader attempts to load the module using an ioctl() to get the interface index
73+
// of a module - it won't have one, but the kernel may load the module. This tends to
74+
// work in docker-in-docker, where the inner-docker may not have "modprobe" or access
75+
// to modules in the host's filesystem.
76+
type ioctlLoader struct{}
77+
78+
func (il ioctlLoader) name() string { return "ioctl" }
79+
80+
func (il ioctlLoader) load(modName string) error {
81+
sd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
82+
if err != nil {
83+
return fmt.Errorf("creating socket for ioctl load of %s: %w", modName, err)
84+
}
85+
defer unix.Close(sd)
86+
87+
// This tends to work, if running with CAP_SYS_MODULE, because...
88+
// https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L457
89+
// https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L371-L372
90+
ifreq, err := unix.NewIfreq(modName)
91+
if err != nil {
92+
return fmt.Errorf("creating ifreq for %s: %w", modName, err)
93+
}
94+
// An error is returned even if the module load is successful. So, ignore it.
95+
_ = unix.IoctlIfreq(sd, unix.SIOCGIFINDEX, ifreq)
96+
return nil
97+
}
98+
99+
// modprobeLoader attempts to load a kernel module using modprobe.
100+
type modprobeLoader struct{}
101+
102+
func (ml modprobeLoader) name() string { return "modprobe" }
103+
104+
func (ml modprobeLoader) load(modName string) error {
105+
out, err := exec.Command("modprobe", "-va", modName).CombinedOutput()
106+
if err != nil {
107+
return fmt.Errorf("modprobe %s failed with message: %q, error: %w", modName, strings.TrimSpace(string(out)), err)
108+
}
109+
return nil
110+
}

‎libnetwork/drivers/bridge/setup_bridgenetfiltering.go‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ import (
77
"errors"
88
"fmt"
99
"os"
10-
"os/exec"
1110
"syscall"
1211

1312
"github.com/containerd/log"
13+
"github.com/docker/docker/internal/modprobe"
1414
)
1515

1616
// setupIPv4BridgeNetFiltering checks whether IPv4 forwarding is enabled and, if
@@ -46,21 +46,17 @@ func setupIPv6BridgeNetFiltering(config *networkConfiguration, _ *bridgeInterfac
4646
}
4747

4848
func loadBridgeNetFilterModule(fullPath string) error {
49-
// br_netfilter implictly loads bridge module upon modprobe
50-
modName := "br_netfilter"
51-
if _, err := os.Stat(fullPath); err != nil {
52-
if out, err := exec.Command("modprobe", "-va", modName).CombinedOutput(); err != nil {
53-
log.G(context.TODO()).WithError(err).Errorf("Running modprobe %s failed with message: %s", modName, out)
54-
return fmt.Errorf("cannot restrict inter-container communication: modprobe %s failed: %w", modName, err)
55-
}
56-
}
57-
return nil
49+
// br_netfilter implicitly loads bridge module upon modprobe
50+
return modprobe.LoadModules(context.TODO(), func() error {
51+
_, err := os.Stat(fullPath)
52+
return err
53+
}, "br_netfilter")
5854
}
5955

6056
// Enable bridge net filtering if not already enabled. See GitHub issue #11404
6157
func enableBridgeNetFiltering(nfParam string) error {
6258
if err := loadBridgeNetFilterModule(nfParam); err != nil {
63-
return fmt.Errorf("loadBridgeNetFilterModule failed: %s", err)
59+
return fmt.Errorf("cannot restrict inter-container communication or run without the userland proxy: %w", err)
6460
}
6561
enabled, err := getKernelBoolParam(nfParam)
6662
if err != nil {

‎libnetwork/ns/init_linux.go‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,12 @@ package ns
22

33
import (
44
"context"
5-
"fmt"
6-
"os/exec"
7-
"strings"
85
"sync"
96
"syscall"
107
"time"
118

129
"github.com/containerd/log"
10+
"github.com/docker/docker/internal/modprobe"
1311
"github.com/docker/docker/internal/nlwrap"
1412
"github.com/vishvananda/netns"
1513
)
@@ -65,12 +63,8 @@ func getSupportedNlFamilies() []int {
6563
fams = append(fams, syscall.NETLINK_XFRM)
6664
}
6765
// NETLINK_NETFILTER test
68-
if err := loadNfConntrackModules(); err != nil {
69-
if checkNfSocket() != nil {
70-
log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
71-
} else {
72-
fams = append(fams, syscall.NETLINK_NETFILTER)
73-
}
66+
if err := modprobe.LoadModules(context.TODO(), checkNfSocket, "nf_conntrack", "nf_conntrack_netlink"); err != nil {
67+
log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
7468
} else {
7569
fams = append(fams, syscall.NETLINK_NETFILTER)
7670
}
@@ -88,16 +82,6 @@ func checkXfrmSocket() error {
8882
return nil
8983
}
9084

91-
func loadNfConntrackModules() error {
92-
if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil {
93-
return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
94-
}
95-
if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil {
96-
return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
97-
}
98-
return nil
99-
}
100-
10185
// API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink)
10286
func checkNfSocket() error {
10387
fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)

0 commit comments

Comments
 (0)