Skip to content

Commit 05462c2

Browse files
mrjanamrjana
committed
Workaround kernel bugs s related to namespaces
This PR attempts to work around bugs present in kernel version 3.18-4.0.1 relating to namespace creation and destruction. This fix attempts to avoid certain systemmcalls to not get in the kkernel bug path as well as lazily garbage collecting the name paths when they are removed. Signed-off-by: Jana Radhakrishnan <[email protected]>
1 parent 6743808 commit 05462c2

3 files changed

Lines changed: 133 additions & 22 deletions

File tree

‎sandbox/namespace_linux.go‎

Lines changed: 111 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,28 @@ import (
44
"fmt"
55
"net"
66
"os"
7+
"os/exec"
8+
"path/filepath"
79
"runtime"
810
"sync"
911
"syscall"
12+
"time"
1013

14+
log "github.com/Sirupsen/logrus"
15+
"github.com/docker/docker/pkg/reexec"
1116
"github.com/vishvananda/netlink"
1217
"github.com/vishvananda/netns"
1318
)
1419

1520
const prefix = "/var/run/docker/netns"
1621

17-
var once sync.Once
22+
var (
23+
once sync.Once
24+
garbagePathMap = make(map[string]bool)
25+
gpmLock sync.Mutex
26+
gpmWg sync.WaitGroup
27+
gpmCleanupPeriod = 60
28+
)
1829

1930
// The networkNamespace type is the linux implementation of the Sandbox
2031
// interface. It represents a linux network namespace, and moves an interface
@@ -26,11 +37,56 @@ type networkNamespace struct {
2637
sync.Mutex
2738
}
2839

40+
func init() {
41+
reexec.Register("netns-create", reexecCreateNamespace)
42+
}
43+
2944
func createBasePath() {
3045
err := os.MkdirAll(prefix, 0644)
3146
if err != nil && !os.IsExist(err) {
3247
panic("Could not create net namespace path directory")
3348
}
49+
50+
// cleanup any stale namespace files if any
51+
cleanupNamespaceFiles()
52+
53+
// Start the garbage collection go routine
54+
go removeUnusedPaths()
55+
}
56+
57+
func removeUnusedPaths() {
58+
for {
59+
time.Sleep(time.Duration(gpmCleanupPeriod) * time.Second)
60+
61+
gpmLock.Lock()
62+
pathList := make([]string, 0, len(garbagePathMap))
63+
for path := range garbagePathMap {
64+
pathList = append(pathList, path)
65+
}
66+
garbagePathMap = make(map[string]bool)
67+
gpmWg.Add(1)
68+
gpmLock.Unlock()
69+
70+
for _, path := range pathList {
71+
os.Remove(path)
72+
}
73+
74+
gpmWg.Done()
75+
}
76+
}
77+
78+
func addToGarbagePaths(path string) {
79+
gpmLock.Lock()
80+
defer gpmLock.Unlock()
81+
82+
garbagePathMap[path] = true
83+
}
84+
85+
func removeFromGarbagePaths(path string) {
86+
gpmLock.Lock()
87+
defer gpmLock.Unlock()
88+
89+
delete(garbagePathMap, path)
3490
}
3591

3692
// GenerateKey generates a sandbox key based on the passed
@@ -55,6 +111,16 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) {
55111
return &networkNamespace{path: key, sinfo: info}, nil
56112
}
57113

114+
func reexecCreateNamespace() {
115+
if len(os.Args) < 2 {
116+
log.Fatal("no namespace path provided")
117+
}
118+
119+
if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil {
120+
log.Fatal(err)
121+
}
122+
}
123+
58124
func createNetworkNamespace(path string, osCreate bool) (*Info, error) {
59125
runtime.LockOSThread()
60126
defer runtime.UnlockOSThread()
@@ -69,46 +135,67 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) {
69135
return nil, err
70136
}
71137

138+
cmd := &exec.Cmd{
139+
Path: reexec.Self(),
140+
Args: append([]string{"netns-create"}, path),
141+
Stdout: os.Stdout,
142+
Stderr: os.Stderr,
143+
}
72144
if osCreate {
73-
defer netns.Set(origns)
74-
newns, err := netns.New()
75-
if err != nil {
76-
return nil, err
77-
}
78-
defer newns.Close()
79-
80-
if err := loopbackUp(); err != nil {
81-
return nil, err
82-
}
145+
cmd.SysProcAttr = &syscall.SysProcAttr{}
146+
cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
83147
}
84-
85-
procNet := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid())
86-
87-
if err := syscall.Mount(procNet, path, "bind", syscall.MS_BIND, ""); err != nil {
88-
return nil, err
148+
if err := cmd.Run(); err != nil {
149+
return nil, fmt.Errorf("namespace creation reexec command failed: %v", err)
89150
}
90151

91152
interfaces := []*Interface{}
92153
info := &Info{Interfaces: interfaces}
93154
return info, nil
94155
}
95156

96-
func cleanupNamespaceFile(path string) {
157+
func cleanupNamespaceFiles() {
158+
filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error {
159+
stat, err := os.Stat(path)
160+
if err != nil {
161+
return err
162+
}
163+
164+
if stat.IsDir() {
165+
return filepath.SkipDir
166+
}
167+
168+
syscall.Unmount(path, syscall.MNT_DETACH)
169+
os.Remove(path)
170+
171+
return nil
172+
})
173+
}
174+
175+
func unmountNamespaceFile(path string) {
97176
if _, err := os.Stat(path); err == nil {
98-
n := &networkNamespace{path: path}
99-
n.Destroy()
177+
syscall.Unmount(path, syscall.MNT_DETACH)
100178
}
101179
}
102180

103181
func createNamespaceFile(path string) (err error) {
104182
var f *os.File
105183

106184
once.Do(createBasePath)
107-
// cleanup namespace file if it already exists because of a previous ungraceful exit.
108-
cleanupNamespaceFile(path)
185+
// Remove it from garbage collection list if present
186+
removeFromGarbagePaths(path)
187+
188+
// If the path is there unmount it first
189+
unmountNamespaceFile(path)
190+
191+
// wait for garbage collection to complete if it is in progress
192+
// before trying to create the file.
193+
gpmWg.Wait()
194+
109195
if f, err = os.Create(path); err == nil {
110196
f.Close()
111197
}
198+
112199
return err
113200
}
114201

@@ -269,5 +356,7 @@ func (n *networkNamespace) Destroy() error {
269356
return err
270357
}
271358

272-
return os.Remove(n.path)
359+
// Stash it into the garbage collection list
360+
addToGarbagePaths(n.path)
361+
return nil
273362
}

‎sandbox/sandbox_linux_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"path/filepath"
77
"runtime"
88
"testing"
9+
"time"
910

1011
"github.com/docker/libnetwork/netutils"
1112
"github.com/vishvananda/netlink"
@@ -31,6 +32,9 @@ func newKey(t *testing.T) (string, error) {
3132
return "", err
3233
}
3334

35+
// Set the rpmCleanupPeriod to be low to make the test run quicker
36+
gpmCleanupPeriod = 2
37+
3438
return name, nil
3539
}
3640

@@ -137,3 +141,10 @@ func verifySandbox(t *testing.T, s Sandbox) {
137141
err)
138142
}
139143
}
144+
145+
func verifyCleanup(t *testing.T, s Sandbox) {
146+
time.Sleep(time.Duration(gpmCleanupPeriod*2) * time.Second)
147+
if _, err := os.Stat(s.Key()); err == nil {
148+
t.Fatalf("The sandbox path %s is not getting cleanup event after twice the cleanup period", s.Key())
149+
}
150+
}

‎sandbox/sandbox_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,19 @@ package sandbox
22

33
import (
44
"net"
5+
"os"
56
"testing"
7+
8+
"github.com/docker/docker/pkg/reexec"
69
)
710

11+
func TestMain(m *testing.M) {
12+
if reexec.Init() {
13+
return
14+
}
15+
os.Exit(m.Run())
16+
}
17+
818
func TestSandboxCreate(t *testing.T) {
919
key, err := newKey(t)
1020
if err != nil {
@@ -44,6 +54,7 @@ func TestSandboxCreate(t *testing.T) {
4454

4555
verifySandbox(t, s)
4656
s.Destroy()
57+
verifyCleanup(t, s)
4758
}
4859

4960
func TestSandboxCreateTwice(t *testing.T) {

0 commit comments

Comments
 (0)