Skip to content

Commit 3ec19ff

Browse files
mrjanamavenugo
mrjana
authored and
mavenugo
committed
Workaround kernel bugs s related to namespaces
This PR attempts to work around bugs present in kernel version 3.18-4.0.1 relating to namespace creation and destruction. This fix attempts to avoid certain systemmcalls to not get in the kkernel bug path as well as lazily garbage collecting the name paths when they are removed. Signed-off-by: Jana Radhakrishnan <[email protected]>
1 parent d1d854e commit 3ec19ff

3 files changed

Lines changed: 133 additions & 22 deletions

File tree

‎libnetwork/sandbox/namespace_linux.go‎

Lines changed: 111 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,29 @@ import (
44
"fmt"
55
"net"
66
"os"
7+
"os/exec"
8+
"path/filepath"
79
"runtime"
810
"sync"
911
"syscall"
12+
"time"
1013

14+
log "github.com/Sirupsen/logrus"
15+
"github.com/docker/docker/pkg/reexec"
1116
"github.com/docker/libnetwork/types"
1217
"github.com/vishvananda/netlink"
1318
"github.com/vishvananda/netns"
1419
)
1520

1621
const prefix = "/var/run/docker/netns"
1722

18-
var once sync.Once
23+
var (
24+
once sync.Once
25+
garbagePathMap = make(map[string]bool)
26+
gpmLock sync.Mutex
27+
gpmWg sync.WaitGroup
28+
gpmCleanupPeriod = 60
29+
)
1930

2031
// The networkNamespace type is the linux implementation of the Sandbox
2132
// interface. It represents a linux network namespace, and moves an interface
@@ -27,11 +38,56 @@ type networkNamespace struct {
2738
sync.Mutex
2839
}
2940

41+
func init() {
42+
reexec.Register("netns-create", reexecCreateNamespace)
43+
}
44+
3045
func createBasePath() {
3146
err := os.MkdirAll(prefix, 0644)
3247
if err != nil && !os.IsExist(err) {
3348
panic("Could not create net namespace path directory")
3449
}
50+
51+
// cleanup any stale namespace files if any
52+
cleanupNamespaceFiles()
53+
54+
// Start the garbage collection go routine
55+
go removeUnusedPaths()
56+
}
57+
58+
func removeUnusedPaths() {
59+
for {
60+
time.Sleep(time.Duration(gpmCleanupPeriod) * time.Second)
61+
62+
gpmLock.Lock()
63+
pathList := make([]string, 0, len(garbagePathMap))
64+
for path := range garbagePathMap {
65+
pathList = append(pathList, path)
66+
}
67+
garbagePathMap = make(map[string]bool)
68+
gpmWg.Add(1)
69+
gpmLock.Unlock()
70+
71+
for _, path := range pathList {
72+
os.Remove(path)
73+
}
74+
75+
gpmWg.Done()
76+
}
77+
}
78+
79+
func addToGarbagePaths(path string) {
80+
gpmLock.Lock()
81+
defer gpmLock.Unlock()
82+
83+
garbagePathMap[path] = true
84+
}
85+
86+
func removeFromGarbagePaths(path string) {
87+
gpmLock.Lock()
88+
defer gpmLock.Unlock()
89+
90+
delete(garbagePathMap, path)
3591
}
3692

3793
// GenerateKey generates a sandbox key based on the passed
@@ -56,6 +112,16 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) {
56112
return &networkNamespace{path: key, sinfo: info}, nil
57113
}
58114

115+
func reexecCreateNamespace() {
116+
if len(os.Args) < 2 {
117+
log.Fatal("no namespace path provided")
118+
}
119+
120+
if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil {
121+
log.Fatal(err)
122+
}
123+
}
124+
59125
func createNetworkNamespace(path string, osCreate bool) (*Info, error) {
60126
runtime.LockOSThread()
61127
defer runtime.UnlockOSThread()
@@ -70,46 +136,67 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) {
70136
return nil, err
71137
}
72138

139+
cmd := &exec.Cmd{
140+
Path: reexec.Self(),
141+
Args: append([]string{"netns-create"}, path),
142+
Stdout: os.Stdout,
143+
Stderr: os.Stderr,
144+
}
73145
if osCreate {
74-
defer netns.Set(origns)
75-
newns, err := netns.New()
76-
if err != nil {
77-
return nil, err
78-
}
79-
defer newns.Close()
80-
81-
if err := loopbackUp(); err != nil {
82-
return nil, err
83-
}
146+
cmd.SysProcAttr = &syscall.SysProcAttr{}
147+
cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
84148
}
85-
86-
procNet := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid())
87-
88-
if err := syscall.Mount(procNet, path, "bind", syscall.MS_BIND, ""); err != nil {
89-
return nil, err
149+
if err := cmd.Run(); err != nil {
150+
return nil, fmt.Errorf("namespace creation reexec command failed: %v", err)
90151
}
91152

92153
interfaces := []*Interface{}
93154
info := &Info{Interfaces: interfaces}
94155
return info, nil
95156
}
96157

97-
func cleanupNamespaceFile(path string) {
158+
func cleanupNamespaceFiles() {
159+
filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error {
160+
stat, err := os.Stat(path)
161+
if err != nil {
162+
return err
163+
}
164+
165+
if stat.IsDir() {
166+
return filepath.SkipDir
167+
}
168+
169+
syscall.Unmount(path, syscall.MNT_DETACH)
170+
os.Remove(path)
171+
172+
return nil
173+
})
174+
}
175+
176+
func unmountNamespaceFile(path string) {
98177
if _, err := os.Stat(path); err == nil {
99-
n := &networkNamespace{path: path}
100-
n.Destroy()
178+
syscall.Unmount(path, syscall.MNT_DETACH)
101179
}
102180
}
103181

104182
func createNamespaceFile(path string) (err error) {
105183
var f *os.File
106184

107185
once.Do(createBasePath)
108-
// cleanup namespace file if it already exists because of a previous ungraceful exit.
109-
cleanupNamespaceFile(path)
186+
// Remove it from garbage collection list if present
187+
removeFromGarbagePaths(path)
188+
189+
// If the path is there unmount it first
190+
unmountNamespaceFile(path)
191+
192+
// wait for garbage collection to complete if it is in progress
193+
// before trying to create the file.
194+
gpmWg.Wait()
195+
110196
if f, err = os.Create(path); err == nil {
111197
f.Close()
112198
}
199+
113200
return err
114201
}
115202

@@ -310,5 +397,7 @@ func (n *networkNamespace) Destroy() error {
310397
return err
311398
}
312399

313-
return os.Remove(n.path)
400+
// Stash it into the garbage collection list
401+
addToGarbagePaths(n.path)
402+
return nil
314403
}

‎libnetwork/sandbox/sandbox_linux_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"path/filepath"
77
"runtime"
88
"testing"
9+
"time"
910

1011
"github.com/docker/libnetwork/netutils"
1112
"github.com/vishvananda/netlink"
@@ -31,6 +32,9 @@ func newKey(t *testing.T) (string, error) {
3132
return "", err
3233
}
3334

35+
// Set the rpmCleanupPeriod to be low to make the test run quicker
36+
gpmCleanupPeriod = 2
37+
3438
return name, nil
3539
}
3640

@@ -146,3 +150,10 @@ func verifySandbox(t *testing.T, s Sandbox) {
146150
err)
147151
}
148152
}
153+
154+
func verifyCleanup(t *testing.T, s Sandbox) {
155+
time.Sleep(time.Duration(gpmCleanupPeriod*2) * time.Second)
156+
if _, err := os.Stat(s.Key()); err == nil {
157+
t.Fatalf("The sandbox path %s is not getting cleanup event after twice the cleanup period", s.Key())
158+
}
159+
}

‎libnetwork/sandbox/sandbox_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,19 @@ package sandbox
22

33
import (
44
"net"
5+
"os"
56
"testing"
7+
8+
"github.com/docker/docker/pkg/reexec"
69
)
710

11+
func TestMain(m *testing.M) {
12+
if reexec.Init() {
13+
return
14+
}
15+
os.Exit(m.Run())
16+
}
17+
818
func TestSandboxCreate(t *testing.T) {
919
key, err := newKey(t)
1020
if err != nil {
@@ -44,6 +54,7 @@ func TestSandboxCreate(t *testing.T) {
4454

4555
verifySandbox(t, s)
4656
s.Destroy()
57+
verifyCleanup(t, s)
4758
}
4859

4960
func TestSandboxCreateTwice(t *testing.T) {

0 commit comments

Comments
 (0)