Skip to content

Commit 367eddb

Browse files
stevvooestevvooe
committed
archive, cio, cmd, linux: use buffer pools
To avoid buffer bloat in long running processes, we try to use buffer pools where possible. This is meant to address shim memory usage issues, but may not be the root cause. Signed-off-by: Stephen J Day <[email protected]> (cherry picked from commit cd72819) Signed-off-by: Stephen J Day <[email protected]>
1 parent a4837f3 commit 367eddb

13 files changed

Lines changed: 96 additions & 38 deletions

File tree

‎archive/tar.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
"github.com/pkg/errors"
2020
)
2121

22-
var bufferPool = &sync.Pool{
22+
var bufPool = &sync.Pool{
2323
New: func() interface{} {
2424
buffer := make([]byte, 32*1024)
2525
return &buffer
@@ -423,9 +423,7 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
423423
}
424424
defer file.Close()
425425

426-
buf := bufferPool.Get().(*[]byte)
427-
n, err := io.CopyBuffer(cw.tw, file, *buf)
428-
bufferPool.Put(buf)
426+
n, err := copyBuffered(context.TODO(), cw.tw, file)
429427
if err != nil {
430428
return errors.Wrap(err, "failed to copy")
431429
}
@@ -578,8 +576,8 @@ func (cw *changeWriter) includeParents(hdr *tar.Header) error {
578576
}
579577

580578
func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) {
581-
buf := bufferPool.Get().(*[]byte)
582-
defer bufferPool.Put(buf)
579+
buf := bufPool.Get().(*[]byte)
580+
defer bufPool.Put(buf)
583581

584582
for {
585583
select {

‎cio/io.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,14 @@ import (
88
"sync"
99
)
1010

11-
// Config holds the io configurations.
11+
var bufPool = sync.Pool{
12+
New: func() interface{} {
13+
buffer := make([]byte, 32<<10)
14+
return &buffer
15+
},
16+
}
17+
18+
// Config holds the IO configurations.
1219
type Config struct {
1320
// Terminal is true if one has been allocated
1421
Terminal bool

‎cio/io_unix.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,9 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
5353
}
5454
set = append(set, f)
5555
go func(w io.WriteCloser) {
56-
io.Copy(w, ioset.in)
56+
p := bufPool.Get().(*[]byte)
57+
defer bufPool.Put(p)
58+
io.CopyBuffer(w, ioset.in, *p)
5759
w.Close()
5860
}(f)
5961

@@ -63,7 +65,9 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
6365
set = append(set, f)
6466
wg.Add(1)
6567
go func(r io.ReadCloser) {
66-
io.Copy(ioset.out, r)
68+
p := bufPool.Get().(*[]byte)
69+
defer bufPool.Put(p)
70+
io.CopyBuffer(ioset.out, r, *p)
6771
r.Close()
6872
wg.Done()
6973
}(f)
@@ -76,7 +80,10 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
7680
if !tty {
7781
wg.Add(1)
7882
go func(r io.ReadCloser) {
79-
io.Copy(ioset.err, r)
83+
p := bufPool.Get().(*[]byte)
84+
defer bufPool.Put(p)
85+
86+
io.CopyBuffer(ioset.err, r, *p)
8087
r.Close()
8188
wg.Done()
8289
}(f)

‎cio/io_windows.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,11 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
4646
log.L.WithError(err).Errorf("failed to accept stdin connection on %s", fifos.In)
4747
return
4848
}
49-
io.Copy(c, ioset.in)
49+
50+
p := bufPool.Get().(*[]byte)
51+
defer bufPool.Put(p)
52+
53+
io.CopyBuffer(c, ioset.in, *p)
5054
c.Close()
5155
l.Close()
5256
}()
@@ -72,7 +76,10 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
7276
log.L.WithError(err).Errorf("failed to accept stdout connection on %s", fifos.Out)
7377
return
7478
}
75-
io.Copy(ioset.out, c)
79+
p := bufPool.Get().(*[]byte)
80+
defer bufPool.Put(p)
81+
82+
io.CopyBuffer(ioset.out, c, *p)
7683
c.Close()
7784
l.Close()
7885
}()
@@ -98,7 +105,10 @@ func copyIO(fifos *FIFOSet, ioset *ioSet, tty bool) (_ *wgCloser, err error) {
98105
log.L.WithError(err).Errorf("failed to accept stderr connection on %s", fifos.Err)
99106
return
100107
}
101-
io.Copy(ioset.err, c)
108+
p := bufPool.Get().(*[]byte)
109+
defer bufPool.Put(p)
110+
111+
io.CopyBuffer(ioset.err, c, *p)
102112
c.Close()
103113
l.Close()
104114
}()

‎cmd/containerd/main.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,15 @@ import (
1010
"os/signal"
1111
"time"
1212

13-
"google.golang.org/grpc/grpclog"
14-
15-
gocontext "golang.org/x/net/context"
16-
1713
"github.com/containerd/containerd/log"
1814
"github.com/containerd/containerd/server"
1915
"github.com/containerd/containerd/sys"
2016
"github.com/containerd/containerd/version"
2117
"github.com/pkg/errors"
2218
"github.com/sirupsen/logrus"
2319
"github.com/urfave/cli"
20+
gocontext "golang.org/x/net/context"
21+
"google.golang.org/grpc/grpclog"
2422
)
2523

2624
const usage = `

‎cmd/containerd/main_unix.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@ import (
77
"os"
88
"runtime"
99

10-
"github.com/sirupsen/logrus"
11-
"golang.org/x/sys/unix"
12-
1310
"github.com/containerd/containerd/log"
1411
"github.com/containerd/containerd/server"
12+
"github.com/sirupsen/logrus"
13+
"golang.org/x/sys/unix"
1514
)
1615

1716
const defaultConfigPath = "/etc/containerd/config.toml"

‎cmd/ctr/commands/shim/io_unix.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ import (
1212
"golang.org/x/sys/unix"
1313
)
1414

15+
var bufPool = sync.Pool{
16+
New: func() interface{} {
17+
buffer := make([]byte, 32<<10)
18+
return &buffer
19+
},
20+
}
21+
1522
func prepareStdio(stdin, stdout, stderr string, console bool) (wg *sync.WaitGroup, err error) {
1623
wg = &sync.WaitGroup{}
1724
ctx := gocontext.Background()
@@ -26,7 +33,9 @@ func prepareStdio(stdin, stdout, stderr string, console bool) (wg *sync.WaitGrou
2633
}
2734
}(f)
2835
go func(w io.WriteCloser) {
29-
io.Copy(w, os.Stdin)
36+
p := bufPool.Get().(*[]byte)
37+
defer bufPool.Put(p)
38+
io.CopyBuffer(w, os.Stdin, *p)
3039
w.Close()
3140
}(f)
3241

‎linux/bundle.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
package linux
44

55
import (
6-
"bytes"
76
"context"
8-
"io"
7+
"io/ioutil"
98
"os"
109
"path/filepath"
1110

@@ -52,12 +51,7 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
5251
if err := os.Mkdir(filepath.Join(path, "rootfs"), 0711); err != nil {
5352
return nil, err
5453
}
55-
f, err := os.Create(filepath.Join(path, configFilename))
56-
if err != nil {
57-
return nil, err
58-
}
59-
defer f.Close()
60-
_, err = io.Copy(f, bytes.NewReader(spec))
54+
err = ioutil.WriteFile(filepath.Join(path, configFilename), spec, 0666)
6155
return &bundle{
6256
id: id,
6357
path: path,

‎linux/proc/io.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,23 @@ import (
1313
runc "github.com/containerd/go-runc"
1414
)
1515

16+
var bufPool = sync.Pool{
17+
New: func() interface{} {
18+
buffer := make([]byte, 32<<10)
19+
return &buffer
20+
},
21+
}
22+
1623
func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, wg, cwg *sync.WaitGroup) error {
1724
for name, dest := range map[string]func(wc io.WriteCloser, rc io.Closer){
1825
stdout: func(wc io.WriteCloser, rc io.Closer) {
1926
wg.Add(1)
2027
cwg.Add(1)
2128
go func() {
2229
cwg.Done()
23-
io.Copy(wc, rio.Stdout())
30+
p := bufPool.Get().(*[]byte)
31+
defer bufPool.Put(p)
32+
io.CopyBuffer(wc, rio.Stdout(), *p)
2433
wg.Done()
2534
wc.Close()
2635
rc.Close()
@@ -31,7 +40,10 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w
3140
cwg.Add(1)
3241
go func() {
3342
cwg.Done()
34-
io.Copy(wc, rio.Stderr())
43+
p := bufPool.Get().(*[]byte)
44+
defer bufPool.Put(p)
45+
46+
io.CopyBuffer(wc, rio.Stderr(), *p)
3547
wg.Done()
3648
wc.Close()
3749
rc.Close()
@@ -59,7 +71,10 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w
5971
cwg.Add(1)
6072
go func() {
6173
cwg.Done()
62-
io.Copy(rio.Stdin(), f)
74+
p := bufPool.Get().(*[]byte)
75+
defer bufPool.Put(p)
76+
77+
io.CopyBuffer(rio.Stdin(), f, *p)
6378
rio.Stdin().Close()
6479
f.Close()
6580
}()

‎linux/proc/utils.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ func copyFile(to, from string) error {
6666
return err
6767
}
6868
defer tt.Close()
69-
_, err = io.Copy(tt, ff)
69+
70+
p := bufPool.Get().(*[]byte)
71+
defer bufPool.Put(p)
72+
_, err = io.CopyBuffer(tt, ff, *p)
7073
return err
7174
}
7275

0 commit comments

Comments
 (0)