Skip to content

Commit cd72819

Browse files
stevvooestevvooe
committed
archive, cio, cmd, linux: use buffer pools
To avoid buffer bloat in long running processes, we try to use buffer pools where possible. This is meant to address shim memory usage issues, but may not be the root cause. Signed-off-by: Stephen J Day <[email protected]>
1 parent aaf930e commit cd72819

14 files changed

Lines changed: 105 additions & 39 deletions

File tree

‎archive/tar.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
"github.com/pkg/errors"
2020
)
2121

22-
var bufferPool = &sync.Pool{
22+
var bufPool = &sync.Pool{
2323
New: func() interface{} {
2424
buffer := make([]byte, 32*1024)
2525
return &buffer
@@ -527,9 +527,7 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e
527527
}
528528
defer file.Close()
529529

530-
buf := bufferPool.Get().(*[]byte)
531-
n, err := io.CopyBuffer(cw.tw, file, *buf)
532-
bufferPool.Put(buf)
530+
n, err := copyBuffered(context.TODO(), cw.tw, file)
533531
if err != nil {
534532
return errors.Wrap(err, "failed to copy")
535533
}
@@ -589,8 +587,8 @@ func (cw *changeWriter) includeParents(hdr *tar.Header) error {
589587
}
590588

591589
func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) {
592-
buf := bufferPool.Get().(*[]byte)
593-
defer bufferPool.Put(buf)
590+
buf := bufPool.Get().(*[]byte)
591+
defer bufPool.Put(buf)
594592

595593
for {
596594
select {

‎archive/tar_windows.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,10 @@ func writeBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
386386
return nil, err
387387
}
388388
}
389+
390+
buf := bufPool.Get().(*[]byte)
391+
defer bufPool.Put(buf)
392+
389393
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
390394
bhdr := winio.BackupHeader{
391395
Id: winio.BackupData,
@@ -395,7 +399,7 @@ func writeBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
395399
if err != nil {
396400
return nil, err
397401
}
398-
_, err = io.Copy(bw, t)
402+
_, err = io.CopyBuffer(bw, t, *buf)
399403
if err != nil {
400404
return nil, err
401405
}
@@ -418,7 +422,7 @@ func writeBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
418422
if err != nil {
419423
return nil, err
420424
}
421-
_, err = io.Copy(bw, t)
425+
_, err = io.CopyBuffer(bw, t, *buf)
422426
if err != nil {
423427
return nil, err
424428
}

‎cio/io.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ import (
88
"sync"
99
)
1010

11+
var bufPool = sync.Pool{
12+
New: func() interface{} {
13+
buffer := make([]byte, 32<<10)
14+
return &buffer
15+
},
16+
}
17+
1118
// Config holds the IO configurations.
1219
type Config struct {
1320
// Terminal is true if one has been allocated

‎cio/io_unix.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,23 +47,32 @@ func copyIO(fifos *FIFOSet, ioset *Streams) (*cio, error) {
4747

4848
if fifos.Stdin != "" {
4949
go func() {
50-
io.Copy(pipes.Stdin, ioset.Stdin)
50+
p := bufPool.Get().(*[]byte)
51+
defer bufPool.Put(p)
52+
53+
io.CopyBuffer(pipes.Stdin, ioset.Stdin, *p)
5154
pipes.Stdin.Close()
5255
}()
5356
}
5457

5558
var wg = &sync.WaitGroup{}
5659
wg.Add(1)
5760
go func() {
58-
io.Copy(ioset.Stdout, pipes.Stdout)
61+
p := bufPool.Get().(*[]byte)
62+
defer bufPool.Put(p)
63+
64+
io.CopyBuffer(ioset.Stdout, pipes.Stdout, *p)
5965
pipes.Stdout.Close()
6066
wg.Done()
6167
}()
6268

6369
if !fifos.Terminal {
6470
wg.Add(1)
6571
go func() {
66-
io.Copy(ioset.Stderr, pipes.Stderr)
72+
p := bufPool.Get().(*[]byte)
73+
defer bufPool.Put(p)
74+
75+
io.CopyBuffer(ioset.Stderr, pipes.Stderr, *p)
6776
pipes.Stderr.Close()
6877
wg.Done()
6978
}()

‎cio/io_windows.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@ func copyIO(fifos *FIFOSet, ioset *Streams) (*cio, error) {
4747
log.L.WithError(err).Errorf("failed to accept stdin connection on %s", fifos.Stdin)
4848
return
4949
}
50-
io.Copy(c, ioset.Stdin)
50+
51+
p := bufPool.Get().(*[]byte)
52+
defer bufPool.Put(p)
53+
54+
io.CopyBuffer(c, ioset.Stdin, *p)
5155
c.Close()
5256
l.Close()
5357
}()
@@ -73,7 +77,11 @@ func copyIO(fifos *FIFOSet, ioset *Streams) (*cio, error) {
7377
log.L.WithError(err).Errorf("failed to accept stdout connection on %s", fifos.Stdout)
7478
return
7579
}
76-
io.Copy(ioset.Stdout, c)
80+
81+
p := bufPool.Get().(*[]byte)
82+
defer bufPool.Put(p)
83+
84+
io.CopyBuffer(ioset.Stdout, c, *p)
7785
c.Close()
7886
l.Close()
7987
}()
@@ -99,7 +107,11 @@ func copyIO(fifos *FIFOSet, ioset *Streams) (*cio, error) {
99107
log.L.WithError(err).Errorf("failed to accept stderr connection on %s", fifos.Stderr)
100108
return
101109
}
102-
io.Copy(ioset.Stderr, c)
110+
111+
p := bufPool.Get().(*[]byte)
112+
defer bufPool.Put(p)
113+
114+
io.CopyBuffer(ioset.Stderr, c, *p)
103115
c.Close()
104116
l.Close()
105117
}()

‎cmd/containerd/main.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,15 @@ import (
1010
"os/signal"
1111
"time"
1212

13-
"google.golang.org/grpc/grpclog"
14-
15-
gocontext "golang.org/x/net/context"
16-
1713
"github.com/containerd/containerd/log"
1814
"github.com/containerd/containerd/server"
1915
"github.com/containerd/containerd/sys"
2016
"github.com/containerd/containerd/version"
2117
"github.com/pkg/errors"
2218
"github.com/sirupsen/logrus"
2319
"github.com/urfave/cli"
20+
gocontext "golang.org/x/net/context"
21+
"google.golang.org/grpc/grpclog"
2422
)
2523

2624
const usage = `

‎cmd/containerd/main_unix.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@ import (
77
"os"
88
"runtime"
99

10-
"github.com/sirupsen/logrus"
11-
"golang.org/x/sys/unix"
12-
1310
"github.com/containerd/containerd/log"
1411
"github.com/containerd/containerd/server"
12+
"github.com/sirupsen/logrus"
13+
"golang.org/x/sys/unix"
1514
)
1615

1716
const defaultConfigPath = "/etc/containerd/config.toml"

‎cmd/ctr/commands/shim/io_unix.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ import (
1212
"golang.org/x/sys/unix"
1313
)
1414

15+
var bufPool = sync.Pool{
16+
New: func() interface{} {
17+
buffer := make([]byte, 32<<10)
18+
return &buffer
19+
},
20+
}
21+
1522
func prepareStdio(stdin, stdout, stderr string, console bool) (wg *sync.WaitGroup, err error) {
1623
wg = &sync.WaitGroup{}
1724
ctx := gocontext.Background()
@@ -26,7 +33,9 @@ func prepareStdio(stdin, stdout, stderr string, console bool) (wg *sync.WaitGrou
2633
}
2734
}(f)
2835
go func(w io.WriteCloser) {
29-
io.Copy(w, os.Stdin)
36+
p := bufPool.Get().(*[]byte)
37+
defer bufPool.Put(p)
38+
io.CopyBuffer(w, os.Stdin, *p)
3039
w.Close()
3140
}(f)
3241

‎linux/bundle.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
package linux
44

55
import (
6-
"bytes"
76
"context"
8-
"io"
7+
"io/ioutil"
98
"os"
109
"path/filepath"
1110

@@ -52,12 +51,7 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
5251
if err := os.Mkdir(filepath.Join(path, "rootfs"), 0711); err != nil {
5352
return nil, err
5453
}
55-
f, err := os.Create(filepath.Join(path, configFilename))
56-
if err != nil {
57-
return nil, err
58-
}
59-
defer f.Close()
60-
_, err = io.Copy(f, bytes.NewReader(spec))
54+
err = ioutil.WriteFile(filepath.Join(path, configFilename), spec, 0666)
6155
return &bundle{
6256
id: id,
6357
path: path,

‎linux/proc/io.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,23 @@ import (
1313
runc "github.com/containerd/go-runc"
1414
)
1515

16+
var bufPool = sync.Pool{
17+
New: func() interface{} {
18+
buffer := make([]byte, 32<<10)
19+
return &buffer
20+
},
21+
}
22+
1623
func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, wg, cwg *sync.WaitGroup) error {
1724
for name, dest := range map[string]func(wc io.WriteCloser, rc io.Closer){
1825
stdout: func(wc io.WriteCloser, rc io.Closer) {
1926
wg.Add(1)
2027
cwg.Add(1)
2128
go func() {
2229
cwg.Done()
23-
io.Copy(wc, rio.Stdout())
30+
p := bufPool.Get().(*[]byte)
31+
defer bufPool.Put(p)
32+
io.CopyBuffer(wc, rio.Stdout(), *p)
2433
wg.Done()
2534
wc.Close()
2635
rc.Close()
@@ -31,7 +40,10 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w
3140
cwg.Add(1)
3241
go func() {
3342
cwg.Done()
34-
io.Copy(wc, rio.Stderr())
43+
p := bufPool.Get().(*[]byte)
44+
defer bufPool.Put(p)
45+
46+
io.CopyBuffer(wc, rio.Stderr(), *p)
3547
wg.Done()
3648
wc.Close()
3749
rc.Close()
@@ -59,7 +71,10 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w
5971
cwg.Add(1)
6072
go func() {
6173
cwg.Done()
62-
io.Copy(rio.Stdin(), f)
74+
p := bufPool.Get().(*[]byte)
75+
defer bufPool.Put(p)
76+
77+
io.CopyBuffer(rio.Stdin(), f, *p)
6378
rio.Stdin().Close()
6479
f.Close()
6580
}()

0 commit comments

Comments
 (0)