Skip to content

Commit 963f161

Browse files
tonistiigitonistiigi
committed
resources: CNI network usage sampling support
Signed-off-by: Tonis Tiigi <[email protected]>
1 parent 6e87e4b commit 963f161

9 files changed

Lines changed: 225 additions & 26 deletions

File tree

‎executor/resources/monitor.go‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"time"
1313

1414
"github.com/moby/buildkit/executor/resources/types"
15+
"github.com/moby/buildkit/util/network"
1516
"golang.org/x/sys/unix"
1617
)
1718

@@ -34,6 +35,7 @@ type cgroupRecord struct {
3435
err error
3536
done chan struct{}
3637
monitor *Monitor
38+
netSampler NetworkSampler
3739
}
3840

3941
func (r *cgroupRecord) Wait() error {
@@ -89,6 +91,13 @@ func (r *cgroupRecord) sample() (*types.Sample, error) {
8991
IOStat: io,
9092
PIDsStat: pids,
9193
}
94+
if r.netSampler != nil {
95+
net, err := r.netSampler.Sample()
96+
if err != nil {
97+
return nil, err
98+
}
99+
sample.NetStat = net
100+
}
92101
return sample, nil
93102
}
94103

@@ -117,24 +126,34 @@ type Monitor struct {
117126
records map[string]*cgroupRecord
118127
}
119128

120-
func (m *Monitor) RecordNamespace(ns string, release func(context.Context) error) (types.Recorder, error) {
129+
type NetworkSampler interface {
130+
Sample() (*network.Sample, error)
131+
}
132+
133+
type RecordOpt struct {
134+
Release func(context.Context) error
135+
NetworkSampler NetworkSampler
136+
}
137+
138+
func (m *Monitor) RecordNamespace(ns string, opt RecordOpt) (types.Recorder, error) {
121139
isClosed := false
122140
select {
123141
case <-m.closed:
124142
isClosed = true
125143
default:
126144
}
127145
if !isCgroupV2 || isClosed {
128-
if err := release(context.TODO()); err != nil {
146+
if err := opt.Release(context.TODO()); err != nil {
129147
return nil, err
130148
}
131149
return &nopRecord{}, nil
132150
}
133151
r := &cgroupRecord{
134-
ns: ns,
135-
release: release,
136-
done: make(chan struct{}),
137-
monitor: m,
152+
ns: ns,
153+
release: opt.Release,
154+
done: make(chan struct{}),
155+
monitor: m,
156+
netSampler: opt.NetworkSampler,
138157
}
139158
m.mu.Lock()
140159
m.records[ns] = r

‎executor/resources/types/types.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package types
22

3-
import "time"
3+
import (
4+
"time"
5+
6+
"github.com/moby/buildkit/util/network"
7+
)
48

59
type Recorder interface {
610
Wait() error
@@ -9,11 +13,12 @@ type Recorder interface {
913

1014
// Sample represents a wrapper for sampled data of cgroupv2 controllers
1115
type Sample struct {
12-
Timestamp time.Time `json:"timestamp"`
13-
CPUStat *CPUStat `json:"cpuStat,omitempty"`
14-
MemoryStat *MemoryStat `json:"memoryStat,omitempty"`
15-
IOStat *IOStat `json:"ioStat,omitempty"`
16-
PIDsStat *PIDsStat `json:"pidsStat,omitempty"`
16+
Timestamp time.Time `json:"timestamp"`
17+
CPUStat *CPUStat `json:"cpuStat,omitempty"`
18+
MemoryStat *MemoryStat `json:"memoryStat,omitempty"`
19+
IOStat *IOStat `json:"ioStat,omitempty"`
20+
PIDsStat *PIDsStat `json:"pidsStat,omitempty"`
21+
NetStat *network.Sample `json:"netStat,omitempty"`
1722
}
1823

1924
// CPUStat represents the sampling state of the cgroupv2 CPU controller

‎executor/runcexecutor/executor.go‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,12 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
174174
if err != nil {
175175
return nil, err
176176
}
177-
defer namespace.Close()
177+
doReleaseNetwork := true
178+
defer func() {
179+
if doReleaseNetwork {
180+
namespace.Close()
181+
}
182+
}()
178183

179184
if meta.NetMode == pb.NetMode_HOST {
180185
bklog.G(ctx).Info("enabling HostNetworking")
@@ -304,19 +309,31 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
304309
}
305310
})
306311
}, true)
312+
313+
releaseContainer := func(ctx context.Context) error {
314+
err := w.runc.Delete(ctx, id, &runc.DeleteOpts{})
315+
err1 := namespace.Close()
316+
if err == nil {
317+
err = err1
318+
}
319+
return err
320+
}
321+
doReleaseNetwork = false
322+
307323
err = exitError(ctx, err)
308324
if err != nil {
309-
w.runc.Delete(context.TODO(), id, &runc.DeleteOpts{})
325+
releaseContainer(context.TODO())
310326
return nil, err
311327
}
312328

313329
cgroupPath := spec.Linux.CgroupsPath
314330
if cgroupPath != "" {
315-
return w.resmon.RecordNamespace(cgroupPath, func(ctx context.Context) error {
316-
return w.runc.Delete(ctx, id, &runc.DeleteOpts{})
331+
return w.resmon.RecordNamespace(cgroupPath, resources.RecordOpt{
332+
Release: releaseContainer,
333+
NetworkSampler: namespace,
317334
})
318335
}
319-
return nil, w.runc.Delete(context.TODO(), id, &runc.DeleteOpts{})
336+
return nil, releaseContainer(context.TODO())
320337
}
321338

322339
func exitError(ctx context.Context, err error) error {

‎util/network/cniprovider/cni.go‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -242,42 +242,97 @@ func (c *cniProvider) newNS(ctx context.Context, hostname string) (*cniNS, error
242242
cni.WithArgs("IgnoreUnknown", "1"))
243243
}
244244

245-
if _, err := c.CNI.Setup(context.TODO(), id, nativeID, nsOpts...); err != nil {
245+
cniRes, err := c.CNI.Setup(context.TODO(), id, nativeID, nsOpts...)
246+
if err != nil {
246247
deleteNetNS(nativeID)
247248
return nil, errors.Wrap(err, "CNI setup error")
248249
}
249250
trace.SpanFromContext(ctx).AddEvent("finished setting up network namespace")
250251
bklog.G(ctx).Debugf("finished setting up network namespace %s", id)
251252

252-
return &cniNS{
253+
vethName := ""
254+
for k := range cniRes.Interfaces {
255+
if strings.HasPrefix(k, "veth") {
256+
if vethName != "" {
257+
// invalid config
258+
vethName = ""
259+
break
260+
}
261+
vethName = k
262+
}
263+
}
264+
265+
ns := &cniNS{
253266
nativeID: nativeID,
254267
id: id,
255268
handle: c.CNI,
256269
opts: nsOpts,
257-
}, nil
270+
vethName: vethName,
271+
}
272+
273+
if ns.vethName != "" {
274+
sample, err := ns.sample()
275+
if err == nil && sample != nil {
276+
ns.canSample = true
277+
ns.offsetSample = sample
278+
}
279+
}
280+
281+
return ns, nil
258282
}
259283

260284
type cniNS struct {
261-
pool *cniPool
262-
handle cni.CNI
263-
id string
264-
nativeID string
265-
opts []cni.NamespaceOpts
266-
lastUsed time.Time
285+
pool *cniPool
286+
handle cni.CNI
287+
id string
288+
nativeID string
289+
opts []cni.NamespaceOpts
290+
lastUsed time.Time
291+
vethName string
292+
canSample bool
293+
offsetSample *network.Sample
294+
prevSample *network.Sample
267295
}
268296

269297
func (ns *cniNS) Set(s *specs.Spec) error {
270298
return setNetNS(s, ns.nativeID)
271299
}
272300

273301
func (ns *cniNS) Close() error {
302+
if ns.prevSample != nil {
303+
ns.offsetSample = ns.prevSample
304+
}
274305
if ns.pool == nil {
275306
return ns.release()
276307
}
277308
ns.pool.put(ns)
278309
return nil
279310
}
280311

312+
func (ns *cniNS) Sample() (*network.Sample, error) {
313+
if !ns.canSample {
314+
return nil, nil
315+
}
316+
s, err := ns.sample()
317+
if err != nil {
318+
return nil, err
319+
}
320+
if s == nil {
321+
return nil, nil
322+
}
323+
if ns.offsetSample != nil {
324+
s.TxBytes -= ns.offsetSample.TxBytes
325+
s.RxBytes -= ns.offsetSample.RxBytes
326+
s.TxPackets -= ns.offsetSample.TxPackets
327+
s.RxPackets -= ns.offsetSample.RxPackets
328+
s.TxErrors -= ns.offsetSample.TxErrors
329+
s.RxErrors -= ns.offsetSample.RxErrors
330+
s.TxDropped -= ns.offsetSample.TxDropped
331+
s.RxDropped -= ns.offsetSample.RxDropped
332+
}
333+
return s, nil
334+
}
335+
281336
func (ns *cniNS) release() error {
282337
bklog.L.Debugf("releasing cni network namespace %s", ns.id)
283338
err := ns.handle.Remove(context.TODO(), ns.id, ns.nativeID, ns.opts...)

‎util/network/cniprovider/cni_linux.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package cniprovider
2+
3+
import (
4+
"path/filepath"
5+
"strconv"
6+
"strings"
7+
"syscall"
8+
9+
"github.com/moby/buildkit/util/network"
10+
"github.com/pkg/errors"
11+
)
12+
13+
func (ns *cniNS) sample() (*network.Sample, error) {
14+
dirfd, err := syscall.Open(filepath.Join("/sys/class/net", ns.vethName, "statistics"), syscall.O_RDONLY, 0)
15+
if err != nil {
16+
if errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ENOTDIR) {
17+
return nil, nil
18+
}
19+
return nil, err
20+
}
21+
defer syscall.Close(dirfd)
22+
23+
buf := make([]byte, 32)
24+
stat := &network.Sample{}
25+
26+
for _, name := range []string{"tx_bytes", "rx_bytes", "tx_packets", "rx_packets", "tx_errors", "rx_errors", "tx_dropped", "rx_dropped"} {
27+
n, err := readFileAt(dirfd, name, buf)
28+
if err != nil {
29+
return nil, errors.Wrapf(err, "failed to read %s", name)
30+
}
31+
switch name {
32+
case "tx_bytes":
33+
stat.TxBytes = n
34+
case "rx_bytes":
35+
stat.RxBytes = n
36+
case "tx_packets":
37+
stat.TxPackets = n
38+
case "rx_packets":
39+
stat.RxPackets = n
40+
case "tx_errors":
41+
stat.TxErrors = n
42+
case "rx_errors":
43+
stat.RxErrors = n
44+
case "tx_dropped":
45+
stat.TxDropped = n
46+
case "rx_dropped":
47+
stat.RxDropped = n
48+
}
49+
}
50+
ns.prevSample = stat
51+
return stat, nil
52+
}
53+
54+
func readFileAt(dirfd int, filename string, buf []byte) (int64, error) {
55+
fd, err := syscall.Openat(dirfd, filename, syscall.O_RDONLY, 0)
56+
if err != nil {
57+
return 0, err
58+
}
59+
defer syscall.Close(fd)
60+
61+
n, err := syscall.Read(fd, buf[:])
62+
if err != nil {
63+
return 0, err
64+
}
65+
nn, err := strconv.ParseInt(strings.TrimSpace(string(buf[:n])), 10, 64)
66+
if err != nil {
67+
return 0, err
68+
}
69+
return nn, nil
70+
}

‎util/network/cniprovider/cni_nolinux.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
//go:build !linux
2+
// +build !linux
3+
4+
package cniprovider
5+
6+
import (
7+
"github.com/moby/buildkit/util/network"
8+
)
9+
10+
func (ns *cniNS) sample() (*network.Sample, error) {
11+
return nil, nil
12+
}

‎util/network/host.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,7 @@ func (h *hostNS) Set(s *specs.Spec) error {
3535
func (h *hostNS) Close() error {
3636
return nil
3737
}
38+
39+
func (h *hostNS) Sample() (*Sample, error) {
40+
return nil, nil
41+
}

‎util/network/network.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,17 @@ import (
77
specs "github.com/opencontainers/runtime-spec/specs-go"
88
)
99

10+
type Sample struct {
11+
RxBytes int64 `json:"rxBytes,omitempty"`
12+
RxPackets int64 `json:"rxPackets,omitempty"`
13+
RxErrors int64 `json:"rxErrors,omitempty"`
14+
RxDropped int64 `json:"rxDropped,omitempty"`
15+
TxBytes int64 `json:"txBytes,omitempty"`
16+
TxPackets int64 `json:"txPackets,omitempty"`
17+
TxErrors int64 `json:"txErrors,omitempty"`
18+
TxDropped int64 `json:"txDropped,omitempty"`
19+
}
20+
1021
// Provider interface for Network
1122
type Provider interface {
1223
io.Closer
@@ -18,4 +29,6 @@ type Namespace interface {
1829
io.Closer
1930
// Set the namespace on the spec
2031
Set(*specs.Spec) error
32+
33+
Sample() (*Sample, error)
2134
}

‎util/network/none.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,7 @@ func (h *noneNS) Set(s *specs.Spec) error {
3131
func (h *noneNS) Close() error {
3232
return nil
3333
}
34+
35+
func (h *noneNS) Sample() (*Sample, error) {
36+
return nil, nil
37+
}

0 commit comments

Comments
 (0)