Skip to content

Commit 9776fb5

Browse files
tonistiigitonistiigi
committed
Add incremental context send support
Signed-off-by: Tonis Tiigi <[email protected]>
1 parent 0c780c8 commit 9776fb5

6 files changed

Lines changed: 287 additions & 69 deletions

File tree

‎cli/command/formatter/disk_usage.go‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@ const (
2929
// DiskUsageContext contains disk usage specific information required by the formatter, encapsulate a Context struct.
3030
type DiskUsageContext struct {
3131
Context
32-
Verbose bool
33-
LayersSize int64
34-
Images []*types.ImageSummary
35-
Containers []*types.Container
36-
Volumes []*types.Volume
32+
Verbose bool
33+
LayersSize int64
34+
Images []*types.ImageSummary
35+
Containers []*types.Container
36+
Volumes []*types.Volume
37+
BuilderSize int64
3738
}
3839

3940
func (ctx *DiskUsageContext) startSubsection(format string) (*template.Template, error) {
@@ -97,6 +98,13 @@ func (ctx *DiskUsageContext) Write() (err error) {
9798
return err
9899
}
99100

101+
err = ctx.contextFormat(tmpl, &diskUsageBuilderContext{
102+
builderSize: ctx.BuilderSize,
103+
})
104+
if err != nil {
105+
return err
106+
}
107+
100108
diskUsageContainersCtx := diskUsageContainersContext{containers: []*types.Container{}}
101109
diskUsageContainersCtx.header = map[string]string{
102110
"Type": typeHeader,
@@ -179,6 +187,9 @@ func (ctx *DiskUsageContext) verboseWrite() (err error) {
179187
}
180188
}
181189
ctx.postFormat(tmpl, newVolumeContext())
190+
191+
// And build cache
192+
fmt.Fprintf(ctx.Output, "\nBuild cache usage: %s\n\n", units.HumanSize(float64(ctx.BuilderSize)))
182193
return
183194
}
184195

@@ -357,3 +368,32 @@ func (c *diskUsageVolumesContext) Reclaimable() string {
357368

358369
return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
359370
}
371+
372+
type diskUsageBuilderContext struct {
373+
HeaderContext
374+
builderSize int64
375+
}
376+
377+
func (c *diskUsageBuilderContext) MarshalJSON() ([]byte, error) {
378+
return marshalJSON(c)
379+
}
380+
381+
func (c *diskUsageBuilderContext) Type() string {
382+
return "Build Cache"
383+
}
384+
385+
func (c *diskUsageBuilderContext) TotalCount() string {
386+
return ""
387+
}
388+
389+
func (c *diskUsageBuilderContext) Active() string {
390+
return ""
391+
}
392+
393+
func (c *diskUsageBuilderContext) Size() string {
394+
return units.HumanSize(float64(c.builderSize))
395+
}
396+
397+
func (c *diskUsageBuilderContext) Reclaimable() string {
398+
return c.Size()
399+
}

‎cli/command/formatter/disk_usage_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ func TestDiskUsageContextFormatWrite(t *testing.T) {
2323
Images 0 0 0B 0B
2424
Containers 0 0 0B 0B
2525
Local Volumes 0 0 0B 0B
26+
Build Cache 0B 0B
2627
`,
2728
},
2829
{
@@ -38,6 +39,9 @@ CONTAINER ID IMAGE COMMAND LOCAL VOLUMES
3839
Local Volumes space usage:
3940
4041
VOLUME NAME LINKS SIZE
42+
43+
Build cache usage: 0B
44+
4145
`,
4246
},
4347
// Errors
@@ -70,6 +74,7 @@ VOLUME NAME LINKS SIZE
7074
Images 0 0 0B 0B
7175
Containers 0 0 0B 0B
7276
Local Volumes 0 0 0B 0B
77+
Build Cache 0B 0B
7378
`,
7479
},
7580
{
@@ -82,6 +87,7 @@ Local Volumes 0 0 0B
8287
Images 0
8388
Containers 0
8489
Local Volumes 0
90+
Build Cache
8591
`,
8692
},
8793
// Raw Format
@@ -109,6 +115,12 @@ active: 0
109115
size: 0B
110116
reclaimable: 0B
111117
118+
type: Build Cache
119+
total:
120+
active:
121+
size: 0B
122+
reclaimable: 0B
123+
112124
`,
113125
},
114126
}

‎cli/command/image/build.go‎

Lines changed: 65 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,23 @@ import (
44
"archive/tar"
55
"bufio"
66
"bytes"
7-
"crypto/rand"
8-
"crypto/sha256"
9-
"encoding/hex"
107
"encoding/json"
118
"fmt"
129
"io"
1310
"io/ioutil"
1411
"os"
15-
"path/filepath"
1612
"regexp"
1713
"runtime"
1814

1915
"github.com/Sirupsen/logrus"
2016
"github.com/docker/cli/cli"
2117
"github.com/docker/cli/cli/command"
2218
"github.com/docker/cli/cli/command/image/build"
23-
cliconfig "github.com/docker/cli/cli/config"
2419
"github.com/docker/cli/opts"
2520
"github.com/docker/distribution/reference"
2621
"github.com/docker/docker/api"
2722
"github.com/docker/docker/api/types"
2823
"github.com/docker/docker/api/types/container"
29-
"github.com/docker/docker/api/types/versions"
30-
"github.com/docker/docker/client/session"
3124
"github.com/docker/docker/pkg/archive"
3225
"github.com/docker/docker/pkg/jsonmessage"
3326
"github.com/docker/docker/pkg/progress"
@@ -69,6 +62,7 @@ type buildOptions struct {
6962
squash bool
7063
target string
7164
imageIDFile string
65+
stream bool
7266
}
7367

7468
// dockerfileFromStdin returns true when the user specified that the Dockerfile
@@ -141,6 +135,10 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
141135
flags.SetAnnotation("squash", "experimental", nil)
142136
flags.SetAnnotation("squash", "version", []string{"1.25"})
143137

138+
flags.BoolVar(&options.stream, "stream", false, "Stream attaches to server to negotiate build context")
139+
flags.SetAnnotation("stream", "experimental", nil)
140+
flags.SetAnnotation("stream", "version", []string{"1.31"})
141+
144142
return cmd
145143
}
146144

@@ -160,10 +158,6 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {
160158
return out.output.WriteProgress(prog)
161159
}
162160

163-
func isSessionSupported(dockerCli *command.DockerCli) bool {
164-
return dockerCli.ServerInfo().HasExperimental && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.30")
165-
}
166-
167161
// nolint: gocyclo
168162
func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
169163
var (
@@ -201,6 +195,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
201195

202196
switch {
203197
case options.contextFromStdin():
198+
// buildCtx is tar archive. if stdin was dockerfile then it is wrapped
204199
buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
205200
case isLocalDir(specifiedContext):
206201
contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, options.dockerfileName)
@@ -224,7 +219,8 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
224219
contextDir = tempDir
225220
}
226221

227-
if buildCtx == nil {
222+
// read from a directory into tar archive
223+
if buildCtx == nil && !options.stream {
228224
excludes, err := build.ReadDockerignore(contextDir)
229225
if err != nil {
230226
return err
@@ -255,14 +251,24 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
255251
}
256252
}
257253

258-
// replace Dockerfile if added dynamically
259-
if dockerfileCtx != nil {
254+
// replace Dockerfile if it was added from stdin and there is archive context
255+
if dockerfileCtx != nil && buildCtx != nil {
260256
buildCtx, relDockerfile, err = build.AddDockerfileToBuildContext(dockerfileCtx, buildCtx)
261257
if err != nil {
262258
return err
263259
}
264260
}
265261

262+
// if streaming and dockerfile was not from stdin then read from file
263+
// to the same reader that is usually stdin
264+
if options.stream && dockerfileCtx == nil {
265+
dockerfileCtx, err = os.Open(relDockerfile)
266+
if err != nil {
267+
return errors.Wrapf(err, "failed to open %s", relDockerfile)
268+
}
269+
defer dockerfileCtx.Close()
270+
}
271+
266272
ctx, cancel := context.WithCancel(context.Background())
267273
defer cancel()
268274

@@ -271,9 +277,19 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
271277
translator := func(ctx context.Context, ref reference.NamedTagged) (reference.Canonical, error) {
272278
return TrustedReference(ctx, dockerCli, ref, nil)
273279
}
274-
// Wrap the tar archive to replace the Dockerfile entry with the rewritten
275-
// Dockerfile which uses trusted pulls.
276-
buildCtx = replaceDockerfileTarWrapper(ctx, buildCtx, relDockerfile, translator, &resolvedTags)
280+
// if there is a tar wrapper, the dockerfile needs to be replaced inside it
281+
if buildCtx != nil {
282+
// Wrap the tar archive to replace the Dockerfile entry with the rewritten
283+
// Dockerfile which uses trusted pulls.
284+
buildCtx = replaceDockerfileTarWrapper(ctx, buildCtx, relDockerfile, translator, &resolvedTags)
285+
} else if dockerfileCtx != nil {
286+
// if there was not archive context still do the possible replacements in Dockerfile
287+
newDockerfile, _, err := rewriteDockerfileFrom(ctx, dockerfileCtx, translator)
288+
if err != nil {
289+
return err
290+
}
291+
dockerfileCtx = ioutil.NopCloser(bytes.NewBuffer(newDockerfile))
292+
}
277293
}
278294

279295
// Setup an upload progress bar
@@ -282,22 +298,41 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
282298
progressOutput = &lastProgressOutput{output: progressOutput}
283299
}
284300

301+
// if up to this point nothing has set the context then we must have have
302+
// another way for sending it(streaming) and set the context to the Dockerfile
285303
if dockerfileCtx != nil && buildCtx == nil {
286304
buildCtx = dockerfileCtx
287305
}
288-
var s *session.Session
289-
if isSessionSupported(dockerCli) {
290-
sharedKey, err := getBuildSharedKey(contextDir)
291-
if err != nil {
292-
return errors.Wrap(err, "failed to get build shared key")
293-
}
294-
s, err = session.NewSession(filepath.Base(contextDir), sharedKey)
295-
if err != nil {
296-
return errors.Wrap(err, "failed to create session")
297-
}
306+
307+
s, err := trySession(dockerCli, contextDir)
308+
if err != nil {
309+
return err
310+
}
311+
312+
var body io.Reader
313+
if buildCtx != nil && !options.stream {
314+
body = progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")
298315
}
299316

300-
var body io.Reader = progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")
317+
// add context stream to the session
318+
if options.stream && s != nil {
319+
syncDone := make(chan error)
320+
if err := addDirToSession(s, contextDir, progressOutput, syncDone); err != nil {
321+
return err
322+
}
323+
324+
buf := newBufferedWriter(syncDone, buildBuff)
325+
defer func() {
326+
select {
327+
case <-buf.flushed:
328+
case <-ctx.Done():
329+
}
330+
}()
331+
buildBuff = buf
332+
333+
remote = clientSessionRemote
334+
body = buildCtx
335+
}
301336

302337
authConfigs, _ := dockerCli.GetAllCredentials()
303338
buildOptions := types.ImageBuildOptions{
@@ -336,7 +371,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
336371
logrus.Debugf("running session: %v", s.UUID())
337372
if err := s.Run(ctx, dockerCli.Client().DialSession); err != nil {
338373
logrus.Error(err)
339-
cancel()
374+
cancel() // cancel progress context
340375
}
341376
}()
342377
buildOptions.SessionID = s.UUID()
@@ -347,6 +382,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
347382
if options.quiet {
348383
fmt.Fprintf(dockerCli.Err(), "%s", progBuff)
349384
}
385+
cancel()
350386
return err
351387
}
352388
defer response.Body.Close()
@@ -414,36 +450,6 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
414450
return nil
415451
}
416452

417-
func getBuildSharedKey(dir string) (string, error) {
418-
// build session is hash of build dir with node based randomness
419-
sessionFile := filepath.Join(cliconfig.Dir(), ".buildsharedkey")
420-
if _, err := os.Lstat(sessionFile); err != nil {
421-
if os.IsNotExist(err) {
422-
b := make([]byte, 32)
423-
if _, err := rand.Read(b); err != nil {
424-
return "", err
425-
}
426-
if err := os.MkdirAll(cliconfig.Dir(), 0600); err != nil {
427-
return "", err
428-
}
429-
if err := ioutil.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil {
430-
return "", err
431-
}
432-
} else {
433-
return "", err
434-
}
435-
}
436-
437-
dt, err := ioutil.ReadFile(sessionFile)
438-
if err != nil {
439-
return "", errors.Wrapf(err, "failed to read %s", sessionFile)
440-
441-
}
442-
443-
s := sha256.Sum256([]byte(fmt.Sprintf("%s:%s", dt, dir)))
444-
return hex.EncodeToString(s[:]), nil // add randomness to force recheck
445-
}
446-
447453
func isLocalDir(c string) bool {
448454
_, err := os.Stat(c)
449455
return err == nil

0 commit comments

Comments
 (0)