Skip to content

Commit 59582a8

Browse files
tonistiigitonistiigi
authored
Merge pull request #2471 from tonistiigi/v0.14.1-picks
[v0.14] cherry picks for v0.14.1
2 parents 171fcbe + a3b1fae commit 59582a8

20 files changed

Lines changed: 331 additions & 179 deletions

File tree

‎build/build.go‎

Lines changed: 41 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -541,7 +541,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
541541
}
542542

543543
if pushNames != "" {
544-
progress.Write(pw, fmt.Sprintf("merging manifest list %s", pushNames), func() error {
544+
err := progress.Write(pw, fmt.Sprintf("merging manifest list %s", pushNames), func() error {
545545
descs := make([]specs.Descriptor, 0, len(res))
546546

547547
for _, r := range res {
@@ -637,6 +637,9 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
637637
}
638638
return nil
639639
})
640+
if err != nil {
641+
return err
642+
}
640643
}
641644
return nil
642645
})
@@ -781,11 +784,11 @@ func calculateChildTargets(reqs map[string][]*reqForNode, opt map[string]Options
781784
}
782785

783786
func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *client.SolveOpt) error {
784-
m := map[string]string{}
787+
m := map[string][]string{}
785788
for k, v := range so.FrontendAttrs {
786789
if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") {
787790
target := resultKey(index, strings.TrimPrefix(v, "target:"))
788-
m[target] = k
791+
m[target] = append(m[target], k)
789792
}
790793
}
791794
if len(m) == 0 {
@@ -800,7 +803,7 @@ func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *c
800803
return err
801804
}
802805

803-
for k, v := range m {
806+
for k, contexts := range m {
804807
r, ok := res[k]
805808
if !ok {
806809
continue
@@ -815,51 +818,54 @@ func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *c
815818
if so.FrontendInputs == nil {
816819
so.FrontendInputs = map[string]llb.State{}
817820
}
818-
if len(rr.Refs) > 0 {
819-
for platform, r := range rr.Refs {
820-
st, err := r.ToState()
821+
822+
for _, v := range contexts {
823+
if len(rr.Refs) > 0 {
824+
for platform, r := range rr.Refs {
825+
st, err := r.ToState()
826+
if err != nil {
827+
return err
828+
}
829+
so.FrontendInputs[k+"::"+platform] = st
830+
so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform
831+
metadata := make(map[string][]byte)
832+
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey+"/"+platform]; ok {
833+
metadata[exptypes.ExporterImageConfigKey] = dt
834+
}
835+
if dt, ok := rr.Metadata["containerimage.buildinfo/"+platform]; ok {
836+
metadata["containerimage.buildinfo"] = dt
837+
}
838+
if len(metadata) > 0 {
839+
dt, err := json.Marshal(metadata)
840+
if err != nil {
841+
return err
842+
}
843+
so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
844+
}
845+
}
846+
delete(so.FrontendAttrs, v)
847+
}
848+
if rr.Ref != nil {
849+
st, err := rr.Ref.ToState()
821850
if err != nil {
822851
return err
823852
}
824-
so.FrontendInputs[k+"::"+platform] = st
825-
so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform
853+
so.FrontendInputs[k] = st
854+
so.FrontendAttrs[v] = "input:" + k
826855
metadata := make(map[string][]byte)
827-
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey+"/"+platform]; ok {
856+
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey]; ok {
828857
metadata[exptypes.ExporterImageConfigKey] = dt
829858
}
830-
if dt, ok := rr.Metadata["containerimage.buildinfo/"+platform]; ok {
859+
if dt, ok := rr.Metadata["containerimage.buildinfo"]; ok {
831860
metadata["containerimage.buildinfo"] = dt
832861
}
833862
if len(metadata) > 0 {
834863
dt, err := json.Marshal(metadata)
835864
if err != nil {
836865
return err
837866
}
838-
so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
839-
}
840-
}
841-
delete(so.FrontendAttrs, v)
842-
}
843-
if rr.Ref != nil {
844-
st, err := rr.Ref.ToState()
845-
if err != nil {
846-
return err
847-
}
848-
so.FrontendInputs[k] = st
849-
so.FrontendAttrs[v] = "input:" + k
850-
metadata := make(map[string][]byte)
851-
if dt, ok := rr.Metadata[exptypes.ExporterImageConfigKey]; ok {
852-
metadata[exptypes.ExporterImageConfigKey] = dt
853-
}
854-
if dt, ok := rr.Metadata["containerimage.buildinfo"]; ok {
855-
metadata["containerimage.buildinfo"] = dt
856-
}
857-
if len(metadata) > 0 {
858-
dt, err := json.Marshal(metadata)
859-
if err != nil {
860-
return err
867+
so.FrontendAttrs["input-metadata:"+k] = string(dt)
861868
}
862-
so.FrontendAttrs["input-metadata:"+k] = string(dt)
863869
}
864870
}
865871
}

‎build/driver.go‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package build
33
import (
44
"context"
55
"fmt"
6+
"sync"
67

78
"github.com/containerd/containerd/platforms"
89
"github.com/docker/buildx/builder"
@@ -46,10 +47,22 @@ func (dp resolvedNode) BuildOpts(ctx context.Context) (gateway.BuildOpts, error)
4647

4748
type matchMaker func(specs.Platform) platforms.MatchComparer
4849

50+
type cachedGroup[T any] struct {
51+
g flightcontrol.Group[T]
52+
cache map[int]T
53+
cacheMu sync.Mutex
54+
}
55+
56+
func newCachedGroup[T any]() cachedGroup[T] {
57+
return cachedGroup[T]{
58+
cache: map[int]T{},
59+
}
60+
}
61+
4962
type nodeResolver struct {
50-
nodes []builder.Node
51-
clients flightcontrol.Group[*client.Client]
52-
opt flightcontrol.Group[gateway.BuildOpts]
63+
nodes []builder.Node
64+
clients cachedGroup[*client.Client]
65+
buildOpts cachedGroup[gateway.BuildOpts]
5366
}
5467

5568
func resolveDrivers(ctx context.Context, nodes []builder.Node, opt map[string]Options, pw progress.Writer) (map[string][]*resolvedNode, error) {
@@ -63,7 +76,9 @@ func resolveDrivers(ctx context.Context, nodes []builder.Node, opt map[string]Op
6376

6477
func newDriverResolver(nodes []builder.Node) *nodeResolver {
6578
r := &nodeResolver{
66-
nodes: nodes,
79+
nodes: nodes,
80+
clients: newCachedGroup[*client.Client](),
81+
buildOpts: newCachedGroup[gateway.BuildOpts](),
6782
}
6883
return r
6984
}
@@ -179,6 +194,7 @@ func (r *nodeResolver) resolve(ctx context.Context, ps []specs.Platform, pw prog
179194
resolver: r,
180195
driverIndex: 0,
181196
})
197+
nodeIdxs = append(nodeIdxs, 0)
182198
} else {
183199
for i, idx := range nodeIdxs {
184200
node := &resolvedNode{
@@ -237,11 +253,24 @@ func (r *nodeResolver) boot(ctx context.Context, idxs []int, pw progress.Writer)
237253
for i, idx := range idxs {
238254
i, idx := i, idx
239255
eg.Go(func() error {
240-
c, err := r.clients.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (*client.Client, error) {
256+
c, err := r.clients.g.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (*client.Client, error) {
241257
if r.nodes[idx].Driver == nil {
242258
return nil, nil
243259
}
244-
return driver.Boot(ctx, baseCtx, r.nodes[idx].Driver, pw)
260+
r.clients.cacheMu.Lock()
261+
c, ok := r.clients.cache[idx]
262+
r.clients.cacheMu.Unlock()
263+
if ok {
264+
return c, nil
265+
}
266+
c, err := driver.Boot(ctx, baseCtx, r.nodes[idx].Driver, pw)
267+
if err != nil {
268+
return nil, err
269+
}
270+
r.clients.cacheMu.Lock()
271+
r.clients.cache[idx] = c
272+
r.clients.cacheMu.Unlock()
273+
return c, nil
245274
})
246275
if err != nil {
247276
return err
@@ -272,14 +301,25 @@ func (r *nodeResolver) opts(ctx context.Context, idxs []int, pw progress.Writer)
272301
continue
273302
}
274303
eg.Go(func() error {
275-
opt, err := r.opt.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (gateway.BuildOpts, error) {
276-
opt := gateway.BuildOpts{}
304+
opt, err := r.buildOpts.g.Do(ctx, fmt.Sprint(idx), func(ctx context.Context) (gateway.BuildOpts, error) {
305+
r.buildOpts.cacheMu.Lock()
306+
opt, ok := r.buildOpts.cache[idx]
307+
r.buildOpts.cacheMu.Unlock()
308+
if ok {
309+
return opt, nil
310+
}
277311
_, err := c.Build(ctx, client.SolveOpt{
278312
Internal: true,
279313
}, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
280314
opt = c.BuildOpts()
281315
return nil, nil
282316
}, nil)
317+
if err != nil {
318+
return gateway.BuildOpts{}, err
319+
}
320+
r.buildOpts.cacheMu.Lock()
321+
r.buildOpts.cache[idx] = opt
322+
r.buildOpts.cacheMu.Unlock()
283323
return opt, err
284324
})
285325
if err != nil {

‎build/opt.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
162162
case 1:
163163
// valid
164164
case 0:
165-
if !noDefaultLoad() {
165+
if !noDefaultLoad() && opt.PrintFunc == nil {
166166
if nodeDriver.IsMobyDriver() {
167167
// backwards compat for docker driver only:
168168
// this ensures the build results in a docker image.

‎cmd/buildx/main.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"context"
45
"fmt"
56
"os"
67

@@ -15,6 +16,7 @@ import (
1516
cliflags "github.com/docker/cli/cli/flags"
1617
"github.com/moby/buildkit/solver/errdefs"
1718
"github.com/moby/buildkit/util/stack"
19+
"go.opentelemetry.io/otel"
1820

1921
//nolint:staticcheck // vendored dependencies may still use this
2022
"github.com/containerd/containerd/pkg/seed"
@@ -38,10 +40,27 @@ func runStandalone(cmd *command.DockerCli) error {
3840
if err := cmd.Initialize(cliflags.NewClientOptions()); err != nil {
3941
return err
4042
}
43+
defer flushMetrics(cmd)
44+
4145
rootCmd := commands.NewRootCmd(os.Args[0], false, cmd)
4246
return rootCmd.Execute()
4347
}
4448

49+
// flushMetrics will manually flush metrics from the configured
50+
// meter provider. This is needed when running in standalone mode
51+
// because the meter provider is initialized by the cli library,
52+
// but the mechanism for forcing it to report is not presently
53+
// exposed and not invoked when run in standalone mode.
54+
// There are plans to fix that in the next release, but this is
55+
// needed temporarily until the API for this is more thorough.
56+
func flushMetrics(cmd *command.DockerCli) {
57+
if mp, ok := cmd.MeterProvider().(command.MeterProvider); ok {
58+
if err := mp.ForceFlush(context.Background()); err != nil {
59+
otel.Handle(err)
60+
}
61+
}
62+
}
63+
4564
func runPlugin(cmd *command.DockerCli) error {
4665
rootCmd := commands.NewRootCmd("buildx", true, cmd)
4766
return plugin.RunPlugin(cmd, rootCmd, manager.Metadata{

‎commands/build.go‎

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -122,27 +122,26 @@ func (o *buildOptions) toControllerOptions() (*controllerapi.BuildOptions, error
122122
}
123123

124124
opts := controllerapi.BuildOptions{
125-
Allow: o.allow,
126-
Annotations: o.annotations,
127-
BuildArgs: buildArgs,
128-
CgroupParent: o.cgroupParent,
129-
ContextPath: o.contextPath,
130-
DockerfileName: o.dockerfileName,
131-
ExtraHosts: o.extraHosts,
132-
Labels: labels,
133-
NetworkMode: o.networkMode,
134-
NoCacheFilter: o.noCacheFilter,
135-
Platforms: o.platforms,
136-
ShmSize: int64(o.shmSize),
137-
Tags: o.tags,
138-
Target: o.target,
139-
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
140-
Builder: o.builder,
141-
NoCache: o.noCache,
142-
Pull: o.pull,
143-
ExportPush: o.exportPush,
144-
ExportLoad: o.exportLoad,
145-
WithProvenanceResponse: len(o.metadataFile) > 0,
125+
Allow: o.allow,
126+
Annotations: o.annotations,
127+
BuildArgs: buildArgs,
128+
CgroupParent: o.cgroupParent,
129+
ContextPath: o.contextPath,
130+
DockerfileName: o.dockerfileName,
131+
ExtraHosts: o.extraHosts,
132+
Labels: labels,
133+
NetworkMode: o.networkMode,
134+
NoCacheFilter: o.noCacheFilter,
135+
Platforms: o.platforms,
136+
ShmSize: int64(o.shmSize),
137+
Tags: o.tags,
138+
Target: o.target,
139+
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
140+
Builder: o.builder,
141+
NoCache: o.noCache,
142+
Pull: o.pull,
143+
ExportPush: o.exportPush,
144+
ExportLoad: o.exportLoad,
146145
}
147146

148147
// TODO: extract env var parsing to a method easily usable by library consumers
@@ -207,6 +206,8 @@ func (o *buildOptions) toControllerOptions() (*controllerapi.BuildOptions, error
207206
return nil, err
208207
}
209208

209+
opts.WithProvenanceResponse = opts.PrintFunc == nil && len(o.metadataFile) > 0
210+
210211
return &opts, nil
211212
}
212213

@@ -268,8 +269,7 @@ func (o *buildOptionsHash) String() string {
268269
}
269270

270271
func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions) (err error) {
271-
mp := dockerCli.MeterProvider(ctx)
272-
defer metricutil.Shutdown(ctx, mp)
272+
mp := dockerCli.MeterProvider()
273273

274274
ctx, end, err := tracing.TraceCurrentCommand(ctx, "build")
275275
if err != nil {
@@ -365,15 +365,14 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
365365
return errors.Wrap(err, "writing image ID file")
366366
}
367367
}
368-
if options.metadataFile != "" {
369-
if err := writeMetadataFile(options.metadataFile, decodeExporterResponse(resp.ExporterResponse)); err != nil {
370-
return err
371-
}
372-
}
373368
if opts.PrintFunc != nil {
374369
if err := printResult(opts.PrintFunc, resp.ExporterResponse); err != nil {
375370
return err
376371
}
372+
} else if options.metadataFile != "" {
373+
if err := writeMetadataFile(options.metadataFile, decodeExporterResponse(resp.ExporterResponse)); err != nil {
374+
return err
375+
}
377376
}
378377
return nil
379378
}

0 commit comments

Comments
 (0)