Skip to content

vendor: update buildkit to v0.11.2 #44686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cpuguy83 merged 3 commits into from
Feb 2, 2023
Merged

vendor: update buildkit to v0.11.2 #44686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
85169a0
vendor: github.com/tonistiigi/fsutil v0.0.0-20221114235510-0127568185cf
thaJeztah Nov 24, 2022
666334b
vendor: update buildkit to v0.11.2
tonistiigi Dec 21, 2022
043dbc0
github: apply patch for CNI test that was incorrectly skipped
tonistiigi Feb 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
8 changes: 5 additions & 3 deletions .github/workflows/buildkit.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,17 @@ jobs:
-
name: BuildKit ref
run: |
# FIXME(thaJeztah) temporarily overriding version to use for tests; remove with the next release of buildkit
# FIXME(tonistiigi) test suite needs patch moby/buildkit#3567
# echo "BUILDKIT_REPO=moby/buildkit" >> $GITHUB_ENV
# echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
echo "BUILDKIT_REF=3a391492c9d0b7428b6dcaa18c5aa3b5951fdacd" >> $GITHUB_ENV
echo "BUILDKIT_REPO=tonistiigi/buildkit" >> $GITHUB_ENV
echo "BUILDKIT_REF=db67180a1a439efb1547ecf5decd4003ec8f621b" >> $GITHUB_ENV
Comment on lines +75 to +76
Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Feb 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self; we can remove this once moby/buildkit#3567 is merged and cherry-picked in the 0.11 branch

working-directory: moby
-
name: Checkout BuildKit ${{ env.BUILDKIT_REF }}
uses: actions/checkout@v3
with:
repository: "moby/buildkit"
repository: ${{ env.BUILDKIT_REPO }}
ref: ${{ env.BUILDKIT_REF }}
path: buildkit
-
Expand Down
22 changes: 21 additions & 1 deletion builder/builder-next/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline"
localremotecache "github.com/moby/buildkit/cache/remotecache/local"
"github.com/moby/buildkit/client"
bkconfig "github.com/moby/buildkit/cmd/buildkitd/config"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/frontend"
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
Expand All @@ -38,6 +39,7 @@ import (
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/worker"
"github.com/pkg/errors"
"go.etcd.io/bbolt"
bolt "go.etcd.io/bbolt"
)

Expand Down Expand Up @@ -157,6 +159,11 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
return nil, err
}

historyDB, err := bbolt.Open(filepath.Join(opt.Root, "history.db"), 0o600, nil)
if err != nil {
return nil, err
}

gcPolicy, err := getGCPolicy(opt.BuilderConfig, root)
if err != nil {
return nil, errors.Wrap(err, "could not get builder GC policy")
Expand Down Expand Up @@ -189,6 +196,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
Transport: rt,
Layers: layers,
Platforms: archutil.SupportedPlatforms(true),
LeaseManager: lm,
}

wc := &worker.Controller{}
Expand All @@ -203,6 +211,14 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
"gateway.v0": gateway.NewGatewayFrontend(wc),
}

var hconf *bkconfig.HistoryConfig
if opt.BuilderConfig.History != nil {
hconf = &bkconfig.HistoryConfig{
MaxAge: opt.BuilderConfig.History.MaxAge,
MaxEntries: opt.BuilderConfig.History.MaxEntries,
}
}

return control.NewController(control.Opt{
SessionManager: opt.SessionManager,
WorkerController: wc,
Expand All @@ -215,7 +231,11 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
"inline": inlineremotecache.ResolveCacheExporterFunc(),
},
Entitlements: getEntitlements(opt.BuilderConfig),
Entitlements: getEntitlements(opt.BuilderConfig),
LeaseManager: lm,
ContentStore: store,
HistoryDB: historyDB,
HistoryConfig: hconf,
})
}

Expand Down
7 changes: 6 additions & 1 deletion builder/builder-next/executor_unix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package buildkit

import (
"context"
"os"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -68,7 +69,7 @@ type bridgeProvider struct {
Root string
}

func (p *bridgeProvider) New() (network.Namespace, error) {
func (p *bridgeProvider) New(ctx context.Context, hostname string) (network.Namespace, error) {
n, err := p.NetworkByName(networkName)
if err != nil {
return nil, err
Expand All @@ -82,6 +83,10 @@ func (p *bridgeProvider) New() (network.Namespace, error) {
return iface, nil
}

func (p *bridgeProvider) Close() error {
return nil
}

type lnInterface struct {
ep *libnetwork.Endpoint
sbx *libnetwork.Sandbox
Expand Down
39 changes: 17 additions & 22 deletions builder/builder-next/exporter/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/docker/docker/reference"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/util/compression"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -103,22 +102,18 @@ func (e *imageExporterInstance) Name() string {
return "exporting to image"
}

func (e *imageExporterInstance) Config() exporter.Config {
return exporter.Config{
Compression: compression.Config{
Type: compression.Default,
},
}
func (e *imageExporterInstance) Config() *exporter.Config {
return exporter.NewConfig()
}

func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source, sessionID string) (map[string]string, error) {
func (e *imageExporterInstance) Export(ctx context.Context, inp *exporter.Source, sessionID string) (map[string]string, exporter.DescriptorReference, error) {
Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This signature is starting to look a bit awkward; I had a quick look, and it seems this is ultimately defined in the BuildKit repository. As it's being updated already, would it make sense to have a struct definition for the returned values (which contains both the map[string]string and the DescriptoReference) work?

I haven't looked very closely, but the existing map[string]string returns this;

return map[string]string{
	exptypes.ExporterImageConfigDigestKey: configDigest.String(),
	exptypes.ExporterImageDigestKey:       id.String(),
}

Which feels like it was intended as a "generic" key/value list (maybe I'm wrong though), so perhaps the map could be a map[string]interface instead, OR ( if the "what should be returned" should be more formal) the map could be discarded and individual fields defined for them.

Copy link
Copy Markdown
Member Author

@tonistiigi tonistiigi Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are for different use cases. Map is for the client to handle metadata while the new field in for buildkit internal post-processing that runs after the exporter. If there are more fields for post-processing in the future, then the new field should be turned into struct instead of adding a 4th var.

if len(inp.Refs) > 1 {
return nil, fmt.Errorf("exporting multiple references to image store is currently unsupported")
return nil, nil, fmt.Errorf("exporting multiple references to image store is currently unsupported")
}

ref := inp.Ref
if ref != nil && len(inp.Refs) == 1 {
return nil, fmt.Errorf("invalid exporter input: Ref and Refs are mutually exclusive")
return nil, nil, fmt.Errorf("invalid exporter input: Ref and Refs are mutually exclusive")
}

// only one loop
Expand All @@ -137,14 +132,14 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source,
case 1:
platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]
if !ok {
return nil, fmt.Errorf("cannot export image, missing platforms mapping")
return nil, nil, fmt.Errorf("cannot export image, missing platforms mapping")
}
var p exptypes.Platforms
if err := json.Unmarshal(platformsBytes, &p); err != nil {
return nil, errors.Wrapf(err, "failed to parse platforms passed to exporter")
return nil, nil, errors.Wrapf(err, "failed to parse platforms passed to exporter")
}
if len(p.Platforms) != len(inp.Refs) {
return nil, errors.Errorf("number of platforms does not match references %d %d", len(p.Platforms), len(inp.Refs))
return nil, nil, errors.Errorf("number of platforms does not match references %d %d", len(p.Platforms), len(inp.Refs))
}
config = inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, p.Platforms[0].ID)]
if v, ok := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, p.Platforms[0].ID)]; ok {
Expand All @@ -157,16 +152,16 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source,
layersDone := oneOffProgress(ctx, "exporting layers")

if err := ref.Finalize(ctx); err != nil {
return nil, layersDone(err)
return nil, nil, layersDone(err)
}

if err := ref.Extract(ctx, nil); err != nil {
return nil, err
return nil, nil, err
}

diffIDs, err := e.opt.Differ.EnsureLayer(ctx, ref.ID())
if err != nil {
return nil, layersDone(err)
return nil, nil, layersDone(err)
}

diffs = make([]digest.Digest, len(diffIDs))
Expand All @@ -181,36 +176,36 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source,
var err error
config, err = emptyImageConfig()
if err != nil {
return nil, err
return nil, nil, err
}
}

history, err := parseHistoryFromConfig(config)
if err != nil {
return nil, err
return nil, nil, err
}

diffs, history = normalizeLayersAndHistory(diffs, history, ref)

config, err = patchImageConfig(config, diffs, history, inp.Metadata[exptypes.ExporterInlineCache], buildInfo)
if err != nil {
return nil, err
return nil, nil, err
}

configDigest := digest.FromBytes(config)

configDone := oneOffProgress(ctx, fmt.Sprintf("writing image %s", configDigest))
id, err := e.opt.ImageStore.Create(config)
if err != nil {
return nil, configDone(err)
return nil, nil, configDone(err)
}
_ = configDone(nil)

if e.opt.ReferenceStore != nil {
for _, targetName := range e.targetNames {
tagDone := oneOffProgress(ctx, "naming to "+targetName.String())
if err := e.opt.ReferenceStore.AddTag(targetName, digest.Digest(id), true); err != nil {
return nil, tagDone(err)
return nil, nil, tagDone(err)
}
_ = tagDone(nil)
}
Expand All @@ -219,5 +214,5 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source,
return map[string]string{
exptypes.ExporterImageConfigDigestKey: configDigest.String(),
exptypes.ExporterImageDigestKey: id.String(),
}, nil
}, nil, nil
}
32 changes: 32 additions & 0 deletions builder/builder-next/worker/worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/rootfs"
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
Expand Down Expand Up @@ -39,13 +40,18 @@ import (
"github.com/moby/buildkit/util/compression"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/version"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
)

func init() {
version.Version = "v0.11.0-rc3"
}

Comment on lines +51 to +54
Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we have a plain var for this, so that it can be through a -X option at compile time?

Copy link
Copy Markdown
Member Author

@tonistiigi tonistiigi Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to set it with -X then I think you can just set the version.Version directly there. Or is there any problem with -X not working with vendored packages?

Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering that indeed; wasn't sure if it would allow setting things in vendor.

Copy link
Copy Markdown
Member

@crazy-max crazy-max Jan 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's fine, maybe also add a comment in vendor.mod:

moby/vendor.mod

Line 54 in b9fe30d

github.com/moby/buildkit v0.10.6

to not forget to also update this.

const labelCreatedAt = "buildkit/createdat"

// LayerAccess provides access to a moby layer from a snapshot
Expand All @@ -63,6 +69,7 @@ type Opt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
CacheManager cache.Manager
LeaseManager leases.Manager
ImageSource *containerimage.Source
DownloadManager *xfer.LayerDownloadManager
V2MetadataService distmetadata.V2MetadataService
Expand Down Expand Up @@ -157,17 +164,42 @@ func (w *Worker) GCPolicy() []client.PruneInfo {
return w.Opt.GCPolicy
}

// BuildkitVersion returns BuildKit version
func (w *Worker) BuildkitVersion() client.BuildkitVersion {
return client.BuildkitVersion{
Package: version.Package,
Version: version.Version + "-moby",
Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still wondering a bit if -moby would already be reflected from the builder type (container vs docker / moby)

Revision: version.Revision,
Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Dec 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This probably needs packaging changes to propagate the version (as the version from the vendored BuildKit code won't be set) 🤔

Copy link
Copy Markdown
Member

@crazy-max crazy-max Jan 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hum yes we might need smth like: crazy-max@59746a8

Copy link
Copy Markdown
Member

@thaJeztah thaJeztah Feb 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@crazy-max remind me to this one 😅

}
}

// Close closes the worker and releases all resources
func (w *Worker) Close() error {
return nil
}

// ContentStore returns content store
func (w *Worker) ContentStore() content.Store {
return w.Opt.ContentStore
}

// LeaseManager returns leases.Manager for the worker
func (w *Worker) LeaseManager() leases.Manager {
return w.Opt.LeaseManager
}

// LoadRef loads a reference by ID
func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.ImmutableRef, error) {
var opts []cache.RefOption
if hidden {
opts = append(opts, cache.NoUpdateLastUsed)
}
if id == "" {
// results can have nil refs if they are optimized out to be equal to scratch,
// i.e. Diff(A,A) == scratch
return nil, nil
}

return w.CacheManager().Get(ctx, id, nil, opts...)
}

Expand Down
11 changes: 9 additions & 2 deletions daemon/config/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ type BuilderGCConfig struct {
DefaultKeepStorage string `json:",omitempty"`
}

// BuilderHistoryConfig contains history config for a buildkit builder
type BuilderHistoryConfig struct {
MaxAge int64 `json:",omitempty"`
MaxEntries int64 `json:",omitempty"`
}

// BuilderEntitlements contains settings to enable/disable entitlements
type BuilderEntitlements struct {
NetworkHost *bool `json:"network-host,omitempty"`
Expand All @@ -68,6 +74,7 @@ type BuilderEntitlements struct {

// BuilderConfig contains config for the builder
type BuilderConfig struct {
GC BuilderGCConfig `json:",omitempty"`
Entitlements BuilderEntitlements `json:",omitempty"`
GC BuilderGCConfig `json:",omitempty"`
Entitlements BuilderEntitlements `json:",omitempty"`
History *BuilderHistoryConfig `json:",omitempty"`
}
4 changes: 2 additions & 2 deletions integration/build/build_session_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ func testBuildWithSession(t *testing.T, client dclient.APIClient, daemonHost str
sess, err := session.NewSession(ctx, "foo1", "foo")
assert.Check(t, err)

fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{
{Dir: dir},
fsProvider := filesync.NewFSSyncProvider(filesync.StaticDirSource{
"": {Dir: dir},
})
sess.Allow(fsProvider)

Expand Down
Loading