Skip to content

Commit 9e8cf10

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #42473 from thaJeztah/unfork_buildkit
revendor BuildKit (master branch)
2 parents 2773f81 + 3eb1257 commit 9e8cf10

233 files changed

Lines changed: 2680 additions & 4925 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎builder/builder-next/worker/worker.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import (
4949
"github.com/pkg/errors"
5050
"github.com/sirupsen/logrus"
5151
bolt "go.etcd.io/bbolt"
52+
"golang.org/x/sync/semaphore"
5253
)
5354

5455
const labelCreatedAt = "buildkit/createdat"
@@ -189,13 +190,15 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm
189190
// ResolveOp converts a LLB vertex into a LLB operation
190191
func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager) (solver.Op, error) {
191192
if baseOp, ok := v.Sys().(*pb.Op); ok {
193+
// TODO do we need to pass a value here? Where should it come from? https://github.com/moby/buildkit/commit/b3cf7c43cfefdfd7a945002c0e76b54e346ab6cf
194+
var parallelism *semaphore.Weighted
192195
switch op := baseOp.Op.(type) {
193196
case *pb.Op_Source:
194-
return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w)
197+
return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, parallelism, sm, w)
195198
case *pb.Op_Exec:
196-
return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager(), sm, w.Opt.MetadataStore, w.Executor(), w)
199+
return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager(), parallelism, sm, w.Opt.MetadataStore, w.Executor(), w)
197200
case *pb.Op_File:
198-
return ops.NewFileOp(v, op, w.CacheManager(), w.Opt.MetadataStore, w)
201+
return ops.NewFileOp(v, op, w.CacheManager(), parallelism, w.Opt.MetadataStore, w)
199202
case *pb.Op_Build:
200203
return ops.NewBuildOp(v, op, s, w)
201204
}

‎builder/dockerfile/copy.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/docker/docker/pkg/streamformatter"
2525
"github.com/docker/docker/pkg/system"
2626
"github.com/docker/docker/pkg/urlutil"
27+
"github.com/moby/buildkit/frontend/dockerfile/instructions"
2728
specs "github.com/opencontainers/image-spec/specs-go/v1"
2829
"github.com/pkg/errors"
2930
)
@@ -107,9 +108,8 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i
107108

108109
}
109110

110-
func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) {
111+
func (o *copier) createCopyInstruction(sourcesAndDest instructions.SourcesAndDest, cmdName string) (copyInstruction, error) {
111112
inst := copyInstruction{cmdName: cmdName}
112-
last := len(args) - 1
113113

114114
// Work in platform-specific filepath semantics
115115
// TODO: This OS switch for paths is NOT correct and should not be supported.
@@ -118,9 +118,9 @@ func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstr
118118
if o.platform != nil {
119119
pathOS = o.platform.OS
120120
}
121-
inst.dest = fromSlash(args[last], pathOS)
121+
inst.dest = fromSlash(sourcesAndDest.DestPath, pathOS)
122122
separator := string(separator(pathOS))
123-
infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest)
123+
infos, err := o.getCopyInfosForSourcePaths(sourcesAndDest.SourcePaths, inst.dest)
124124
if err != nil {
125125
return inst, errors.Wrapf(err, "%s failed", cmdName)
126126
}

‎builder/dockerfile/dispatchers_test.go‎

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -547,11 +547,19 @@ func TestRunIgnoresHealthcheck(t *testing.T) {
547547
assert.NilError(t, err)
548548

549549
expectedTest := []string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"}
550-
cmd := &instructions.HealthCheckCommand{
551-
Health: &container.HealthConfig{
552-
Test: expectedTest,
550+
healthint, err := instructions.ParseInstruction(&parser.Node{
551+
Original: `HEALTHCHECK CMD curl -f http://localhost/ || exit 1`,
552+
Value: "healthcheck",
553+
Next: &parser.Node{
554+
Value: "cmd",
555+
Next: &parser.Node{
556+
Value: `curl -f http://localhost/ || exit 1`,
557+
},
553558
},
554-
}
559+
})
560+
assert.NilError(t, err)
561+
cmd := healthint.(*instructions.HealthCheckCommand)
562+
555563
assert.NilError(t, dispatch(sb, cmd))
556564
assert.Assert(t, sb.state.runConfig.Healthcheck != nil)
557565

@@ -562,12 +570,11 @@ func TestRunIgnoresHealthcheck(t *testing.T) {
562570
}
563571

564572
sb.state.buildArgs.AddArg("one", strPtr("two"))
565-
run := &instructions.RunCommand{
566-
ShellDependantCmdLine: instructions.ShellDependantCmdLine{
567-
CmdLine: strslice.StrSlice{"echo foo"},
568-
PrependShell: true,
569-
},
570-
}
573+
runint, err := instructions.ParseInstruction(&parser.Node{Original: `RUN echo foo`, Value: "run"})
574+
assert.NilError(t, err)
575+
run := runint.(*instructions.RunCommand)
576+
run.PrependShell = true
577+
571578
assert.NilError(t, dispatch(sb, run))
572579
assert.Check(t, is.DeepEqual(expectedTest, sb.state.runConfig.Healthcheck.Test))
573580
}
@@ -579,24 +586,33 @@ func TestDispatchUnsupportedOptions(t *testing.T) {
579586
sb.state.operatingSystem = runtime.GOOS
580587

581588
t.Run("ADD with chmod", func(t *testing.T) {
582-
cmd := &instructions.AddCommand{SourcesAndDest: []string{".", "."}, Chmod: "0655"}
589+
cmd := &instructions.AddCommand{
590+
SourcesAndDest: instructions.SourcesAndDest{
591+
SourcePaths: []string{"."},
592+
DestPath: ".",
593+
},
594+
Chmod: "0655",
595+
}
583596
err := dispatch(sb, cmd)
584597
assert.Error(t, err, "the --chmod option requires BuildKit. Refer to https://docs.docker.com/go/buildkit/ to learn how to build images with BuildKit enabled")
585598
})
586599

587600
t.Run("COPY with chmod", func(t *testing.T) {
588-
cmd := &instructions.CopyCommand{SourcesAndDest: []string{".", "."}, Chmod: "0655"}
601+
cmd := &instructions.CopyCommand{
602+
SourcesAndDest: instructions.SourcesAndDest{
603+
SourcePaths: []string{"."},
604+
DestPath: ".",
605+
},
606+
Chmod: "0655",
607+
}
589608
err := dispatch(sb, cmd)
590609
assert.Error(t, err, "the --chmod option requires BuildKit. Refer to https://docs.docker.com/go/buildkit/ to learn how to build images with BuildKit enabled")
591610
})
592611

593612
t.Run("RUN with unsupported options", func(t *testing.T) {
594-
cmd := &instructions.RunCommand{
595-
ShellDependantCmdLine: instructions.ShellDependantCmdLine{
596-
CmdLine: strslice.StrSlice{"echo foo"},
597-
PrependShell: true,
598-
},
599-
}
613+
runint, err := instructions.ParseInstruction(&parser.Node{Original: `RUN echo foo`, Value: "run"})
614+
assert.NilError(t, err)
615+
cmd := runint.(*instructions.RunCommand)
600616

601617
// classic builder "RUN" currently doesn't support any flags, but testing
602618
// both "known" flags and "bogus" flags for completeness, and in case

‎builder/dockerfile/evaluator_test.go‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,66 +32,62 @@ func TestDispatch(t *testing.T) {
3232
{
3333
name: "ADD multiple files to file",
3434
cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
35-
"file1.txt",
36-
"file2.txt",
37-
"test",
35+
SourcePaths: []string{"file1.txt", "file2.txt"},
36+
DestPath: "test",
3837
}},
3938
expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
4039
files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
4140
},
4241
{
4342
name: "Wildcard ADD multiple files to file",
4443
cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
45-
"file*.txt",
46-
"test",
44+
SourcePaths: []string{"file*.txt"},
45+
DestPath: "test",
4746
}},
4847
expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
4948
files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
5049
},
5150
{
5251
name: "COPY multiple files to file",
5352
cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
54-
"file1.txt",
55-
"file2.txt",
56-
"test",
53+
SourcePaths: []string{"file1.txt", "file2.txt"},
54+
DestPath: "test",
5755
}},
5856
expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
5957
files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
6058
},
6159
{
6260
name: "ADD multiple files to file with whitespace",
6361
cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
64-
"test file1.txt",
65-
"test file2.txt",
66-
"test",
62+
SourcePaths: []string{"test file1.txt", "test file2.txt"},
63+
DestPath: "test",
6764
}},
6865
expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
6966
files: map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
7067
},
7168
{
7269
name: "COPY multiple files to file with whitespace",
7370
cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
74-
"test file1.txt",
75-
"test file2.txt",
76-
"test",
71+
SourcePaths: []string{"test file1.txt", "test file2.txt"},
72+
DestPath: "test",
7773
}},
7874
expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
7975
files: map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
8076
},
8177
{
8278
name: "COPY wildcard no files",
8379
cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
84-
"file*.txt",
85-
"/tmp/",
80+
SourcePaths: []string{"file*.txt"},
81+
DestPath: "/tmp/",
8682
}},
8783
expectedError: "COPY failed: no source files were specified",
8884
files: nil,
8985
},
9086
{
9187
name: "COPY url",
9288
cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
93-
"https://index.docker.io/robots.txt",
94-
"/",
89+
SourcePaths: []string{"https://index.docker.io/robots.txt"},
90+
DestPath: "/",
9591
}},
9692
expectedError: "source can't be a URL for COPY",
9793
files: nil,

‎daemon/daemon.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ import (
4242
"github.com/docker/docker/daemon/logger"
4343
"github.com/docker/docker/daemon/network"
4444
"github.com/docker/docker/errdefs"
45-
bkconfig "github.com/moby/buildkit/cmd/buildkitd/config"
4645
"github.com/moby/buildkit/util/resolver"
4746
"github.com/sirupsen/logrus"
4847

@@ -160,7 +159,7 @@ func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
160159
var (
161160
registryKey = "docker.io"
162161
mirrors = make([]string, len(daemon.configStore.Mirrors))
163-
m = map[string]bkconfig.RegistryConfig{}
162+
m = map[string]resolver.RegistryConfig{}
164163
)
165164
// must trim "https://" or "http://" prefix
166165
for i, v := range daemon.configStore.Mirrors {
@@ -170,11 +169,11 @@ func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
170169
mirrors[i] = v
171170
}
172171
// set mirrors for default registry
173-
m[registryKey] = bkconfig.RegistryConfig{Mirrors: mirrors}
172+
m[registryKey] = resolver.RegistryConfig{Mirrors: mirrors}
174173

175174
for _, v := range daemon.configStore.InsecureRegistries {
176175
u, err := url.Parse(v)
177-
c := bkconfig.RegistryConfig{}
176+
c := resolver.RegistryConfig{}
178177
if err == nil {
179178
v = u.Host
180179
t := true
@@ -198,7 +197,7 @@ func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
198197
if fis, err := ioutil.ReadDir(certsDir); err == nil {
199198
for _, fi := range fis {
200199
if _, ok := m[fi.Name()]; !ok {
201-
m[fi.Name()] = bkconfig.RegistryConfig{
200+
m[fi.Name()] = resolver.RegistryConfig{
202201
TLSConfigDir: []string{filepath.Join(certsDir, fi.Name())},
203202
}
204203
}

‎vendor.conf‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ github.com/moby/sys b0f1fd7235275d01bd35cc4421e8
1818
github.com/creack/pty 2a38352e8b4d7ab6c336eef107e42a55e72e7fbc # v1.1.11
1919
github.com/sirupsen/logrus 6699a89a232f3db797f2e280639854bbc4b89725 # v1.7.0
2020
github.com/tchap/go-patricia a7f0089c6f496e8e70402f61733606daa326cac5 # v2.3.0
21-
golang.org/x/net 6772e930b67bb09bf22262c7378e7d2f67cf59d1
21+
golang.org/x/net e18ecbb051101a46fc263334b127c89bc7bff7ea
2222
golang.org/x/sys d19ff857e887eacb631721f188c7d365c2331456
2323
github.com/docker/go-units 519db1ee28dcc9fd2474ae59fca29a810482bfb1 # v0.4.0
2424
github.com/docker/go-connections 7395e3f8aa162843a74ed6d48e79627d9792ac55 # v0.4.0
@@ -29,11 +29,11 @@ github.com/syndtr/gocapability 42c35b4376354fd554efc7ad35e0
2929

3030
github.com/RackSec/srslog a4725f04ec91af1a91b380da679d6e0c2f061e59
3131
github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6721b191b0369 # v0.3.8
32-
golang.org/x/sync 6e8e738ad208923de99951fe0b48239bfd864f28
32+
golang.org/x/sync 036812b2e83c0ddf193dd5a34e034151da389d09
3333

3434
# buildkit
35-
github.com/moby/buildkit 7e03277b32d4f0150bed0e081d4253b3a8557f13 https://github.com/cpuguy83/buildkit.git # v0.8.3-3-g244e8cde + libnetwork changes
36-
github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
35+
github.com/moby/buildkit 9f254e18360a24c2ae47b26f772c3c89533bcbb7 # master / v0.9.0-dev
36+
github.com/tonistiigi/fsutil d72af97c0eaf93c1d20360e3cb9c63c223675b83
3737
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
3838
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
3939
github.com/opentracing/opentracing-go d34af3eaa63c4d08ab54863a4bdd0daa45212e12 # v1.2.0
@@ -63,7 +63,6 @@ github.com/vishvananda/netlink f049be6f391489d3f374498fe0c8
6363
github.com/moby/ipvs 4566ccea0e08d68e9614c3e7a64a23b850c4bb35 # v1.0.1
6464
github.com/urfave/cli a65b733b303f0055f8d324d805f393cd3e7a7904
6565

66-
github.com/BurntSushi/toml 3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005 # v0.3.1
6766
github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374
6867
github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d
6968
github.com/coreos/etcd 2c834459e1aab78a5d5219c7dfe42335fc4b617a # v3.3.25
@@ -151,8 +150,8 @@ github.com/golang/protobuf 84668698ea25b64748563aa20726
151150
github.com/cloudflare/cfssl 5d63dbd981b5c408effbb58c442d54761ff94fbd # 1.3.2
152151
github.com/fernet/fernet-go 9eac43b88a5efb8651d24de9b68e87567e029736
153152
github.com/google/certificate-transparency-go 37a384cd035e722ea46e55029093e26687138edf # v1.0.20
154-
golang.org/x/crypto c1f2f97bffc9c53fc40a1a28a5b460094c0050d9
155-
golang.org/x/time 555d28b269f0569763d25dbe1a237ae74c6bcc82
153+
golang.org/x/crypto 0c34fe9e7dc2486962ef9867e3edb3503537209f
154+
golang.org/x/time 3af7569d3a1e776fc2a3c1cec133b43105ea9c2e
156155
github.com/hashicorp/go-memdb cb9a474f84cc5e41b273b20c6927680b2a8776ad
157156
github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
158157
github.com/hashicorp/golang-lru 7f827b33c0f158ec5dfbba01bb0b14a4541fd81d # v0.5.3

‎vendor/github.com/BurntSushi/toml/COPYING‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

0 commit comments

Comments
 (0)