Skip to content

Commit 7ff02fb

Browse files
tonistiigitonistiigi
authored
Merge pull request #6173 from tonistiigi/git-qs-addopt
add keep-git-dir and submodules controls for Git querystring URLs
2 parents 0c716fd + 04dc5dd commit 7ff02fb

12 files changed

Lines changed: 215 additions & 32 deletions

File tree

‎client/llb/source.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,11 @@ func Git(url, fragment string, opts ...GitOption) State {
341341
addCap(&gi.Constraints, pb.CapSourceGitChecksum)
342342
}
343343

344+
if gi.SkipSubmodules {
345+
attrs[pb.AttrGitSkipSubmodules] = "true"
346+
addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules)
347+
}
348+
344349
addCap(&gi.Constraints, pb.CapSourceGit)
345350

346351
source := NewSource("git://"+id, attrs, gi.Constraints)
@@ -367,6 +372,7 @@ type GitInfo struct {
367372
Checksum string
368373
Ref string
369374
SubDir string
375+
SkipSubmodules bool
370376
}
371377

372378
func GitRef(v string) GitOption {
@@ -381,6 +387,12 @@ func GitSubDir(v string) GitOption {
381387
})
382388
}
383389

390+
func GitSkipSubmodules() GitOption {
391+
return gitOptionFunc(func(gi *GitInfo) {
392+
gi.SkipSubmodules = true
393+
})
394+
}
395+
384396
func KeepGitDir() GitOption {
385397
return gitOptionFunc(func(gi *GitInfo) {
386398
gi.KeepGitDir = true

‎frontend/dockerfile/dfgitutil/git_ref.go‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package dfgitutil
33

44
import (
55
"net/url"
6+
"strconv"
67
"strings"
78

89
cerrdefs "github.com/containerd/errdefs"
@@ -49,6 +50,12 @@ type GitRef struct {
4950
// Discouraged, although not deprecated.
5051
// Instead, consider using an encrypted TCP connection such as "[email protected]/foo/bar.git" or "https://github.com/foo/bar.git".
5152
UnencryptedTCP bool
53+
54+
// KeepGitDir is true for URL that controls whether to keep the .git directory.
55+
KeepGitDir *bool
56+
57+
// Submodules is true for URL that controls whether to fetch git submodules.
58+
Submodules *bool
5259
}
5360

5461
// ParseGitRef parses a git ref.
@@ -121,11 +128,14 @@ func (gf *GitRef) loadQuery(query url.Values) error {
121128
var tag, branch string
122129
for k, v := range query {
123130
switch len(v) {
124-
case 0:
125-
return errors.Errorf("query %q has no value", k)
126-
case 1:
127-
if v[0] == "" {
128-
return errors.Errorf("query %q has no value", k)
131+
case 0, 1:
132+
if len(v) == 0 || v[0] == "" {
133+
switch k {
134+
case "submodules", "keep-git-dir":
135+
v = nil
136+
default:
137+
return errors.Errorf("query %q has no value", k)
138+
}
129139
}
130140
// NOP
131141
default:
@@ -148,6 +158,30 @@ func (gf *GitRef) loadQuery(query url.Values) error {
148158
gf.SubDir = v[0]
149159
case "checksum", "commit":
150160
gf.Checksum = v[0]
161+
case "keep-git-dir":
162+
var vv bool
163+
if len(v) == 0 {
164+
vv = true
165+
} else {
166+
var err error
167+
vv, err = strconv.ParseBool(v[0])
168+
if err != nil {
169+
return errors.Errorf("invalid keep-git-dir value: %q", v[0])
170+
}
171+
}
172+
gf.KeepGitDir = &vv
173+
case "submodules":
174+
var vv bool
175+
if len(v) == 0 {
176+
vv = true
177+
} else {
178+
var err error
179+
vv, err = strconv.ParseBool(v[0])
180+
if err != nil {
181+
return errors.Errorf("invalid submodules value: %q", v[0])
182+
}
183+
}
184+
gf.Submodules = &vv
151185
default:
152186
return errors.Errorf("unexpected query %q", k)
153187
}

‎frontend/dockerfile/dockerfile2llb/convert.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1520,7 +1520,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
15201520
llb.WithCustomName(pgName),
15211521
llb.GitRef(gitRef.Ref),
15221522
}
1523-
if cfg.keepGitDir {
1523+
if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil {
1524+
if *cfg.keepGitDir != *gitRef.KeepGitDir {
1525+
return errors.New("inconsistent keep-git-dir configuration")
1526+
}
1527+
}
1528+
if gitRef.KeepGitDir != nil {
1529+
cfg.keepGitDir = gitRef.KeepGitDir
1530+
}
1531+
if cfg.keepGitDir != nil && *cfg.keepGitDir {
15241532
gitOptions = append(gitOptions, llb.KeepGitDir())
15251533
}
15261534
if cfg.checksum != "" && gitRef.Checksum != "" {
@@ -1537,6 +1545,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
15371545
if gitRef.SubDir != "" {
15381546
gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir))
15391547
}
1548+
if gitRef.Submodules != nil && !*gitRef.Submodules {
1549+
gitOptions = append(gitOptions, llb.GitSkipSubmodules())
1550+
}
15401551

15411552
st := llb.Git(gitRef.Remote, "", gitOptions...)
15421553
opts := append([]llb.CopyOption{&llb.CopyInfo{
@@ -1711,7 +1722,7 @@ type copyConfig struct {
17111722
chown string
17121723
chmod string
17131724
link bool
1714-
keepGitDir bool
1725+
keepGitDir *bool
17151726
checksum string
17161727
parents bool
17171728
location []parser.Range

‎frontend/dockerfile/dockerfile_addgit_test.go‎

Lines changed: 102 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -356,8 +356,25 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) {
356356
integration.SkipOnPlatform(t, "windows")
357357
f := getFrontend(t, sb)
358358

359-
gitDir, err := os.MkdirTemp("", "buildkit")
359+
subModDir := t.TempDir()
360+
defer os.RemoveAll(subModDir)
361+
362+
err := runShell(subModDir, []string{
363+
"git init",
364+
"git config --local user.email test",
365+
"git config --local user.name test",
366+
"echo 123 >file",
367+
"git add file",
368+
"git commit -m initial",
369+
"git update-server-info",
370+
}...)
360371
require.NoError(t, err)
372+
373+
subModServer := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(subModDir))))
374+
defer subModServer.Close()
375+
submodServerURL := subModServer.URL
376+
377+
gitDir := t.TempDir()
361378
defer os.RemoveAll(gitDir)
362379
err = runShell(gitDir, []string{
363380
"git init",
@@ -368,13 +385,20 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) {
368385
require.NoError(t, err)
369386

370387
err = os.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte(`
388+
FROM scratch AS withgit
389+
COPY .git/HEAD out
390+
391+
FROM scratch as withsubmod
392+
COPY submod/file out
393+
371394
FROM scratch
372395
COPY foo out
373396
`), 0600)
374397
require.NoError(t, err)
375398

376399
err = runShell(gitDir, []string{
377-
"git add Dockerfile foo",
400+
"git submodule add " + submodServerURL + "/.git submod",
401+
"git add Dockerfile foo submod",
378402
"git commit -m initial",
379403
"git tag v0.0.1",
380404
"git branch base",
@@ -426,6 +450,7 @@ COPY foo out
426450
type tcase struct {
427451
name string
428452
url string
453+
target string
429454
expectOut string
430455
expectErr string
431456
}
@@ -528,15 +553,73 @@ COPY foo out
528553
url: serverURL + "/.git?subdir=sub&ref=feature",
529554
expectOut: "subfeature\n",
530555
},
556+
{
557+
name: "withgit",
558+
url: serverURL + "/.git?keep-git-dir=true",
559+
expectOut: commitHashLatest + "\n",
560+
target: "withgit",
561+
},
562+
{
563+
name: "withgitandtag",
564+
url: serverURL + "/.git?tag=v0.0.2&keep-git-dir=true",
565+
expectOut: commitHashV2 + "\n",
566+
target: "withgit",
567+
},
568+
{
569+
name: "withgit-default",
570+
url: serverURL + "/.git",
571+
expectErr: ".git/HEAD\": not found",
572+
target: "withgit",
573+
},
574+
{
575+
name: "withgit-valueless",
576+
url: serverURL + "/.git?keep-git-dir&submodules",
577+
expectOut: commitHashLatest + "\n",
578+
target: "withgit",
579+
},
580+
{
581+
name: "withgit-forbidden",
582+
url: serverURL + "/.git?keep-git-dir=false",
583+
expectErr: ".git/HEAD\": not found",
584+
target: "withgit",
585+
},
586+
{
587+
name: "withsubmod",
588+
url: serverURL + "/.git",
589+
expectOut: "123\n",
590+
target: "withsubmod",
591+
},
592+
{
593+
name: "withsubmodset",
594+
url: serverURL + "/.git?submodules=true",
595+
expectOut: "123\n",
596+
target: "withsubmod",
597+
},
598+
{
599+
name: "withsubmodempty",
600+
url: serverURL + "/.git?submodules",
601+
expectOut: "123\n",
602+
target: "withsubmod",
603+
},
604+
{
605+
name: "withoutsubmod",
606+
url: serverURL + "/.git?submodules=false",
607+
expectErr: "submod/file\": not found",
608+
target: "withsubmod",
609+
},
531610
}
532611

533612
for _, tc := range tcases {
534613
t.Run("context_"+tc.name, func(t *testing.T) {
535614
dest := t.TempDir()
615+
attrs := map[string]string{
616+
"context": tc.url,
617+
}
618+
if tc.target != "" {
619+
attrs["target"] = tc.target
620+
}
536621
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
537-
FrontendAttrs: map[string]string{
538-
"context": tc.url,
539-
},
622+
FrontendAttrs: attrs,
540623
Exports: []client.ExportEntry{
541624
{
542625
Type: client.ExporterLocal,
@@ -566,15 +649,28 @@ COPY foo out
566649

567650
for _, tc := range tcases {
568651
dockerfile2 := fmt.Sprintf(`
569-
FROM scratch
652+
FROM scratch AS main
570653
ADD %s /repo/
654+
655+
FROM scratch as withsubmod
656+
COPY --from=main /repo/submod/file /repo/foo
657+
658+
FROM scratch AS withgit
659+
COPY --from=main /repo/.git/HEAD /repo/foo
660+
661+
FROM main
571662
`, tc.url)
572663
inDir := integration.Tmpdir(t,
573664
fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600),
574665
)
575666
t.Run("add_"+tc.name, func(t *testing.T) {
576667
dest := t.TempDir()
668+
attrs := map[string]string{}
669+
if tc.target != "" {
670+
attrs["target"] = tc.target
671+
}
577672
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
673+
FrontendAttrs: attrs,
578674
Exports: []client.ExportEntry{
579675
{
580676
Type: client.ExporterLocal,

‎frontend/dockerfile/instructions/commands.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ type AddCommand struct {
244244
Chmod string
245245
Link bool
246246
ExcludePatterns []string
247-
KeepGitDir bool // whether to keep .git dir, only meaningful for git sources
247+
KeepGitDir *bool // whether to keep .git dir, only meaningful for git sources
248248
Checksum string
249249
Unpack *bool
250250
}

‎frontend/dockerfile/instructions/parse.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,13 +354,19 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
354354
unpack = &b
355355
}
356356

357+
var keepGit *bool
358+
if _, ok := req.flags.used["keep-git-dir"]; ok {
359+
b := flKeepGitDir.Value == "true"
360+
keepGit = &b
361+
}
362+
357363
return &AddCommand{
358364
withNameAndCode: newWithNameAndCode(req),
359365
SourcesAndDest: *sourcesAndDest,
360366
Chown: flChown.Value,
361367
Chmod: flChmod.Value,
362368
Link: flLink.Value == "true",
363-
KeepGitDir: flKeepGitDir.Value == "true",
369+
KeepGitDir: keepGit,
364370
Checksum: flChecksum.Value,
365371
ExcludePatterns: stringValuesFromFlagIfPossible(flExcludes),
366372
Unpack: unpack,

‎frontend/dockerui/context.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
6969
bctx.dockerfileLocalName = v
7070
}
7171

72-
keepGit := false
72+
var keepGit *bool
7373
if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
74-
keepGit = v
74+
keepGit = &v
7575
}
7676
if st, ok, err := DetectGitContext(opts[localNameContext], keepGit); ok {
7777
if err != nil {
@@ -143,7 +143,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
143143
return bctx, nil
144144
}
145145

146-
func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
146+
func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) {
147147
g, isGit, err := dfgitutil.ParseGitRef(ref)
148148
if err != nil {
149149
return nil, isGit, err
@@ -152,7 +152,10 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
152152
llb.GitRef(g.Ref),
153153
WithInternalName("load git source " + ref),
154154
}
155-
if keepGit {
155+
if g.KeepGitDir != nil && *g.KeepGitDir {
156+
gitOpts = append(gitOpts, llb.KeepGitDir())
157+
}
158+
if keepGit != nil && *keepGit {
156159
gitOpts = append(gitOpts, llb.KeepGitDir())
157160
}
158161
if g.SubDir != "" {
@@ -161,6 +164,9 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
161164
if g.Checksum != "" {
162165
gitOpts = append(gitOpts, llb.GitChecksum(g.Checksum))
163166
}
167+
if g.Submodules != nil && !*g.Submodules {
168+
gitOpts = append(gitOpts, llb.GitSkipSubmodules())
169+
}
164170

165171
st := llb.Git(g.Remote, "", gitOpts...)
166172
return &st, true, nil

‎frontend/dockerui/namedcontext.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
138138
}
139139
return &st, &img, nil
140140
case "git":
141-
st, ok, err := DetectGitContext(nc.input, true)
141+
st, ok, err := DetectGitContext(nc.input, nil)
142142
if !ok {
143143
return nil, nil, errors.Errorf("invalid git context %s", nc.input)
144144
}
@@ -147,7 +147,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
147147
}
148148
return st, nil, nil
149149
case "http", "https":
150-
st, ok, err := DetectGitContext(nc.input, true)
150+
st, ok, err := DetectGitContext(nc.input, nil)
151151
if !ok {
152152
httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
153153
st = &httpst

0 commit comments

Comments
 (0)