moby · tonistiigi · Jun 16, 2022 · Apr 11, 2022 · tonistiigi · Jun 5, 2022
diff --git a/client/client_test.go b/client/client_test.go
@@ -15,6 +15,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -25,6 +26,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
 	ctderrdefs "github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/namespaces"
@@ -142,6 +144,7 @@ func TestIntegration(t *testing.T) {
 		testFileOpInputSwap,
 		testRelativeMountpoint,
 		testLocalSourceDiffer,
+		testOCILayoutSource,
 		testBuildExportZstd,
 		testPullZstdImage,
 		testMergeOp,
@@ -1594,6 +1597,106 @@ func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffT
 	}
 }
 
+func testOCILayoutSource(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	// create a tempdir where we will store the OCI layout
+	dir, err := os.MkdirTemp("", "buildkit-oci-layout")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	// make an image that is exported there
+	busybox := llb.Image("busybox:latest")
+	st := llb.Scratch()
+
+	run := func(cmd string) {
+		st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+	}
+
+	run(`sh -c "echo -n first > foo"`)
+	run(`sh -c "echo -n second > bar"`)
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	outW := bytes.NewBuffer(nil)
+	attrs := map[string]string{}
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:   ExporterOCI,
+				Attrs:  attrs,
+				Output: fixedWriteCloser(nopWriteCloser{outW}),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	// extract the tar stream to the directory as OCI layout
+	m, err := testutil.ReadTarToMap(outW.Bytes(), false)
+	require.NoError(t, err)
+
+	for filename, content := range m {
+		fullFilename := path.Join(dir, filename)
+		err = os.MkdirAll(path.Dir(fullFilename), 0755)
+		require.NoError(t, err)
+		if content.Header.FileInfo().IsDir() {
+			err = os.MkdirAll(fullFilename, 0755)
+			require.NoError(t, err)
+		} else {
+			err = os.WriteFile(fullFilename, content.Data, 0644)
+			require.NoError(t, err)
+		}
+	}
+
+	var index ocispecs.Index
+	err = json.Unmarshal(m["index.json"].Data, &index)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(index.Manifests))
+	digest := index.Manifests[0].Digest
+
+	store, err := local.NewStore(dir)
+	require.NoError(t, err)
+
+	// reference the OCI Layout in a build
+	// note that the key does not need to be the directory name, just something unique.
+	// since we are doing just one build with one remote here, we can give it any old ID,
+	// even something really imaginative, like "one"
+	csID := "one"
+	st = llb.OCILayout(csID, digest)
+
+	def, err = st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir, err := os.MkdirTemp("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		OCIStores: map[string]content.Store{
+			csID: store,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("first"), dt)
+
+	dt, err = os.ReadFile(filepath.Join(destDir, "bar"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("second"), dt)
+}
+
 func testFileOpRmWildcard(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

diff --git a/client/llb/resolver.go b/client/llb/resolver.go
@@ -34,8 +34,17 @@ type ImageMetaResolver interface {
 	ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (digest.Digest, []byte, error)
 }
 
+type ResolverType int
+
+const (
+	ResolverTypeRegistry ResolverType = iota
+	ResolverTypeOCILayout
+)
+
 type ResolveImageConfigOpt struct {
-	Platform    *ocispecs.Platform
-	ResolveMode string
-	LogName     string
+	Platform     *ocispecs.Platform
+	ResolveMode  string
+	LogName      string
+	ResolverType // default is ResolverTypeRegistry
+	SessionID    string
 }
diff --git a/client/llb/source.go b/client/llb/source.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	_ "crypto/sha256" // for opencontainers/go-digest
 	"encoding/json"
+	"fmt"
 	"os"
 	"strconv"
 	"strings"
@@ -132,8 +133,9 @@ func Image(ref string, opts ...ImageOption) State {
 					p = c.Platform
 				}
 				_, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
-					Platform:    p,
-					ResolveMode: info.resolveMode.String(),
+					Platform:     p,
+					ResolveMode:  info.resolveMode.String(),
+					ResolverType: ResolverTypeRegistry,
 				})
 				if err != nil {
 					return State{}, err
@@ -147,8 +149,9 @@ func Image(ref string, opts ...ImageOption) State {
 				p = c.Platform
 			}
 			dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
-				Platform:    p,
-				ResolveMode: info.resolveMode.String(),
+				Platform:     p,
+				ResolveMode:  info.resolveMode.String(),
+				ResolverType: ResolverTypeRegistry,
 			})
 			if err != nil {
 				return State{}, err
@@ -452,6 +455,57 @@ func Differ(t DiffType, required bool) LocalOption {
 	})
 }
 
+func OCILayout(contentStoreID string, dig digest.Digest, opts ...OCILayoutOption) State {
+	gi := &OCILayoutInfo{}
+
+	for _, o := range opts {
+		o.SetOCILayoutOption(gi)
+	}
+	attrs := map[string]string{}
+	if gi.sessionID != "" {
+		attrs[pb.AttrOCILayoutSessionID] = gi.sessionID
+		addCap(&gi.Constraints, pb.CapSourceOCILayoutSessionID)
+	}
+
+	if ll := gi.layerLimit; ll != nil {
+		attrs[pb.AttrOCILayoutLayerLimit] = strconv.FormatInt(int64(*ll), 10)
+		addCap(&gi.Constraints, pb.CapSourceOCILayoutLayerLimit)
+	}
+
+	addCap(&gi.Constraints, pb.CapSourceOCILayout)
+
+	source := NewSource(fmt.Sprintf("oci-layout://%s@%s", contentStoreID, dig), attrs, gi.Constraints)
+	return NewState(source.Output())
+}
+
+type OCILayoutOption interface {
+	SetOCILayoutOption(*OCILayoutInfo)
+}
+
+type ociLayoutOptionFunc func(*OCILayoutInfo)
+
+func (fn ociLayoutOptionFunc) SetOCILayoutOption(li *OCILayoutInfo) {
+	fn(li)
+}
+
+func OCISessionID(id string) OCILayoutOption {
+	return ociLayoutOptionFunc(func(oi *OCILayoutInfo) {
+		oi.sessionID = id
+	})
+}
+
+func OCILayerLimit(limit int) OCILayoutOption {
+	return ociLayoutOptionFunc(func(oi *OCILayoutInfo) {
+		oi.layerLimit = &limit
+	})
+}
+
+type OCILayoutInfo struct {
+	constraintsWrapper
+	sessionID  string
+	layerLimit *int
+}
+
 type DiffType string
 
 const (

diff --git a/client/llb/state.go b/client/llb/state.go
@@ -455,6 +455,7 @@ type ConstraintsOpt interface {
 	HTTPOption
 	ImageOption
 	GitOption
+	OCILayoutOption
 }
 
 type constraintsOptFunc func(m *Constraints)
@@ -471,6 +472,10 @@ func (fn constraintsOptFunc) SetLocalOption(li *LocalInfo) {
 	li.applyConstraints(fn)
 }
 
+func (fn constraintsOptFunc) SetOCILayoutOption(oi *OCILayoutInfo) {
+	oi.applyConstraints(fn)
+}
+
 func (fn constraintsOptFunc) SetHTTPOption(hi *HTTPInfo) {
 	hi.applyConstraints(fn)
 }

diff --git a/client/solve.go b/client/solve.go
@@ -33,6 +33,7 @@ import (
 type SolveOpt struct {
 	Exports               []ExportEntry
 	LocalDirs             map[string]string
+	OCIStores             map[string]content.Store
 	SharedKey             string
 	Frontend              string
 	FrontendAttrs         map[string]string
@@ -158,8 +159,27 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			}
 		}
 
-		if len(cacheOpt.contentStores) > 0 {
-			s.Allow(sessioncontent.NewAttachable(cacheOpt.contentStores))
+		// this is a new map that contains both cacheOpt stores and OCILayout stores
+		contentStores := make(map[string]content.Store, len(cacheOpt.contentStores)+len(opt.OCIStores))
+		// copy over the stores references from cacheOpt
+		for key, store := range cacheOpt.contentStores {
+			contentStores[key] = store
+		}
+		// copy over the stores references from ociLayout opts
+		for key, store := range opt.OCIStores {
+			// conflicts are not allowed
+			if _, ok := contentStores[key]; ok {
+				// we probably should check if the store is identical, but given that
+				// https://pkg.go.dev/github.com/containerd/containerd/content#Store
+				// is just an interface, composing 4 others, that is rather hard to do.
+				// For a future iteration.
+				return nil, errors.Errorf("contentStore key %s exists in both cache and OCI layouts", key)
+			}
+			contentStores[key] = store
+		}
+
+		if len(contentStores) > 0 {
+			s.Allow(sessioncontent.NewAttachable(contentStores))
 		}
 
 		eg.Go(func() error {

diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go
@@ -189,6 +189,7 @@ func buildAction(clicontext *cli.Context) error {
 		// LocalDirs is set later
 		Frontend: clicontext.String("frontend"),
 		// FrontendAttrs is set later
+		// OCILayouts is set later
 		CacheExports:        cacheExports,
 		CacheImports:        cacheImports,
 		Session:             attachable,
@@ -205,6 +206,11 @@ func buildAction(clicontext *cli.Context) error {
 		return errors.Wrap(err, "invalid local")
 	}
 
+	solveOpt.OCIStores, err = build.ParseOCILayout(clicontext.StringSlice("oci-layout"))
+	if err != nil {
+		return errors.Wrap(err, "invalid oci-layout")
+	}
+
 	var def *llb.Definition
 	if clicontext.String("frontend") == "" {
 		if fi, _ := os.Stdin.Stat(); (fi.Mode() & os.ModeCharDevice) != 0 {

diff --git a/cmd/buildctl/build/ocilayout.go b/cmd/buildctl/build/ocilayout.go
@@ -0,0 +1,27 @@
+package build
+
+import (
+	"strings"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
+	"github.com/pkg/errors"
+)
+
+// ParseOCILayout parses --oci-layout
+func ParseOCILayout(layouts []string) (map[string]content.Store, error) {
+	contentStores := make(map[string]content.Store)
+	for _, idAndDir := range layouts {
+		parts := strings.SplitN(idAndDir, "=", 2)
+		if len(parts) != 2 {
+			return nil, errors.Errorf("oci-layout option must be 'id=path/to/layout', instead had invalid %s", idAndDir)
+		}
+		cs, err := local.NewStore(parts[1])
+		if err != nil {
+			return nil, errors.Wrapf(err, "oci-layout context at %s failed to initialize", parts[1])
+		}
+		contentStores[parts[0]] = cs
+	}
+
+	return contentStores, nil
+}