Skip to content
This repository was archived by the owner on Jan 15, 2026. It is now read-only.

Commit 90c4fea

Browse files
wkingwking
committed
image: Add image-layout directory based CAS and ref engines
These are much nicer than the tar engines (hooray atomic renames :), so switch the manifest tests tests back to using the directory-backed engines. I do with there was a paginated, callback-based directory lister we could use instead of ioutils.ReadDir. On the other hand, by the time directories get big enough for that to matter we may be sharding them anyway. Signed-off-by: W. Trevor King <[email protected]>
1 parent 5001873 commit 90c4fea

File tree

12 files changed

+492
-36
lines changed

12 files changed

+492
-36
lines changed

‎cmd/oci-image-init/image_layout.go‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,33 @@ import (
2323
"golang.org/x/net/context"
2424
)
2525

26-
type imageLayout struct{}
26+
type imageLayout struct {
27+
backend string
28+
}
2729

2830
func newImageLayoutCmd() *cobra.Command {
29-
state := &imageLayout{}
31+
state := &imageLayout{
32+
backend: "dir",
33+
}
3034

31-
return &cobra.Command{
35+
cmd := &cobra.Command{
3236
Use: "image-layout PATH",
3337
Short: "Initialize an OCI image-layout repository",
3438
Run: state.Run,
3539
}
40+
41+
cmd.Flags().StringVar(
42+
&state.backend, "type", "dir",
43+
"Select the image-backend. Choices: dir, tar. Defaults to dir.",
44+
)
45+
46+
return cmd
3647
}
3748

3849
func (state *imageLayout) Run(cmd *cobra.Command, args []string) {
50+
var err error
3951
if len(args) != 1 {
40-
if err := cmd.Usage(); err != nil {
52+
if err = cmd.Usage(); err != nil {
4153
fmt.Fprintln(os.Stderr, err)
4254
}
4355
os.Exit(1)
@@ -47,7 +59,14 @@ func (state *imageLayout) Run(cmd *cobra.Command, args []string) {
4759

4860
ctx := context.Background()
4961

50-
err := layout.CreateTarFile(ctx, path)
62+
switch state.backend {
63+
case "dir":
64+
err = layout.CreateDir(ctx, path)
65+
case "tar":
66+
err = layout.CreateTarFile(ctx, path)
67+
default:
68+
err = fmt.Errorf("unrecognized type: %q", state.backend)
69+
}
5170
if err != nil {
5271
fmt.Fprintln(os.Stderr, err)
5372
os.Exit(1)

‎cmd/oci-image-init/oci-image-init-image-layout.1.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ oci-image-init-image-layout \- Initialize an OCI image-layout repository
1818
**--help**
1919
Print usage statement
2020

21+
**--type**
22+
Select the image-layout backend.
23+
Choices: dir, tar.
24+
Defaults to dir.
25+
2126
# SEE ALSO
2227

2328
**oci-image-init**(1), **oci-cas**(1), ***oci-refs**(1)

‎image/cas/layout/dir.go‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
// Copyright 2016 The Linux Foundation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package layout
16+
17+
import (
18+
"crypto/sha256"
19+
"fmt"
20+
"io"
21+
"io/ioutil"
22+
"os"
23+
"path/filepath"
24+
25+
"github.com/opencontainers/image-tools/image/cas"
26+
"github.com/opencontainers/image-tools/image/layout"
27+
"golang.org/x/net/context"
28+
)
29+
30+
// DirEngine is a cas.Engine backed by a directory.
31+
type DirEngine struct {
32+
path string
33+
temp string
34+
}
35+
36+
// NewDirEngine returns a new DirEngine.
37+
func NewDirEngine(ctx context.Context, path string) (eng cas.Engine, err error) {
38+
engine := &DirEngine{
39+
path: path,
40+
}
41+
42+
err = layout.CheckDirVersion(ctx, engine.path)
43+
if err != nil {
44+
return nil, err
45+
}
46+
47+
engine.temp = filepath.Join(path, "tmp")
48+
err = os.MkdirAll(engine.temp, 0777)
49+
if err != nil {
50+
return nil, err
51+
}
52+
53+
return engine, nil
54+
}
55+
56+
// Put adds a new blob to the store.
57+
func (engine *DirEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) {
58+
hash := sha256.New()
59+
algorithm := "sha256"
60+
61+
var file *os.File
62+
file, err = ioutil.TempFile(engine.temp, "blob-")
63+
if err != nil {
64+
return "", err
65+
}
66+
defer func() {
67+
if err != nil {
68+
os.Remove(file.Name())
69+
}
70+
}()
71+
defer file.Close()
72+
73+
hashingWriter := io.MultiWriter(file, hash)
74+
io.Copy(hashingWriter, reader)
75+
file.Close()
76+
77+
digest = fmt.Sprintf("%s:%x", algorithm, hash.Sum(nil))
78+
targetName, err := blobPath(digest, string(os.PathSeparator))
79+
if err != nil {
80+
return "", err
81+
}
82+
83+
path := filepath.Join(engine.path, targetName)
84+
err = os.MkdirAll(filepath.Dir(path), 0777)
85+
if err != nil {
86+
return "", err
87+
}
88+
89+
err = os.Rename(file.Name(), path)
90+
if err != nil {
91+
return "", err
92+
}
93+
94+
return digest, nil
95+
}
96+
97+
// Get returns a reader for retrieving a blob from the store.
98+
func (engine *DirEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) {
99+
targetName, err := blobPath(digest, string(os.PathSeparator))
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
return os.Open(filepath.Join(engine.path, targetName))
105+
}
106+
107+
// Delete removes a blob from the store.
108+
func (engine *DirEngine) Delete(ctx context.Context, digest string) (err error) {
109+
targetName, err := blobPath(digest, string(os.PathSeparator))
110+
if err != nil {
111+
return err
112+
}
113+
114+
err = os.Remove(filepath.Join(engine.path, targetName))
115+
if os.IsNotExist(err) {
116+
return nil
117+
}
118+
return err
119+
}
120+
121+
// Close releases resources held by the engine.
122+
func (engine *DirEngine) Close() (err error) {
123+
return nil
124+
}

‎image/cas/layout/main.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
package layout
2020

2121
import (
22+
"fmt"
2223
"os"
24+
"strings"
2325

2426
"github.com/opencontainers/image-tools/image/cas"
2527
"golang.org/x/net/context"
@@ -28,10 +30,29 @@ import (
2830
// NewEngine instantiates an engine with the appropriate backend (tar,
2931
// HTTP, ...).
3032
func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) {
33+
engine, err = NewDirEngine(ctx, path)
34+
if err == nil {
35+
return engine, err
36+
}
37+
3138
file, err := os.OpenFile(path, os.O_RDWR, 0)
32-
if err != nil {
33-
return nil, err
39+
if err == nil {
40+
return NewTarEngine(ctx, file)
41+
}
42+
43+
return nil, fmt.Errorf("unrecognized engine at %q", path)
44+
}
45+
46+
// blobPath returns the PATH to the DIGEST blob. SEPARATOR selects
47+
// the path separator used between components.
48+
func blobPath(digest string, separator string) (path string, err error) {
49+
fields := strings.SplitN(digest, ":", 2)
50+
if len(fields) != 2 {
51+
return "", fmt.Errorf("invalid digest: %q, %v", digest, fields)
3452
}
53+
algorithm := fields[0]
54+
hash := fields[1]
3555

36-
return NewTarEngine(ctx, file)
56+
components := []string{".", "blobs", algorithm, hash}
57+
return strings.Join(components, separator), nil
3758
}

‎image/cas/layout/tar.go‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"io"
2424
"io/ioutil"
2525
"os"
26-
"strings"
2726

2827
"github.com/opencontainers/image-tools/image/cas"
2928
"github.com/opencontainers/image-tools/image/layout"
@@ -64,7 +63,10 @@ func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest stri
6463

6564
_, err = engine.Get(ctx, digest)
6665
if err == os.ErrNotExist {
67-
targetName := fmt.Sprintf("./blobs/%s/%s", algorithm, hexHash)
66+
targetName, err := blobPath(digest, "/")
67+
if err != nil {
68+
return "", err
69+
}
6870
reader = bytes.NewReader(data)
6971
err = layout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size)
7072
if err != nil {
@@ -79,14 +81,10 @@ func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest stri
7981

8082
// Get returns a reader for retrieving a blob from the store.
8183
func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) {
82-
fields := strings.SplitN(digest, ":", 2)
83-
if len(fields) != 2 {
84-
return nil, fmt.Errorf("invalid digest: %q, %v", digest, fields)
84+
targetName, err := blobPath(digest, "/")
85+
if err != nil {
86+
return nil, err
8587
}
86-
algorithm := fields[0]
87-
hash := fields[1]
88-
89-
targetName := fmt.Sprintf("./blobs/%s/%s", algorithm, hash)
9088

9189
_, tarReader, err := layout.TarEntryByName(ctx, engine.file, targetName)
9290
if err != nil {

‎image/layout/dir.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Copyright 2016 The Linux Foundation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package layout
16+
17+
import (
18+
"encoding/json"
19+
"errors"
20+
"fmt"
21+
"os"
22+
"path/filepath"
23+
24+
"golang.org/x/net/context"
25+
)
26+
27+
// CheckDirVersion checks the oci-layout entry in an image-layout
28+
// directory and returns an error if oci-layout is missing or has
29+
// unrecognized content.
30+
func CheckDirVersion(ctx context.Context, path string) (err error) {
31+
file, err := os.Open(filepath.Join(path, "oci-layout"))
32+
if os.IsNotExist(err) {
33+
return errors.New("oci-layout not found")
34+
}
35+
if err != nil {
36+
return err
37+
}
38+
defer file.Close()
39+
40+
return CheckVersion(ctx, file)
41+
}
42+
43+
// CreateDir creates a new image-layout directory at the given path.
44+
func CreateDir(ctx context.Context, path string) (err error) {
45+
err = os.MkdirAll(path, 0777)
46+
if err != nil {
47+
return err
48+
}
49+
50+
file, err := os.OpenFile(
51+
filepath.Join(path, "oci-layout"),
52+
os.O_WRONLY|os.O_CREATE|os.O_EXCL,
53+
0666,
54+
)
55+
if err != nil {
56+
return err
57+
}
58+
defer file.Close()
59+
60+
imageLayoutVersion := ImageLayoutVersion{
61+
Version: "1.0.0",
62+
}
63+
imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion)
64+
if err != nil {
65+
return err
66+
}
67+
n, err := file.Write(imageLayoutVersionBytes)
68+
if err != nil {
69+
return err
70+
}
71+
if n < len(imageLayoutVersionBytes) {
72+
return fmt.Errorf("wrote %d of %d bytes", n, len(imageLayoutVersionBytes))
73+
}
74+
75+
err = os.MkdirAll(filepath.Join(path, "blobs"), 0777)
76+
if err != nil {
77+
return err
78+
}
79+
80+
return os.MkdirAll(filepath.Join(path, "refs"), 0777)
81+
}

‎image/layout/layout.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,32 @@
1515
// Package layout defines utility code shared by refs/layout and cas/layout.
1616
package layout
1717

18+
import (
19+
"encoding/json"
20+
"fmt"
21+
"io"
22+
23+
"golang.org/x/net/context"
24+
)
25+
1826
// ImageLayoutVersion represents the oci-version content for the image
1927
// layout format.
2028
type ImageLayoutVersion struct {
2129
Version string `json:"imageLayoutVersion"`
2230
}
31+
32+
// CheckVersion checks an oci-layout reader and returns an error if it
33+
// has unrecognized content.
34+
func CheckVersion(ctx context.Context, reader io.Reader) (err error) {
35+
decoder := json.NewDecoder(reader)
36+
var version ImageLayoutVersion
37+
err = decoder.Decode(&version)
38+
if err != nil {
39+
return err
40+
}
41+
if version.Version != "1.0.0" {
42+
return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version)
43+
}
44+
45+
return nil
46+
}

0 commit comments

Comments
 (0)