Skip to content

Commit aa96c31

Browse files
author
Alfred Landrum
committed
Let graphdrivers declare diff stream fidelity
This allows graphdrivers to declare that they can reproduce the original diff stream for a layer. If they do so, the layer store will not use tar-split processing, but will still verify the digest on layer export. This makes it easier to experiment with non-default diff formats. Signed-off-by: Alfred Landrum <[email protected]>
1 parent b47c50c commit aa96c31

6 files changed

Lines changed: 128 additions & 35 deletions

File tree

‎daemon/graphdriver/driver.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,23 @@ type Driver interface {
112112
DiffDriver
113113
}
114114

115+
// Capabilities defines a list of capabilities a driver may implement.
116+
// These capabilities are not required; however, they do determine how a
117+
// graphdriver can be used.
118+
type Capabilities struct {
119+
// Flags that this driver is capable of reproducing exactly equivalent
120+
// diffs for read-only layers. If set, clients can rely on the driver
121+
// for consistent tar streams, and avoid extra processing to account
122+
// for potential differences (eg: the layer store's use of tar-split).
123+
ReproducesExactDiffs bool
124+
}
125+
126+
// CapabilityDriver is the interface for layered file system drivers that
127+
// can report on their Capabilities.
128+
type CapabilityDriver interface {
129+
Capabilities() Capabilities
130+
}
131+
115132
// DiffGetterDriver is the interface for layered file system drivers that
116133
// provide a specialized function for getting file contents for tar-split.
117134
type DiffGetterDriver interface {

‎daemon/graphdriver/plugin.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,6 @@ func newPluginDriver(name string, pl plugingetter.CompatPlugin, config Options)
3838
}
3939
}
4040
}
41-
proxy := &graphDriverProxy{name, pl}
41+
proxy := &graphDriverProxy{name, pl, Capabilities{}}
4242
return proxy, proxy.Init(filepath.Join(home, name), config.DriverOptions, config.UIDMaps, config.GIDMaps)
4343
}

‎daemon/graphdriver/proxy.go‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ import (
99
"github.com/docker/docker/pkg/archive"
1010
"github.com/docker/docker/pkg/idtools"
1111
"github.com/docker/docker/pkg/plugingetter"
12+
"github.com/docker/docker/pkg/plugins"
1213
)
1314

1415
type graphDriverProxy struct {
1516
name string
1617
p plugingetter.CompatPlugin
18+
caps Capabilities
1719
}
1820

1921
type graphDriverRequest struct {
@@ -24,13 +26,14 @@ type graphDriverRequest struct {
2426
}
2527

2628
type graphDriverResponse struct {
27-
Err string `json:",omitempty"`
28-
Dir string `json:",omitempty"`
29-
Exists bool `json:",omitempty"`
30-
Status [][2]string `json:",omitempty"`
31-
Changes []archive.Change `json:",omitempty"`
32-
Size int64 `json:",omitempty"`
33-
Metadata map[string]string `json:",omitempty"`
29+
Err string `json:",omitempty"`
30+
Dir string `json:",omitempty"`
31+
Exists bool `json:",omitempty"`
32+
Status [][2]string `json:",omitempty"`
33+
Changes []archive.Change `json:",omitempty"`
34+
Size int64 `json:",omitempty"`
35+
Metadata map[string]string `json:",omitempty"`
36+
Capabilities Capabilities `json:",omitempty"`
3437
}
3538

3639
type graphDriverInitRequest struct {
@@ -60,13 +63,33 @@ func (d *graphDriverProxy) Init(home string, opts []string, uidMaps, gidMaps []i
6063
if ret.Err != "" {
6164
return errors.New(ret.Err)
6265
}
66+
caps, err := d.fetchCaps()
67+
if err != nil {
68+
return err
69+
}
70+
d.caps = caps
6371
return nil
6472
}
6573

74+
func (d *graphDriverProxy) fetchCaps() (Capabilities, error) {
75+
args := &graphDriverRequest{}
76+
var ret graphDriverResponse
77+
if err := d.p.Client().Call("GraphDriver.Capabilities", args, &ret); err != nil {
78+
if !plugins.IsNotFound(err) {
79+
return Capabilities{}, err
80+
}
81+
}
82+
return ret.Capabilities, nil
83+
}
84+
6685
func (d *graphDriverProxy) String() string {
6786
return d.name
6887
}
6988

89+
func (d *graphDriverProxy) Capabilities() Capabilities {
90+
return d.caps
91+
}
92+
7093
func (d *graphDriverProxy) CreateReadWrite(id, parent string, opts *CreateOpts) error {
7194
return d.create("GraphDriver.CreateReadWrite", id, parent, opts)
7295
}

‎docs/extend/plugins_graphdriver.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,29 @@ The request also includes a list of UID and GID mappings, structed as follows:
8484
Respond with a non-empty string error if an error occurred.
8585

8686

87+
### /GraphDriver.Capabilities
88+
89+
**Request**:
90+
```json
91+
{}
92+
```
93+
94+
Get behavioral characteristics of the graph driver. If a plugin does not handle
95+
this request, the engine will use default values for all capabilities.
96+
97+
**Response**:
98+
```json
99+
{
100+
"ReproducesExactDiffs": false,
101+
}
102+
```
103+
104+
Respond with values of capabilities:
105+
106+
* **ReproducesExactDiffs** Defaults to false. Flags that this driver is capable
107+
of reproducing exactly equivalent diffs for read-only filesystem layers.
108+
109+
87110
### /GraphDriver.Create
88111

89112
**Request**:

‎layer/layer_store.go‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type layerStore struct {
3434

3535
mounts map[string]*mountedLayer
3636
mountL sync.Mutex
37+
38+
useTarSplit bool
3739
}
3840

3941
// StoreOptions are the options used to create a new Store instance
@@ -74,11 +76,17 @@ func NewStoreFromOptions(options StoreOptions) (Store, error) {
7476
// metadata store and graph driver. The metadata store will be used to restore
7577
// the Store.
7678
func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver) (Store, error) {
79+
caps := graphdriver.Capabilities{}
80+
if capDriver, ok := driver.(graphdriver.CapabilityDriver); ok {
81+
caps = capDriver.Capabilities()
82+
}
83+
7784
ls := &layerStore{
78-
store: store,
79-
driver: driver,
80-
layerMap: map[ChainID]*roLayer{},
81-
mounts: map[string]*mountedLayer{},
85+
store: store,
86+
driver: driver,
87+
layerMap: map[ChainID]*roLayer{},
88+
mounts: map[string]*mountedLayer{},
89+
useTarSplit: !caps.ReproducesExactDiffs,
8290
}
8391

8492
ids, mounts, err := store.List()
@@ -207,18 +215,21 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
207215
digester := digest.Canonical.Digester()
208216
tr := io.TeeReader(ts, digester.Hash())
209217

210-
tsw, err := tx.TarSplitWriter(true)
211-
if err != nil {
212-
return err
213-
}
214-
metaPacker := storage.NewJSONPacker(tsw)
215-
defer tsw.Close()
218+
rdr := tr
219+
if ls.useTarSplit {
220+
tsw, err := tx.TarSplitWriter(true)
221+
if err != nil {
222+
return err
223+
}
224+
metaPacker := storage.NewJSONPacker(tsw)
225+
defer tsw.Close()
216226

217-
// we're passing nil here for the file putter, because the ApplyDiff will
218-
// handle the extraction of the archive
219-
rdr, err := asm.NewInputTarStream(tr, metaPacker, nil)
220-
if err != nil {
221-
return err
227+
// we're passing nil here for the file putter, because the ApplyDiff will
228+
// handle the extraction of the archive
229+
rdr, err = asm.NewInputTarStream(tr, metaPacker, nil)
230+
if err != nil {
231+
return err
232+
}
222233
}
223234

224235
applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, rdr)
@@ -640,6 +651,34 @@ func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc Mou
640651
return initID, nil
641652
}
642653

654+
func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
655+
if !ls.useTarSplit {
656+
var parentCacheID string
657+
if rl.parent != nil {
658+
parentCacheID = rl.parent.cacheID
659+
}
660+
661+
return ls.driver.Diff(rl.cacheID, parentCacheID)
662+
}
663+
664+
r, err := ls.store.TarSplitReader(rl.chainID)
665+
if err != nil {
666+
return nil, err
667+
}
668+
669+
pr, pw := io.Pipe()
670+
go func() {
671+
err := ls.assembleTarTo(rl.cacheID, r, nil, pw)
672+
if err != nil {
673+
pw.CloseWithError(err)
674+
} else {
675+
pw.Close()
676+
}
677+
}()
678+
679+
return pr, nil
680+
}
681+
643682
func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error {
644683
diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver)
645684
if !ok {

‎layer/ro_layer.go‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,25 +24,16 @@ type roLayer struct {
2424
// TarStream for roLayer guarantees that the data that is produced is the exact
2525
// data that the layer was registered with.
2626
func (rl *roLayer) TarStream() (io.ReadCloser, error) {
27-
r, err := rl.layerStore.store.TarSplitReader(rl.chainID)
27+
rc, err := rl.layerStore.getTarStream(rl)
2828
if err != nil {
2929
return nil, err
3030
}
3131

32-
pr, pw := io.Pipe()
33-
go func() {
34-
err := rl.layerStore.assembleTarTo(rl.cacheID, r, nil, pw)
35-
if err != nil {
36-
pw.CloseWithError(err)
37-
} else {
38-
pw.Close()
39-
}
40-
}()
41-
rc, err := newVerifiedReadCloser(pr, digest.Digest(rl.diffID))
32+
vrc, err := newVerifiedReadCloser(rc, digest.Digest(rl.diffID))
4233
if err != nil {
4334
return nil, err
4435
}
45-
return rc, nil
36+
return vrc, nil
4637
}
4738

4839
// TarStreamFrom does not make any guarantees to the correctness of the produced

0 commit comments

Comments
 (0)