Skip to content

Commit d73e62f

Browse files
sipsmasipsma
committed
Add initial MergeOp implementation.
This consists of just the base MergeOp with support for merging LLB results that include deletions using hardlinks as the efficient path and copies as fallback. Signed-off-by: Erik Sipsma <[email protected]>
1 parent 9321ec2 commit d73e62f

39 files changed

Lines changed: 3518 additions & 568 deletions

‎api/services/control/control.pb.go‎

Lines changed: 147 additions & 90 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/control/control.proto‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,14 @@ message UsageRecord {
4141
bool Mutable = 2;
4242
bool InUse = 3;
4343
int64 Size = 4;
44-
string Parent = 5;
44+
string Parent = 5 [deprecated=true];
4545
google.protobuf.Timestamp CreatedAt = 6 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
4646
google.protobuf.Timestamp LastUsedAt = 7 [(gogoproto.stdtime) = true];
4747
int64 UsageCount = 8;
4848
string Description = 9;
4949
string RecordType = 10;
5050
bool Shared = 11;
51+
repeated string Parents = 12;
5152
}
5253

5354
message SolveRequest {

‎cache/blobs.go‎

Lines changed: 172 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -54,170 +54,177 @@ type compressor func(dest io.Writer, requiredMediaType string) (io.WriteCloser,
5454

5555
func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool, compressionType compression.Type, forceCompression bool, s session.Group) error {
5656
eg, ctx := errgroup.WithContext(ctx)
57-
if sr.parent != nil {
57+
switch sr.kind() {
58+
case Merge:
59+
for _, parent := range sr.mergeParents {
60+
parent := parent
61+
eg.Go(func() error {
62+
return computeBlobChain(ctx, parent, createIfNeeded, compressionType, forceCompression, s)
63+
})
64+
}
65+
case Layer:
5866
eg.Go(func() error {
59-
return computeBlobChain(ctx, sr.parent, createIfNeeded, compressionType, forceCompression, s)
67+
return computeBlobChain(ctx, sr.layerParent, createIfNeeded, compressionType, forceCompression, s)
6068
})
61-
}
62-
63-
eg.Go(func() error {
64-
_, err := g.Do(ctx, fmt.Sprintf("%s-%t", sr.ID(), createIfNeeded), func(ctx context.Context) (interface{}, error) {
65-
if sr.getBlob() != "" {
66-
return nil, nil
67-
}
68-
if !createIfNeeded {
69-
return nil, errors.WithStack(ErrNoBlobs)
70-
}
69+
fallthrough
70+
case BaseLayer:
71+
eg.Go(func() error {
72+
_, err := g.Do(ctx, fmt.Sprintf("%s-%t", sr.ID(), createIfNeeded), func(ctx context.Context) (interface{}, error) {
73+
if sr.getBlob() != "" {
74+
return nil, nil
75+
}
76+
if !createIfNeeded {
77+
return nil, errors.WithStack(ErrNoBlobs)
78+
}
7179

72-
var mediaType string
73-
var compressorFunc compressor
74-
var finalize func(context.Context, content.Store) (map[string]string, error)
75-
switch compressionType {
76-
case compression.Uncompressed:
77-
mediaType = ocispecs.MediaTypeImageLayer
78-
case compression.Gzip:
79-
mediaType = ocispecs.MediaTypeImageLayerGzip
80-
case compression.EStargz:
81-
compressorFunc, finalize = compressEStargz()
82-
mediaType = ocispecs.MediaTypeImageLayerGzip
83-
case compression.Zstd:
84-
compressorFunc = zstdWriter
85-
mediaType = ocispecs.MediaTypeImageLayer + "+zstd"
86-
default:
87-
return nil, errors.Errorf("unknown layer compression type: %q", compressionType)
88-
}
80+
var mediaType string
81+
var compressorFunc compressor
82+
var finalize func(context.Context, content.Store) (map[string]string, error)
83+
switch compressionType {
84+
case compression.Uncompressed:
85+
mediaType = ocispecs.MediaTypeImageLayer
86+
case compression.Gzip:
87+
mediaType = ocispecs.MediaTypeImageLayerGzip
88+
case compression.EStargz:
89+
compressorFunc, finalize = compressEStargz()
90+
mediaType = ocispecs.MediaTypeImageLayerGzip
91+
case compression.Zstd:
92+
compressorFunc = zstdWriter
93+
mediaType = ocispecs.MediaTypeImageLayer + "+zstd"
94+
default:
95+
return nil, errors.Errorf("unknown layer compression type: %q", compressionType)
96+
}
8997

90-
var lower []mount.Mount
91-
if sr.parent != nil {
92-
m, err := sr.parent.Mount(ctx, true, s)
98+
var lower []mount.Mount
99+
if sr.layerParent != nil {
100+
m, err := sr.layerParent.Mount(ctx, true, s)
101+
if err != nil {
102+
return nil, err
103+
}
104+
var release func() error
105+
lower, release, err = m.Mount()
106+
if err != nil {
107+
return nil, err
108+
}
109+
if release != nil {
110+
defer release()
111+
}
112+
}
113+
m, err := sr.Mount(ctx, true, s)
93114
if err != nil {
94115
return nil, err
95116
}
96-
var release func() error
97-
lower, release, err = m.Mount()
117+
upper, release, err := m.Mount()
98118
if err != nil {
99119
return nil, err
100120
}
101121
if release != nil {
102122
defer release()
103123
}
104-
}
105-
m, err := sr.Mount(ctx, true, s)
106-
if err != nil {
107-
return nil, err
108-
}
109-
upper, release, err := m.Mount()
110-
if err != nil {
111-
return nil, err
112-
}
113-
if release != nil {
114-
defer release()
115-
}
116-
var desc ocispecs.Descriptor
117-
118-
// Determine differ and error/log handling according to the platform, envvar and the snapshotter.
119-
var enableOverlay, fallback, logWarnOnErr bool
120-
if forceOvlStr := os.Getenv("BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF"); forceOvlStr != "" {
121-
enableOverlay, err = strconv.ParseBool(forceOvlStr)
122-
if err != nil {
123-
return nil, errors.Wrapf(err, "invalid boolean in BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF")
124-
}
125-
fallback = false // prohibit fallback on debug
126-
} else if !isTypeWindows(sr) {
127-
enableOverlay, fallback = true, true
128-
switch sr.cm.ManagerOpt.Snapshotter.Name() {
129-
case "overlayfs", "stargz":
130-
// overlayfs-based snapshotters should support overlay diff. so print warn log on failure.
131-
logWarnOnErr = true
132-
case "fuse-overlayfs":
133-
// not supported with fuse-overlayfs snapshotter which doesn't provide overlayfs mounts.
134-
// TODO: add support for fuse-overlayfs
135-
enableOverlay = false
124+
var desc ocispecs.Descriptor
125+
126+
// Determine differ and error/log handling according to the platform, envvar and the snapshotter.
127+
var enableOverlay, fallback, logWarnOnErr bool
128+
if forceOvlStr := os.Getenv("BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF"); forceOvlStr != "" {
129+
enableOverlay, err = strconv.ParseBool(forceOvlStr)
130+
if err != nil {
131+
return nil, errors.Wrapf(err, "invalid boolean in BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF")
132+
}
133+
fallback = false // prohibit fallback on debug
134+
} else if !isTypeWindows(sr) {
135+
enableOverlay, fallback = true, true
136+
switch sr.cm.Snapshotter.Name() {
137+
case "overlayfs", "stargz":
138+
// overlayfs-based snapshotters should support overlay diff. so print warn log on failure.
139+
logWarnOnErr = true
140+
case "fuse-overlayfs":
141+
// not supported with fuse-overlayfs snapshotter which doesn't provide overlayfs mounts.
142+
// TODO: add support for fuse-overlayfs
143+
enableOverlay = false
144+
}
136145
}
137-
}
138-
if enableOverlay {
139-
computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc)
140-
if !ok || err != nil {
141-
if !fallback {
142-
if !ok {
143-
return nil, errors.Errorf("overlay mounts not detected (lower=%+v,upper=%+v)", lower, upper)
146+
if enableOverlay {
147+
computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc)
148+
if !ok || err != nil {
149+
if !fallback {
150+
if !ok {
151+
return nil, errors.Errorf("overlay mounts not detected (lower=%+v,upper=%+v)", lower, upper)
152+
}
153+
if err != nil {
154+
return nil, errors.Wrapf(err, "failed to compute overlay diff")
155+
}
144156
}
145-
if err != nil {
146-
return nil, errors.Wrapf(err, "failed to compute overlay diff")
157+
if logWarnOnErr {
158+
logrus.Warnf("failed to compute blob by overlay differ (ok=%v): %v", ok, err)
147159
}
148160
}
149-
if logWarnOnErr {
150-
logrus.Warnf("failed to compute blob by overlay differ (ok=%v): %v", ok, err)
161+
if ok {
162+
desc = computed
151163
}
152164
}
153-
if ok {
154-
desc = computed
165+
166+
if desc.Digest == "" {
167+
desc, err = sr.cm.Differ.Compare(ctx, lower, upper,
168+
diff.WithMediaType(mediaType),
169+
diff.WithReference(sr.ID()),
170+
diff.WithCompressor(compressorFunc),
171+
)
172+
if err != nil {
173+
return nil, err
174+
}
155175
}
156-
}
157176

158-
if desc.Digest == "" {
159-
desc, err = sr.cm.Differ.Compare(ctx, lower, upper,
160-
diff.WithMediaType(mediaType),
161-
diff.WithReference(sr.ID()),
162-
diff.WithCompressor(compressorFunc),
163-
)
177+
if desc.Annotations == nil {
178+
desc.Annotations = map[string]string{}
179+
}
180+
if finalize != nil {
181+
a, err := finalize(ctx, sr.cm.ContentStore)
182+
if err != nil {
183+
return nil, errors.Wrapf(err, "failed to finalize compression")
184+
}
185+
for k, v := range a {
186+
desc.Annotations[k] = v
187+
}
188+
}
189+
info, err := sr.cm.ContentStore.Info(ctx, desc.Digest)
164190
if err != nil {
165191
return nil, err
166192
}
167-
}
168193

169-
if desc.Annotations == nil {
170-
desc.Annotations = map[string]string{}
171-
}
172-
if finalize != nil {
173-
a, err := finalize(ctx, sr.cm.ContentStore)
174-
if err != nil {
175-
return nil, errors.Wrapf(err, "failed to finalize compression")
194+
if diffID, ok := info.Labels[containerdUncompressed]; ok {
195+
desc.Annotations[containerdUncompressed] = diffID
196+
} else if mediaType == ocispecs.MediaTypeImageLayer {
197+
desc.Annotations[containerdUncompressed] = desc.Digest.String()
198+
} else {
199+
return nil, errors.Errorf("unknown layer compression type")
176200
}
177-
for k, v := range a {
178-
desc.Annotations[k] = v
179-
}
180-
}
181201

182-
info, err := sr.cm.ContentStore.Info(ctx, desc.Digest)
202+
if err := sr.setBlob(ctx, compressionType, desc); err != nil {
203+
return nil, err
204+
}
205+
return nil, nil
206+
})
183207
if err != nil {
184-
return nil, err
208+
return err
185209
}
186210

187-
if diffID, ok := info.Labels[containerdUncompressed]; ok {
188-
desc.Annotations[containerdUncompressed] = diffID
189-
} else if mediaType == ocispecs.MediaTypeImageLayer {
190-
desc.Annotations[containerdUncompressed] = desc.Digest.String()
191-
} else {
192-
return nil, errors.Errorf("unknown layer compression type")
193-
}
194-
195-
if err := sr.setBlob(ctx, compressionType, desc); err != nil {
196-
return nil, err
211+
if forceCompression {
212+
if err := ensureCompression(ctx, sr, compressionType, s); err != nil {
213+
return errors.Wrapf(err, "failed to ensure compression type of %q", compressionType)
214+
}
197215
}
198-
199-
return nil, nil
216+
return nil
200217
})
201-
if err != nil {
202-
return err
203-
}
204-
if forceCompression {
205-
if err := ensureCompression(ctx, sr, compressionType, s); err != nil {
206-
return errors.Wrapf(err, "failed to ensure compression type of %q", compressionType)
207-
}
208-
}
209-
return nil
210-
})
218+
}
211219

212220
if err := eg.Wait(); err != nil {
213221
return err
214222
}
215-
return sr.setChains(ctx)
223+
return sr.computeChainMetadata(ctx)
216224
}
217225

218226
// setBlob associates a blob with the cache record.
219227
// A lease must be held for the blob when calling this function
220-
// Caller should call Info() for knowing what current values are actually set
221228
func (sr *immutableRef) setBlob(ctx context.Context, compressionType compression.Type, desc ocispecs.Descriptor) error {
222229
if _, ok := leases.FromContext(ctx); !ok {
223230
return errors.Errorf("missing lease requirement for setBlob")
@@ -267,9 +274,9 @@ func (sr *immutableRef) setBlob(ctx context.Context, compressionType compression
267274
return nil
268275
}
269276

270-
func (sr *immutableRef) setChains(ctx context.Context) error {
277+
func (sr *immutableRef) computeChainMetadata(ctx context.Context) error {
271278
if _, ok := leases.FromContext(ctx); !ok {
272-
return errors.Errorf("missing lease requirement for setChains")
279+
return errors.Errorf("missing lease requirement for computeChainMetadata")
273280
}
274281

275282
sr.mu.Lock()
@@ -281,13 +288,37 @@ func (sr *immutableRef) setChains(ctx context.Context) error {
281288

282289
var chainIDs []digest.Digest
283290
var blobChainIDs []digest.Digest
284-
if sr.parent != nil {
285-
chainIDs = append(chainIDs, digest.Digest(sr.parent.getChainID()))
286-
blobChainIDs = append(blobChainIDs, digest.Digest(sr.parent.getBlobChainID()))
291+
292+
// Blobs should be set the actual layers in the ref's chain, no
293+
// any merge refs.
294+
layerChain := sr.layerChain()
295+
var layerParent *cacheRecord
296+
switch sr.kind() {
297+
case Merge:
298+
layerParent = layerChain[len(layerChain)-1].cacheRecord
299+
case Layer:
300+
// skip the last layer in the chain, which is this ref itself
301+
layerParent = layerChain[len(layerChain)-2].cacheRecord
302+
}
303+
if layerParent != nil {
304+
if parentChainID := layerParent.getChainID(); parentChainID != "" {
305+
chainIDs = append(chainIDs, parentChainID)
306+
} else {
307+
return errors.Errorf("failed to set chain for reference with non-addressable parent")
308+
}
309+
if parentBlobChainID := layerParent.getBlobChainID(); parentBlobChainID != "" {
310+
blobChainIDs = append(blobChainIDs, parentBlobChainID)
311+
} else {
312+
return errors.Errorf("failed to set blobchain for reference with non-addressable parent")
313+
}
314+
}
315+
316+
switch sr.kind() {
317+
case Layer, BaseLayer:
318+
diffID := digest.Digest(sr.getDiffID())
319+
chainIDs = append(chainIDs, diffID)
320+
blobChainIDs = append(blobChainIDs, imagespecidentity.ChainID([]digest.Digest{digest.Digest(sr.getBlob()), diffID}))
287321
}
288-
diffID := digest.Digest(sr.getDiffID())
289-
chainIDs = append(chainIDs, diffID)
290-
blobChainIDs = append(blobChainIDs, imagespecidentity.ChainID([]digest.Digest{digest.Digest(sr.getBlob()), diffID}))
291322

292323
chainID := imagespecidentity.ChainID(chainIDs)
293324
blobChainID := imagespecidentity.ChainID(blobChainIDs)
@@ -304,8 +335,15 @@ func isTypeWindows(sr *immutableRef) bool {
304335
if sr.GetLayerType() == "windows" {
305336
return true
306337
}
307-
if parent := sr.parent; parent != nil {
308-
return isTypeWindows(parent)
338+
switch sr.kind() {
339+
case Merge:
340+
for _, p := range sr.mergeParents {
341+
if isTypeWindows(p) {
342+
return true
343+
}
344+
}
345+
case Layer:
346+
return isTypeWindows(sr.layerParent)
309347
}
310348
return false
311349
}

0 commit comments

Comments
 (0)