Skip to content

Commit cec72ef

Browse files
mxpvmxpv
committed
devmapper: add snapshotter
Signed-off-by: Maksym Pavlenko <[email protected]>
1 parent 3a75882 commit cec72ef

2 files changed

Lines changed: 470 additions & 0 deletions

File tree

‎snapshots/devmapper/snapshotter.go‎

Lines changed: 392 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,392 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package devmapper
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"os/exec"
24+
"path/filepath"
25+
"strings"
26+
"sync"
27+
28+
"github.com/containerd/containerd/log"
29+
"github.com/containerd/containerd/mount"
30+
"github.com/containerd/containerd/snapshots"
31+
"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
32+
"github.com/containerd/containerd/snapshots/storage"
33+
"github.com/hashicorp/go-multierror"
34+
"github.com/pkg/errors"
35+
"github.com/sirupsen/logrus"
36+
)
37+
38+
const (
39+
metadataFileName = "metadata.db"
40+
fsTypeExt4 = "ext4"
41+
)
42+
43+
type closeFunc func() error
44+
45+
// Snapshotter implements containerd's snapshotter (https://godoc.org/github.com/containerd/containerd/snapshots#Snapshotter)
46+
// based on Linux device-mapper targets.
47+
type Snapshotter struct {
48+
store *storage.MetaStore
49+
pool *PoolDevice
50+
config *Config
51+
cleanupFn []closeFunc
52+
closeOnce sync.Once
53+
}
54+
55+
// NewSnapshotter creates new device mapper snapshotter.
56+
// Internally it creates thin-pool device (or reloads if it's already exists) and
57+
// initializes a database file for metadata.
58+
func NewSnapshotter(ctx context.Context, config *Config) (*Snapshotter, error) {
59+
// Make sure snapshotter configuration valid before running
60+
if err := config.parse(); err != nil {
61+
return nil, err
62+
}
63+
64+
if err := config.Validate(); err != nil {
65+
return nil, err
66+
}
67+
68+
var cleanupFn []closeFunc
69+
70+
if err := os.MkdirAll(config.RootPath, 0750); err != nil && !os.IsExist(err) {
71+
return nil, errors.Wrapf(err, "failed to create root directory: %s", config.RootPath)
72+
}
73+
74+
store, err := storage.NewMetaStore(filepath.Join(config.RootPath, metadataFileName))
75+
if err != nil {
76+
return nil, errors.Wrap(err, "failed to create metastore")
77+
}
78+
79+
cleanupFn = append(cleanupFn, store.Close)
80+
81+
poolDevice, err := NewPoolDevice(ctx, config)
82+
if err != nil {
83+
return nil, err
84+
}
85+
86+
cleanupFn = append(cleanupFn, poolDevice.Close)
87+
88+
return &Snapshotter{
89+
store: store,
90+
config: config,
91+
pool: poolDevice,
92+
cleanupFn: cleanupFn,
93+
}, nil
94+
}
95+
96+
// Stat returns the info for an active or committed snapshot from store
97+
func (s *Snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
98+
log.G(ctx).WithField("key", key).Debug("stat")
99+
100+
var (
101+
info snapshots.Info
102+
err error
103+
)
104+
105+
err = s.withTransaction(ctx, false, func(ctx context.Context) error {
106+
_, info, _, err = storage.GetInfo(ctx, key)
107+
return err
108+
})
109+
110+
return info, err
111+
}
112+
113+
// Update updates an existing snapshot info's data
114+
func (s *Snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
115+
log.G(ctx).Debugf("update: %s", strings.Join(fieldpaths, ", "))
116+
117+
var err error
118+
err = s.withTransaction(ctx, true, func(ctx context.Context) error {
119+
info, err = storage.UpdateInfo(ctx, info, fieldpaths...)
120+
return err
121+
})
122+
123+
return info, err
124+
}
125+
126+
// Usage not yet implemented
127+
func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
128+
log.G(ctx).WithField("key", key).Debug("usage")
129+
130+
return snapshots.Usage{}, errors.New("usage not implemented")
131+
}
132+
133+
// Mounts return the list of mounts for the active or view snapshot
134+
func (s *Snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) {
135+
log.G(ctx).WithField("key", key).Debug("mounts")
136+
137+
var (
138+
snap storage.Snapshot
139+
err error
140+
)
141+
142+
err = s.withTransaction(ctx, false, func(ctx context.Context) error {
143+
snap, err = storage.GetSnapshot(ctx, key)
144+
return err
145+
})
146+
147+
return s.buildMounts(snap), nil
148+
}
149+
150+
// Prepare creates thin device for an active snapshot identified by key
151+
func (s *Snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
152+
log.G(ctx).WithFields(logrus.Fields{"key": key, "parent": parent}).Debug("prepare")
153+
154+
var (
155+
mounts []mount.Mount
156+
err error
157+
)
158+
159+
err = s.withTransaction(ctx, true, func(ctx context.Context) error {
160+
mounts, err = s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts...)
161+
return err
162+
})
163+
164+
return mounts, err
165+
}
166+
167+
// View creates readonly thin device for the given snapshot key
168+
func (s *Snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
169+
log.G(ctx).WithFields(logrus.Fields{"key": key, "parent": parent}).Debug("prepare")
170+
171+
var (
172+
mounts []mount.Mount
173+
err error
174+
)
175+
176+
err = s.withTransaction(ctx, true, func(ctx context.Context) error {
177+
mounts, err = s.createSnapshot(ctx, snapshots.KindView, key, parent, opts...)
178+
return err
179+
})
180+
181+
return mounts, err
182+
}
183+
184+
// Commit marks an active snapshot as committed in meta store.
185+
// Block device unmount operation captures snapshot changes by itself, so no
186+
// additional actions needed within Commit operation.
187+
func (s *Snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
188+
log.G(ctx).WithFields(logrus.Fields{"name": name, "key": key}).Debug("commit")
189+
190+
return s.withTransaction(ctx, true, func(ctx context.Context) error {
191+
_, err := storage.CommitActive(ctx, key, name, snapshots.Usage{}, opts...)
192+
return err
193+
})
194+
}
195+
196+
// Remove removes thin device and snapshot metadata by key
197+
func (s *Snapshotter) Remove(ctx context.Context, key string) error {
198+
log.G(ctx).WithField("key", key).Debug("remove")
199+
200+
return s.withTransaction(ctx, true, func(ctx context.Context) error {
201+
return s.removeDevice(ctx, key)
202+
})
203+
}
204+
205+
func (s *Snapshotter) removeDevice(ctx context.Context, key string) error {
206+
snapID, _, err := storage.Remove(ctx, key)
207+
if err != nil {
208+
return err
209+
}
210+
211+
deviceName := s.getDeviceName(snapID)
212+
if err := s.pool.RemoveDevice(ctx, deviceName); err != nil {
213+
log.G(ctx).WithError(err).Errorf("failed to remove device")
214+
return err
215+
}
216+
217+
return nil
218+
}
219+
220+
// Walk iterates through all metadata Info for the stored snapshots and calls the provided function for each.
221+
func (s *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
222+
log.G(ctx).Debug("walk")
223+
return s.withTransaction(ctx, false, func(ctx context.Context) error {
224+
return storage.WalkInfo(ctx, fn)
225+
})
226+
}
227+
228+
// ResetPool deactivates and deletes all thin devices in thin-pool.
229+
// Used for cleaning pool after benchmarking.
230+
func (s *Snapshotter) ResetPool(ctx context.Context) error {
231+
names, err := s.pool.metadata.GetDeviceNames(ctx)
232+
if err != nil {
233+
return err
234+
}
235+
236+
var result *multierror.Error
237+
for _, name := range names {
238+
if err := s.pool.RemoveDevice(ctx, name); err != nil {
239+
result = multierror.Append(result, err)
240+
}
241+
}
242+
243+
return result.ErrorOrNil()
244+
}
245+
246+
// Close releases devmapper snapshotter resources.
247+
// All subsequent Close calls will be ignored.
248+
func (s *Snapshotter) Close() error {
249+
log.L.Debug("close")
250+
251+
var result *multierror.Error
252+
s.closeOnce.Do(func() {
253+
for _, fn := range s.cleanupFn {
254+
if err := fn(); err != nil {
255+
result = multierror.Append(result, err)
256+
}
257+
}
258+
})
259+
260+
return result.ErrorOrNil()
261+
}
262+
263+
func (s *Snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
264+
snap, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
265+
if err != nil {
266+
return nil, err
267+
}
268+
269+
if len(snap.ParentIDs) == 0 {
270+
deviceName := s.getDeviceName(snap.ID)
271+
log.G(ctx).Debugf("creating new thin device '%s'", deviceName)
272+
273+
err := s.pool.CreateThinDevice(ctx, deviceName, s.config.BaseImageSizeBytes)
274+
if err != nil {
275+
log.G(ctx).WithError(err).Errorf("failed to create thin device for snapshot %s", snap.ID)
276+
return nil, err
277+
}
278+
279+
if err := s.mkfs(ctx, deviceName); err != nil {
280+
return nil, err
281+
}
282+
} else {
283+
parentDeviceName := s.getDeviceName(snap.ParentIDs[0])
284+
snapDeviceName := s.getDeviceName(snap.ID)
285+
log.G(ctx).Debugf("creating snapshot device '%s' from '%s'", snapDeviceName, parentDeviceName)
286+
287+
err := s.pool.CreateSnapshotDevice(ctx, parentDeviceName, snapDeviceName, s.config.BaseImageSizeBytes)
288+
if err != nil {
289+
log.G(ctx).WithError(err).Errorf("failed to create snapshot device from parent %s", parentDeviceName)
290+
return nil, err
291+
}
292+
}
293+
294+
mounts := s.buildMounts(snap)
295+
296+
// Remove default directories not expected by the container image
297+
_ = mount.WithTempMount(ctx, mounts, func(root string) error {
298+
return os.Remove(filepath.Join(root, "lost+found"))
299+
})
300+
301+
return mounts, nil
302+
}
303+
304+
// mkfs creates ext4 filesystem on the given devmapper device
305+
func (s *Snapshotter) mkfs(ctx context.Context, deviceName string) error {
306+
args := []string{
307+
"-E",
308+
// We don't want any zeroing in advance when running mkfs on thin devices (see "man mkfs.ext4")
309+
"nodiscard,lazy_itable_init=0,lazy_journal_init=0",
310+
dmsetup.GetFullDevicePath(deviceName),
311+
}
312+
313+
log.G(ctx).Debugf("mkfs.ext4 %s", strings.Join(args, " "))
314+
output, err := exec.Command("mkfs.ext4", args...).CombinedOutput()
315+
if err != nil {
316+
log.G(ctx).WithError(err).Errorf("failed to write fs:\n%s", string(output))
317+
return err
318+
}
319+
320+
log.G(ctx).Debugf("mkfs:\n%s", string(output))
321+
return nil
322+
}
323+
324+
func (s *Snapshotter) getDeviceName(snapID string) string {
325+
// Add pool name as prefix to avoid collisions with devices from other pools
326+
return fmt.Sprintf("%s-snap-%s", s.config.PoolName, snapID)
327+
}
328+
329+
func (s *Snapshotter) getDevicePath(snap storage.Snapshot) string {
330+
name := s.getDeviceName(snap.ID)
331+
return dmsetup.GetFullDevicePath(name)
332+
}
333+
334+
func (s *Snapshotter) buildMounts(snap storage.Snapshot) []mount.Mount {
335+
var options []string
336+
337+
if snap.Kind != snapshots.KindActive {
338+
options = append(options, "ro")
339+
}
340+
341+
mounts := []mount.Mount{
342+
{
343+
Source: s.getDevicePath(snap),
344+
Type: fsTypeExt4,
345+
Options: options,
346+
},
347+
}
348+
349+
return mounts
350+
}
351+
352+
// withTransaction wraps fn callback with containerd's meta store transaction.
353+
// If callback returns an error or transaction is not writable, database transaction will be discarded.
354+
func (s *Snapshotter) withTransaction(ctx context.Context, writable bool, fn func(ctx context.Context) error) error {
355+
ctx, trans, err := s.store.TransactionContext(ctx, writable)
356+
if err != nil {
357+
return err
358+
}
359+
360+
var result *multierror.Error
361+
362+
err = fn(ctx)
363+
if err != nil {
364+
result = multierror.Append(result, err)
365+
}
366+
367+
// Always rollback if transaction is not writable
368+
if err != nil || !writable {
369+
if terr := trans.Rollback(); terr != nil {
370+
log.G(ctx).WithError(terr).Error("failed to rollback transaction")
371+
result = multierror.Append(result, errors.Wrap(terr, "rollback failed"))
372+
}
373+
} else {
374+
if terr := trans.Commit(); terr != nil {
375+
log.G(ctx).WithError(terr).Error("failed to commit transaction")
376+
result = multierror.Append(result, errors.Wrap(terr, "commit failed"))
377+
}
378+
}
379+
380+
if err := result.ErrorOrNil(); err != nil {
381+
log.G(ctx).WithError(err).Debug("snapshotter error")
382+
383+
// Unwrap if just one error
384+
if len(result.Errors) == 1 {
385+
return result.Errors[0]
386+
}
387+
388+
return err
389+
}
390+
391+
return nil
392+
}

0 commit comments

Comments
 (0)