Skip to content

Commit 67b54c6

Browse files
author
Wei Fu
committed
Support >= 128 layers in overlayfs snapshots
Auto-detect longest common dir in lowerdir option and compact it if the option size is hitting one page size. If does, Use chdir + CLONE to do mount thing to avoid hitting one page argument buffer in linux kernel mount. Signed-off-by: Wei Fu <[email protected]>
1 parent 26e2dd6 commit 67b54c6

7 files changed

+673
-2
lines changed

‎mount/mount_linux.go

+153-2
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,41 @@
1717
package mount
1818

1919
import (
20+
"fmt"
21+
"os"
22+
"path"
2023
"strings"
2124
"time"
2225

26+
"github.com/containerd/containerd/sys"
2327
"github.com/pkg/errors"
2428
"golang.org/x/sys/unix"
2529
)
2630

31+
var pagesize = 4096
32+
33+
func init() {
34+
pagesize = os.Getpagesize()
35+
}
36+
2737
// Mount to the provided target path
2838
func (m *Mount) Mount(target string) error {
29-
flags, data := parseMountOptions(m.Options)
39+
var (
40+
chdir string
41+
options = m.Options
42+
)
43+
44+
// avoid hitting one page limit of mount argument buffer
45+
//
46+
// NOTE: 512 is a buffer during pagesize check.
47+
if m.Type == "overlay" && optionsSize(options) >= pagesize-512 {
48+
chdir, options = compactLowerdirOption(options)
49+
}
50+
51+
flags, data := parseMountOptions(options)
52+
if len(data) > pagesize {
53+
return errors.Errorf("mount options is too long")
54+
}
3055

3156
// propagation types.
3257
const ptypes = unix.MS_SHARED | unix.MS_PRIVATE | unix.MS_SLAVE | unix.MS_UNBINDABLE
@@ -38,7 +63,7 @@ func (m *Mount) Mount(target string) error {
3863
if flags&unix.MS_REMOUNT == 0 || data != "" {
3964
// Initial call applying all non-propagation flags for mount
4065
// or remount with changed data
41-
if err := unix.Mount(m.Source, target, m.Type, uintptr(oflags), data); err != nil {
66+
if err := mountAt(chdir, m.Source, target, m.Type, uintptr(oflags), data); err != nil {
4267
return err
4368
}
4469
}
@@ -155,3 +180,129 @@ func parseMountOptions(options []string) (int, string) {
155180
}
156181
return flag, strings.Join(data, ",")
157182
}
183+
184+
// compactLowerdirOption updates overlay lowdir option and returns the common
185+
// dir among all the lowdirs.
186+
func compactLowerdirOption(opts []string) (string, []string) {
187+
idx, dirs := findOverlayLowerdirs(opts)
188+
if idx == -1 || len(dirs) == 1 {
189+
// no need to compact if there is only one lowerdir
190+
return "", opts
191+
}
192+
193+
// find out common dir
194+
commondir := longestCommonPrefix(dirs)
195+
if commondir == "" {
196+
return "", opts
197+
}
198+
199+
// NOTE: the snapshot id is based on digits.
200+
// in order to avoid to get snapshots/x, should be back to parent dir.
201+
// however, there is assumption that the common dir is ${root}/io.containerd.v1.overlayfs/snapshots.
202+
commondir = path.Dir(commondir)
203+
if commondir == "/" {
204+
return "", opts
205+
}
206+
commondir = commondir + "/"
207+
208+
newdirs := make([]string, 0, len(dirs))
209+
for _, dir := range dirs {
210+
newdirs = append(newdirs, dir[len(commondir):])
211+
}
212+
213+
newopts := copyOptions(opts)
214+
newopts = append(newopts[:idx], newopts[idx+1:]...)
215+
newopts = append(newopts, fmt.Sprintf("lowerdir=%s", strings.Join(newdirs, ":")))
216+
return commondir, newopts
217+
}
218+
219+
// findOverlayLowerdirs returns the index of lowerdir in mount's options and
220+
// all the lowerdir target.
221+
func findOverlayLowerdirs(opts []string) (int, []string) {
222+
var (
223+
idx = -1
224+
prefix = "lowerdir="
225+
)
226+
227+
for i, opt := range opts {
228+
if strings.HasPrefix(opt, prefix) {
229+
idx = i
230+
break
231+
}
232+
}
233+
234+
if idx == -1 {
235+
return -1, nil
236+
}
237+
return idx, strings.Split(opts[idx][len(prefix):], ":")
238+
}
239+
240+
// longestCommonPrefix finds the longest common prefix in the string slice.
241+
func longestCommonPrefix(strs []string) string {
242+
if len(strs) == 0 {
243+
return ""
244+
} else if len(strs) == 1 {
245+
return strs[0]
246+
}
247+
248+
// find out the min/max value by alphabetical order
249+
min, max := strs[0], strs[0]
250+
for _, str := range strs[1:] {
251+
if min > str {
252+
min = str
253+
}
254+
if max < str {
255+
max = str
256+
}
257+
}
258+
259+
// find out the common part between min and max
260+
for i := 0; i < len(min) && i < len(max); i++ {
261+
if min[i] != max[i] {
262+
return min[:i]
263+
}
264+
}
265+
return min
266+
}
267+
268+
// copyOptions copies the options.
269+
func copyOptions(opts []string) []string {
270+
if len(opts) == 0 {
271+
return nil
272+
}
273+
274+
acopy := make([]string, len(opts))
275+
copy(acopy, opts)
276+
return acopy
277+
}
278+
279+
// optionsSize returns the byte size of options of mount.
280+
func optionsSize(opts []string) int {
281+
size := 0
282+
for _, opt := range opts {
283+
size += len(opt)
284+
}
285+
return size
286+
}
287+
288+
func mountAt(chdir string, source, target, fstype string, flags uintptr, data string) error {
289+
if chdir == "" {
290+
return unix.Mount(source, target, fstype, flags, data)
291+
}
292+
293+
f, err := os.Open(chdir)
294+
if err != nil {
295+
return errors.Wrap(err, "failed to mountat")
296+
}
297+
defer f.Close()
298+
299+
fs, err := f.Stat()
300+
if err != nil {
301+
return errors.Wrap(err, "failed to mountat")
302+
}
303+
304+
if !fs.IsDir() {
305+
return errors.Wrap(errors.Errorf("%s is not dir", chdir), "failed to mountat")
306+
}
307+
return errors.Wrap(sys.FMountat(f.Fd(), source, target, fstype, flags, data), "failed to mountat")
308+
}

‎mount/mount_linux_test.go

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// +build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package mount
20+
21+
import (
22+
"reflect"
23+
"testing"
24+
)
25+
26+
func TestLongestCommonPrefix(t *testing.T) {
27+
tcases := []struct {
28+
in []string
29+
expected string
30+
}{
31+
{[]string{}, ""},
32+
{[]string{"foo"}, "foo"},
33+
{[]string{"foo", "bar"}, ""},
34+
{[]string{"foo", "foo"}, "foo"},
35+
{[]string{"foo", "foobar"}, "foo"},
36+
{[]string{"foo", "", "foobar"}, ""},
37+
}
38+
39+
for i, tc := range tcases {
40+
if got := longestCommonPrefix(tc.in); got != tc.expected {
41+
t.Fatalf("[%d case] expected (%s), but got (%s)", i+1, tc.expected, got)
42+
}
43+
}
44+
}
45+
46+
func TestCompactLowerdirOption(t *testing.T) {
47+
tcases := []struct {
48+
opts []string
49+
commondir string
50+
newopts []string
51+
}{
52+
// no lowerdir or only one
53+
{
54+
[]string{"workdir=a"},
55+
"",
56+
[]string{"workdir=a"},
57+
},
58+
{
59+
[]string{"workdir=a", "lowerdir=b"},
60+
"",
61+
[]string{"workdir=a", "lowerdir=b"},
62+
},
63+
64+
// >= 2 lowerdir
65+
{
66+
[]string{"lowerdir=/snapshots/1/fs:/snapshots/10/fs"},
67+
"/snapshots/",
68+
[]string{"lowerdir=1/fs:10/fs"},
69+
},
70+
{
71+
[]string{"lowerdir=/snapshots/1/fs:/snapshots/10/fs:/snapshots/2/fs"},
72+
"/snapshots/",
73+
[]string{"lowerdir=1/fs:10/fs:2/fs"},
74+
},
75+
76+
// if common dir is /
77+
{
78+
[]string{"lowerdir=/snapshots/1/fs:/other_snapshots/1/fs"},
79+
"",
80+
[]string{"lowerdir=/snapshots/1/fs:/other_snapshots/1/fs"},
81+
},
82+
}
83+
84+
for i, tc := range tcases {
85+
dir, opts := compactLowerdirOption(tc.opts)
86+
if dir != tc.commondir {
87+
t.Fatalf("[%d case] expected common dir (%s), but got (%s)", i+1, tc.commondir, dir)
88+
}
89+
90+
if !reflect.DeepEqual(opts, tc.newopts) {
91+
t.Fatalf("[%d case] expected options (%v), but got (%v)", i+1, tc.newopts, opts)
92+
}
93+
}
94+
}

‎snapshots/testsuite/testsuite.go

+93
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.
6363
t.Run("StatInWalk", makeTest(name, snapshotterFn, checkStatInWalk))
6464
t.Run("CloseTwice", makeTest(name, snapshotterFn, closeTwice))
6565
t.Run("RootPermission", makeTest(name, snapshotterFn, checkRootPermission))
66+
67+
t.Run("128LayersMount", makeTest(name, snapshotterFn, check128LayersMount))
6668
}
6769

6870
func makeTest(name string, snapshotterFn func(ctx context.Context, root string) (snapshots.Snapshotter, func() error, error), fn func(ctx context.Context, t *testing.T, snapshotter snapshots.Snapshotter, work string)) func(t *testing.T) {
@@ -860,3 +862,94 @@ func checkRootPermission(ctx context.Context, t *testing.T, snapshotter snapshot
860862
t.Fatalf("expected 0755, got 0%o", mode)
861863
}
862864
}
865+
866+
func check128LayersMount(ctx context.Context, t *testing.T, snapshotter snapshots.Snapshotter, work string) {
867+
lowestApply := fstest.Apply(
868+
fstest.CreateFile("/bottom", []byte("way at the bottom\n"), 0777),
869+
fstest.CreateFile("/overwriteme", []byte("FIRST!\n"), 0777),
870+
fstest.CreateDir("/ADDHERE", 0755),
871+
fstest.CreateDir("/ONLYME", 0755),
872+
fstest.CreateFile("/ONLYME/bottom", []byte("bye!\n"), 0777),
873+
)
874+
875+
appliers := []fstest.Applier{lowestApply}
876+
for i := 1; i <= 127; i++ {
877+
appliers = append(appliers, fstest.Apply(
878+
fstest.CreateFile("/overwriteme", []byte(fmt.Sprintf("%d WAS HERE!\n", i)), 0777),
879+
fstest.CreateFile(fmt.Sprintf("/ADDHERE/file-%d", i), []byte("same\n"), 0755),
880+
fstest.RemoveAll("/ONLYME"),
881+
fstest.CreateDir("/ONLYME", 0755),
882+
fstest.CreateFile(fmt.Sprintf("/ONLYME/file-%d", i), []byte("only me!\n"), 0777),
883+
))
884+
}
885+
886+
flat := filepath.Join(work, "flat")
887+
if err := os.MkdirAll(flat, 0777); err != nil {
888+
t.Fatalf("failed to create flat dir(%s): %+v", flat, err)
889+
}
890+
891+
// NOTE: add gc labels to avoid snapshots get removed by gc...
892+
parent := ""
893+
for i, applier := range appliers {
894+
preparing := filepath.Join(work, fmt.Sprintf("prepare-layer-%d", i))
895+
if err := os.MkdirAll(preparing, 0777); err != nil {
896+
t.Fatalf("[layer %d] failed to create preparing dir(%s): %+v", i, preparing, err)
897+
}
898+
899+
mounts, err := snapshotter.Prepare(ctx, preparing, parent, opt)
900+
if err != nil {
901+
t.Fatalf("[layer %d] failed to get mount info: %+v", i, err)
902+
}
903+
904+
if err := mount.All(mounts, preparing); err != nil {
905+
t.Fatalf("[layer %d] failed to mount on the target(%s): %+v", i, preparing, err)
906+
}
907+
908+
if err := fstest.CheckDirectoryEqual(preparing, flat); err != nil {
909+
testutil.Unmount(t, preparing)
910+
t.Fatalf("[layer %d] preparing doesn't equal to flat before apply: %+v", i, err)
911+
}
912+
913+
if err := applier.Apply(flat); err != nil {
914+
testutil.Unmount(t, preparing)
915+
t.Fatalf("[layer %d] failed to apply on flat dir: %+v", i, err)
916+
}
917+
918+
if err = applier.Apply(preparing); err != nil {
919+
testutil.Unmount(t, preparing)
920+
t.Fatalf("[layer %d] failed to apply on preparing dir: %+v", i, err)
921+
}
922+
923+
if err := fstest.CheckDirectoryEqual(preparing, flat); err != nil {
924+
testutil.Unmount(t, preparing)
925+
t.Fatalf("[layer %d] preparing doesn't equal to flat after apply: %+v", i, err)
926+
}
927+
928+
testutil.Unmount(t, preparing)
929+
930+
parent = filepath.Join(work, fmt.Sprintf("committed-%d", i))
931+
if err := snapshotter.Commit(ctx, parent, preparing, opt); err != nil {
932+
t.Fatalf("[layer %d] failed to commit the preparing: %+v", i, err)
933+
}
934+
935+
}
936+
937+
view := filepath.Join(work, "fullview")
938+
if err := os.MkdirAll(view, 0777); err != nil {
939+
t.Fatalf("failed to create fullview dir(%s): %+v", view, err)
940+
}
941+
942+
mounts, err := snapshotter.View(ctx, view, parent, opt)
943+
if err != nil {
944+
t.Fatalf("failed to get view's mount info: %+v", err)
945+
}
946+
947+
if err := mount.All(mounts, view); err != nil {
948+
t.Fatalf("failed to mount on the target(%s): %+v", view, err)
949+
}
950+
defer testutil.Unmount(t, view)
951+
952+
if err := fstest.CheckDirectoryEqual(view, flat); err != nil {
953+
t.Fatalf("fullview should equal to flat: %+v", err)
954+
}
955+
}

0 commit comments

Comments
 (0)