Skip to content

Commit 1d9893e

Browse files
estespestesp
authored
Merge pull request #169 from dmcgowan/fix-usage-block-size
Fix incorrect usage calculation
2 parents 91328d7 + 363153d commit 1d9893e

4 files changed

Lines changed: 301 additions & 4 deletions

File tree

‎fs/du_test.go‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fs
18+
19+
import (
20+
"context"
21+
"io"
22+
"io/ioutil"
23+
"math/rand"
24+
"os"
25+
"path/filepath"
26+
"runtime"
27+
"testing"
28+
29+
"github.com/containerd/continuity/fs/fstest"
30+
"github.com/pkg/errors"
31+
)
32+
33+
func TestUsage(t *testing.T) {
34+
align, dirs, err := getTmpAlign()
35+
if err != nil {
36+
t.Fatal(err)
37+
}
38+
39+
type testCase struct {
40+
name string
41+
fs fstest.Applier
42+
size int64
43+
}
44+
testCases := []testCase{
45+
{
46+
name: "SingleSmallFile",
47+
fs: fstest.Apply(
48+
fstest.CreateDir("/dir", 0755),
49+
fstest.CreateRandomFile("/dir/file", 1, 5, 0644),
50+
),
51+
size: dirs(2) + align(5),
52+
},
53+
{
54+
name: "MultipleSmallFile",
55+
fs: fstest.Apply(
56+
fstest.CreateDir("/dir", 0755),
57+
fstest.CreateRandomFile("/dir/file1", 2, 5, 0644),
58+
fstest.CreateRandomFile("/dir/file2", 3, 5, 0644),
59+
),
60+
size: dirs(2) + align(5)*2,
61+
},
62+
{
63+
name: "BiggerFiles",
64+
fs: fstest.Apply(
65+
fstest.CreateDir("/dir", 0755),
66+
fstest.CreateRandomFile("/dir/file1", 4, 5, 0644),
67+
fstest.CreateRandomFile("/dir/file2", 5, 1024, 0644),
68+
fstest.CreateRandomFile("/dir/file3", 6, 50*1024, 0644),
69+
),
70+
size: dirs(2) + align(5) + align(1024) + align(50*1024),
71+
},
72+
}
73+
if runtime.GOOS != "windows" {
74+
testCases = append(testCases, []testCase{
75+
{
76+
name: "SparseFiles",
77+
fs: fstest.Apply(
78+
fstest.CreateDir("/dir", 0755),
79+
fstest.CreateRandomFile("/dir/file1", 7, 5, 0644),
80+
createSparseFile("/dir/sparse1", 8, 0644, 5, 1024*1024, 5),
81+
createSparseFile("/dir/sparse2", 9, 0644, 0, 1024*1024),
82+
createSparseFile("/dir/sparse2", 10, 0644, 0, 1024*1024*1024, 1024),
83+
),
84+
size: dirs(2) + align(5)*3 + align(1024),
85+
},
86+
{
87+
name: "Hardlinks",
88+
fs: fstest.Apply(
89+
fstest.CreateDir("/dir", 0755),
90+
fstest.CreateRandomFile("/dir/file1", 11, 60*1024, 0644),
91+
fstest.Link("/dir/file1", "/dir/link1"),
92+
),
93+
size: dirs(2) + align(60*1024),
94+
},
95+
{
96+
name: "HardlinkSparefile",
97+
fs: fstest.Apply(
98+
fstest.CreateDir("/dir", 0755),
99+
createSparseFile("/dir/file1", 10, 0644, 30*1024, 1024*1024*1024, 30*1024),
100+
fstest.Link("/dir/file1", "/dir/link1"),
101+
),
102+
size: dirs(2) + align(30*1024)*2,
103+
},
104+
}...)
105+
}
106+
107+
for i := range testCases {
108+
tc := testCases[i]
109+
t.Run(tc.name, func(t *testing.T) {
110+
t.Parallel()
111+
usage, err := fsUsage(tc.fs)
112+
if err != nil {
113+
t.Fatal(err)
114+
}
115+
if usage.Size != tc.size {
116+
t.Fatalf("Wrong usage size %d, expected %d", usage.Size, tc.size)
117+
}
118+
})
119+
}
120+
}
121+
122+
func fsUsage(fs fstest.Applier) (Usage, error) {
123+
t1, err := ioutil.TempDir("", "usage-")
124+
if err != nil {
125+
return Usage{}, errors.Wrap(err, "failed to create temp dir")
126+
}
127+
defer os.RemoveAll(t1)
128+
129+
if err := fs.Apply(t1); err != nil {
130+
return Usage{}, errors.Wrap(err, "failed to apply base filesystem")
131+
}
132+
133+
return DiskUsage(context.Background(), t1)
134+
}
135+
136+
// createSparseFile creates a sparse file filled with random
137+
// bytes for data parts
138+
// The parse alternate data length, hole length, data length, ....
139+
// To start a file as sparse, give an initial data length of 0
140+
func createSparseFile(name string, seed int64, perm os.FileMode, parts ...int64) fstest.Applier {
141+
return sparseFile{
142+
name: name,
143+
seed: seed,
144+
parts: parts,
145+
perm: perm,
146+
}
147+
}
148+
149+
type sparseFile struct {
150+
name string
151+
seed int64
152+
parts []int64
153+
perm os.FileMode
154+
}
155+
156+
func (sf sparseFile) Apply(root string) (retErr error) {
157+
fullPath := filepath.Join(root, sf.name)
158+
f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, sf.perm)
159+
if err != nil {
160+
return err
161+
}
162+
defer func() {
163+
err := f.Close()
164+
if err != nil && retErr == nil {
165+
retErr = err
166+
}
167+
}()
168+
169+
rr := rand.New(rand.NewSource(sf.seed))
170+
171+
parts := sf.parts
172+
for len(parts) > 0 {
173+
// Write content
174+
if parts[0] > 0 {
175+
_, err = io.Copy(f, io.LimitReader(rr, parts[0]))
176+
if err != nil {
177+
return err
178+
}
179+
}
180+
parts = parts[1:]
181+
182+
if len(parts) > 0 {
183+
if parts[0] != 0 {
184+
f.Seek(parts[0], io.SeekCurrent)
185+
}
186+
parts = parts[1:]
187+
}
188+
}
189+
return os.Chmod(fullPath, sf.perm)
190+
}

‎fs/du_unix.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ import (
2525
"syscall"
2626
)
2727

28+
// blocksUnitSize is the unit used by `st_blocks` in `stat` in bytes.
29+
// See https://man7.org/linux/man-pages/man2/stat.2.html
30+
// st_blocks
31+
// This field indicates the number of blocks allocated to the
32+
// file, in 512-byte units. (This may be smaller than
33+
// st_size/512 when the file has holes.)
34+
const blocksUnitSize = 512
35+
2836
type inode struct {
2937
// TODO(stevvooe): Can probably reduce memory usage by not tracking
3038
// device, but we can leave this right for now.
@@ -63,8 +71,7 @@ func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
6371
inoKey := newInode(stat)
6472
if _, ok := inodes[inoKey]; !ok {
6573
inodes[inoKey] = struct{}{}
66-
// on arm64 stat.Blksize is int32
67-
size += stat.Blocks * int64(stat.Blksize) // nolint: unconvert
74+
size += stat.Blocks * blocksUnitSize
6875
}
6976

7077
return nil
@@ -95,8 +102,7 @@ func diffUsage(ctx context.Context, a, b string) (Usage, error) {
95102
inoKey := newInode(stat)
96103
if _, ok := inodes[inoKey]; !ok {
97104
inodes[inoKey] = struct{}{}
98-
// on arm64 stat.Blksize is int32
99-
size += stat.Blocks * int64(stat.Blksize) // nolint: unconvert
105+
size += stat.Blocks * blocksUnitSize
100106
}
101107

102108
return nil

‎fs/du_unix_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// +build !windows
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package fs
20+
21+
import (
22+
"io/ioutil"
23+
"os"
24+
"syscall"
25+
26+
"github.com/pkg/errors"
27+
)
28+
29+
func getBsize(root string) (int64, error) {
30+
var s syscall.Statfs_t
31+
if err := syscall.Statfs(root, &s); err != nil {
32+
return 0, err
33+
}
34+
35+
return int64(s.Bsize), nil // nolint: unconvert
36+
}
37+
38+
// getTmpAlign returns filesystem specific size alignment functions
39+
// first: aligns filesize to file usage based on blocks
40+
// second: determines directory usage based on directory count (assumes small directories)
41+
func getTmpAlign() (func(int64) int64, func(int64) int64, error) {
42+
t1, err := ioutil.TempDir("", "compute-align-")
43+
if err != nil {
44+
return nil, nil, errors.Wrap(err, "failed to create temp dir")
45+
}
46+
defer os.RemoveAll(t1)
47+
48+
bsize, err := getBsize(t1)
49+
if err != nil {
50+
return nil, nil, errors.Wrap(err, "failed to get bsize")
51+
}
52+
53+
align := func(size int64) int64 {
54+
// Align to blocks
55+
aligned := (size / bsize) * bsize
56+
57+
// Add next block if has remainder
58+
if size%bsize > 0 {
59+
aligned += bsize
60+
}
61+
62+
return aligned
63+
}
64+
65+
fi, err := os.Stat(t1)
66+
if err != nil {
67+
return nil, nil, errors.Wrap(err, "failed to stat directory")
68+
}
69+
70+
dirSize := fi.Sys().(*syscall.Stat_t).Blocks * blocksUnitSize
71+
dirs := func(count int64) int64 {
72+
return count * dirSize
73+
}
74+
75+
return align, dirs, nil
76+
}

‎fs/du_windows_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fs
18+
19+
func getTmpAlign() (func(int64) int64, func(int64) int64, error) {
20+
return func(s int64) int64 {
21+
return s
22+
}, func(c int64) int64 {
23+
return c * 4096
24+
}, nil
25+
}

0 commit comments

Comments
 (0)