Skip to content

Commit b97555e

Browse files
dmcgowandmcgowan
committed
Fix incorrect usage calculation
Set the size based on the correct calculation of blocks and the fixed size unit. Add tests which align the size with the filesystem block. Signed-off-by: Derek McGowan <[email protected]>
1 parent 91328d7 commit b97555e

4 files changed

Lines changed: 284 additions & 4 deletions

File tree

‎fs/du_test.go‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fs
18+
19+
import (
20+
"context"
21+
"io"
22+
"io/ioutil"
23+
"math/rand"
24+
"os"
25+
"path/filepath"
26+
"runtime"
27+
"testing"
28+
29+
"github.com/containerd/continuity/fs/fstest"
30+
"github.com/pkg/errors"
31+
)
32+
33+
func TestUsage(t *testing.T) {
34+
align, err := getTmpAlign()
35+
if err != nil {
36+
t.Fatal(err)
37+
}
38+
39+
type testCase struct {
40+
name string
41+
fs fstest.Applier
42+
size int64
43+
}
44+
testCases := []testCase{
45+
{
46+
name: "SingleSmallFile",
47+
fs: fstest.Apply(
48+
fstest.CreateDir("/dir", 0755),
49+
fstest.CreateRandomFile("/dir/file", 1, 5, 0644),
50+
),
51+
size: align(5),
52+
},
53+
{
54+
name: "MultipleSmallFile",
55+
fs: fstest.Apply(
56+
fstest.CreateDir("/dir", 0755),
57+
fstest.CreateRandomFile("/dir/file1", 2, 5, 0644),
58+
fstest.CreateRandomFile("/dir/file2", 3, 5, 0644),
59+
),
60+
size: align(5) * 2,
61+
},
62+
{
63+
name: "BiggerFiles",
64+
fs: fstest.Apply(
65+
fstest.CreateDir("/dir", 0755),
66+
fstest.CreateRandomFile("/dir/file1", 4, 5, 0644),
67+
fstest.CreateRandomFile("/dir/file2", 5, 1024, 0644),
68+
fstest.CreateRandomFile("/dir/file3", 6, 50*1024, 0644),
69+
),
70+
size: align(5) + align(1024) + align(50*1024),
71+
},
72+
}
73+
if runtime.GOOS != "windows" {
74+
testCases = append(testCases, []testCase{
75+
{
76+
name: "SparseFiles",
77+
fs: fstest.Apply(
78+
fstest.CreateDir("/dir", 0755),
79+
fstest.CreateRandomFile("/dir/file1", 7, 5, 0644),
80+
createSparseFile("/dir/sparse1", 8, 0644, 5, 1024*1024, 5),
81+
createSparseFile("/dir/sparse2", 9, 0644, 0, 1024*1024),
82+
createSparseFile("/dir/sparse2", 10, 0644, 0, 1024*1024*1024, 1024),
83+
),
84+
size: align(5)*3 + align(1024),
85+
},
86+
{
87+
name: "Hardlinks",
88+
fs: fstest.Apply(
89+
fstest.CreateDir("/dir", 0755),
90+
fstest.CreateRandomFile("/dir/file1", 11, 60*1024, 0644),
91+
fstest.Link("/dir/file1", "/dir/link1"),
92+
),
93+
size: align(60 * 1024),
94+
},
95+
{
96+
name: "HardlinkSparefile",
97+
fs: fstest.Apply(
98+
fstest.CreateDir("/dir", 0755),
99+
createSparseFile("/dir/file1", 10, 0644, 30*1024, 1024*1024*1024, 30*1024),
100+
fstest.Link("/dir/file1", "/dir/link1"),
101+
),
102+
size: align(30*1024) * 2,
103+
},
104+
}...)
105+
}
106+
107+
for i := range testCases {
108+
tc := testCases[i]
109+
t.Run(tc.name, func(t *testing.T) {
110+
t.Parallel()
111+
usage, err := fsUsage(tc.fs)
112+
if err != nil {
113+
t.Fatal(err)
114+
}
115+
if usage.Size != tc.size {
116+
t.Fatalf("Wrong usage size %d, expected %d", usage.Size, tc.size)
117+
}
118+
})
119+
}
120+
}
121+
122+
func fsUsage(fs fstest.Applier) (Usage, error) {
123+
t1, err := ioutil.TempDir("", "usage-")
124+
if err != nil {
125+
return Usage{}, errors.Wrap(err, "failed to create temp dir")
126+
}
127+
defer os.RemoveAll(t1)
128+
129+
if err := fs.Apply(t1); err != nil {
130+
return Usage{}, errors.Wrap(err, "failed to apply base filesystem")
131+
}
132+
133+
return DiskUsage(context.Background(), t1)
134+
}
135+
136+
// createSparseFile creates a sparse file filled with random
137+
// bytes for data parts
138+
// The parse alternate data length, hole length, data length, ....
139+
// To start a file as sparse, give an initial data length of 0
140+
func createSparseFile(name string, seed int64, perm os.FileMode, parts ...int64) fstest.Applier {
141+
return sparseFile{
142+
name: name,
143+
seed: seed,
144+
parts: parts,
145+
perm: perm,
146+
}
147+
}
148+
149+
type sparseFile struct {
150+
name string
151+
seed int64
152+
parts []int64
153+
perm os.FileMode
154+
}
155+
156+
func (sf sparseFile) Apply(root string) (retErr error) {
157+
fullPath := filepath.Join(root, sf.name)
158+
f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, sf.perm)
159+
if err != nil {
160+
return err
161+
}
162+
defer func() {
163+
err := f.Close()
164+
if err != nil && retErr == nil {
165+
retErr = err
166+
}
167+
}()
168+
169+
rr := rand.New(rand.NewSource(sf.seed))
170+
171+
parts := sf.parts
172+
for len(parts) > 0 {
173+
// Write content
174+
if parts[0] > 0 {
175+
_, err = io.Copy(f, io.LimitReader(rr, parts[0]))
176+
if err != nil {
177+
return err
178+
}
179+
}
180+
parts = parts[1:]
181+
182+
if len(parts) > 0 {
183+
if parts[0] != 0 {
184+
f.Seek(parts[0], io.SeekCurrent)
185+
}
186+
parts = parts[1:]
187+
}
188+
}
189+
return os.Chmod(fullPath, sf.perm)
190+
}

‎fs/du_unix.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ import (
2525
"syscall"
2626
)
2727

28+
// blocksUnitSize is the unit used by `st_blocks` in `stat` in bytes.
29+
// See https://man7.org/linux/man-pages/man2/stat.2.html
30+
// st_blocks
31+
// This field indicates the number of blocks allocated to the
32+
// file, in 512-byte units. (This may be smaller than
33+
// st_size/512 when the file has holes.)
34+
const blocksUnitSize = 512
35+
2836
type inode struct {
2937
// TODO(stevvooe): Can probably reduce memory usage by not tracking
3038
// device, but we can leave this right for now.
@@ -63,8 +71,7 @@ func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
6371
inoKey := newInode(stat)
6472
if _, ok := inodes[inoKey]; !ok {
6573
inodes[inoKey] = struct{}{}
66-
// on arm64 stat.Blksize is int32
67-
size += stat.Blocks * int64(stat.Blksize) // nolint: unconvert
74+
size += stat.Blocks * blocksUnitSize
6875
}
6976

7077
return nil
@@ -95,8 +102,7 @@ func diffUsage(ctx context.Context, a, b string) (Usage, error) {
95102
inoKey := newInode(stat)
96103
if _, ok := inodes[inoKey]; !ok {
97104
inodes[inoKey] = struct{}{}
98-
// on arm64 stat.Blksize is int32
99-
size += stat.Blocks * int64(stat.Blksize) // nolint: unconvert
105+
size += stat.Blocks * blocksUnitSize
100106
}
101107

102108
return nil

‎fs/du_unix_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// +build !windows
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package fs
20+
21+
import (
22+
"io/ioutil"
23+
"os"
24+
"syscall"
25+
26+
"github.com/pkg/errors"
27+
)
28+
29+
func getBsize(root string) (int64, error) {
30+
var s syscall.Statfs_t
31+
if err := syscall.Statfs(root, &s); err != nil {
32+
return 0, err
33+
}
34+
35+
return int64(s.Bsize), nil // nolint: unconvert
36+
}
37+
38+
func getTmpAlign() (func(int64) int64, error) {
39+
t1, err := ioutil.TempDir("", "compute-align-")
40+
if err != nil {
41+
return nil, errors.Wrap(err, "failed to create temp dir")
42+
}
43+
defer os.RemoveAll(t1)
44+
45+
bsize, err := getBsize(t1)
46+
if err != nil {
47+
return nil, errors.Wrap(err, "failed to get bsize")
48+
}
49+
50+
return func(size int64) int64 {
51+
// Align to blocks
52+
aligned := (size / bsize) * bsize
53+
54+
// Add next block if has remainder
55+
if size%bsize > 0 {
56+
aligned += bsize
57+
}
58+
59+
return aligned
60+
}, nil
61+
}

‎fs/du_windows_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fs
18+
19+
func getTmpAlign() (func(int64) int64, error) {
20+
return func(s int64) int64 {
21+
return s
22+
}, nil
23+
}

0 commit comments

Comments
 (0)