Skip to content

Commit ef10c9d

Browse files
committed
feat: parallel cuts with private views of chisel's cache
cuts of one run used to go one after another because two chisels that miss the same blob at once write to the same <digest>.tmp and corrupt each other (canonical/chisel#340). a worker now runs chisel with XDG_CACHE_HOME pointed at a private view of the cache: a directory of hard links to every shared blob, seeded once per worker, with the blobs each cut adds linked back into the shared cache afterwards. a lone worker uses the shared cache directly. the scheduler is one unit per (package, arch) pulled by a pool of -j workers, the lead arch cut alone first and the rest after, so the slice list and the arch: all blobs are warm before the fan-out. -j defaults to -1, one worker per cpu; 0 and anything below -1 are usage errors. a package named twice is cut once, since two workers on one rootfs dir raced each other. the cli tests now point XDG_CACHE_HOME at a temp dir, as Main seeded views of the real cache before.
1 parent 9f56b6b commit ef10c9d

7 files changed

Lines changed: 610 additions & 32 deletions

File tree

‎internal/cache/cache.go‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
// Package cache gives each concurrently running chisel a private view of
2+
// chisel's blob cache. chisel writes a missed blob to a fixed <digest>.tmp
3+
// name with no locking, so two chisels missing the same blob at once corrupt
4+
// each other (canonical/chisel#340). A view is a directory seeded with hard
5+
// links to every blob of the shared cache; chisel runs with XDG_CACHE_HOME
6+
// pointed at it, writes its misses there, and the new blobs are linked back
7+
// into the shared cache afterwards. Blobs are content-addressed and chisel
8+
// only ever renames a complete one into place, so sharing inodes is safe.
9+
package cache
10+
11+
import (
12+
"errors"
13+
"io"
14+
"io/fs"
15+
"os"
16+
"path/filepath"
17+
"regexp"
18+
"sync"
19+
"syscall"
20+
"time"
21+
)
22+
23+
// Dir is where the chisel binary keeps its cache, by chisel's own rule:
24+
// $XDG_CACHE_HOME/chisel, else $HOME/.cache/chisel. It is "" when neither
25+
// variable is set; chisel then uses a fresh temp dir per run and there is
26+
// nothing to share.
27+
func Dir() string {
28+
if d := os.Getenv("XDG_CACHE_HOME"); d != "" {
29+
return filepath.Join(d, "chisel")
30+
}
31+
if h := os.Getenv("HOME"); h != "" {
32+
return filepath.Join(h, ".cache", "chisel")
33+
}
34+
return ""
35+
}
36+
37+
// View is a private copy of the shared cache for one chisel at a time.
38+
type View struct {
39+
Home string // the XDG_CACHE_HOME to run chisel with
40+
shared string
41+
dir string // Home/chisel
42+
copy bool // shared is on another filesystem: no links, merge by copying
43+
known map[string]bool // <kind>/<name> seeded or merged already
44+
}
45+
46+
// Linking tens of thousands of blobs is syscall-bound; a few goroutines
47+
// roughly halve the wall time on macOS and linux, more gain nothing.
48+
const linkers = 4
49+
50+
// A blob's name is its digest in hex; anything else in a digest directory is
51+
// a temp file or a lock.
52+
var blob = regexp.MustCompile(`^[0-9a-f]+$`)
53+
54+
// New makes a view under <parent of shared>/chisel-slice-coverage, so on the
55+
// same filesystem, seeded with every blob of shared. Views a crashed run left
56+
// behind for more than a day are removed on the way.
57+
func New(shared string) (*View, error) {
58+
base := filepath.Join(filepath.Dir(shared), "chisel-slice-coverage")
59+
if err := os.MkdirAll(base, 0755); err != nil {
60+
return nil, err
61+
}
62+
sweep(base)
63+
home, err := os.MkdirTemp(base, "view-*")
64+
if err != nil {
65+
return nil, err
66+
}
67+
v := &View{Home: home, shared: shared, dir: filepath.Join(home, "chisel"), known: map[string]bool{}}
68+
if err := v.seed(); err != nil {
69+
_ = os.RemoveAll(home)
70+
return nil, err
71+
}
72+
return v, nil
73+
}
74+
75+
// seed hard-links every blob of the shared cache into the view. On another
76+
// filesystem the first link fails and the view starts empty.
77+
func (v *View) seed() error {
78+
var names []string
79+
kinds := map[string]bool{}
80+
err := eachBlob(v.shared, func(kind, name string) error {
81+
if !kinds[kind] {
82+
kinds[kind] = true
83+
if err := os.MkdirAll(filepath.Join(v.dir, kind), 0755); err != nil {
84+
return err
85+
}
86+
}
87+
names = append(names, filepath.Join(kind, name))
88+
return nil
89+
})
90+
if err != nil {
91+
return err
92+
}
93+
if len(names) > 0 {
94+
err := os.Link(filepath.Join(v.shared, names[0]), filepath.Join(v.dir, names[0]))
95+
if errors.Is(err, syscall.EXDEV) {
96+
v.copy = true
97+
return nil
98+
}
99+
if err != nil {
100+
return err
101+
}
102+
names = names[1:]
103+
}
104+
var wg sync.WaitGroup
105+
var mu sync.Mutex
106+
var first error
107+
next := make(chan string)
108+
for i := 0; i < linkers; i++ {
109+
wg.Add(1)
110+
go func() {
111+
defer wg.Done()
112+
for name := range next {
113+
if err := os.Link(filepath.Join(v.shared, name), filepath.Join(v.dir, name)); err != nil {
114+
mu.Lock()
115+
if first == nil {
116+
first = err
117+
}
118+
mu.Unlock()
119+
}
120+
}
121+
}()
122+
}
123+
for _, name := range names {
124+
next <- name
125+
}
126+
close(next)
127+
wg.Wait()
128+
for _, name := range names {
129+
v.known[name] = true
130+
}
131+
return first
132+
}
133+
134+
// Merge publishes every blob the view gained since it was seeded or last
135+
// merged into the shared cache. A blob already there is left alone, which is
136+
// also how two views merging the same blob at once settle it.
137+
func (v *View) Merge() error {
138+
return eachBlob(v.dir, func(kind, name string) error {
139+
if v.known[filepath.Join(kind, name)] {
140+
return nil
141+
}
142+
v.known[filepath.Join(kind, name)] = true
143+
dst := filepath.Join(v.shared, kind, name)
144+
if _, err := os.Lstat(dst); err == nil {
145+
return nil
146+
}
147+
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
148+
return err
149+
}
150+
src := filepath.Join(v.dir, kind, name)
151+
if v.copy {
152+
return copyFile(src, dst)
153+
}
154+
err := os.Link(src, dst)
155+
if errors.Is(err, fs.ErrExist) {
156+
return nil
157+
}
158+
return err
159+
})
160+
}
161+
162+
// Remove deletes the view. Blobs not merged are lost.
163+
func (v *View) Remove() error {
164+
return os.RemoveAll(v.Home)
165+
}
166+
167+
// eachBlob calls fn for every blob under root, as <digest kind dir>/<name>.
168+
// A missing root is an empty cache.
169+
func eachBlob(root string, fn func(kind, name string) error) error {
170+
kinds, err := os.ReadDir(root)
171+
if errors.Is(err, fs.ErrNotExist) {
172+
return nil
173+
}
174+
if err != nil {
175+
return err
176+
}
177+
for _, k := range kinds {
178+
if !k.IsDir() {
179+
continue
180+
}
181+
entries, err := os.ReadDir(filepath.Join(root, k.Name()))
182+
if err != nil {
183+
return err
184+
}
185+
for _, e := range entries {
186+
if !e.Type().IsRegular() || !blob.MatchString(e.Name()) {
187+
continue
188+
}
189+
if err := fn(k.Name(), e.Name()); err != nil {
190+
return err
191+
}
192+
}
193+
}
194+
return nil
195+
}
196+
197+
// copyFile writes src to dst through a temp file in dst's directory, so dst
198+
// is never seen incomplete.
199+
func copyFile(src, dst string) error {
200+
in, err := os.Open(src)
201+
if err != nil {
202+
return err
203+
}
204+
defer in.Close()
205+
tmp, err := os.CreateTemp(filepath.Dir(dst), filepath.Base(dst)+".view.*")
206+
if err != nil {
207+
return err
208+
}
209+
if _, err := io.Copy(tmp, in); err != nil {
210+
_ = tmp.Close()
211+
_ = os.Remove(tmp.Name())
212+
return err
213+
}
214+
if err := tmp.Close(); err != nil {
215+
_ = os.Remove(tmp.Name())
216+
return err
217+
}
218+
return os.Rename(tmp.Name(), dst)
219+
}
220+
221+
// sweep removes views under base that nothing has touched for a day.
222+
func sweep(base string) {
223+
entries, err := os.ReadDir(base)
224+
if err != nil {
225+
return
226+
}
227+
stale := time.Now().Add(-24 * time.Hour)
228+
for _, e := range entries {
229+
info, err := e.Info()
230+
if err != nil || !e.IsDir() || !info.ModTime().Before(stale) {
231+
continue
232+
}
233+
if ok, _ := filepath.Match("view-*", e.Name()); ok {
234+
_ = os.RemoveAll(filepath.Join(base, e.Name()))
235+
}
236+
}
237+
}

‎internal/cache/cache_test.go‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package cache
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
"time"
8+
9+
"github.com/lczyk/assert"
10+
)
11+
12+
func write(t *testing.T, p, content string) {
13+
t.Helper()
14+
assert.NoError(t, os.MkdirAll(filepath.Dir(p), 0755))
15+
assert.NoError(t, os.WriteFile(p, []byte(content), 0644))
16+
}
17+
18+
func sameFile(t *testing.T, a, b string) bool {
19+
t.Helper()
20+
fa, err := os.Stat(a)
21+
assert.NoError(t, err)
22+
fb, err := os.Stat(b)
23+
assert.NoError(t, err)
24+
return os.SameFile(fa, fb)
25+
}
26+
27+
func TestDir(t *testing.T) {
28+
t.Setenv("XDG_CACHE_HOME", "/x")
29+
t.Setenv("HOME", "/h")
30+
assert.Equal(t, Dir(), "/x/chisel")
31+
t.Setenv("XDG_CACHE_HOME", "")
32+
assert.Equal(t, Dir(), "/h/.cache/chisel")
33+
t.Setenv("HOME", "")
34+
assert.Equal(t, Dir(), "")
35+
}
36+
37+
func TestViewSeedsMergesAndRemoves(t *testing.T) {
38+
shared := filepath.Join(t.TempDir(), "chisel")
39+
write(t, filepath.Join(shared, "sha256", "aa"), "a")
40+
write(t, filepath.Join(shared, "sha512", "bb"), "b")
41+
write(t, filepath.Join(shared, "sha256", "cc.tmp"), "half")
42+
write(t, filepath.Join(shared, "releases", ".lock"), "")
43+
44+
v, err := New(shared)
45+
assert.NoError(t, err)
46+
assert.Equal(t, filepath.Dir(filepath.Dir(v.Home)), filepath.Dir(shared))
47+
view := filepath.Join(v.Home, "chisel")
48+
assert.That(t, sameFile(t, filepath.Join(shared, "sha256", "aa"), filepath.Join(view, "sha256", "aa")))
49+
assert.That(t, sameFile(t, filepath.Join(shared, "sha512", "bb"), filepath.Join(view, "sha512", "bb")))
50+
_, err = os.Stat(filepath.Join(view, "sha256", "cc.tmp"))
51+
assert.That(t, os.IsNotExist(err))
52+
_, err = os.Stat(filepath.Join(view, "releases"))
53+
assert.That(t, os.IsNotExist(err))
54+
55+
// chisel fetched dd into the view, left a temp file, and someone else
56+
// published ee in the shared cache meanwhile.
57+
write(t, filepath.Join(view, "sha256", "dd"), "d")
58+
write(t, filepath.Join(view, "sha256", "ff.tmp"), "half")
59+
write(t, filepath.Join(shared, "sha256", "ee"), "e")
60+
assert.NoError(t, v.Merge())
61+
assert.That(t, sameFile(t, filepath.Join(shared, "sha256", "dd"), filepath.Join(view, "sha256", "dd")))
62+
_, err = os.Stat(filepath.Join(shared, "sha256", "ff.tmp"))
63+
assert.That(t, os.IsNotExist(err))
64+
// Merging again is a no-op, and a blob both sides have is left alone.
65+
write(t, filepath.Join(view, "sha256", "ee"), "e2")
66+
assert.NoError(t, v.Merge())
67+
b, err := os.ReadFile(filepath.Join(shared, "sha256", "ee"))
68+
assert.NoError(t, err)
69+
assert.Equal(t, string(b), "e")
70+
71+
assert.NoError(t, v.Remove())
72+
_, err = os.Stat(v.Home)
73+
assert.That(t, os.IsNotExist(err))
74+
b, err = os.ReadFile(filepath.Join(shared, "sha256", "dd"))
75+
assert.NoError(t, err)
76+
assert.Equal(t, string(b), "d")
77+
}
78+
79+
func TestViewOfEmptyCache(t *testing.T) {
80+
shared := filepath.Join(t.TempDir(), "chisel")
81+
v, err := New(shared)
82+
assert.NoError(t, err)
83+
write(t, filepath.Join(v.Home, "chisel", "sha256", "aa"), "a")
84+
assert.NoError(t, v.Merge())
85+
_, err = os.Stat(filepath.Join(shared, "sha256", "aa"))
86+
assert.NoError(t, err)
87+
assert.NoError(t, v.Remove())
88+
}
89+
90+
func TestNewSweepsStaleViews(t *testing.T) {
91+
shared := filepath.Join(t.TempDir(), "chisel")
92+
base := filepath.Join(filepath.Dir(shared), "chisel-slice-coverage")
93+
stale := filepath.Join(base, "view-stale")
94+
fresh := filepath.Join(base, "view-fresh")
95+
other := filepath.Join(base, "other")
96+
for _, d := range []string{stale, fresh, other} {
97+
assert.NoError(t, os.MkdirAll(d, 0755))
98+
}
99+
old := time.Now().Add(-48 * time.Hour)
100+
assert.NoError(t, os.Chtimes(stale, old, old))
101+
assert.NoError(t, os.Chtimes(other, old, old))
102+
103+
v, err := New(shared)
104+
assert.NoError(t, err)
105+
defer v.Remove()
106+
_, err = os.Stat(stale)
107+
assert.That(t, os.IsNotExist(err))
108+
_, err = os.Stat(fresh)
109+
assert.NoError(t, err)
110+
_, err = os.Stat(other)
111+
assert.NoError(t, err)
112+
}
113+
114+
func TestCopyMerge(t *testing.T) {
115+
// The copying fallback, as used when the shared cache is on another
116+
// filesystem, is exercised directly since a test cannot cross one.
117+
shared := filepath.Join(t.TempDir(), "chisel")
118+
v, err := New(shared)
119+
assert.NoError(t, err)
120+
defer v.Remove()
121+
v.copy = true
122+
write(t, filepath.Join(v.Home, "chisel", "sha256", "aa"), "a")
123+
assert.NoError(t, v.Merge())
124+
b, err := os.ReadFile(filepath.Join(shared, "sha256", "aa"))
125+
assert.NoError(t, err)
126+
assert.Equal(t, string(b), "a")
127+
assert.That(t, !sameFile(t, filepath.Join(shared, "sha256", "aa"), filepath.Join(v.Home, "chisel", "sha256", "aa")))
128+
entries, err := os.ReadDir(filepath.Join(shared, "sha256"))
129+
assert.NoError(t, err)
130+
assert.Len(t, entries, 1)
131+
}

0 commit comments

Comments
 (0)