Skip to content

Commit a6f1417

Browse files
committed
feat: set the cache expiry with CHISEL_CACHE_EXPIRY
CHISEL_CACHE_EXPIRY sets how many days a cache entry can go unused, 7 by default, and 0 disables expiry. Any other value logs a warning and leaves the cache alone, since deleting is the one thing that can't be undone. debug check-release-archives fetches every package of a release, so it now expires the cache too. find and info only refresh the release directory, which is replaced in place and does not grow. The command tests point XDG_CACHE_HOME at a temporary directory, so they never expire the cache of whoever runs them.
1 parent c053a58 commit a6f1417

9 files changed

Lines changed: 187 additions & 18 deletions

File tree

‎cmd/chisel/cmd_cut.go‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ to create a new filesystem tree in the root location.
2020
2121
By default it fetches the slices for the same Ubuntu version as the
2222
current host, unless the --release flag is used.
23+
24+
Packages and archive indexes are cached, and cache entries that have not
25+
been used for 7 days are removed after the cut. Set CHISEL_CACHE_EXPIRY
26+
to a number of days to change that, or to 0 to keep all entries.
2327
`
2428

2529
var cutDescs = map[string]string{
@@ -29,9 +33,6 @@ var cutDescs = map[string]string{
2933
"ignore": "Conditions to ignore (e.g. unmaintained, unstable)",
3034
}
3135

32-
// cacheExpiry is how long a cache entry can go unused before a cut removes it.
33-
const cacheExpiry = 7 * 24 * time.Hour
34-
3536
type cmdCut struct {
3637
Release string `long:"release" value-name:"<dir>"`
3738
RootDir string `long:"root" value-name:"<dir>" required:"yes"`
@@ -134,11 +135,6 @@ func (cmd *cmdCut) Execute(args []string) error {
134135
return err
135136
}
136137

137-
// Reading an entry refreshes its timestamp, so this only drops what no
138-
// cut has used for a while. Failing to clean up is not a failed cut.
139-
cc := &cache.Cache{Dir: cacheDir}
140-
if err := cc.Expire(cacheExpiry); err != nil {
141-
logf("Warning: cannot expire cache entries: %v", err)
142-
}
138+
expireCache(cacheDir)
143139
return nil
144140
}

‎cmd/chisel/cmd_debug_check_release_archives.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ func (cmd *cmdDebugCheckReleaseArchives) Execute(args []string) error {
6262
return err
6363
}
6464

65+
cacheDir := cache.DefaultDir("chisel")
6566
archives := make(map[string]archive.Archive)
6667
for archiveName, archiveInfo := range release.Archives {
6768
openArchive, err := archiveOpen(&archive.Options{
@@ -71,7 +72,7 @@ func (cmd *cmdDebugCheckReleaseArchives) Execute(args []string) error {
7172
Suites: archiveInfo.Suites,
7273
Components: archiveInfo.Components,
7374
Pro: archiveInfo.Pro,
74-
CacheDir: cache.DefaultDir("chisel"),
75+
CacheDir: cacheDir,
7576
PubKeys: archiveInfo.PubKeys,
7677
Maintained: archiveInfo.Maintained,
7778
OldRelease: archiveInfo.OldRelease,
@@ -89,6 +90,7 @@ func (cmd *cmdDebugCheckReleaseArchives) Execute(args []string) error {
8990
if err != nil {
9091
return err
9192
}
93+
expireCache(cacheDir)
9294

9395
var issues []any
9496
type pathConflict struct {

‎cmd/chisel/cmd_debug_check_release_archives_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"path/filepath"
66
"slices"
77
"strings"
8+
"time"
89

910
. "gopkg.in/check.v1"
1011
"gopkg.in/yaml.v3"
@@ -520,6 +521,16 @@ func (s *ChiselSuite) TestRun(c *C) {
520521
})
521522
defer restore()
522523

524+
// A cache entry no command has used for a while.
525+
stale := filepath.Join(os.Getenv("XDG_CACHE_HOME"), "chisel", "sha256", strings.Repeat("0", 64))
526+
err = os.MkdirAll(filepath.Dir(stale), 0755)
527+
c.Assert(err, IsNil)
528+
err = os.WriteFile(stale, nil, 0644)
529+
c.Assert(err, IsNil)
530+
lastUsed := time.Now().Add(-8 * 24 * time.Hour)
531+
err = os.Chtimes(stale, lastUsed, lastUsed)
532+
c.Assert(err, IsNil)
533+
523534
cliArgs := []string{"debug", "check-release-archives", "--release", releaseDir}
524535
if test.arch != "" {
525536
cliArgs = slices.Concat(cliArgs, []string{"--arch", test.arch})
@@ -531,6 +542,9 @@ func (s *ChiselSuite) TestRun(c *C) {
531542
} else {
532543
c.Assert(err, IsNil)
533544
}
545+
// Once the packages are fetched, the cache is expired.
546+
_, err = os.Stat(stale)
547+
c.Assert(os.IsNotExist(err), Equals, true)
534548
if test.stdout != "" {
535549
test.stdout = string(testutil.Reindent(test.stdout))
536550
test.stdout = strings.TrimSpace(test.stdout) + "\n"

‎cmd/chisel/export_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ func FakeIsStdinTTY(t bool) (restore func()) {
2222

2323
var FindSlices = findSlices
2424

25+
var CacheExpiry = cacheExpiry
26+
27+
var ExpireCache = expireCache
28+
2529
func FakeArchiveOpen(f func(_ *archive.Options) (archive.Archive, error)) (restore func()) {
2630
oldArchiveOpen := archiveOpen
2731
archiveOpen = f

‎cmd/chisel/helpers.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@ import (
44
"fmt"
55
"os"
66
"regexp"
7+
"strconv"
78
"strings"
9+
"time"
810

11+
"github.com/canonical/chisel/internal/cache"
912
"github.com/canonical/chisel/internal/setup"
1013
)
1114

@@ -69,3 +72,43 @@ func obtainRelease(releaseStr string) (release *setup.Release, err error) {
6972
}
7073
return release, nil
7174
}
75+
76+
// defaultCacheExpiry is how long a cache entry can go unused before it is
77+
// removed, unless CHISEL_CACHE_EXPIRY says otherwise.
78+
const defaultCacheExpiry = 7 * 24 * time.Hour
79+
80+
// cacheExpiry returns how long a cache entry can go unused before it is
81+
// removed. CHISEL_CACHE_EXPIRY sets it as a number of days, and 0 disables
82+
// expiry.
83+
func cacheExpiry() (time.Duration, error) {
84+
value := os.Getenv("CHISEL_CACHE_EXPIRY")
85+
if value == "" {
86+
return defaultCacheExpiry, nil
87+
}
88+
// Limit it to 16 bits, about 179 years, so the duration cannot overflow.
89+
days, err := strconv.ParseUint(value, 10, 16)
90+
if err != nil {
91+
return 0, fmt.Errorf("invalid CHISEL_CACHE_EXPIRY %q, expected a number of days", value)
92+
}
93+
return time.Duration(days) * 24 * time.Hour, nil
94+
}
95+
96+
// expireCache removes the entries of the cache in dir that have not been
97+
// used for the time given by cacheExpiry. It is meant to run once a command
98+
// is done with the cache.
99+
func expireCache(dir string) {
100+
expiry, err := cacheExpiry()
101+
if err != nil {
102+
logf("Warning: %v; the cache is not expired", err)
103+
return
104+
}
105+
if expiry == 0 {
106+
return
107+
}
108+
// Reading an entry refreshes its timestamp, so this only drops what no
109+
// command has used for a while.
110+
cc := &cache.Cache{Dir: dir}
111+
if err := cc.Expire(expiry); err != nil {
112+
logf("Warning: cannot expire cache entries: %v", err)
113+
}
114+
}

‎cmd/chisel/helpers_test.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package main_test
2+
3+
import (
4+
"bytes"
5+
"log"
6+
"os"
7+
"path/filepath"
8+
"time"
9+
10+
. "gopkg.in/check.v1"
11+
12+
chisel "github.com/canonical/chisel/cmd/chisel"
13+
)
14+
15+
var cacheExpiryTests = []struct {
16+
value string
17+
expiry time.Duration
18+
err string
19+
}{
20+
{value: "", expiry: 7 * 24 * time.Hour},
21+
{value: "0", expiry: 0},
22+
{value: "1", expiry: 24 * time.Hour},
23+
{value: "30", expiry: 30 * 24 * time.Hour},
24+
{value: "65535", expiry: 65535 * 24 * time.Hour},
25+
{value: "65536", err: `invalid CHISEL_CACHE_EXPIRY "65536", expected a number of days`},
26+
{value: "-1", err: `invalid CHISEL_CACHE_EXPIRY "-1", expected a number of days`},
27+
{value: "7d", err: `invalid CHISEL_CACHE_EXPIRY "7d", expected a number of days`},
28+
{value: "1.5", err: `invalid CHISEL_CACHE_EXPIRY "1.5", expected a number of days`},
29+
{value: " 7", err: `invalid CHISEL_CACHE_EXPIRY " 7", expected a number of days`},
30+
}
31+
32+
func (s *ChiselSuite) TestCacheExpiry(c *C) {
33+
for _, test := range cacheExpiryTests {
34+
c.Logf("CHISEL_CACHE_EXPIRY=%q", test.value)
35+
restore := fakeEnv("CHISEL_CACHE_EXPIRY", test.value)
36+
expiry, err := chisel.CacheExpiry()
37+
restore()
38+
if test.err != "" {
39+
c.Assert(err, ErrorMatches, test.err)
40+
continue
41+
}
42+
c.Assert(err, IsNil)
43+
c.Assert(expiry, Equals, test.expiry)
44+
}
45+
}
46+
47+
var expireCacheTests = []struct {
48+
value string
49+
expired bool
50+
log string
51+
}{
52+
{value: "", expired: true},
53+
{value: "7", expired: true},
54+
{value: "30", expired: false},
55+
{value: "0", expired: false},
56+
{value: "never", expired: false, log: "Warning: invalid CHISEL_CACHE_EXPIRY \"never\", expected a number of days; the cache is not expired\n"},
57+
}
58+
59+
func (s *ChiselSuite) TestExpireCache(c *C) {
60+
var logs bytes.Buffer
61+
chisel.SetLogger(log.New(&logs, "", 0))
62+
defer chisel.SetLogger(nil)
63+
64+
for _, test := range expireCacheTests {
65+
c.Logf("CHISEL_CACHE_EXPIRY=%q", test.value)
66+
logs.Reset()
67+
68+
dir := c.MkDir()
69+
entry := filepath.Join(dir, "sha256", "5b41362bc82b7f3d56edc5a306db22105707d01ff4819e26faef9724a2d406c9")
70+
err := os.MkdirAll(filepath.Dir(entry), 0755)
71+
c.Assert(err, IsNil)
72+
err = os.WriteFile(entry, []byte("data1"), 0644)
73+
c.Assert(err, IsNil)
74+
lastUsed := time.Now().Add(-8 * 24 * time.Hour)
75+
err = os.Chtimes(entry, lastUsed, lastUsed)
76+
c.Assert(err, IsNil)
77+
78+
restore := fakeEnv("CHISEL_CACHE_EXPIRY", test.value)
79+
chisel.ExpireCache(dir)
80+
restore()
81+
82+
_, err = os.Stat(entry)
83+
c.Assert(os.IsNotExist(err), Equals, test.expired)
84+
c.Assert(logs.String(), Equals, test.log)
85+
}
86+
}

‎cmd/chisel/main_test.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ func (s *BaseChiselSuite) SetUpTest(c *C) {
4444

4545
s.AddCleanup(chisel.FakeIsStdoutTTY(false))
4646
s.AddCleanup(chisel.FakeIsStdinTTY(false))
47+
48+
// Commands expire the cache, so keep them away from the real one.
49+
s.AddCleanup(fakeEnv("XDG_CACHE_HOME", c.MkDir()))
4750
}
4851

4952
func (s *BaseChiselSuite) TearDownTest(c *C) {
@@ -75,6 +78,18 @@ func fakeVersion(v string) (restore func()) {
7578
return func() { cmd.Version = old }
7679
}
7780

81+
func fakeEnv(name, value string) (restore func()) {
82+
old, ok := os.LookupEnv(name)
83+
os.Setenv(name, value)
84+
return func() {
85+
if ok {
86+
os.Setenv(name, old)
87+
} else {
88+
os.Unsetenv(name)
89+
}
90+
}
91+
}
92+
7893
type ChiselSuite struct {
7994
BaseChiselSuite
8095
}

‎tests/AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ The `tests/` directory contains the Spread integration test suite. It exercises
1717
# Directory
1818

1919
- `basic/` - Core slice extraction scenario verifying that files are correctly written to the target root filesystem and that mutation scripts are applied.
20-
- `cache-expiry/` - Verifies that a cut expires cache entries unused for a week and keeps the ones it reads.
20+
- `cache-expiry/` - Verifies that a cut expires cache entries unused for a week, keeps the ones it reads and files Chisel did not write, and that `CHISEL_CACHE_EXPIRY=0` disables expiry.
2121
- `find/` - Tests for the `chisel find` command, covering search by slice name and package.
2222
- `info/` - Tests for the `chisel info` command, verifying detailed slice metadata output.
2323
- `debug-check-release-archives/` - Tests detection of package-content conflicts that cannot be identified from slice definitions alone.

‎tests/cache-expiry/task.yaml‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
summary: Cache entries that no cut has used for a week are expired
1+
summary: Cut expires cache entries unused for a week, unless told otherwise
22

33
execute: |
44
cache_dir=${XDG_CACHE_HOME:-$HOME/.cache}/chisel
@@ -9,12 +9,13 @@ execute: |
99
EXTRA_OPTIONS="--ignore=unmaintained "
1010
fi
1111
12-
mkdir -p ${rootfs_folder}_1 ${rootfs_folder}_2
12+
mkdir -p ${rootfs_folder}_1 ${rootfs_folder}_2 ${rootfs_folder}_3
1313
chisel cut --release ${OS}-${RELEASE} --root ${rootfs_folder}_1 \
1414
$EXTRA_OPTIONS base-passwd_data
1515
1616
# Find the cached base-passwd deb (under the strongest digest the archive
17-
# offers) and backdate it, and add a stale entry that no cut uses.
17+
# offers) and backdate it. Add a stale entry that no cut uses, and a file
18+
# Chisel did not write.
1819
used=""
1920
for f in ${cache_dir}/sha*/*; do
2021
if [ "$(dpkg-deb -f "$f" Package 2>/dev/null)" = "base-passwd" ]; then
@@ -24,13 +25,21 @@ execute: |
2425
test -n "$used"
2526
stale=${cache_dir}/sha256/$(echo stale | sha256sum | cut -d' ' -f1)
2627
echo stale > $stale
27-
touch -d "8 days ago" $used $stale
28+
foreign=${cache_dir}/sha256/notes
29+
echo notes > $foreign
30+
touch -d "8 days ago" $used $stale $foreign
2831
29-
chisel cut --release ${OS}-${RELEASE} --root ${rootfs_folder}_2 \
32+
# Nothing is expired when expiry is disabled.
33+
CHISEL_CACHE_EXPIRY=0 chisel cut --release ${OS}-${RELEASE} \
34+
--root ${rootfs_folder}_2 $EXTRA_OPTIONS base-passwd_data
35+
test -e $stale
36+
37+
chisel cut --release ${OS}-${RELEASE} --root ${rootfs_folder}_3 \
3038
$EXTRA_OPTIONS base-passwd_data
3139
32-
# The stale entry is gone. The deb was used again, so it is kept, with a
33-
# fresh timestamp.
40+
# The stale entry is gone, and the file Chisel did not write is kept. The
41+
# deb was used again, so it is kept, with a fresh timestamp.
3442
test ! -e $stale
43+
test -e $foreign
3544
test -e $used
3645
test $(( $(date +%s) - $(stat -c %Y $used) )) -lt 3600

0 commit comments

Comments
 (0)