Skip to content

Commit 91b1acb

Browse files
committed
Fall back to btrfs when ZFS is unavailable
ZFS is preferred, but WSL, SUSE, Arch, Fedora and others may ship without it. Instead of keying the fallback off WSL, pick the first driver LXD reports as supported (ZFS, then Btrfs), and fail the system check with a clear error if the pool is missing and neither is available. Record the pool's driver atomically, since it may be set after the daemon starts serving. Adopt an existing pool's driver if it is ZFS or Btrfs and reject anything else, recreate a missing pool on the preferred supported driver, and serialize ensureBackendReady so concurrent callers don't race to create the pool. Update the docs to describe the detection-based fallback. Closes: canonical#742 Signed-off-by: Lincoln Wallace <[email protected]> wip Signed-off-by: Lincoln Wallace <[email protected]>
1 parent c7f1844 commit 91b1acb

4 files changed

Lines changed: 190 additions & 29 deletions

File tree

‎internal/workshop/lxd/export_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,21 @@ var (
2323
HandleImageUpdate = handleImageUpdate
2424
CheckServerVersion = checkVersion
2525
GenerateCNAME = generateCNAME
26+
PreferredDriver = preferredDriver
27+
CheckStoragePool = checkStoragePool
28+
AdoptPoolDriver = adoptPoolDriver
29+
PoolDriver = poolDriver
2630
)
2731

32+
// MockPoolDriver clears the recorded storage pool driver and returns a
33+
// function restoring it.
34+
func MockPoolDriver() (restore func()) {
35+
old := storagePoolDriver.Swap(nil)
36+
return func() {
37+
storagePoolDriver.Store(old)
38+
}
39+
}
40+
2841
func MockFirewallChecker(f func(string) string) func() {
2942
old := firewallChecker
3043
firewallChecker = f

‎internal/workshop/lxd/lxd_backend.go‎

Lines changed: 82 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ import (
3030
"slices"
3131
"strconv"
3232
"strings"
33+
"sync"
34+
"sync/atomic"
3335
"time"
3436

3537
lxd "github.com/canonical/lxd/client"
@@ -82,16 +84,38 @@ const (
8284
)
8385

8486
var (
85-
storagePoolDriver = "zfs"
87+
// storageDrivers lists the supported pool drivers, most preferred first.
88+
storageDrivers = []string{"zfs", "btrfs"}
89+
90+
storagePoolDriver atomic.Pointer[string]
91+
92+
// backendReadyMu serializes ensureBackendReady.
93+
backendReadyMu sync.Mutex
8694

8795
workshopFormatsChecked = false
8896
)
8997

90-
func init() {
91-
if osutil.IsWSL() {
92-
storagePoolDriver = "btrfs"
98+
// poolDriver returns the storage pool driver, or "" if unknown.
99+
func poolDriver() string {
100+
if driver := storagePoolDriver.Load(); driver != nil {
101+
return *driver
102+
}
103+
return ""
104+
}
105+
106+
// adoptPoolDriver records driver as the storage pool driver if supported.
107+
func adoptPoolDriver(driver string) error {
108+
if !slices.Contains(storageDrivers, driver) {
109+
return fmt.Errorf("cannot use storage pool %q: unsupported driver %q (expected one of %s)", storagePool, driver, strings.Join(storageDrivers, ", "))
93110
}
111+
if old := poolDriver(); old != "" && old != driver {
112+
logger.Noticef("Storage pool %q changed driver from %q to %q", storagePool, old, driver)
113+
}
114+
storagePoolDriver.Store(&driver)
115+
return nil
116+
}
94117

118+
func init() {
95119
// Order matters: capabilities (version and storage) must be validated
96120
// before ensureBackendReady attempts to create the storage pool and
97121
// network. Registering it as a check also lets the daemon recover after
@@ -102,8 +126,7 @@ func init() {
102126
syscheck.RegisterCheck(checkStorageSpace)
103127
}
104128

105-
type Backend struct {
106-
}
129+
type Backend struct{}
107130

108131
var _ workshop.Backend = (*Backend)(nil)
109132

@@ -174,19 +197,35 @@ func checkVersion(version string) error {
174197
return nil
175198
}
176199

177-
func checkStorageDriver(drivers []api.ServerStorageDriverInfo) error {
178-
hasDriver := func(driver api.ServerStorageDriverInfo) bool {
179-
return driver.Name == storagePoolDriver
200+
// preferredDriver returns the first supported storage driver in order of
201+
// preference (ZFS, then btrfs), or an error if none is supported.
202+
func preferredDriver(supported []api.ServerStorageDriverInfo) (string, error) {
203+
for _, driver := range storageDrivers {
204+
if slices.ContainsFunc(supported, func(d api.ServerStorageDriverInfo) bool {
205+
return d.Name == driver
206+
}) {
207+
return driver, nil
208+
}
180209
}
181-
if slices.ContainsFunc(drivers, hasDriver) {
210+
return "", fmt.Errorf("suitable storage backend not found: none of %s is available", strings.Join(storageDrivers, ", "))
211+
}
212+
213+
// checkStoragePool checks that the storage pool is usable or can be created.
214+
func checkStoragePool(conn lxd.InstanceServer, supported []api.ServerStorageDriverInfo) error {
215+
if pool, _, err := conn.GetStoragePool(storagePool); err == nil {
216+
// LXD still returns a pool it failed to initialize (e.g. its driver's
217+
// kernel module is gone), and retries until it recovers.
218+
if pool.Status == api.StoragePoolStatusUnvailable {
219+
return fmt.Errorf("%q storage pool unavailable: LXD cannot initialize it with the %q driver", storagePool, pool.Driver)
220+
}
182221
return nil
222+
} else if !api.StatusErrorCheck(err, http.StatusNotFound) {
223+
return fmt.Errorf("%q storage pool unavailable: %w", storagePool, err)
183224
}
184225

185-
// The LXD error message when creating a pool is:
186-
// Error: Error loading "zfs" module: Failed to run: modprobe -b zfs:
187-
// exit status 1 (modprobe: FATAL: Module zfs not found ...)
188-
// We keep the first part for consistency, the rest doesn't add much.
189-
return fmt.Errorf(`suitable storage backend not found: error loading %q module`, storagePoolDriver)
226+
// The pool is missing; make sure it can be created.
227+
_, err := preferredDriver(supported)
228+
return err
190229
}
191230

192231
// checkStorageSpace puts the daemon into degraded mode when the workshop
@@ -250,7 +289,7 @@ func checkServerCapabilities() error {
250289

251290
workshop.WorkshopVMsSupportSDKs.Store(checkVMsSupportSDKs(conn))
252291

253-
return checkStorageDriver(info.Environment.StorageSupportedDrivers)
292+
return checkStoragePool(conn, info.Environment.StorageSupportedDrivers)
254293
}
255294

256295
// vmDiskShiftSupported checks whether LXD supports mounting shifted SDK
@@ -315,22 +354,33 @@ func New() (*Backend, error) {
315354
// ensureBackendReady creates the LXD storage pool and network the backend
316355
// relies on, it is idempotent.
317356
func ensureBackendReady() error {
357+
backendReadyMu.Lock()
358+
defer backendReadyMu.Unlock()
359+
318360
// TODO: consider separating the storage pool or network for different users.
319361
conn, err := lxd.ConnectLXDUnix("", nil)
320362
if err != nil {
321363
return ErrorLxdBackend(err)
322364
}
323365
defer conn.Disconnect()
324366

325-
// Create LXD storage pool if it doesn't exist.
326-
pools, err := conn.GetStoragePools()
327-
if err != nil {
328-
return err
329-
}
330-
if idx := slices.IndexFunc(pools, func(p api.StoragePool) bool { return p.Name == storagePool }); idx < 0 {
367+
// Create the LXD storage pool if it doesn't exist, and record its driver so
368+
// pool operations can reuse it.
369+
existingPool, _, err := conn.GetStoragePool(storagePool)
370+
if api.StatusErrorCheck(err, http.StatusNotFound) {
371+
// Pick from the drivers LXD supports now, as they may have changed
372+
// since the pool was last created.
373+
info, _, err := conn.GetServer()
374+
if err != nil {
375+
return err
376+
}
377+
driver, err := preferredDriver(info.Environment.StorageSupportedDrivers)
378+
if err != nil {
379+
return err
380+
}
331381
req := api.StoragePoolsPost{
332382
Name: storagePool,
333-
Driver: storagePoolDriver,
383+
Driver: driver,
334384
}
335385
op, err := conn.CreateStoragePool(req)
336386
if err != nil {
@@ -339,6 +389,9 @@ func ensureBackendReady() error {
339389
if err := op.Wait(); err != nil {
340390
return err
341391
}
392+
if err := adoptPoolDriver(driver); err != nil {
393+
return err
394+
}
342395

343396
// Ensure the new pool has enough total space available.
344397
pool, etag, err := conn.GetStoragePool(storagePool)
@@ -369,8 +422,10 @@ func ensureBackendReady() error {
369422

370423
logger.Noticef("On ensureBackendReady: set storage pool to the minimal size: %dGiB", storagePoolMinimalGiB)
371424
}
372-
} else if pools[idx].Driver != storagePoolDriver {
373-
return fmt.Errorf("storage pool %q already exists with a different driver: %q (expected %q)", storagePool, pools[idx].Driver, storagePoolDriver)
425+
} else if err != nil {
426+
return err
427+
} else if err := adoptPoolDriver(existingPool.Driver); err != nil {
428+
return err
374429
}
375430

376431
network, etag, err := conn.GetNetwork(networkName)
@@ -1014,7 +1069,7 @@ func (s *Backend) execCommand(conn lxd.InstanceServer, ctx context.Context, name
10141069
}
10151070

10161071
opmeta := op.Get()
1017-
var env = map[string]string{}
1072+
env := map[string]string{}
10181073
for k, v := range opmeta.Metadata["environment"].(map[string]any) {
10191074
if value, ok := v.(string); ok {
10201075
env[k] = value
@@ -1038,7 +1093,7 @@ func (s *Backend) execCommand(conn lxd.InstanceServer, ctx context.Context, name
10381093
// unconditionally right after the operation has exited, so it will not be
10391094
// blocked if we are here
10401095
<-done
1041-
var status = int(op.Get().Metadata["return"].(float64))
1096+
status := int(op.Get().Metadata["return"].(float64))
10421097
if status != 0 {
10431098
return &workshop.ErrExec{Status: status}
10441099
}

‎internal/workshop/lxd/lxd_backend_snapshots.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,7 @@ func mergeDevices(source map[string]map[string]string, sdks []sdk.ContentID, w s
731731
return k != "root"
732732
})
733733

734-
if storagePoolDriver == "zfs" {
734+
if poolDriver() == "zfs" {
735735
root := maps.Clone(source["root"])
736736
if source == nil || root == nil {
737737
return fmt.Errorf("internal error: %q workshop has no rootfs", w)
@@ -862,7 +862,7 @@ func (s *Backend) copyInstance(src, dst lxd.InstanceServer, srcName, dstName str
862862

863863
req := *srcInst
864864

865-
if storagePoolDriver == "zfs" {
865+
if poolDriver() == "zfs" {
866866
req.Devices = maps.Clone(req.Devices)
867867
root := maps.Clone(req.Devices["root"])
868868
if req.Devices == nil || root == nil {

‎internal/workshop/lxd/lxd_backend_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ import (
1818
"crypto/sha3"
1919
"encoding/hex"
2020
"fmt"
21+
"net/http"
2122
"testing"
2223

24+
lxd "github.com/canonical/lxd/client"
25+
"github.com/canonical/lxd/shared/api"
2326
"gopkg.in/check.v1"
2427

2528
"github.com/canonical/workshop/internal/sdk"
@@ -262,3 +265,93 @@ func (f *LxdBeTests) TestCheckLxdVersion(c *check.C) {
262265
err = lxdbackend.CheckServerVersion("6.7.9")
263266
c.Assert(err, check.ErrorMatches, `(?s).*LXD server version.*is not supported.*`)
264267
}
268+
269+
func driverInfo(names ...string) []api.ServerStorageDriverInfo {
270+
infos := make([]api.ServerStorageDriverInfo, 0, len(names))
271+
for _, n := range names {
272+
infos = append(infos, api.ServerStorageDriverInfo{Name: n})
273+
}
274+
return infos
275+
}
276+
277+
// preferredDriver keeps ZFS whenever LXD reports it, falls back to Btrfs
278+
// otherwise, and errors when neither is supported. It trusts LXD's
279+
// supported-drivers list (which LXD builds by trying to load each driver's
280+
// module) rather than probing modules.
281+
func (f *LxdBeTests) TestPreferredDriver(c *check.C) {
282+
driver, err := lxdbackend.PreferredDriver(driverInfo("zfs", "btrfs", "dir"))
283+
c.Check(err, check.IsNil)
284+
c.Check(driver, check.Equals, "zfs")
285+
286+
driver, err = lxdbackend.PreferredDriver(driverInfo("btrfs", "zfs"))
287+
c.Check(err, check.IsNil)
288+
c.Check(driver, check.Equals, "zfs")
289+
290+
driver, err = lxdbackend.PreferredDriver(driverInfo("btrfs", "dir"))
291+
c.Check(err, check.IsNil)
292+
c.Check(driver, check.Equals, "btrfs")
293+
294+
driver, err = lxdbackend.PreferredDriver(driverInfo("dir"))
295+
c.Check(err, check.ErrorMatches, "suitable storage backend not found.*")
296+
c.Check(driver, check.Equals, "")
297+
298+
driver, err = lxdbackend.PreferredDriver(nil)
299+
c.Check(err, check.ErrorMatches, "suitable storage backend not found.*")
300+
c.Check(driver, check.Equals, "")
301+
}
302+
303+
// poolServer is an lxd.InstanceServer whose GetStoragePool returns pool, or
304+
// err if set.
305+
type poolServer struct {
306+
lxd.InstanceServer
307+
pool api.StoragePool
308+
err error
309+
}
310+
311+
func (s poolServer) GetStoragePool(string) (*api.StoragePool, string, error) {
312+
if s.err != nil {
313+
return nil, "", s.err
314+
}
315+
return &s.pool, "", nil
316+
}
317+
318+
func (f *LxdBeTests) TestCheckStoragePool(c *check.C) {
319+
notFound := api.StatusErrorf(http.StatusNotFound, "not found")
320+
321+
// An existing pool doesn't need a supported driver to be listed.
322+
created := api.StoragePool{Driver: "zfs", Status: api.StoragePoolStatusCreated}
323+
c.Check(lxdbackend.CheckStoragePool(poolServer{pool: created}, nil), check.IsNil)
324+
325+
// An existing pool LXD failed to initialize is unusable.
326+
unavailable := api.StoragePool{Driver: "zfs", Status: api.StoragePoolStatusUnvailable}
327+
err := lxdbackend.CheckStoragePool(poolServer{pool: unavailable}, driverInfo("btrfs"))
328+
c.Check(err, check.ErrorMatches, `"workshop" storage pool unavailable: LXD cannot initialize it with the "zfs" driver`)
329+
330+
// A missing pool is fine as long as it can be created.
331+
c.Check(lxdbackend.CheckStoragePool(poolServer{err: notFound}, driverInfo("btrfs")), check.IsNil)
332+
333+
// A missing pool with no usable driver can't be created.
334+
err = lxdbackend.CheckStoragePool(poolServer{err: notFound}, driverInfo("dir"))
335+
c.Check(err, check.ErrorMatches, "suitable storage backend not found.*")
336+
337+
// Any other error means the pool can't be checked.
338+
err = lxdbackend.CheckStoragePool(poolServer{err: fmt.Errorf("connection reset")}, driverInfo("zfs"))
339+
c.Check(err, check.ErrorMatches, `"workshop" storage pool unavailable: connection reset`)
340+
}
341+
342+
func (f *LxdBeTests) TestAdoptPoolDriver(c *check.C) {
343+
defer lxdbackend.MockPoolDriver()()
344+
c.Check(lxdbackend.PoolDriver(), check.Equals, "")
345+
346+
c.Assert(lxdbackend.AdoptPoolDriver("zfs"), check.IsNil)
347+
c.Check(lxdbackend.PoolDriver(), check.Equals, "zfs")
348+
349+
// A pool recreated on another supported driver is adopted.
350+
c.Assert(lxdbackend.AdoptPoolDriver("btrfs"), check.IsNil)
351+
c.Check(lxdbackend.PoolDriver(), check.Equals, "btrfs")
352+
353+
// An unsupported driver is rejected and the recorded one is kept.
354+
err := lxdbackend.AdoptPoolDriver("dir")
355+
c.Check(err, check.ErrorMatches, `cannot use storage pool "workshop": unsupported driver "dir" \(expected one of zfs, btrfs\)`)
356+
c.Check(lxdbackend.PoolDriver(), check.Equals, "btrfs")
357+
}

0 commit comments

Comments
 (0)