containerd: include present content size in disk usage calculations

jsternberg · jsternberg · commit f07242f6d709 · 2025-05-15T09:46:51.000-05:00
The present content size is included in the image size usage and should
be included in the total size that the layer takes up on disk.

This prevents an issue where the reclaimable amount reported by the CLI
was a negative number.

This also updates the `/system/df` endpoint to use a new type that
computes information that was previously computed by the CLI. Computing
these in the server should require less work from the CLI and ensure
the calculations are more accurate because the CLI doesn't have to
reconstruct the numbers.

Signed-off-by: Jonathan A. Sternberg &lt;jonathan.sternberg@docker.com&gt;
diff --git a/api/server/router/system/backend.go b/api/server/router/system/backend.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/build"
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/registry"
@@ -29,7 +30,7 @@ type DiskUsageOptions struct {
 type Backend interface {
 	SystemInfo(context.Context) (*system.Info, error)
 	SystemVersion(context.Context) (types.Version, error)
-	SystemDiskUsage(ctx context.Context, opts DiskUsageOptions) (*types.DiskUsage, error)
+	SystemDiskUsage(ctx context.Context, opts DiskUsageOptions) (*system.DiskUsage, error)
 	SubscribeToEvents(since, until time.Time, ef filters.Args) ([]events.Message, chan interface{})
 	UnsubscribeFromEvents(chan interface{})
 	AuthenticateToRegistry(ctx context.Context, authConfig *registry.AuthConfig) (string, string, error)
@@ -43,7 +44,7 @@ type ClusterBackend interface {
 
 // BuildBackend provides build specific system information.
 type BuildBackend interface {
-	DiskUsage(context.Context) ([]*types.BuildCache, error)
+	DiskUsage(context.Context) ([]*build.CacheRecord, error)
 }
 
 // StatusProvider provides methods to get the swarm status of the current node.
diff --git a/api/server/router/system/system_routes.go b/api/server/router/system/system_routes.go
@@ -14,6 +14,7 @@ import (
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/server/router/build"
 	"github.com/docker/docker/api/types"
+	buildtypes "github.com/docker/docker/api/types/build"
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/registry"
@@ -170,7 +171,7 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 
 	eg, ctx := errgroup.WithContext(ctx)
 
-	var systemDiskUsage *types.DiskUsage
+	var systemDiskUsage *system.DiskUsage
 	if getContainers || getImages || getVolumes {
 		eg.Go(func() error {
 			var err error
@@ -183,7 +184,7 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 		})
 	}
 
-	var buildCache []*types.BuildCache
+	var buildCache []*buildtypes.CacheRecord
 	if getBuildCache {
 		eg.Go(func() error {
 			var err error
@@ -194,7 +195,7 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 			if buildCache == nil {
 				// Ensure empty `BuildCache` field is represented as empty JSON array(`[]`)
 				// instead of `null` to be consistent with `Images`, `Containers` etc.
-				buildCache = []*types.BuildCache{}
+				buildCache = []*buildtypes.CacheRecord{}
 			}
 			return nil
 		})
@@ -219,23 +220,42 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 			b.Parent = "" //nolint:staticcheck // ignore SA1019 (Parent field is deprecated)
 		}
 	}
-	if versions.LessThan(version, "1.44") {
-		for _, b := range systemDiskUsage.Images {
+	if versions.LessThan(version, "1.44") && systemDiskUsage != nil && systemDiskUsage.Images != nil {
+		for _, b := range systemDiskUsage.Images.Items {
 			b.VirtualSize = b.Size //nolint:staticcheck // ignore SA1019: field is deprecated, but still set on API < v1.44.
 		}
 	}
 
-	du := types.DiskUsage{
-		BuildCache:  buildCache,
-		BuilderSize: builderSize,
+	du := system.DiskUsage{}
+	if getBuildCache {
+		du.BuildCache = &buildtypes.CacheDiskUsage{
+			TotalSize: builderSize,
+			Items:     buildCache,
+		}
 	}
 	if systemDiskUsage != nil {
-		du.LayersSize = systemDiskUsage.LayersSize
 		du.Images = systemDiskUsage.Images
 		du.Containers = systemDiskUsage.Containers
 		du.Volumes = systemDiskUsage.Volumes
 	}
-	return httputils.WriteJSON(w, http.StatusOK, du)
+
+	// Use the old struct for the API return value.
+	var v types.DiskUsage
+	if du.Images != nil {
+		v.LayersSize = du.Images.TotalSize
+		v.Images = du.Images.Items
+	}
+	if du.Containers != nil {
+		v.Containers = du.Containers.Items
+	}
+	if du.Volumes != nil {
+		v.Volumes = du.Volumes.Items
+	}
+	if du.BuildCache != nil {
+		v.BuildCache = du.BuildCache.Items
+	}
+	v.BuilderSize = builderSize
+	return httputils.WriteJSON(w, http.StatusOK, v)
 }
 
 type invalidRequestError struct {
diff --git a/api/types/build/cache.go b/api/types/build/cache.go
@@ -0,0 +1,30 @@
+package build
+
+import "time"
+
+// CacheRecord contains information about a build cache record.
+type CacheRecord struct {
+	// ID is the unique ID of the build cache record.
+	ID string
+	// Parent is the ID of the parent build cache record.
+	//
+	// Deprecated: deprecated in API v1.42 and up, as it was deprecated in BuildKit; use Parents instead.
+	Parent string `json:"Parent,omitempty"`
+	// Parents is the list of parent build cache record IDs.
+	Parents []string `json:" Parents,omitempty"`
+	// Type is the cache record type.
+	Type string
+	// Description is a description of the build-step that produced the build cache.
+	Description string
+	// InUse indicates if the build cache is in use.
+	InUse bool
+	// Shared indicates if the build cache is shared.
+	Shared bool
+	// Size is the amount of disk space used by the build cache (in bytes).
+	Size int64
+	// CreatedAt is the date and time at which the build cache was created.
+	CreatedAt time.Time
+	// LastUsedAt is the date and time at which the build cache was last used.
+	LastUsedAt *time.Time
+	UsageCount int
+}
diff --git a/api/types/build/disk_usage.go b/api/types/build/disk_usage.go
@@ -0,0 +1,8 @@
+package build
+
+// CacheDiskUsage contains disk usage for the build cache.
+type CacheDiskUsage struct {
+	TotalSize   int64
+	Reclaimable int64
+	Items       []*CacheRecord
+}
diff --git a/api/types/container/disk_usage.go b/api/types/container/disk_usage.go
@@ -0,0 +1,8 @@
+package container
+
+// DiskUsage contains disk usage for containers.
+type DiskUsage struct {
+	TotalSize   int64
+	Reclaimable int64
+	Items       []*Summary
+}
diff --git a/api/types/image/disk_usage.go b/api/types/image/disk_usage.go
@@ -0,0 +1,8 @@
+package image
+
+// DiskUsage contains disk usage for images.
+type DiskUsage struct {
+	TotalSize   int64
+	Reclaimable int64
+	Items       []*Summary
+}
diff --git a/api/types/system/disk_usage.go b/api/types/system/disk_usage.go
@@ -0,0 +1,17 @@
+package system
+
+import (
+	"github.com/docker/docker/api/types/build"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/api/types/volume"
+)
+
+// DiskUsage contains response of Engine API for API 1.49 and greater:
+// GET "/system/df"
+type DiskUsage struct {
+	Images     *image.DiskUsage
+	Containers *container.DiskUsage
+	Volumes    *volume.DiskUsage
+	BuildCache *build.CacheDiskUsage
+}
diff --git a/api/types/types.go b/api/types/types.go
@@ -1,8 +1,7 @@
 package types // import "github.com/docker/docker/api/types"
 
 import (
-	"time"
-
+	"github.com/docker/docker/api/types/build"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/image"
@@ -141,31 +140,9 @@ type BuildResult struct {
 }
 
 // BuildCache contains information about a build cache record.
-type BuildCache struct {
-	// ID is the unique ID of the build cache record.
-	ID string
-	// Parent is the ID of the parent build cache record.
-	//
-	// Deprecated: deprecated in API v1.42 and up, as it was deprecated in BuildKit; use Parents instead.
-	Parent string `json:"Parent,omitempty"`
-	// Parents is the list of parent build cache record IDs.
-	Parents []string `json:" Parents,omitempty"`
-	// Type is the cache record type.
-	Type string
-	// Description is a description of the build-step that produced the build cache.
-	Description string
-	// InUse indicates if the build cache is in use.
-	InUse bool
-	// Shared indicates if the build cache is shared.
-	Shared bool
-	// Size is the amount of disk space used by the build cache (in bytes).
-	Size int64
-	// CreatedAt is the date and time at which the build cache was created.
-	CreatedAt time.Time
-	// LastUsedAt is the date and time at which the build cache was last used.
-	LastUsedAt *time.Time
-	UsageCount int
-}
+//
+// Deprecated: deprecated in API 1.49. Use [build.CacheRecord] instead.
+type BuildCache = build.CacheRecord
 
 // BuildCachePruneOptions hold parameters to prune the build cache
 type BuildCachePruneOptions struct {
diff --git a/api/types/volume/disk_usage.go b/api/types/volume/disk_usage.go
@@ -0,0 +1,8 @@
+package volume
+
+// DiskUsage contains disk usage for volumes.
+type DiskUsage struct {
+	TotalSize   int64
+	Reclaimable int64
+	Items       []*Volume
+}
diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go
@@ -14,6 +14,7 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
+	"github.com/docker/docker/api/types/build"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	timetypes "github.com/docker/docker/api/types/time"
@@ -149,15 +150,15 @@ func (b *Builder) Cancel(ctx context.Context, id string) error {
 }
 
 // DiskUsage returns a report about space used by build cache
-func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
+func (b *Builder) DiskUsage(ctx context.Context) ([]*build.CacheRecord, error) {
 	duResp, err := b.controller.DiskUsage(ctx, &controlapi.DiskUsageRequest{})
 	if err != nil {
 		return nil, err
 	}
 
-	var items []*types.BuildCache
+	var items []*build.CacheRecord
 	for _, r := range duResp.Record {
-		items = append(items, &types.BuildCache{
+		items = append(items, &build.CacheRecord{
 			ID:          r.ID,
 			Parent:      r.Parent, //nolint:staticcheck // ignore SA1019 (Parent field is deprecated)
 			Parents:     r.Parents,
diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go
@@ -21,6 +21,8 @@ import (
 	"github.com/docker/docker/distribution"
 	"github.com/docker/docker/errdefs"
 	"github.com/moby/sys/user"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -127,21 +129,51 @@ func (i *ImageService) StorageDriver() string {
 	return i.snapshotter
 }
 
+// ImageDiskUsage returns the number of bytes used by content and layer stores
+// called from disk_usage.go
+func (i *ImageService) ImageDiskUsage(ctx context.Context) (int64, error) {
+	diskUsage, err := i.layerDiskUsage(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	// Include the size of content size from the images.
+	imgs, err := i.images.List(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	visitedImages := make(map[digest.Digest]struct{})
+	for _, img := range imgs {
+		if err := i.walkPresentChildren(ctx, img.Target, func(ctx context.Context, desc ocispec.Descriptor) error {
+			if _, ok := visitedImages[desc.Digest]; ok {
+				return nil
+			}
+			visitedImages[desc.Digest] = struct{}{}
+
+			diskUsage += desc.Size
+			return nil
+		}); err != nil {
+			return 0, err
+		}
+	}
+	return diskUsage, nil
+}
+
 // LayerDiskUsage returns the number of bytes used by layer stores
 // called from disk_usage.go
-func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) {
-	var allLayersSize int64
+func (i *ImageService) layerDiskUsage(ctx context.Context) (allLayersSize int64, err error) {
 	// TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273
 	snapshotter := i.client.SnapshotService(i.snapshotter)
-	snapshotter.Walk(ctx, func(ctx context.Context, info snapshots.Info) error {
+	err = snapshotter.Walk(ctx, func(ctx context.Context, info snapshots.Info) error {
 		usage, err := snapshotter.Usage(ctx, info.Name)
 		if err != nil {
 			return err
 		}
 		allLayersSize += usage.Size
 		return nil
 	})
-	return allLayersSize, nil
+	return allLayersSize, err
 }
 
 // UpdateConfig values
diff --git a/daemon/daemon.go b/daemon/daemon.go
@@ -36,7 +36,7 @@ import (
 	networktypes "github.com/docker/docker/api/types/network"
 	registrytypes "github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/api/types/volume"
+	volumetypes "github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/container"
 	executorpkg "github.com/docker/docker/daemon/cluster/executor"
@@ -131,9 +131,9 @@ type Daemon struct {
 	seccompProfile     []byte
 	seccompProfilePath string
 
-	usageContainers singleflight.Group[struct{}, []*containertypes.Summary]
+	usageContainers singleflight.Group[struct{}, *containertypes.DiskUsage]
 	usageImages     singleflight.Group[struct{}, []*imagetypes.Summary]
-	usageVolumes    singleflight.Group[struct{}, []*volume.Volume]
+	usageVolumes    singleflight.Group[struct{}, *volumetypes.DiskUsage]
 	usageLayer      singleflight.Group[struct{}, int64]
 
 	pruneRunning atomic.Bool
diff --git a/daemon/disk_usage.go b/daemon/disk_usage.go
diff --git a/daemon/image_service.go b/daemon/image_service.go
diff --git a/daemon/images/service.go b/daemon/images/service.go
diff --git a/integration/system/disk_usage_test.go b/integration/system/disk_usage_test.go
