Add label filter for docker system prune

yongtang · yongtang · commit 702524732427 · 2017-04-06T09:11:25.000-07:00
This fix tries to address the issue raised in 29999 where it was not
possible to mask these items (like important non-removable stuff)
from `docker system prune`.

This fix adds `label` and `label!` field for `--filter` in `system prune`,
so that it is possible to selectively prune items like:
```
$ docker container prune --filter label=foo

$ docker container prune --filter label!=bar
```

Additional unit tests and integration tests have been added.

This fix fixes 29999.

Signed-off-by: Yong Tang &lt;yong.tang.github@outlook.com&gt;
diff --git a/api/server/router/volume/volume_routes.go b/api/server/router/volume/volume_routes.go
@@ -72,7 +72,12 @@ func (v *volumeRouter) postVolumesPrune(ctx context.Context, w http.ResponseWrit
 		return err
 	}
 
-	pruneReport, err := v.backend.VolumesPrune(filters.Args{})
+	pruneFilters, err := filters.FromParam(r.Form.Get("filters"))
+	if err != nil {
+		return err
+	}
+
+	pruneReport, err := v.backend.VolumesPrune(pruneFilters)
 	if err != nil {
 		return err
 	}
diff --git a/cli/command/container/prune.go b/cli/command/container/prune.go
@@ -49,7 +49,7 @@ const warning = `WARNING! This will remove all stopped containers.
 Are you sure you want to continue?`
 
 func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
-	pruneFilters := opts.filter.Value()
+	pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
 
 	if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
 		return
diff --git a/cli/command/image/prune.go b/cli/command/image/prune.go
@@ -58,6 +58,7 @@ Are you sure you want to continue?`
 func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
 	pruneFilters := opts.filter.Value()
 	pruneFilters.Add("dangling", fmt.Sprintf("%v", !opts.all))
+	pruneFilters = command.PruneFilters(dockerCli, pruneFilters)
 
 	warning := danglingWarning
 	if opts.all {
diff --git a/cli/command/network/prune.go b/cli/command/network/prune.go
@@ -48,7 +48,7 @@ const warning = `WARNING! This will remove all networks not used by at least one
 Are you sure you want to continue?`
 
 func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (output string, err error) {
-	pruneFilters := opts.filter.Value()
+	pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
 
 	if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
 		return
diff --git a/cli/command/prune/prune.go b/cli/command/prune/prune.go
@@ -37,7 +37,7 @@ func RunContainerPrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uin
 
 // RunVolumePrune executes a prune command for volumes
 func RunVolumePrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uint64, string, error) {
-	return volume.RunPrune(dockerCli)
+	return volume.RunPrune(dockerCli, filter)
 }
 
 // RunImagePrune executes a prune command for images
diff --git a/cli/command/utils.go b/cli/command/utils.go
@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/pkg/system"
 )
 
@@ -85,3 +86,34 @@ func PromptForConfirmation(ins *InStream, outs *OutStream, message string) bool
 	answer, _, _ := reader.ReadLine()
 	return strings.ToLower(string(answer)) == "y"
 }
+
+// PruneFilters returns consolidated prune filters obtained from config.json and cli
+func PruneFilters(dockerCli Cli, pruneFilters filters.Args) filters.Args {
+	if dockerCli.ConfigFile() == nil {
+		return pruneFilters
+	}
+	for _, f := range dockerCli.ConfigFile().PruneFilters {
+		parts := strings.SplitN(f, "=", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		if parts[0] == "label" {
+			// CLI label filter supersede config.json.
+			// If CLI label filter conflict with config.json,
+			// skip adding label! filter in config.json.
+			if pruneFilters.Include("label!") && pruneFilters.ExactMatch("label!", parts[1]) {
+				continue
+			}
+		} else if parts[0] == "label!" {
+			// CLI label! filter supersede config.json.
+			// If CLI label! filter conflict with config.json,
+			// skip adding label filter in config.json.
+			if pruneFilters.Include("label") && pruneFilters.ExactMatch("label", parts[1]) {
+				continue
+			}
+		}
+		pruneFilters.Add(parts[0], parts[1])
+	}
+
+	return pruneFilters
+}
diff --git a/cli/command/volume/prune.go b/cli/command/volume/prune.go
@@ -3,21 +3,22 @@ package volume
 import (
 	"fmt"
 
-	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
+	"github.com/docker/docker/opts"
 	units "github.com/docker/go-units"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context"
 )
 
 type pruneOptions struct {
-	force bool
+	force  bool
+	filter opts.FilterOpt
 }
 
 // NewPruneCommand returns a new cobra prune command for volumes
 func NewPruneCommand(dockerCli command.Cli) *cobra.Command {
-	var opts pruneOptions
+	opts := pruneOptions{filter: opts.NewFilterOpt()}
 
 	cmd := &cobra.Command{
 		Use:   "prune [OPTIONS]",
@@ -39,6 +40,7 @@ func NewPruneCommand(dockerCli command.Cli) *cobra.Command {
 
 	flags := cmd.Flags()
 	flags.BoolVarP(&opts.force, "force", "f", false, "Do not prompt for confirmation")
+	flags.Var(&opts.filter, "filter", "Provide filter values (e.g. 'label=<label>')")
 
 	return cmd
 }
@@ -47,11 +49,13 @@ const warning = `WARNING! This will remove all volumes not used by at least one
 Are you sure you want to continue?`
 
 func runPrune(dockerCli command.Cli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
+	pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
+
 	if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
 		return
 	}
 
-	report, err := dockerCli.Client().VolumesPrune(context.Background(), filters.Args{})
+	report, err := dockerCli.Client().VolumesPrune(context.Background(), pruneFilters)
 	if err != nil {
 		return
 	}
@@ -69,6 +73,6 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) (spaceReclaimed uint64,
 
 // RunPrune calls the Volume Prune API
 // This returns the amount of space reclaimed and a detailed output string
-func RunPrune(dockerCli *command.DockerCli) (uint64, string, error) {
-	return runPrune(dockerCli, pruneOptions{force: true})
+func RunPrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uint64, string, error) {
+	return runPrune(dockerCli, pruneOptions{force: true, filter: filter})
 }
diff --git a/cli/config/configfile/file.go b/cli/config/configfile/file.go
@@ -39,6 +39,7 @@ type ConfigFile struct {
 	TasksFormat          string                      `json:"tasksFormat,omitempty"`
 	SecretFormat         string                      `json:"secretFormat,omitempty"`
 	NodesFormat          string                      `json:"nodesFormat,omitempty"`
+	PruneFilters         []string                    `json:"pruneFilters,omitempty"`
 }
 
 // LegacyLoadFromReader reads the non-nested configuration data given and sets up the
diff --git a/client/container_prune_test.go b/client/container_prune_test.go
@@ -40,6 +40,11 @@ func TestContainersPrune(t *testing.T) {
 	danglingUntilFilters.Add("dangling", "true")
 	danglingUntilFilters.Add("until", "2016-12-15T14:00")
 
+	labelFilters := filters.NewArgs()
+	labelFilters.Add("dangling", "true")
+	labelFilters.Add("label", "label1=foo")
+	labelFilters.Add("label", "label2!=bar")
+
 	listCases := []struct {
 		filters             filters.Args
 		expectedQueryParams map[string]string
@@ -76,6 +81,14 @@ func TestContainersPrune(t *testing.T) {
 				"filters": `{"dangling":{"false":true}}`,
 			},
 		},
+		{
+			filters: labelFilters,
+			expectedQueryParams: map[string]string{
+				"until":   "",
+				"filter":  "",
+				"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
+			},
+		},
 	}
 	for _, listCase := range listCases {
 		client := &Client{
diff --git a/client/image_prune_test.go b/client/image_prune_test.go
@@ -36,6 +36,11 @@ func TestImagesPrune(t *testing.T) {
 	noDanglingFilters := filters.NewArgs()
 	noDanglingFilters.Add("dangling", "false")
 
+	labelFilters := filters.NewArgs()
+	labelFilters.Add("dangling", "true")
+	labelFilters.Add("label", "label1=foo")
+	labelFilters.Add("label", "label2!=bar")
+
 	listCases := []struct {
 		filters             filters.Args
 		expectedQueryParams map[string]string
@@ -64,6 +69,14 @@ func TestImagesPrune(t *testing.T) {
 				"filters": `{"dangling":{"false":true}}`,
 			},
 		},
+		{
+			filters: labelFilters,
+			expectedQueryParams: map[string]string{
+				"until":   "",
+				"filter":  "",
+				"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
+			},
+		},
 	}
 	for _, listCase := range listCases {
 		client := &Client{
diff --git a/client/network_prune_test.go b/client/network_prune_test.go
@@ -38,6 +38,11 @@ func TestNetworksPrune(t *testing.T) {
 	noDanglingFilters := filters.NewArgs()
 	noDanglingFilters.Add("dangling", "false")
 
+	labelFilters := filters.NewArgs()
+	labelFilters.Add("dangling", "true")
+	labelFilters.Add("label", "label1=foo")
+	labelFilters.Add("label", "label2!=bar")
+
 	listCases := []struct {
 		filters             filters.Args
 		expectedQueryParams map[string]string
@@ -66,6 +71,14 @@ func TestNetworksPrune(t *testing.T) {
 				"filters": `{"dangling":{"false":true}}`,
 			},
 		},
+		{
+			filters: labelFilters,
+			expectedQueryParams: map[string]string{
+				"until":   "",
+				"filter":  "",
+				"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
+			},
+		},
 	}
 	for _, listCase := range listCases {
 		client := &Client{
diff --git a/daemon/prune.go b/daemon/prune.go
@@ -34,6 +34,9 @@ func (daemon *Daemon) ContainersPrune(pruneFilters filters.Args) (*types.Contain
 			if !until.IsZero() && c.Created.After(until) {
 				continue
 			}
+			if !matchLabels(pruneFilters, c.Config.Labels) {
+				continue
+			}
 			cSize, _ := daemon.getSize(c.ID)
 			// TODO: sets RmLink to true?
 			err := daemon.ContainerRm(c.ID, &types.ContainerRmConfig{})
@@ -60,6 +63,12 @@ func (daemon *Daemon) VolumesPrune(pruneFilters filters.Args) (*types.VolumesPru
 		refs := daemon.volumes.Refs(v)
 
 		if len(refs) == 0 {
+			detailedVolume, ok := v.(volume.DetailedVolume)
+			if ok {
+				if !matchLabels(pruneFilters, detailedVolume.Labels()) {
+					return nil
+				}
+			}
 			vSize, err := directory.Size(v.Path())
 			if err != nil {
 				logrus.Warnf("could not determine size of volume %s: %v", name, err)
@@ -122,6 +131,9 @@ func (daemon *Daemon) ImagesPrune(pruneFilters filters.Args) (*types.ImagesPrune
 		if !until.IsZero() && img.Created.After(until) {
 			continue
 		}
+		if !matchLabels(pruneFilters, img.Config.Labels) {
+			continue
+		}
 		topImages[id] = img
 	}
 
@@ -200,6 +212,9 @@ func (daemon *Daemon) localNetworksPrune(pruneFilters filters.Args) *types.Netwo
 		if !until.IsZero() && nw.Info().Created().After(until) {
 			return false
 		}
+		if !matchLabels(pruneFilters, nw.Info().Labels()) {
+			return false
+		}
 		nwName := nw.Name()
 		if runconfig.IsPreDefinedNetwork(nwName) {
 			return false
@@ -243,6 +258,9 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
 		if !until.IsZero() && nw.Created.After(until) {
 			continue
 		}
+		if !matchLabels(pruneFilters, nw.Labels) {
+			continue
+		}
 		// https://github.com/docker/docker/issues/24186
 		// `docker network inspect` unfortunately displays ONLY those containers that are local to that node.
 		// So we try to remove it anyway and check the error
@@ -266,12 +284,10 @@ func (daemon *Daemon) NetworksPrune(pruneFilters filters.Args) (*types.NetworksP
 		return nil, err
 	}
 
-	clusterRep, err := daemon.clusterNetworksPrune(pruneFilters)
-	if err != nil {
-		return nil, fmt.Errorf("could not remove cluster networks: %s", err)
-	}
 	rep := &types.NetworksPruneReport{}
-	rep.NetworksDeleted = append(rep.NetworksDeleted, clusterRep.NetworksDeleted...)
+	if clusterRep, err := daemon.clusterNetworksPrune(pruneFilters); err == nil {
+		rep.NetworksDeleted = append(rep.NetworksDeleted, clusterRep.NetworksDeleted...)
+	}
 
 	localRep := daemon.localNetworksPrune(pruneFilters)
 	rep.NetworksDeleted = append(rep.NetworksDeleted, localRep.NetworksDeleted...)
@@ -298,3 +314,17 @@ func getUntilFromPruneFilters(pruneFilters filters.Args) (time.Time, error) {
 	until = time.Unix(seconds, nanoseconds)
 	return until, nil
 }
+
+func matchLabels(pruneFilters filters.Args, labels map[string]string) bool {
+	if !pruneFilters.MatchKVList("label", labels) {
+		return false
+	}
+	// By default MatchKVList will return true if field (like 'label!') does not exist
+	// So we have to add additional Include("label!") check
+	if pruneFilters.Include("label!") {
+		if pruneFilters.MatchKVList("label!", labels) {
+			return false
+		}
+	}
+	return true
+}
diff --git a/daemon/volumes.go b/daemon/volumes.go
@@ -295,9 +295,12 @@ func (daemon *Daemon) traverseLocalVolumes(fn func(volume.Volume) error) error {
 
 	for _, v := range vols {
 		name := v.Name()
-		_, err := daemon.volumes.Get(name)
+		vol, err := daemon.volumes.Get(name)
 		if err != nil {
 			logrus.Warnf("failed to retrieve volume %s from store: %v", name, err)
+		} else {
+			// daemon.volumes.Get will return DetailedVolume
+			v = vol
 		}
 
 		err = fn(v)
diff --git a/integration-cli/docker_cli_prune_unix_test.go b/integration-cli/docker_cli_prune_unix_test.go

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,12 @@ func (v *volumeRouter) postVolumesPrune(ctx context.Context, w http.ResponseWrit`
`72`	`72`	`return err`
`73`	`73`	`}`
`74`	`74`
`75`		`- pruneReport, err := v.backend.VolumesPrune(filters.Args{})`
	`75`	`+ pruneFilters, err := filters.FromParam(r.Form.Get("filters"))`
	`76`	`+ if err != nil {`
	`77`	`+ return err`
	`78`	`+ }`
	`79`	`+`
	`80`	`+ pruneReport, err := v.backend.VolumesPrune(pruneFilters)`
`76`	`81`	`if err != nil {`
`77`	`82`	`return err`
`78`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ func RunContainerPrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uin`
`37`	`37`
`38`	`38`	`// RunVolumePrune executes a prune command for volumes`
`39`	`39`	`func RunVolumePrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uint64, string, error) {`
`40`		`- return volume.RunPrune(dockerCli)`
	`40`	`+ return volume.RunPrune(dockerCli, filter)`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`// RunImagePrune executes a prune command for images`
Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ type ConfigFile struct {`
`39`	`39`	TasksFormat string `json:"tasksFormat,omitempty"`
`40`	`40`	SecretFormat string `json:"secretFormat,omitempty"`
`41`	`41`	NodesFormat string `json:"nodesFormat,omitempty"`
	`42`	+ PruneFilters []string `json:"pruneFilters,omitempty"`
`42`	`43`	`}`
`43`	`44`
`44`	`45`	`// LegacyLoadFromReader reads the non-nested configuration data given and sets up the`