Skip to content

Commit da1065f

Browse files
smarterclaytonsmarterclayton
committed
Support api chunking in kubectl get
This enables chunking in the resource builder to make it easy to retrieve resources in pages and visit partial result sets. This adds `--chunk-size` to `kubectl get` only so that users can get comfortable with the use of chunking in beta. Future changes will enable chunking for all CLI commands so that bulk actions can be performed more efficiently.
1 parent 0b1da1f commit da1065f

File tree

9 files changed

+112
-38
lines changed

9 files changed

+112
-38
lines changed

‎hack/make-rules/test-cmd-util.sh

+12-1
Original file line numberDiff line numberDiff line change
@@ -1345,7 +1345,7 @@ run_kubectl_get_tests() {
13451345
fi
13461346

13471347
### Test kubectl get all
1348-
output_message=$(kubectl --v=6 --namespace default get all 2>&1 "${kube_flags[@]}")
1348+
output_message=$(kubectl --v=6 --namespace default get all --experimental-chunk-size=0 2>&1 "${kube_flags[@]}")
13491349
# Post-condition: Check if we get 200 OK from all the url(s)
13501350
kube::test::if_has_string "${output_message}" "/api/v1/namespaces/default/pods 200 OK"
13511351
kube::test::if_has_string "${output_message}" "/api/v1/namespaces/default/replicationcontrollers 200 OK"
@@ -1356,6 +1356,17 @@ run_kubectl_get_tests() {
13561356
kube::test::if_has_string "${output_message}" "/apis/extensions/v1beta1/namespaces/default/deployments 200 OK"
13571357
kube::test::if_has_string "${output_message}" "/apis/extensions/v1beta1/namespaces/default/replicasets 200 OK"
13581358

1359+
### Test kubectl get chunk size
1360+
output_message=$(kubectl --v=6 get clusterrole --experimental-chunk-size=10 2>&1 "${kube_flags[@]}")
1361+
# Post-condition: Check if we get a limit and continue
1362+
kube::test::if_has_string "${output_message}" "/clusterroles?limit=10 200 OK"
1363+
kube::test::if_has_string "${output_message}" "/v1/clusterroles?continue="
1364+
1365+
### Test kubectl get chunk size defaults to 500
1366+
output_message=$(kubectl --v=6 get clusterrole 2>&1 "${kube_flags[@]}")
1367+
# Post-condition: Check if we get a limit and continue
1368+
kube::test::if_has_string "${output_message}" "/clusterroles?limit=500 200 OK"
1369+
13591370
### Test --allow-missing-template-keys
13601371
# Pre-condition: no POD exists
13611372
create_and_use_new_namespace

‎pkg/kubectl/cmd/apply.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,15 @@ func (p *pruner) prune(namespace string, mapping *meta.RESTMapping, shortOutput,
462462
return err
463463
}
464464

465-
objList, err := resource.NewHelper(c, mapping).List(namespace, mapping.GroupVersionKind.Version, p.selector, false, includeUninitialized)
465+
objList, err := resource.NewHelper(c, mapping).List(
466+
namespace,
467+
mapping.GroupVersionKind.Version,
468+
false,
469+
&metav1.ListOptions{
470+
LabelSelector: p.selector,
471+
IncludeUninitialized: includeUninitialized,
472+
},
473+
)
466474
if err != nil {
467475
return err
468476
}

‎pkg/kubectl/cmd/get.go

+4
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ type GetOptions struct {
5151

5252
IgnoreNotFound bool
5353
Raw string
54+
ChunkSize int64
5455
}
5556

5657
var (
@@ -137,6 +138,7 @@ func NewCmdGet(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Comman
137138
cmd.Flags().Bool("show-kind", false, "If present, list the resource type for the requested object(s).")
138139
cmd.Flags().Bool("all-namespaces", false, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.")
139140
cmd.Flags().BoolVar(&options.IgnoreNotFound, "ignore-not-found", false, "Treat \"resource not found\" as a successful retrieval.")
141+
cmd.Flags().Int64Var(&options.ChunkSize, "experimental-chunk-size", 500, "Return large lists in chunks rather than all at once. Pass 0 to disable.")
140142
cmd.Flags().StringSliceP("label-columns", "L", []string{}, "Accepts a comma separated list of labels that are going to be presented as columns. Names are case-sensitive. You can also use multiple flag options like -L label1 -L label2...")
141143
cmd.Flags().Bool("export", false, "If true, use 'export' for the resources. Exported resources are stripped of cluster-specific information.")
142144
addOpenAPIPrintColumnFlags(cmd)
@@ -223,6 +225,7 @@ func RunGet(f cmdutil.Factory, out, errOut io.Writer, cmd *cobra.Command, args [
223225
FilenameParam(enforceNamespace, &options.FilenameOptions).
224226
SelectorParam(selector).
225227
ExportParam(export).
228+
RequestChunksOf(options.ChunkSize).
226229
IncludeUninitialized(includeUninitialized).
227230
ResourceTypeOrNameArgs(true, args...).
228231
SingleResourceType().
@@ -325,6 +328,7 @@ func RunGet(f cmdutil.Factory, out, errOut io.Writer, cmd *cobra.Command, args [
325328
FilenameParam(enforceNamespace, &options.FilenameOptions).
326329
SelectorParam(selector).
327330
ExportParam(export).
331+
RequestChunksOf(options.ChunkSize).
328332
IncludeUninitialized(includeUninitialized).
329333
ResourceTypeOrNameArgs(true, args...).
330334
ContinueOnError().

‎pkg/kubectl/resource/builder.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ type Builder struct {
5656
selector *string
5757
selectAll bool
5858
includeUninitialized bool
59+
limitChunks int64
5960

6061
resources []string
6162

@@ -355,6 +356,14 @@ func (b *Builder) RequireNamespace() *Builder {
355356
return b
356357
}
357358

359+
// RequestChunksOf attempts to load responses from the server in batches of size limit
360+
// to avoid long delays loading and transferring very large lists. If unset defaults to
361+
// no chunking.
362+
func (b *Builder) RequestChunksOf(chunkSize int64) *Builder {
363+
b.limitChunks = chunkSize
364+
return b
365+
}
366+
358367
// SelectEverythingParam
359368
func (b *Builder) SelectAllParam(selectAll bool) *Builder {
360369
if selectAll && b.selector != nil {
@@ -653,7 +662,7 @@ func (b *Builder) visitBySelector() *Result {
653662
if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
654663
selectorNamespace = ""
655664
}
656-
visitors = append(visitors, NewSelector(client, mapping, selectorNamespace, *b.selector, b.export, b.includeUninitialized))
665+
visitors = append(visitors, NewSelector(client, mapping, selectorNamespace, *b.selector, b.export, b.includeUninitialized, b.limitChunks))
657666
}
658667
if b.continueOnError {
659668
result.visitor = EagerVisitorList(visitors)

‎pkg/kubectl/resource/helper.go

+2-8
Original file line numberDiff line numberDiff line change
@@ -63,21 +63,15 @@ func (m *Helper) Get(namespace, name string, export bool) (runtime.Object, error
6363
return req.Do().Get()
6464
}
6565

66-
// TODO: add field selector
67-
func (m *Helper) List(namespace, apiVersion string, selector string, export, includeUninitialized bool) (runtime.Object, error) {
66+
func (m *Helper) List(namespace, apiVersion string, export bool, options *metav1.ListOptions) (runtime.Object, error) {
6867
req := m.RESTClient.Get().
6968
NamespaceIfScoped(namespace, m.NamespaceScoped).
7069
Resource(m.Resource).
71-
VersionedParams(&metav1.ListOptions{
72-
LabelSelector: selector,
73-
}, metav1.ParameterCodec)
70+
VersionedParams(options, metav1.ParameterCodec)
7471
if export {
7572
// TODO: I should be part of ListOptions
7673
req.Param("export", strconv.FormatBool(export))
7774
}
78-
if includeUninitialized {
79-
req.Param("includeUninitialized", strconv.FormatBool(includeUninitialized))
80-
}
8175
return req.Do().Get()
8276
}
8377

‎pkg/kubectl/resource/helper_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ func TestHelperList(t *testing.T) {
358358
RESTClient: client,
359359
NamespaceScoped: true,
360360
}
361-
obj, err := modifier.List("bar", api.Registry.GroupOrDie(api.GroupName).GroupVersion.String(), "foo=baz", false, false)
361+
obj, err := modifier.List("bar", api.Registry.GroupOrDie(api.GroupName).GroupVersion.String(), false, &metav1.ListOptions{LabelSelector: "foo=baz"})
362362
if (err != nil) != test.Err {
363363
t.Errorf("unexpected error: %t %v", test.Err, err)
364364
}

‎pkg/kubectl/resource/selector.go

+54-26
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"k8s.io/apimachinery/pkg/api/errors"
2323
"k8s.io/apimachinery/pkg/api/meta"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2425
"k8s.io/apimachinery/pkg/watch"
2526
)
2627

@@ -32,53 +33,80 @@ type Selector struct {
3233
Selector string
3334
Export bool
3435
IncludeUninitialized bool
36+
LimitChunks int64
3537
}
3638

3739
// NewSelector creates a resource selector which hides details of getting items by their label selector.
38-
func NewSelector(client RESTClient, mapping *meta.RESTMapping, namespace string, selector string, export, includeUninitialized bool) *Selector {
40+
func NewSelector(client RESTClient, mapping *meta.RESTMapping, namespace string, selector string, export, includeUninitialized bool, limitChunks int64) *Selector {
3941
return &Selector{
4042
Client: client,
4143
Mapping: mapping,
4244
Namespace: namespace,
4345
Selector: selector,
4446
Export: export,
4547
IncludeUninitialized: includeUninitialized,
48+
LimitChunks: limitChunks,
4649
}
4750
}
4851

49-
// Visit implements Visitor
52+
// Visit implements Visitor and uses request chunking by default.
5053
func (r *Selector) Visit(fn VisitorFunc) error {
51-
list, err := NewHelper(r.Client, r.Mapping).List(r.Namespace, r.ResourceMapping().GroupVersionKind.GroupVersion().String(), r.Selector, r.Export, r.IncludeUninitialized)
52-
if err != nil {
53-
if errors.IsBadRequest(err) || errors.IsNotFound(err) {
54-
if se, ok := err.(*errors.StatusError); ok {
55-
// modify the message without hiding this is an API error
54+
var continueToken string
55+
for {
56+
list, err := NewHelper(r.Client, r.Mapping).List(
57+
r.Namespace,
58+
r.ResourceMapping().GroupVersionKind.GroupVersion().String(),
59+
r.Export,
60+
&metav1.ListOptions{
61+
LabelSelector: r.Selector,
62+
IncludeUninitialized: r.IncludeUninitialized,
63+
Limit: r.LimitChunks,
64+
Continue: continueToken,
65+
},
66+
)
67+
if err != nil {
68+
if errors.IsResourceExpired(err) {
69+
return err
70+
}
71+
if errors.IsBadRequest(err) || errors.IsNotFound(err) {
72+
if se, ok := err.(*errors.StatusError); ok {
73+
// modify the message without hiding this is an API error
74+
if len(r.Selector) == 0 {
75+
se.ErrStatus.Message = fmt.Sprintf("Unable to list %q: %v", r.Mapping.Resource, se.ErrStatus.Message)
76+
} else {
77+
se.ErrStatus.Message = fmt.Sprintf("Unable to find %q that match the selector %q: %v", r.Mapping.Resource, r.Selector, se.ErrStatus.Message)
78+
}
79+
return se
80+
}
5681
if len(r.Selector) == 0 {
57-
se.ErrStatus.Message = fmt.Sprintf("Unable to list %q: %v", r.Mapping.Resource, se.ErrStatus.Message)
58-
} else {
59-
se.ErrStatus.Message = fmt.Sprintf("Unable to find %q that match the selector %q: %v", r.Mapping.Resource, r.Selector, se.ErrStatus.Message)
82+
return fmt.Errorf("Unable to list %q: %v", r.Mapping.Resource, err)
6083
}
61-
return se
62-
}
63-
if len(r.Selector) == 0 {
64-
return fmt.Errorf("Unable to list %q: %v", r.Mapping.Resource, err)
65-
} else {
6684
return fmt.Errorf("Unable to find %q that match the selector %q: %v", r.Mapping.Resource, r.Selector, err)
6785
}
86+
if err := fn(nil, err); err != nil {
87+
return err
88+
}
89+
continue
6890
}
69-
return err
70-
}
71-
accessor := r.Mapping.MetadataAccessor
72-
resourceVersion, _ := accessor.ResourceVersion(list)
73-
info := &Info{
74-
Client: r.Client,
75-
Mapping: r.Mapping,
76-
Namespace: r.Namespace,
91+
accessor := r.Mapping.MetadataAccessor
92+
resourceVersion, _ := accessor.ResourceVersion(list)
93+
nextContinueToken, _ := accessor.Continue(list)
94+
info := &Info{
95+
Client: r.Client,
96+
Mapping: r.Mapping,
97+
Namespace: r.Namespace,
7798

78-
Object: list,
79-
ResourceVersion: resourceVersion,
99+
Object: list,
100+
ResourceVersion: resourceVersion,
101+
}
102+
if err := fn(info, nil); err != nil {
103+
return err
104+
}
105+
if len(nextContinueToken) == 0 {
106+
return nil
107+
}
108+
continueToken = nextContinueToken
80109
}
81-
return fn(info, nil)
82110
}
83111

84112
func (r *Selector) Watch(resourceVersion string) (watch.Interface, error) {

‎staging/src/k8s.io/apimachinery/pkg/api/meta/interfaces.go

+3
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ type MetadataAccessor interface {
7575
Annotations(obj runtime.Object) (map[string]string, error)
7676
SetAnnotations(obj runtime.Object, annotations map[string]string) error
7777

78+
Continue(obj runtime.Object) (string, error)
79+
SetContinue(obj runtime.Object, c string) error
80+
7881
runtime.ResourceVersioner
7982
}
8083

‎staging/src/k8s.io/apimachinery/pkg/api/meta/meta.go

+17
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,23 @@ func (resourceAccessor) SetResourceVersion(obj runtime.Object, version string) e
367367
return nil
368368
}
369369

370+
func (resourceAccessor) Continue(obj runtime.Object) (string, error) {
371+
accessor, err := ListAccessor(obj)
372+
if err != nil {
373+
return "", err
374+
}
375+
return accessor.GetContinue(), nil
376+
}
377+
378+
func (resourceAccessor) SetContinue(obj runtime.Object, version string) error {
379+
accessor, err := ListAccessor(obj)
380+
if err != nil {
381+
return err
382+
}
383+
accessor.SetContinue(version)
384+
return nil
385+
}
386+
370387
// extractFromOwnerReference extracts v to o. v is the OwnerReferences field of an object.
371388
func extractFromOwnerReference(v reflect.Value, o *metav1.OwnerReference) error {
372389
if err := runtime.Field(v, "APIVersion", &o.APIVersion); err != nil {

0 commit comments

Comments
 (0)