chunking.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"reflect"
    23  	"time"
    24  
    25  	"github.com/onsi/ginkgo/v2"
    26  	"github.com/onsi/gomega"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/util/wait"
    32  	"k8s.io/apiserver/pkg/storage/storagebackend"
    33  	"k8s.io/client-go/util/workqueue"
    34  	"k8s.io/kubernetes/test/e2e/framework"
    35  	admissionapi "k8s.io/pod-security-admission/api"
    36  )
    37  
    38  const numberOfTotalResources = 400
    39  
    40  var _ = SIGDescribe("Servers with support for API chunking", func() {
    41  	f := framework.NewDefaultFramework("chunking")
    42  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    43  
    44  	ginkgo.BeforeEach(func(ctx context.Context) {
    45  		ns := f.Namespace.Name
    46  		c := f.ClientSet
    47  		client := c.CoreV1().PodTemplates(ns)
    48  		ginkgo.By("creating a large number of resources")
    49  		workqueue.ParallelizeUntil(ctx, 20, numberOfTotalResources, func(i int) {
    50  			for tries := 3; tries >= 0; tries-- {
    51  				_, err := client.Create(ctx, &v1.PodTemplate{
    52  					ObjectMeta: metav1.ObjectMeta{
    53  						Name: fmt.Sprintf("template-%04d", i),
    54  					},
    55  					Template: v1.PodTemplateSpec{
    56  						Spec: v1.PodSpec{
    57  							Containers: []v1.Container{
    58  								{Name: "test", Image: "test2"},
    59  							},
    60  						},
    61  					},
    62  				}, metav1.CreateOptions{})
    63  				if err == nil {
    64  					return
    65  				}
    66  				framework.Logf("Got an error creating template %d: %v", i, err)
    67  			}
    68  			framework.Failf("Unable to create template %d, exiting", i)
    69  		})
    70  	})
    71  
    72  	/*
    73  		Release: v1.29
    74  		Testname: API Chunking, server should return chunks of results for list calls
    75  		Description: Create a large number of PodTemplates. Attempt to retrieve the first chunk with limit set;
    76  		the server MUST return the chunk of the size not exceeding the limit with RemainingItems set in the response.
    77  		Attempt to retrieve the remaining items by providing the received continuation token and limit;
    78  		the server MUST return the remaining items in chunks of the size not exceeding the limit, with appropriately
    79  		set RemainingItems field in the response and with the ResourceVersion returned in the first response.
    80  		Attempt to list all objects at once without setting the limit; the server MUST return all items in a single
    81  		response.
    82  	*/
    83  	framework.ConformanceIt("should return chunks of results for list calls", func(ctx context.Context) {
    84  		ns := f.Namespace.Name
    85  		c := f.ClientSet
    86  		client := c.CoreV1().PodTemplates(ns)
    87  		ginkgo.By("retrieving those results in paged fashion several times")
    88  		for i := 0; i < 3; i++ {
    89  			opts := metav1.ListOptions{}
    90  			found := 0
    91  			var lastRV string
    92  			for {
    93  				// With numberOfTotalResources=400, we want to ensure that both
    94  				// number of items per page and number of pages are non-trivial.
    95  				opts.Limit = 17
    96  				list, err := client.List(ctx, opts)
    97  				framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit)
    98  				framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, list.Continue)
    99  				gomega.Expect(len(list.Items)).To(gomega.BeNumerically("<=", opts.Limit))
   100  
   101  				if len(lastRV) == 0 {
   102  					lastRV = list.ResourceVersion
   103  				}
   104  				gomega.Expect(list.ResourceVersion).To(gomega.Equal(lastRV))
   105  				if list.GetContinue() == "" {
   106  					gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil())
   107  				} else {
   108  					gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil())
   109  					gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources))
   110  				}
   111  				for _, item := range list.Items {
   112  					gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found)))
   113  					found++
   114  				}
   115  				if len(list.Continue) == 0 {
   116  					break
   117  				}
   118  				opts.Continue = list.Continue
   119  			}
   120  			gomega.Expect(found).To(gomega.BeNumerically("==", numberOfTotalResources))
   121  		}
   122  
   123  		ginkgo.By("retrieving those results all at once")
   124  		opts := metav1.ListOptions{Limit: numberOfTotalResources + 1}
   125  		list, err := client.List(ctx, opts)
   126  		framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit)
   127  		gomega.Expect(list.Items).To(gomega.HaveLen(numberOfTotalResources))
   128  	})
   129  
   130  	/*
   131  		Release: v1.29
   132  		Testname: API Chunking, server should support continue listing from the last key even if the original version has been compacted away
   133  		Description: Create a large number of PodTemplates. Attempt to retrieve the first chunk with limit set;
   134  		the server MUST return the chunk of the size not exceeding the limit with RemainingItems set in the response.
   135  		Attempt to retrieve the second page until the continuation token expires; the server MUST return a
   136  		continuation token for inconsistent list continuation.
   137  		Attempt to retrieve the second page with the received inconsistent list continuation token; the server
   138  		MUST return the number of items not exceeding the limit, a new continuation token and appropriately set
   139  		RemainingItems field in the response.
   140  		Attempt to retrieve the remaining pages by passing the received continuation token; the server
   141  		MUST return the remaining items in chunks of the size not exceeding the limit, with appropriately
   142  		set RemainingItems field in the response and with the ResourceVersion returned as part of the inconsistent list.
   143  	*/
   144  	framework.ConformanceIt("should support continue listing from the last key if the original version has been compacted away, though the list is inconsistent", f.WithSlow(), func(ctx context.Context) {
   145  		ns := f.Namespace.Name
   146  		c := f.ClientSet
   147  		client := c.CoreV1().PodTemplates(ns)
   148  
   149  		ginkgo.By("retrieving the first page")
   150  		oneTenth := int64(numberOfTotalResources / 10)
   151  		opts := metav1.ListOptions{}
   152  		opts.Limit = oneTenth
   153  		list, err := client.List(ctx, opts)
   154  		framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit)
   155  		firstToken := list.Continue
   156  		firstRV := list.ResourceVersion
   157  		if list.GetContinue() == "" {
   158  			gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil())
   159  		} else {
   160  			gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil())
   161  			gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items)).To(gomega.BeNumerically("==", numberOfTotalResources))
   162  		}
   163  		framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, firstToken)
   164  
   165  		ginkgo.By("retrieving the second page until the token expires")
   166  		opts.Continue = firstToken
   167  		var inconsistentToken string
   168  		wait.Poll(20*time.Second, 2*storagebackend.DefaultCompactInterval, func() (bool, error) {
   169  			_, err := client.List(ctx, opts)
   170  			if err == nil {
   171  				framework.Logf("Token %s has not expired yet", firstToken)
   172  				return false, nil
   173  			}
   174  			if err != nil && !apierrors.IsResourceExpired(err) {
   175  				return false, err
   176  			}
   177  			framework.Logf("got error %s", err)
   178  			status, ok := err.(apierrors.APIStatus)
   179  			if !ok {
   180  				return false, fmt.Errorf("expect error to implement the APIStatus interface, got %v", reflect.TypeOf(err))
   181  			}
   182  			inconsistentToken = status.Status().ListMeta.Continue
   183  			if len(inconsistentToken) == 0 {
   184  				return false, fmt.Errorf("expect non empty continue token")
   185  			}
   186  			framework.Logf("Retrieved inconsistent continue %s", inconsistentToken)
   187  			return true, nil
   188  		})
   189  
   190  		ginkgo.By("retrieving the second page again with the token received with the error message")
   191  		opts.Continue = inconsistentToken
   192  		list, err = client.List(ctx, opts)
   193  		framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given inconsistent continue token %s and limit: %d", ns, opts.Continue, opts.Limit)
   194  		gomega.Expect(list.ResourceVersion).ToNot(gomega.Equal(firstRV))
   195  		gomega.Expect(len(list.Items)).To(gomega.BeNumerically("==", opts.Limit))
   196  		found := int(oneTenth)
   197  
   198  		if list.GetContinue() == "" {
   199  			gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil())
   200  		} else {
   201  			gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil())
   202  			gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources))
   203  		}
   204  		for _, item := range list.Items {
   205  			gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found)))
   206  			found++
   207  		}
   208  
   209  		ginkgo.By("retrieving all remaining pages")
   210  		opts.Continue = list.Continue
   211  		lastRV := list.ResourceVersion
   212  		for {
   213  			list, err := client.List(ctx, opts)
   214  			framework.ExpectNoError(err, "failed to list pod templates in namespace: %s, given limit: %d", ns, opts.Limit)
   215  			if list.GetContinue() == "" {
   216  				gomega.Expect(list.GetRemainingItemCount()).To(gomega.BeNil())
   217  			} else {
   218  				gomega.Expect(list.GetRemainingItemCount()).ToNot(gomega.BeNil())
   219  				gomega.Expect(int(*list.GetRemainingItemCount()) + len(list.Items) + found).To(gomega.BeNumerically("==", numberOfTotalResources))
   220  			}
   221  			framework.Logf("Retrieved %d/%d results with rv %s and continue %s", len(list.Items), opts.Limit, list.ResourceVersion, list.Continue)
   222  			gomega.Expect(len(list.Items)).To(gomega.BeNumerically("<=", opts.Limit))
   223  			gomega.Expect(list.ResourceVersion).To(gomega.Equal(lastRV))
   224  			for _, item := range list.Items {
   225  				gomega.Expect(item.Name).To(gomega.Equal(fmt.Sprintf("template-%04d", found)))
   226  				found++
   227  			}
   228  			if len(list.Continue) == 0 {
   229  				break
   230  			}
   231  			opts.Continue = list.Continue
   232  		}
   233  		gomega.Expect(found).To(gomega.BeNumerically("==", numberOfTotalResources))
   234  	})
   235  })
   236
View as plain text