controller_revision.go

Documentation: k8s.io/kubernetes/test/e2e/apps

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apps
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/onsi/ginkgo/v2"
    25  	"github.com/onsi/gomega"
    26  
    27  	appsv1 "k8s.io/api/apps/v1"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/labels"
    30  	"k8s.io/apimachinery/pkg/runtime"
    31  	"k8s.io/apimachinery/pkg/types"
    32  	utilrand "k8s.io/apimachinery/pkg/util/rand"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	clientset "k8s.io/client-go/kubernetes"
    35  	"k8s.io/client-go/kubernetes/scheme"
    36  	"k8s.io/client-go/util/retry"
    37  	extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions"
    38  	"k8s.io/kubernetes/pkg/controller"
    39  	labelsutil "k8s.io/kubernetes/pkg/util/labels"
    40  	"k8s.io/kubernetes/test/e2e/framework"
    41  	e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset"
    42  	e2eresource "k8s.io/kubernetes/test/e2e/framework/resource"
    43  	admissionapi "k8s.io/pod-security-admission/api"
    44  	"k8s.io/utils/pointer"
    45  )
    46  
    47  const (
    48  	controllerRevisionRetryPeriod  = 1 * time.Second
    49  	controllerRevisionRetryTimeout = 1 * time.Minute
    50  )
    51  
    52  // This test must be run in serial because it assumes the Daemon Set pods will
    53  // always get scheduled.  If we run other tests in parallel, this may not
    54  // happen.  In the future, running in parallel may work if we have an eviction
    55  // model which lets the DS controller kick out other pods to make room.
    56  // See https://issues.k8s.io/21767 for more details
    57  var _ = SIGDescribe("ControllerRevision", framework.WithSerial(), func() {
    58  	var f *framework.Framework
    59  
    60  	ginkgo.AfterEach(func(ctx context.Context) {
    61  		// Clean up
    62  		daemonsets, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).List(ctx, metav1.ListOptions{})
    63  		framework.ExpectNoError(err, "unable to dump DaemonSets")
    64  		if daemonsets != nil && len(daemonsets.Items) > 0 {
    65  			for _, ds := range daemonsets.Items {
    66  				ginkgo.By(fmt.Sprintf("Deleting DaemonSet %q", ds.Name))
    67  				framework.ExpectNoError(e2eresource.DeleteResourceAndWaitForGC(ctx, f.ClientSet, extensionsinternal.Kind("DaemonSet"), f.Namespace.Name, ds.Name))
    68  				err = wait.PollUntilContextTimeout(ctx, dsRetryPeriod, dsRetryTimeout, true, checkRunningOnNoNodes(f, &ds))
    69  				framework.ExpectNoError(err, "error waiting for daemon pod to be reaped")
    70  			}
    71  		}
    72  		if daemonsets, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).List(ctx, metav1.ListOptions{}); err == nil {
    73  			framework.Logf("daemonset: %s", runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...), daemonsets))
    74  		} else {
    75  			framework.Logf("unable to dump daemonsets: %v", err)
    76  		}
    77  		if pods, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(ctx, metav1.ListOptions{}); err == nil {
    78  			framework.Logf("pods: %s", runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...), pods))
    79  		} else {
    80  			framework.Logf("unable to dump pods: %v", err)
    81  		}
    82  		err = clearDaemonSetNodeLabels(ctx, f.ClientSet)
    83  		framework.ExpectNoError(err)
    84  	})
    85  
    86  	f = framework.NewDefaultFramework("controllerrevisions")
    87  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    88  
    89  	image := WebserverImage
    90  	dsName := "e2e-" + utilrand.String(5) + "-daemon-set"
    91  
    92  	var ns string
    93  	var c clientset.Interface
    94  
    95  	ginkgo.BeforeEach(func(ctx context.Context) {
    96  		ns = f.Namespace.Name
    97  
    98  		c = f.ClientSet
    99  
   100  		updatedNS, err := patchNamespaceAnnotations(ctx, c, ns)
   101  		framework.ExpectNoError(err)
   102  
   103  		ns = updatedNS.Name
   104  
   105  		err = clearDaemonSetNodeLabels(ctx, c)
   106  		framework.ExpectNoError(err)
   107  	})
   108  
   109  	/*
   110  		Release: v1.25
   111  		Testname: ControllerRevision, resource lifecycle
   112  		Description: Creating a DaemonSet MUST succeed. Listing all
   113  		ControllerRevisions with a label selector MUST find only one.
   114  		After patching the ControllerRevision with a new label, the label
   115  		MUST be found. Creating a new ControllerRevision for the DaemonSet
   116  		MUST succeed. Listing the ControllerRevisions by label selector
   117  		MUST find only two. Deleting a ControllerRevision MUST succeed.
   118  		Listing the ControllerRevisions by label selector MUST find only
   119  		one. After updating the ControllerRevision with a new label, the
   120  		label MUST be found. Patching the DaemonSet MUST succeed. Listing the
   121  		ControllerRevisions by label selector MUST find only two. Deleting
   122  		a collection of ControllerRevision via a label selector MUST succeed.
   123  		Listing the ControllerRevisions by label selector MUST find only one.
   124  		The current ControllerRevision revision MUST be 3.
   125  	*/
   126  	framework.ConformanceIt("should manage the lifecycle of a ControllerRevision", func(ctx context.Context) {
   127  		csAppsV1 := f.ClientSet.AppsV1()
   128  
   129  		dsLabel := map[string]string{"daemonset-name": dsName}
   130  		dsLabelSelector := labels.SelectorFromSet(dsLabel).String()
   131  
   132  		ginkgo.By(fmt.Sprintf("Creating DaemonSet %q", dsName))
   133  		testDaemonset, err := csAppsV1.DaemonSets(ns).Create(ctx, newDaemonSetWithLabel(dsName, image, dsLabel), metav1.CreateOptions{})
   134  		framework.ExpectNoError(err)
   135  
   136  		ginkgo.By("Check that daemon pods launch on every node of the cluster.")
   137  		err = wait.PollUntilContextTimeout(ctx, dsRetryPeriod, dsRetryTimeout, true, checkRunningOnAllNodes(f, testDaemonset))
   138  		framework.ExpectNoError(err, "error waiting for daemon pod to start")
   139  		err = e2edaemonset.CheckDaemonStatus(ctx, f, dsName)
   140  		framework.ExpectNoError(err)
   141  
   142  		ginkgo.By(fmt.Sprintf("Confirm DaemonSet %q successfully created with %q label", dsName, dsLabelSelector))
   143  		dsList, err := csAppsV1.DaemonSets("").List(ctx, metav1.ListOptions{LabelSelector: dsLabelSelector})
   144  		framework.ExpectNoError(err, "failed to list Daemon Sets")
   145  		gomega.Expect(dsList.Items).To(gomega.HaveLen(1), "filtered list wasn't found")
   146  
   147  		ds, err := c.AppsV1().DaemonSets(ns).Get(ctx, dsName, metav1.GetOptions{})
   148  		framework.ExpectNoError(err)
   149  
   150  		// Listing across all namespaces to verify api endpoint: listAppsV1ControllerRevisionForAllNamespaces
   151  		ginkgo.By(fmt.Sprintf("Listing all ControllerRevisions with label %q", dsLabelSelector))
   152  		revs, err := csAppsV1.ControllerRevisions("").List(ctx, metav1.ListOptions{LabelSelector: dsLabelSelector})
   153  		framework.ExpectNoError(err, "Failed to list ControllerRevision: %v", err)
   154  		gomega.Expect(revs.Items).To(gomega.HaveLen(1), "Failed to find any controllerRevisions")
   155  
   156  		// Locate the current ControllerRevision from the list
   157  		var initialRevision *appsv1.ControllerRevision
   158  
   159  		rev := revs.Items[0]
   160  		oref := rev.OwnerReferences[0]
   161  		if oref.Kind == "DaemonSet" && oref.UID == ds.UID {
   162  			framework.Logf("Located ControllerRevision: %q", rev.Name)
   163  			initialRevision, err = csAppsV1.ControllerRevisions(ns).Get(ctx, rev.Name, metav1.GetOptions{})
   164  			framework.ExpectNoError(err, "failed to lookup ControllerRevision: %v", err)
   165  			gomega.Expect(initialRevision).NotTo(gomega.BeNil(), "failed to lookup ControllerRevision: %v", initialRevision)
   166  		}
   167  
   168  		ginkgo.By(fmt.Sprintf("Patching ControllerRevision %q", initialRevision.Name))
   169  		payload := "{\"metadata\":{\"labels\":{\"" + initialRevision.Name + "\":\"patched\"}}}"
   170  		patchedControllerRevision, err := csAppsV1.ControllerRevisions(ns).Patch(ctx, initialRevision.Name, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{})
   171  		framework.ExpectNoError(err, "failed to patch ControllerRevision %s in namespace %s", initialRevision.Name, ns)
   172  		gomega.Expect(patchedControllerRevision.Labels).To(gomega.HaveKeyWithValue(initialRevision.Name, "patched"), "Did not find 'patched' label for this ControllerRevision. Current labels: %v", patchedControllerRevision.Labels)
   173  		framework.Logf("%s has been patched", patchedControllerRevision.Name)
   174  
   175  		ginkgo.By("Create a new ControllerRevision")
   176  		ds.Spec.Template.Spec.TerminationGracePeriodSeconds = pointer.Int64(1)
   177  		newHash, newName := hashAndNameForDaemonSet(ds)
   178  		newRevision := &appsv1.ControllerRevision{
   179  			ObjectMeta: metav1.ObjectMeta{
   180  				Name:            newName,
   181  				Namespace:       ds.Namespace,
   182  				Labels:          labelsutil.CloneAndAddLabel(ds.Spec.Template.Labels, appsv1.DefaultDaemonSetUniqueLabelKey, newHash),
   183  				Annotations:     ds.Annotations,
   184  				OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ds, appsv1.SchemeGroupVersion.WithKind("DaemonSet"))},
   185  			},
   186  			Data:     initialRevision.Data,
   187  			Revision: initialRevision.Revision + 1,
   188  		}
   189  		newControllerRevision, err := csAppsV1.ControllerRevisions(ns).Create(ctx, newRevision, metav1.CreateOptions{})
   190  		framework.ExpectNoError(err, "Failed to create ControllerRevision: %v", err)
   191  		framework.Logf("Created ControllerRevision: %s", newControllerRevision.Name)
   192  
   193  		ginkgo.By("Confirm that there are two ControllerRevisions")
   194  		err = wait.PollUntilContextTimeout(ctx, controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, true, checkControllerRevisionListQuantity(f, dsLabelSelector, 2))
   195  		framework.ExpectNoError(err, "failed to count required ControllerRevisions")
   196  
   197  		ginkgo.By(fmt.Sprintf("Deleting ControllerRevision %q", initialRevision.Name))
   198  		err = csAppsV1.ControllerRevisions(ns).Delete(ctx, initialRevision.Name, metav1.DeleteOptions{})
   199  		framework.ExpectNoError(err, "Failed to delete ControllerRevision: %v", err)
   200  
   201  		ginkgo.By("Confirm that there is only one ControllerRevision")
   202  		err = wait.PollUntilContextTimeout(ctx, controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, true, checkControllerRevisionListQuantity(f, dsLabelSelector, 1))
   203  		framework.ExpectNoError(err, "failed to count required ControllerRevisions")
   204  
   205  		listControllerRevisions, err := csAppsV1.ControllerRevisions(ns).List(ctx, metav1.ListOptions{})
   206  		currentControllerRevision := listControllerRevisions.Items[0]
   207  
   208  		ginkgo.By(fmt.Sprintf("Updating ControllerRevision %q", currentControllerRevision.Name))
   209  		var updatedControllerRevision *appsv1.ControllerRevision
   210  
   211  		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   212  			updatedControllerRevision, err = csAppsV1.ControllerRevisions(ns).Get(ctx, currentControllerRevision.Name, metav1.GetOptions{})
   213  			framework.ExpectNoError(err, "Unable to get ControllerRevision %s", currentControllerRevision.Name)
   214  			updatedControllerRevision.Labels[currentControllerRevision.Name] = "updated"
   215  			updatedControllerRevision, err = csAppsV1.ControllerRevisions(ns).Update(ctx, updatedControllerRevision, metav1.UpdateOptions{})
   216  			return err
   217  		})
   218  		framework.ExpectNoError(err, "failed to update ControllerRevision in namespace: %s", ns)
   219  		gomega.Expect(updatedControllerRevision.Labels).To(gomega.HaveKeyWithValue(currentControllerRevision.Name, "updated"), "Did not find 'updated' label for this ControllerRevision. Current labels: %v", updatedControllerRevision.Labels)
   220  		framework.Logf("%s has been updated", updatedControllerRevision.Name)
   221  
   222  		ginkgo.By("Generate another ControllerRevision by patching the Daemonset")
   223  		patch := fmt.Sprintf(`{"spec":{"template":{"spec":{"terminationGracePeriodSeconds": %d}}},"updateStrategy":{"type":"RollingUpdate"}}`, 1)
   224  
   225  		_, err = c.AppsV1().DaemonSets(ns).Patch(ctx, dsName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
   226  		framework.ExpectNoError(err, "error patching daemon set")
   227  
   228  		ginkgo.By("Confirm that there are two ControllerRevisions")
   229  		err = wait.PollUntilContextTimeout(ctx, controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, true, checkControllerRevisionListQuantity(f, dsLabelSelector, 2))
   230  		framework.ExpectNoError(err, "failed to count required ControllerRevisions")
   231  
   232  		updatedLabel := map[string]string{updatedControllerRevision.Name: "updated"}
   233  		updatedLabelSelector := labels.SelectorFromSet(updatedLabel).String()
   234  
   235  		ginkgo.By(fmt.Sprintf("Removing a ControllerRevision via 'DeleteCollection' with labelSelector: %q", updatedLabelSelector))
   236  		err = csAppsV1.ControllerRevisions(ns).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: updatedLabelSelector})
   237  		framework.ExpectNoError(err, "Failed to delete ControllerRevision: %v", err)
   238  
   239  		ginkgo.By("Confirm that there is only one ControllerRevision")
   240  		err = wait.PollUntilContextTimeout(ctx, controllerRevisionRetryPeriod, controllerRevisionRetryTimeout, true, checkControllerRevisionListQuantity(f, dsLabelSelector, 1))
   241  		framework.ExpectNoError(err, "failed to count required ControllerRevisions")
   242  
   243  		list, err := csAppsV1.ControllerRevisions(ns).List(ctx, metav1.ListOptions{})
   244  		framework.ExpectNoError(err, "failed to list ControllerRevision")
   245  		gomega.Expect(list.Items[0].Revision).To(gomega.Equal(int64(3)), "failed to find the expected revision for the Controller")
   246  		framework.Logf("ControllerRevision %q has revision %d", list.Items[0].Name, list.Items[0].Revision)
   247  	})
   248  })
   249  
   250  func checkControllerRevisionListQuantity(f *framework.Framework, label string, quantity int) func(ctx context.Context) (bool, error) {
   251  	return func(ctx context.Context) (bool, error) {
   252  		var err error
   253  
   254  		framework.Logf("Requesting list of ControllerRevisions to confirm quantity")
   255  
   256  		list, err := f.ClientSet.AppsV1().ControllerRevisions(f.Namespace.Name).List(ctx, metav1.ListOptions{
   257  			LabelSelector: label})
   258  		if err != nil {
   259  			return false, err
   260  		}
   261  
   262  		if len(list.Items) != quantity {
   263  			return false, nil
   264  		}
   265  		framework.Logf("Found %d ControllerRevisions", quantity)
   266  		return true, nil
   267  	}
   268  }
   269  
   270  func hashAndNameForDaemonSet(ds *appsv1.DaemonSet) (string, string) {
   271  	hash := fmt.Sprint(controller.ComputeHash(&ds.Spec.Template, ds.Status.CollisionCount))
   272  	name := ds.Name + "-" + hash
   273  	return hash, name
   274  }
   275
View as plain text