Source file src/k8s.io/kubernetes/test/integration/deployment/deployment_test.go

Documentation: k8s.io/kubernetes/test/integration/deployment

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package deployment
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"testing"
    24  
    25  	apps "k8s.io/api/apps/v1"
    26  	v1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/util/intstr"
    29  	"k8s.io/apimachinery/pkg/util/uuid"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	"k8s.io/client-go/util/retry"
    32  	"k8s.io/klog/v2/ktesting"
    33  	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
    34  	"k8s.io/kubernetes/test/integration/framework"
    35  	testutil "k8s.io/kubernetes/test/utils"
    36  	"k8s.io/utils/ptr"
    37  )
    38  
    39  func TestNewDeployment(t *testing.T) {
    40  	_, ctx := ktesting.NewTestContext(t)
    41  	ctx, cancel := context.WithCancel(ctx)
    42  	defer cancel()
    43  
    44  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
    45  	defer closeFn()
    46  	name := "test-new-deployment"
    47  
    48  	ns := framework.CreateNamespaceOrDie(c, name, t)
    49  	defer framework.DeleteNamespaceOrDie(c, ns, t)
    50  
    51  	replicas := int32(20)
    52  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
    53  	tester.deployment.Spec.MinReadySeconds = 4
    54  
    55  	tester.deployment.Annotations = map[string]string{"test": "should-copy-to-replica-set", v1.LastAppliedConfigAnnotation: "should-not-copy-to-replica-set"}
    56  	var err error
    57  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
    58  	if err != nil {
    59  		t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
    60  	}
    61  
    62  	// Start informer and controllers
    63  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
    64  	defer stopControllers()
    65  
    66  	// Wait for the Deployment to be updated to revision 1
    67  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
    68  		t.Fatal(err)
    69  	}
    70  
    71  	// Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
    72  	// Use soft check because this deployment was just created and rolling update strategy might be violated.
    73  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
    74  		t.Fatal(err)
    75  	}
    76  
    77  	// Check new RS annotations
    78  	newRS, err := tester.expectNewReplicaSet()
    79  	if err != nil {
    80  		t.Fatal(err)
    81  	}
    82  	if newRS.Annotations["test"] != "should-copy-to-replica-set" {
    83  		t.Errorf("expected new ReplicaSet annotations copied from Deployment %s, got: %v", tester.deployment.Name, newRS.Annotations)
    84  	}
    85  	if newRS.Annotations[v1.LastAppliedConfigAnnotation] != "" {
    86  		t.Errorf("expected new ReplicaSet last-applied annotation not copied from Deployment %s", tester.deployment.Name)
    87  	}
    88  
    89  	// New RS should contain pod-template-hash in its selector, label, and template label
    90  	rsHash, err := checkRSHashLabels(newRS)
    91  	if err != nil {
    92  		t.Error(err)
    93  	}
    94  
    95  	// All pods targeted by the deployment should contain pod-template-hash in their labels
    96  	selector, err := metav1.LabelSelectorAsSelector(tester.deployment.Spec.Selector)
    97  	if err != nil {
    98  		t.Fatalf("failed to parse deployment %s selector: %v", name, err)
    99  	}
   100  	pods, err := c.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()})
   101  	if err != nil {
   102  		t.Fatalf("failed to list pods of deployment %s: %v", name, err)
   103  	}
   104  	if len(pods.Items) != int(replicas) {
   105  		t.Errorf("expected %d pods, got %d pods", replicas, len(pods.Items))
   106  	}
   107  	podHash, err := checkPodsHashLabel(pods)
   108  	if err != nil {
   109  		t.Error(err)
   110  	}
   111  	if rsHash != podHash {
   112  		t.Errorf("found mismatching pod-template-hash value: rs hash = %s whereas pod hash = %s", rsHash, podHash)
   113  	}
   114  }
   115  
   116  // Deployments should support roll out, roll back, and roll over.
   117  // TODO: drop the rollback portions of this test when extensions/v1beta1 is no longer served
   118  // and rollback endpoint is no longer supported.
   119  func TestDeploymentRollingUpdate(t *testing.T) {
   120  	_, ctx := ktesting.NewTestContext(t)
   121  	ctx, cancel := context.WithCancel(ctx)
   122  	defer cancel()
   123  
   124  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   125  	defer closeFn()
   126  
   127  	name := "test-rolling-update-deployment"
   128  	ns := framework.CreateNamespaceOrDie(c, name, t)
   129  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   130  
   131  	// Start informer and controllers
   132  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   133  	defer stopControllers()
   134  
   135  	replicas := int32(20)
   136  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
   137  	tester.deployment.Spec.MinReadySeconds = 4
   138  	quarter := intstr.FromString("25%")
   139  	tester.deployment.Spec.Strategy.RollingUpdate = &apps.RollingUpdateDeployment{
   140  		MaxUnavailable: &quarter,
   141  		MaxSurge:       &quarter,
   142  	}
   143  
   144  	// Create a deployment.
   145  	var err error
   146  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   147  	if err != nil {
   148  		t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
   149  	}
   150  	oriImage := tester.deployment.Spec.Template.Spec.Containers[0].Image
   151  	if err := tester.waitForDeploymentRevisionAndImage("1", oriImage); err != nil {
   152  		t.Fatal(err)
   153  	}
   154  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   155  		t.Fatal(err)
   156  	}
   157  
   158  	// 1. Roll out a new image.
   159  	image := "new-image"
   160  	if oriImage == image {
   161  		t.Fatalf("bad test setup, deployment %s roll out with the same image", tester.deployment.Name)
   162  	}
   163  	imageFn := func(update *apps.Deployment) {
   164  		update.Spec.Template.Spec.Containers[0].Image = image
   165  	}
   166  	tester.deployment, err = tester.updateDeployment(imageFn)
   167  	if err != nil {
   168  		t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
   169  	}
   170  	if err := tester.waitForDeploymentRevisionAndImage("2", image); err != nil {
   171  		t.Fatal(err)
   172  	}
   173  	if err := tester.waitForDeploymentCompleteAndCheckRollingAndMarkPodsReady(); err != nil {
   174  		t.Fatal(err)
   175  	}
   176  
   177  	// 2. Roll over a deployment before the previous rolling update finishes.
   178  	image = "dont-finish"
   179  	imageFn = func(update *apps.Deployment) {
   180  		update.Spec.Template.Spec.Containers[0].Image = image
   181  	}
   182  	tester.deployment, err = tester.updateDeployment(imageFn)
   183  	if err != nil {
   184  		t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
   185  	}
   186  	if err := tester.waitForDeploymentRevisionAndImage("3", image); err != nil {
   187  		t.Fatal(err)
   188  	}
   189  
   190  	// We don't mark pods as ready so that rollout won't finish.
   191  	// Before the rollout finishes, trigger another rollout.
   192  	image = "rollover"
   193  	imageFn = func(update *apps.Deployment) {
   194  		update.Spec.Template.Spec.Containers[0].Image = image
   195  	}
   196  	tester.deployment, err = tester.updateDeployment(imageFn)
   197  	if err != nil {
   198  		t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
   199  	}
   200  	if err := tester.waitForDeploymentRevisionAndImage("4", image); err != nil {
   201  		t.Fatal(err)
   202  	}
   203  	if err := tester.waitForDeploymentCompleteAndCheckRollingAndMarkPodsReady(); err != nil {
   204  		t.Fatal(err)
   205  	}
   206  	_, allOldRSs, err := testutil.GetOldReplicaSets(tester.deployment, c)
   207  	if err != nil {
   208  		t.Fatalf("failed retrieving old replicasets of deployment %s: %v", tester.deployment.Name, err)
   209  	}
   210  	for _, oldRS := range allOldRSs {
   211  		if *oldRS.Spec.Replicas != 0 {
   212  			t.Errorf("expected old replicaset %s of deployment %s to have 0 replica, got %d", oldRS.Name, tester.deployment.Name, *oldRS.Spec.Replicas)
   213  		}
   214  	}
   215  }
   216  
   217  // selectors are IMMUTABLE for all API versions except apps/v1beta1 and extensions/v1beta1
   218  func TestDeploymentSelectorImmutability(t *testing.T) {
   219  	closeFn, c := dcSimpleSetup(t)
   220  	defer closeFn()
   221  
   222  	name := "test-deployment-selector-immutability"
   223  	ns := framework.CreateNamespaceOrDie(c, name, t)
   224  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   225  
   226  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, int32(20))}
   227  	var err error
   228  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   229  	if err != nil {
   230  		t.Fatalf("failed to create apps/v1 deployment %s: %v", tester.deployment.Name, err)
   231  	}
   232  
   233  	// test to ensure apps/v1 selector is immutable
   234  	deploymentAppsV1, err := c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
   235  	if err != nil {
   236  		t.Fatalf("failed to get apps/v1 deployment %s: %v", name, err)
   237  	}
   238  	newSelectorLabels := map[string]string{"name_apps_v1": "test_apps_v1"}
   239  	deploymentAppsV1.Spec.Selector.MatchLabels = newSelectorLabels
   240  	deploymentAppsV1.Spec.Template.Labels = newSelectorLabels
   241  	_, err = c.AppsV1().Deployments(ns.Name).Update(context.TODO(), deploymentAppsV1, metav1.UpdateOptions{})
   242  	if err == nil {
   243  		t.Fatalf("failed to provide validation error when changing immutable selector when updating apps/v1 deployment %s", deploymentAppsV1.Name)
   244  	}
   245  	expectedErrType := "Invalid value"
   246  	expectedErrDetail := "field is immutable"
   247  	if !strings.Contains(err.Error(), expectedErrType) || !strings.Contains(err.Error(), expectedErrDetail) {
   248  		t.Errorf("error message does not match, expected type: %s, expected detail: %s, got: %s", expectedErrType, expectedErrDetail, err.Error())
   249  	}
   250  }
   251  
   252  // Paused deployment should not start new rollout
   253  func TestPausedDeployment(t *testing.T) {
   254  	_, ctx := ktesting.NewTestContext(t)
   255  	ctx, cancel := context.WithCancel(ctx)
   256  	defer cancel()
   257  
   258  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   259  	defer closeFn()
   260  
   261  	name := "test-paused-deployment"
   262  	ns := framework.CreateNamespaceOrDie(c, name, t)
   263  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   264  
   265  	replicas := int32(1)
   266  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
   267  	tester.deployment.Spec.Paused = true
   268  	tgps := int64(1)
   269  	tester.deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &tgps
   270  
   271  	var err error
   272  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   273  	if err != nil {
   274  		t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
   275  	}
   276  
   277  	// Start informer and controllers
   278  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   279  	defer stopControllers()
   280  
   281  	// Verify that the paused deployment won't create new replica set.
   282  	if err := tester.expectNoNewReplicaSet(); err != nil {
   283  		t.Fatal(err)
   284  	}
   285  
   286  	// Resume the deployment
   287  	tester.deployment, err = tester.updateDeployment(resumeFn)
   288  	if err != nil {
   289  		t.Fatalf("failed to resume deployment %s: %v", tester.deployment.Name, err)
   290  	}
   291  
   292  	// Wait for the controller to notice the resume.
   293  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
   294  		t.Fatal(err)
   295  	}
   296  
   297  	// Wait for the Deployment to be updated to revision 1
   298  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
   299  		t.Fatal(err)
   300  	}
   301  
   302  	// Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
   303  	// Use soft check because this deployment was just created and rolling update strategy might be violated.
   304  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   305  		t.Fatal(err)
   306  	}
   307  
   308  	// A new replicaset should be created.
   309  	if _, err := tester.expectNewReplicaSet(); err != nil {
   310  		t.Fatal(err)
   311  	}
   312  
   313  	// Pause the deployment.
   314  	// The paused deployment shouldn't trigger a new rollout.
   315  	tester.deployment, err = tester.updateDeployment(pauseFn)
   316  	if err != nil {
   317  		t.Fatalf("failed to pause deployment %s: %v", tester.deployment.Name, err)
   318  	}
   319  
   320  	// Wait for the controller to notice the pause.
   321  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
   322  		t.Fatal(err)
   323  	}
   324  
   325  	// Update the deployment template
   326  	newTGPS := int64(0)
   327  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   328  		update.Spec.Template.Spec.TerminationGracePeriodSeconds = &newTGPS
   329  	})
   330  	if err != nil {
   331  		t.Fatalf("failed updating template of deployment %s: %v", tester.deployment.Name, err)
   332  	}
   333  
   334  	// Wait for the controller to notice the rollout.
   335  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
   336  		t.Fatal(err)
   337  	}
   338  
   339  	// Verify that the paused deployment won't create new replica set.
   340  	if err := tester.expectNoNewReplicaSet(); err != nil {
   341  		t.Fatal(err)
   342  	}
   343  
   344  	_, allOldRs, err := testutil.GetOldReplicaSets(tester.deployment, c)
   345  	if err != nil {
   346  		t.Fatalf("failed retrieving old replicasets of deployment %s: %v", tester.deployment.Name, err)
   347  	}
   348  	if len(allOldRs) != 1 {
   349  		t.Errorf("expected an old replica set, got %v", allOldRs)
   350  	}
   351  	if *allOldRs[0].Spec.Template.Spec.TerminationGracePeriodSeconds == newTGPS {
   352  		t.Errorf("TerminationGracePeriodSeconds on the replica set should be %d, got %d", tgps, newTGPS)
   353  	}
   354  }
   355  
   356  // Paused deployment can be scaled
   357  func TestScalePausedDeployment(t *testing.T) {
   358  	_, ctx := ktesting.NewTestContext(t)
   359  	ctx, cancel := context.WithCancel(ctx)
   360  	defer cancel()
   361  
   362  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   363  	defer closeFn()
   364  
   365  	name := "test-scale-paused-deployment"
   366  	ns := framework.CreateNamespaceOrDie(c, name, t)
   367  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   368  
   369  	replicas := int32(1)
   370  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
   371  	tgps := int64(1)
   372  	tester.deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &tgps
   373  
   374  	var err error
   375  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   376  	if err != nil {
   377  		t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
   378  	}
   379  
   380  	// Start informer and controllers
   381  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   382  	defer stopControllers()
   383  
   384  	// Wait for the Deployment to be updated to revision 1
   385  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
   386  		t.Fatal(err)
   387  	}
   388  
   389  	// Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
   390  	// Use soft check because this deployment was just created and rolling update strategy might be violated.
   391  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   392  		t.Fatal(err)
   393  	}
   394  
   395  	// A new replicaset should be created.
   396  	if _, err := tester.expectNewReplicaSet(); err != nil {
   397  		t.Fatal(err)
   398  	}
   399  
   400  	// Pause the deployment.
   401  	tester.deployment, err = tester.updateDeployment(pauseFn)
   402  	if err != nil {
   403  		t.Fatalf("failed to pause deployment %s: %v", tester.deployment.Name, err)
   404  	}
   405  
   406  	// Wait for the controller to notice the scale.
   407  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
   408  		t.Fatal(err)
   409  	}
   410  
   411  	// Scale the paused deployment.
   412  	newReplicas := int32(10)
   413  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   414  		update.Spec.Replicas = &newReplicas
   415  	})
   416  	if err != nil {
   417  		t.Fatalf("failed updating deployment %s: %v", tester.deployment.Name, err)
   418  	}
   419  
   420  	// Wait for the controller to notice the scale.
   421  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
   422  		t.Fatal(err)
   423  	}
   424  
   425  	// Verify that the new replicaset is scaled.
   426  	rs, err := tester.expectNewReplicaSet()
   427  	if err != nil {
   428  		t.Fatal(err)
   429  	}
   430  	if *rs.Spec.Replicas != newReplicas {
   431  		t.Errorf("expected new replicaset replicas = %d, got %d", newReplicas, *rs.Spec.Replicas)
   432  	}
   433  
   434  	// Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
   435  	// Use soft check because this deployment was just scaled and rolling update strategy might be violated.
   436  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   437  		t.Fatal(err)
   438  	}
   439  }
   440  
   441  // Deployment rollout shouldn't be blocked on hash collisions
   442  func TestDeploymentHashCollision(t *testing.T) {
   443  	_, ctx := ktesting.NewTestContext(t)
   444  	ctx, cancel := context.WithCancel(ctx)
   445  	defer cancel()
   446  
   447  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   448  	defer closeFn()
   449  
   450  	name := "test-hash-collision-deployment"
   451  	ns := framework.CreateNamespaceOrDie(c, name, t)
   452  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   453  
   454  	replicas := int32(1)
   455  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
   456  
   457  	var err error
   458  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   459  	if err != nil {
   460  		t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
   461  	}
   462  
   463  	// Start informer and controllers
   464  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   465  	defer stopControllers()
   466  
   467  	// Wait for the Deployment to be updated to revision 1
   468  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
   469  		t.Fatal(err)
   470  	}
   471  
   472  	// Mock a hash collision
   473  	newRS, err := testutil.GetNewReplicaSet(tester.deployment, c)
   474  	if err != nil {
   475  		t.Fatalf("failed getting new replicaset of deployment %s: %v", tester.deployment.Name, err)
   476  	}
   477  	if newRS == nil {
   478  		t.Fatalf("unable to find new replicaset of deployment %s", tester.deployment.Name)
   479  	}
   480  	_, err = tester.updateReplicaSet(newRS.Name, func(update *apps.ReplicaSet) {
   481  		*update.Spec.Template.Spec.TerminationGracePeriodSeconds = int64(5)
   482  	})
   483  	if err != nil {
   484  		t.Fatalf("failed updating replicaset %s template: %v", newRS.Name, err)
   485  	}
   486  
   487  	// Expect deployment collision counter to increment
   488  	if err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
   489  		d, err := c.AppsV1().Deployments(ns.Name).Get(context.TODO(), tester.deployment.Name, metav1.GetOptions{})
   490  		if err != nil {
   491  			return false, nil
   492  		}
   493  		return d.Status.CollisionCount != nil && *d.Status.CollisionCount == int32(1), nil
   494  	}); err != nil {
   495  		t.Fatalf("Failed to increment collision counter for deployment %q: %v", tester.deployment.Name, err)
   496  	}
   497  
   498  	// Expect a new ReplicaSet to be created
   499  	if err := tester.waitForDeploymentRevisionAndImage("2", fakeImage); err != nil {
   500  		t.Fatal(err)
   501  	}
   502  }
   503  
   504  func checkRSHashLabels(rs *apps.ReplicaSet) (string, error) {
   505  	hash := rs.Labels[apps.DefaultDeploymentUniqueLabelKey]
   506  	selectorHash := rs.Spec.Selector.MatchLabels[apps.DefaultDeploymentUniqueLabelKey]
   507  	templateLabelHash := rs.Spec.Template.Labels[apps.DefaultDeploymentUniqueLabelKey]
   508  
   509  	if hash != selectorHash || selectorHash != templateLabelHash {
   510  		return "", fmt.Errorf("mismatching hash value found in replicaset %s: %#v", rs.Name, rs)
   511  	}
   512  	if len(hash) == 0 {
   513  		return "", fmt.Errorf("unexpected replicaset %s missing required pod-template-hash labels", rs.Name)
   514  	}
   515  
   516  	if !strings.HasSuffix(rs.Name, hash) {
   517  		return "", fmt.Errorf("unexpected replicaset %s name suffix doesn't match hash %s", rs.Name, hash)
   518  	}
   519  
   520  	return hash, nil
   521  }
   522  
   523  func checkPodsHashLabel(pods *v1.PodList) (string, error) {
   524  	if len(pods.Items) == 0 {
   525  		return "", fmt.Errorf("no pods given")
   526  	}
   527  	var hash string
   528  	for _, pod := range pods.Items {
   529  		podHash := pod.Labels[apps.DefaultDeploymentUniqueLabelKey]
   530  		if len(podHash) == 0 {
   531  			return "", fmt.Errorf("found pod %s missing pod-template-hash label: %#v", pod.Name, pods)
   532  		}
   533  		// Save the first valid hash
   534  		if len(hash) == 0 {
   535  			hash = podHash
   536  		}
   537  		if podHash != hash {
   538  			return "", fmt.Errorf("found pod %s with mismatching pod-template-hash value %s: %#v", pod.Name, podHash, pods)
   539  		}
   540  	}
   541  	return hash, nil
   542  }
   543  
   544  // Deployment should have a timeout condition when it fails to progress after given deadline.
   545  func TestFailedDeployment(t *testing.T) {
   546  	_, ctx := ktesting.NewTestContext(t)
   547  	ctx, cancel := context.WithCancel(ctx)
   548  	defer cancel()
   549  
   550  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   551  	defer closeFn()
   552  
   553  	name := "test-failed-deployment"
   554  	ns := framework.CreateNamespaceOrDie(c, name, t)
   555  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   556  
   557  	deploymentName := "progress-check"
   558  	replicas := int32(1)
   559  	three := int32(3)
   560  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
   561  	tester.deployment.Spec.ProgressDeadlineSeconds = &three
   562  	var err error
   563  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   564  	if err != nil {
   565  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
   566  	}
   567  
   568  	// Start informer and controllers
   569  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   570  	defer stopControllers()
   571  
   572  	if err = tester.waitForDeploymentUpdatedReplicasGTE(replicas); err != nil {
   573  		t.Fatal(err)
   574  	}
   575  
   576  	// Pods are not marked as Ready, therefore the deployment progress will eventually timeout after progressDeadlineSeconds has passed.
   577  	// Wait for the deployment to have a progress timeout condition.
   578  	if err = tester.waitForDeploymentWithCondition(deploymentutil.TimedOutReason, apps.DeploymentProgressing); err != nil {
   579  		t.Fatal(err)
   580  	}
   581  
   582  	// Manually mark pods as Ready and wait for deployment to complete.
   583  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   584  		t.Fatalf("deployment %q fails to have its status becoming valid: %v", deploymentName, err)
   585  	}
   586  
   587  	// Wait for the deployment to have a progress complete condition.
   588  	if err = tester.waitForDeploymentWithCondition(deploymentutil.NewRSAvailableReason, apps.DeploymentProgressing); err != nil {
   589  		t.Fatal(err)
   590  	}
   591  }
   592  
   593  func TestOverlappingDeployments(t *testing.T) {
   594  	_, ctx := ktesting.NewTestContext(t)
   595  	ctx, cancel := context.WithCancel(ctx)
   596  	defer cancel()
   597  
   598  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   599  	defer closeFn()
   600  
   601  	name := "test-overlapping-deployments"
   602  	ns := framework.CreateNamespaceOrDie(c, name, t)
   603  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   604  
   605  	replicas := int32(1)
   606  	firstDeploymentName := "first-deployment"
   607  	secondDeploymentName := "second-deployment"
   608  	testers := []*deploymentTester{
   609  		{t: t, c: c, deployment: newDeployment(firstDeploymentName, ns.Name, replicas)},
   610  		{t: t, c: c, deployment: newDeployment(secondDeploymentName, ns.Name, replicas)},
   611  	}
   612  
   613  	// Start informer and controllers
   614  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   615  	defer stopControllers()
   616  
   617  	// Create 2 deployments with overlapping selectors
   618  	var err error
   619  	var rss []*apps.ReplicaSet
   620  	for _, tester := range testers {
   621  		tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   622  		dname := tester.deployment.Name
   623  		if err != nil {
   624  			t.Fatalf("failed to create deployment %q: %v", dname, err)
   625  		}
   626  		// Wait for the deployment to be updated to revision 1
   627  		if err = tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
   628  			t.Fatalf("failed to update deployment %q to revision 1: %v", dname, err)
   629  		}
   630  		// Make sure the deployment completes while manually marking its pods as ready at the same time
   631  		if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   632  			t.Fatalf("deployment %q failed to complete: %v", dname, err)
   633  		}
   634  		// Get replicaset of the deployment
   635  		newRS, err := tester.getNewReplicaSet()
   636  		if err != nil {
   637  			t.Fatalf("failed to get new replicaset of deployment %q: %v", dname, err)
   638  		}
   639  		if newRS == nil {
   640  			t.Fatalf("unable to find new replicaset of deployment %q", dname)
   641  		}
   642  		// Store the replicaset for future usage
   643  		rss = append(rss, newRS)
   644  	}
   645  
   646  	// Both deployments should proceed independently, so their respective replicaset should not be the same replicaset
   647  	if rss[0].UID == rss[1].UID {
   648  		t.Fatalf("overlapping deployments should not share the same replicaset")
   649  	}
   650  
   651  	// Scale only the first deployment by 1
   652  	newReplicas := replicas + 1
   653  	testers[0].deployment, err = testers[0].updateDeployment(func(update *apps.Deployment) {
   654  		update.Spec.Replicas = &newReplicas
   655  	})
   656  	if err != nil {
   657  		t.Fatalf("failed updating deployment %q: %v", firstDeploymentName, err)
   658  	}
   659  
   660  	// Make sure the deployment completes after scaling
   661  	if err := testers[0].waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   662  		t.Fatalf("deployment %q failed to complete after scaling: %v", firstDeploymentName, err)
   663  	}
   664  
   665  	// Verify replicaset of both deployments has updated number of replicas
   666  	for i, tester := range testers {
   667  		rs, err := c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), rss[i].Name, metav1.GetOptions{})
   668  		if err != nil {
   669  			t.Fatalf("failed to get replicaset %q: %v", rss[i].Name, err)
   670  		}
   671  		if *rs.Spec.Replicas != *tester.deployment.Spec.Replicas {
   672  			t.Errorf("expected replicaset %q of deployment %q has %d replicas, but found %d replicas", rs.Name, firstDeploymentName, *tester.deployment.Spec.Replicas, *rs.Spec.Replicas)
   673  		}
   674  	}
   675  }
   676  
   677  // Deployment should not block rollout when updating spec replica number and template at the same time.
   678  func TestScaledRolloutDeployment(t *testing.T) {
   679  	logger, ctx := ktesting.NewTestContext(t)
   680  	ctx, cancel := context.WithCancel(ctx)
   681  	defer cancel()
   682  
   683  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   684  	defer closeFn()
   685  
   686  	name := "test-scaled-rollout-deployment"
   687  	ns := framework.CreateNamespaceOrDie(c, name, t)
   688  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   689  
   690  	// Start informer and controllers
   691  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   692  	defer stopControllers()
   693  
   694  	// Create a deployment with rolling update strategy, max surge = 3, and max unavailable = 2
   695  	var err error
   696  	replicas := int32(10)
   697  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
   698  	tester.deployment.Spec.Strategy.RollingUpdate.MaxSurge = ptr.To(intstr.FromInt32(3))
   699  	tester.deployment.Spec.Strategy.RollingUpdate.MaxUnavailable = ptr.To(intstr.FromInt32(2))
   700  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   701  	if err != nil {
   702  		t.Fatalf("failed to create deployment %q: %v", name, err)
   703  	}
   704  	if err = tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
   705  		t.Fatal(err)
   706  	}
   707  	if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   708  		t.Fatalf("deployment %q failed to complete: %v", name, err)
   709  	}
   710  
   711  	// Record current replicaset before starting new rollout
   712  	firstRS, err := tester.expectNewReplicaSet()
   713  	if err != nil {
   714  		t.Fatal(err)
   715  	}
   716  
   717  	// Update the deployment with another new image but do not mark the pods as ready to block new replicaset
   718  	fakeImage2 := "fakeimage2"
   719  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   720  		update.Spec.Template.Spec.Containers[0].Image = fakeImage2
   721  	})
   722  	if err != nil {
   723  		t.Fatalf("failed updating deployment %q: %v", name, err)
   724  	}
   725  	if err = tester.waitForDeploymentRevisionAndImage("2", fakeImage2); err != nil {
   726  		t.Fatal(err)
   727  	}
   728  
   729  	// Verify the deployment has minimum available replicas after 2nd rollout
   730  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
   731  	if err != nil {
   732  		t.Fatalf("failed to get deployment %q: %v", name, err)
   733  	}
   734  	minAvailableReplicas := deploymentutil.MinAvailable(tester.deployment)
   735  	if tester.deployment.Status.AvailableReplicas < minAvailableReplicas {
   736  		t.Fatalf("deployment %q does not have minimum number of available replicas after 2nd rollout", name)
   737  	}
   738  
   739  	// Wait for old replicaset of 1st rollout to have desired replicas
   740  	firstRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), firstRS.Name, metav1.GetOptions{})
   741  	if err != nil {
   742  		t.Fatalf("failed to get replicaset %q: %v", firstRS.Name, err)
   743  	}
   744  	if err = tester.waitRSStable(firstRS); err != nil {
   745  		t.Fatal(err)
   746  	}
   747  
   748  	// Wait for new replicaset of 2nd rollout to have desired replicas
   749  	secondRS, err := tester.expectNewReplicaSet()
   750  	if err != nil {
   751  		t.Fatal(err)
   752  	}
   753  	if err = tester.waitRSStable(secondRS); err != nil {
   754  		t.Fatal(err)
   755  	}
   756  
   757  	// Scale up the deployment and update its image to another new image simultaneously (this time marks all pods as ready)
   758  	newReplicas := int32(20)
   759  	fakeImage3 := "fakeimage3"
   760  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   761  		update.Spec.Replicas = &newReplicas
   762  		update.Spec.Template.Spec.Containers[0].Image = fakeImage3
   763  	})
   764  	if err != nil {
   765  		t.Fatalf("failed updating deployment %q: %v", name, err)
   766  	}
   767  	if err = tester.waitForDeploymentRevisionAndImage("3", fakeImage3); err != nil {
   768  		t.Fatal(err)
   769  	}
   770  	if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   771  		t.Fatalf("deployment %q failed to complete: %v", name, err)
   772  	}
   773  
   774  	// Verify every replicaset has correct desiredReplicas annotation after 3rd rollout
   775  	thirdRS, err := testutil.GetNewReplicaSet(tester.deployment, c)
   776  	if err != nil {
   777  		t.Fatalf("failed getting new revision 3 replicaset for deployment %q: %v", name, err)
   778  	}
   779  	rss := []*apps.ReplicaSet{firstRS, secondRS, thirdRS}
   780  	for _, curRS := range rss {
   781  		curRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), curRS.Name, metav1.GetOptions{})
   782  		if err != nil {
   783  			t.Fatalf("failed to get replicaset when checking desired replicas annotation: %v", err)
   784  		}
   785  		desired, ok := deploymentutil.GetDesiredReplicasAnnotation(logger, curRS)
   786  		if !ok {
   787  			t.Fatalf("failed to retrieve desiredReplicas annotation for replicaset %q", curRS.Name)
   788  		}
   789  		if desired != *(tester.deployment.Spec.Replicas) {
   790  			t.Fatalf("unexpected desiredReplicas annotation for replicaset %q: expected %d, got %d", curRS.Name, *(tester.deployment.Spec.Replicas), desired)
   791  		}
   792  	}
   793  
   794  	// Update the deployment with another new image but do not mark the pods as ready to block new replicaset
   795  	fakeImage4 := "fakeimage4"
   796  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   797  		update.Spec.Template.Spec.Containers[0].Image = fakeImage4
   798  	})
   799  	if err != nil {
   800  		t.Fatalf("failed updating deployment %q: %v", name, err)
   801  	}
   802  	if err = tester.waitForDeploymentRevisionAndImage("4", fakeImage4); err != nil {
   803  		t.Fatal(err)
   804  	}
   805  
   806  	// Verify the deployment has minimum available replicas after 4th rollout
   807  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
   808  	if err != nil {
   809  		t.Fatalf("failed to get deployment %q: %v", name, err)
   810  	}
   811  	minAvailableReplicas = deploymentutil.MinAvailable(tester.deployment)
   812  	if tester.deployment.Status.AvailableReplicas < minAvailableReplicas {
   813  		t.Fatalf("deployment %q does not have minimum number of available replicas after 4th rollout", name)
   814  	}
   815  
   816  	// Wait for old replicaset of 3rd rollout to have desired replicas
   817  	thirdRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), thirdRS.Name, metav1.GetOptions{})
   818  	if err != nil {
   819  		t.Fatalf("failed to get replicaset %q: %v", thirdRS.Name, err)
   820  	}
   821  	if err = tester.waitRSStable(thirdRS); err != nil {
   822  		t.Fatal(err)
   823  	}
   824  
   825  	// Wait for new replicaset of 4th rollout to have desired replicas
   826  	fourthRS, err := tester.expectNewReplicaSet()
   827  	if err != nil {
   828  		t.Fatal(err)
   829  	}
   830  	if err = tester.waitRSStable(fourthRS); err != nil {
   831  		t.Fatal(err)
   832  	}
   833  
   834  	// Scale down the deployment and update its image to another new image simultaneously (this time marks all pods as ready)
   835  	newReplicas = int32(5)
   836  	fakeImage5 := "fakeimage5"
   837  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   838  		update.Spec.Replicas = &newReplicas
   839  		update.Spec.Template.Spec.Containers[0].Image = fakeImage5
   840  	})
   841  	if err != nil {
   842  		t.Fatalf("failed updating deployment %q: %v", name, err)
   843  	}
   844  	if err = tester.waitForDeploymentRevisionAndImage("5", fakeImage5); err != nil {
   845  		t.Fatal(err)
   846  	}
   847  	if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
   848  		t.Fatalf("deployment %q failed to complete: %v", name, err)
   849  	}
   850  
   851  	// Verify every replicaset has correct desiredReplicas annotation after 5th rollout
   852  	fifthRS, err := testutil.GetNewReplicaSet(tester.deployment, c)
   853  	if err != nil {
   854  		t.Fatalf("failed getting new revision 5 replicaset for deployment %q: %v", name, err)
   855  	}
   856  	rss = []*apps.ReplicaSet{thirdRS, fourthRS, fifthRS}
   857  	for _, curRS := range rss {
   858  		curRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), curRS.Name, metav1.GetOptions{})
   859  		if err != nil {
   860  			t.Fatalf("failed to get replicaset when checking desired replicas annotation: %v", err)
   861  		}
   862  		desired, ok := deploymentutil.GetDesiredReplicasAnnotation(logger, curRS)
   863  		if !ok {
   864  			t.Fatalf("failed to retrieve desiredReplicas annotation for replicaset %q", curRS.Name)
   865  		}
   866  		if desired != *(tester.deployment.Spec.Replicas) {
   867  			t.Fatalf("unexpected desiredReplicas annotation for replicaset %q: expected %d, got %d", curRS.Name, *(tester.deployment.Spec.Replicas), desired)
   868  		}
   869  	}
   870  }
   871  
   872  func TestSpecReplicasChange(t *testing.T) {
   873  	_, ctx := ktesting.NewTestContext(t)
   874  	ctx, cancel := context.WithCancel(ctx)
   875  	defer cancel()
   876  
   877  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   878  	defer closeFn()
   879  
   880  	name := "test-spec-replicas-change"
   881  	ns := framework.CreateNamespaceOrDie(c, name, t)
   882  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   883  
   884  	deploymentName := "deployment"
   885  	replicas := int32(1)
   886  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
   887  	tester.deployment.Spec.Strategy.Type = apps.RecreateDeploymentStrategyType
   888  	tester.deployment.Spec.Strategy.RollingUpdate = nil
   889  	var err error
   890  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   891  	if err != nil {
   892  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
   893  	}
   894  
   895  	// Start informer and controllers
   896  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   897  	defer stopControllers()
   898  
   899  	// Scale up/down deployment and verify its replicaset has matching .spec.replicas
   900  	if err = tester.scaleDeployment(2); err != nil {
   901  		t.Fatal(err)
   902  	}
   903  	if err = tester.scaleDeployment(0); err != nil {
   904  		t.Fatal(err)
   905  	}
   906  	if err = tester.scaleDeployment(1); err != nil {
   907  		t.Fatal(err)
   908  	}
   909  
   910  	// Add a template annotation change to test deployment's status does update
   911  	// without .spec.replicas change
   912  	var oldGeneration int64
   913  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   914  		oldGeneration = update.Generation
   915  		update.Spec.RevisionHistoryLimit = ptr.To[int32](4)
   916  	})
   917  	if err != nil {
   918  		t.Fatalf("failed updating deployment %q: %v", tester.deployment.Name, err)
   919  	}
   920  
   921  	savedGeneration := tester.deployment.Generation
   922  	if savedGeneration == oldGeneration {
   923  		t.Fatalf("Failed to verify .Generation has incremented for deployment %q", deploymentName)
   924  	}
   925  	if err = tester.waitForObservedDeployment(savedGeneration); err != nil {
   926  		t.Fatal(err)
   927  	}
   928  }
   929  
   930  func TestDeploymentAvailableCondition(t *testing.T) {
   931  	_, ctx := ktesting.NewTestContext(t)
   932  	ctx, cancel := context.WithCancel(ctx)
   933  	defer cancel()
   934  
   935  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
   936  	defer closeFn()
   937  
   938  	name := "test-deployment-available-condition"
   939  	ns := framework.CreateNamespaceOrDie(c, name, t)
   940  	defer framework.DeleteNamespaceOrDie(c, ns, t)
   941  
   942  	deploymentName := "deployment"
   943  	replicas := int32(10)
   944  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
   945  	// Assign a high value to the deployment's minReadySeconds
   946  	tester.deployment.Spec.MinReadySeconds = 3600
   947  	// progressDeadlineSeconds must be greater than minReadySeconds
   948  	tester.deployment.Spec.ProgressDeadlineSeconds = ptr.To[int32](7200)
   949  	var err error
   950  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
   951  	if err != nil {
   952  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
   953  	}
   954  
   955  	// Start informer and controllers
   956  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
   957  	defer stopControllers()
   958  
   959  	// Wait for the deployment to be observed by the controller and has at least specified number of updated replicas
   960  	if err = tester.waitForDeploymentUpdatedReplicasGTE(replicas); err != nil {
   961  		t.Fatal(err)
   962  	}
   963  
   964  	// Wait for the deployment to have MinimumReplicasUnavailable reason because the pods are not marked as ready
   965  	if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasUnavailable, apps.DeploymentAvailable); err != nil {
   966  		t.Fatal(err)
   967  	}
   968  
   969  	// Verify all replicas fields of DeploymentStatus have desired counts
   970  	if err = tester.checkDeploymentStatusReplicasFields(10, 10, 0, 0, 10); err != nil {
   971  		t.Fatal(err)
   972  	}
   973  
   974  	// Mark the pods as ready without waiting for the deployment to complete
   975  	if err = tester.markUpdatedPodsReadyWithoutComplete(); err != nil {
   976  		t.Fatal(err)
   977  	}
   978  
   979  	// Wait for number of ready replicas to equal number of replicas.
   980  	if err = tester.waitForReadyReplicas(); err != nil {
   981  		t.Fatal(err)
   982  	}
   983  
   984  	// Wait for the deployment to still have MinimumReplicasUnavailable reason within minReadySeconds period
   985  	if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasUnavailable, apps.DeploymentAvailable); err != nil {
   986  		t.Fatal(err)
   987  	}
   988  
   989  	// Verify all replicas fields of DeploymentStatus have desired counts
   990  	if err = tester.checkDeploymentStatusReplicasFields(10, 10, 10, 0, 10); err != nil {
   991  		t.Fatal(err)
   992  	}
   993  
   994  	// Update the deployment's minReadySeconds to a small value
   995  	tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
   996  		update.Spec.MinReadySeconds = 1
   997  	})
   998  	if err != nil {
   999  		t.Fatalf("failed updating deployment %q: %v", deploymentName, err)
  1000  	}
  1001  
  1002  	// Wait for the deployment to notice minReadySeconds has changed
  1003  	if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  1004  		t.Fatal(err)
  1005  	}
  1006  
  1007  	// Wait for the deployment to have MinimumReplicasAvailable reason after minReadySeconds period
  1008  	if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasAvailable, apps.DeploymentAvailable); err != nil {
  1009  		t.Fatal(err)
  1010  	}
  1011  
  1012  	// Verify all replicas fields of DeploymentStatus have desired counts
  1013  	if err = tester.checkDeploymentStatusReplicasFields(10, 10, 10, 10, 0); err != nil {
  1014  		t.Fatal(err)
  1015  	}
  1016  }
  1017  
  1018  // Wait for deployment to automatically patch incorrect ControllerRef of RS
  1019  func testRSControllerRefPatch(t *testing.T, tester *deploymentTester, rs *apps.ReplicaSet, ownerReference *metav1.OwnerReference, expectedOwnerReferenceNum int) {
  1020  	ns := rs.Namespace
  1021  	rsClient := tester.c.AppsV1().ReplicaSets(ns)
  1022  	rs, err := tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  1023  		update.OwnerReferences = []metav1.OwnerReference{*ownerReference}
  1024  	})
  1025  	if err != nil {
  1026  		t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  1027  	}
  1028  
  1029  	if err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1030  		newRS, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1031  		if err != nil {
  1032  			return false, err
  1033  		}
  1034  		return metav1.GetControllerOf(newRS) != nil, nil
  1035  	}); err != nil {
  1036  		t.Fatalf("failed to wait for controllerRef of the replicaset %q to become nil: %v", rs.Name, err)
  1037  	}
  1038  
  1039  	newRS, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1040  	if err != nil {
  1041  		t.Fatalf("failed to obtain replicaset %q: %v", rs.Name, err)
  1042  	}
  1043  	controllerRef := metav1.GetControllerOf(newRS)
  1044  	if controllerRef.UID != tester.deployment.UID {
  1045  		t.Fatalf("controllerRef of replicaset %q has a different UID: Expected %v, got %v", newRS.Name, tester.deployment.UID, controllerRef.UID)
  1046  	}
  1047  	ownerReferenceNum := len(newRS.GetOwnerReferences())
  1048  	if ownerReferenceNum != expectedOwnerReferenceNum {
  1049  		t.Fatalf("unexpected number of owner references for replicaset %q: Expected %d, got %d", newRS.Name, expectedOwnerReferenceNum, ownerReferenceNum)
  1050  	}
  1051  }
  1052  
  1053  func TestGeneralReplicaSetAdoption(t *testing.T) {
  1054  	_, ctx := ktesting.NewTestContext(t)
  1055  	ctx, cancel := context.WithCancel(ctx)
  1056  	defer cancel()
  1057  
  1058  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
  1059  	defer closeFn()
  1060  
  1061  	name := "test-general-replicaset-adoption"
  1062  	ns := framework.CreateNamespaceOrDie(c, name, t)
  1063  	defer framework.DeleteNamespaceOrDie(c, ns, t)
  1064  
  1065  	deploymentName := "deployment"
  1066  	replicas := int32(1)
  1067  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  1068  	var err error
  1069  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  1070  	if err != nil {
  1071  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  1072  	}
  1073  
  1074  	// Start informer and controllers
  1075  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
  1076  	defer stopControllers()
  1077  
  1078  	// Wait for the Deployment to be updated to revision 1
  1079  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  1080  		t.Fatal(err)
  1081  	}
  1082  
  1083  	// Ensure the deployment completes while marking its pods as ready simultaneously
  1084  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  1085  		t.Fatal(err)
  1086  	}
  1087  
  1088  	// Get replicaset of the deployment
  1089  	rs, err := testutil.GetNewReplicaSet(tester.deployment, c)
  1090  	if err != nil {
  1091  		t.Fatalf("failed to get replicaset of deployment %q: %v", deploymentName, err)
  1092  	}
  1093  	if rs == nil {
  1094  		t.Fatalf("unable to find replicaset of deployment %q", deploymentName)
  1095  	}
  1096  
  1097  	// When the only OwnerReference of the RS points to another type of API object such as statefulset
  1098  	// with Controller=false, the deployment should add a second OwnerReference (ControllerRef) pointing to itself
  1099  	// with Controller=true
  1100  	var falseVar = false
  1101  	ownerReference := metav1.OwnerReference{UID: uuid.NewUUID(), APIVersion: "apps/v1", Kind: "StatefulSet", Name: deploymentName, Controller: &falseVar}
  1102  	testRSControllerRefPatch(t, tester, rs, &ownerReference, 2)
  1103  
  1104  	// When the only OwnerReference of the RS points to the deployment with Controller=false,
  1105  	// the deployment should set Controller=true for the only OwnerReference
  1106  	ownerReference = metav1.OwnerReference{UID: tester.deployment.UID, APIVersion: "apps/v1", Kind: "Deployment", Name: deploymentName, Controller: &falseVar}
  1107  	testRSControllerRefPatch(t, tester, rs, &ownerReference, 1)
  1108  }
  1109  
  1110  func testScalingUsingScaleSubresource(t *testing.T, tester *deploymentTester, replicas int32) {
  1111  	ns := tester.deployment.Namespace
  1112  	deploymentName := tester.deployment.Name
  1113  	deploymentClient := tester.c.AppsV1().Deployments(ns)
  1114  	deployment, err := deploymentClient.Get(context.TODO(), deploymentName, metav1.GetOptions{})
  1115  	if err != nil {
  1116  		t.Fatalf("Failed to obtain deployment %q: %v", deploymentName, err)
  1117  	}
  1118  	scale, err := tester.c.AppsV1().Deployments(ns).GetScale(context.TODO(), deploymentName, metav1.GetOptions{})
  1119  	if err != nil {
  1120  		t.Fatalf("Failed to obtain scale subresource for deployment %q: %v", deploymentName, err)
  1121  	}
  1122  	if scale.Spec.Replicas != *deployment.Spec.Replicas {
  1123  		t.Fatalf("Scale subresource for deployment %q does not match .Spec.Replicas: expected %d, got %d", deploymentName, *deployment.Spec.Replicas, scale.Spec.Replicas)
  1124  	}
  1125  
  1126  	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
  1127  		scale, err := tester.c.AppsV1().Deployments(ns).GetScale(context.TODO(), deploymentName, metav1.GetOptions{})
  1128  		if err != nil {
  1129  			return err
  1130  		}
  1131  		scale.Spec.Replicas = replicas
  1132  		_, err = tester.c.AppsV1().Deployments(ns).UpdateScale(context.TODO(), deploymentName, scale, metav1.UpdateOptions{})
  1133  		return err
  1134  	}); err != nil {
  1135  		t.Fatalf("Failed to set .Spec.Replicas of scale subresource for deployment %q: %v", deploymentName, err)
  1136  	}
  1137  
  1138  	deployment, err = deploymentClient.Get(context.TODO(), deploymentName, metav1.GetOptions{})
  1139  	if err != nil {
  1140  		t.Fatalf("Failed to obtain deployment %q: %v", deploymentName, err)
  1141  	}
  1142  	if *deployment.Spec.Replicas != replicas {
  1143  		t.Fatalf(".Spec.Replicas of deployment %q does not match its scale subresource: expected %d, got %d", deploymentName, replicas, *deployment.Spec.Replicas)
  1144  	}
  1145  }
  1146  
  1147  func TestDeploymentScaleSubresource(t *testing.T) {
  1148  	_, ctx := ktesting.NewTestContext(t)
  1149  	ctx, cancel := context.WithCancel(ctx)
  1150  	defer cancel()
  1151  
  1152  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
  1153  	defer closeFn()
  1154  
  1155  	name := "test-deployment-scale-subresource"
  1156  	ns := framework.CreateNamespaceOrDie(c, name, t)
  1157  	defer framework.DeleteNamespaceOrDie(c, ns, t)
  1158  
  1159  	deploymentName := "deployment"
  1160  	replicas := int32(2)
  1161  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  1162  	var err error
  1163  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  1164  	if err != nil {
  1165  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  1166  	}
  1167  
  1168  	// Start informer and controllers
  1169  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
  1170  	defer stopControllers()
  1171  
  1172  	// Wait for the Deployment to be updated to revision 1
  1173  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  1174  		t.Fatal(err)
  1175  	}
  1176  
  1177  	// Ensure the deployment completes while marking its pods as ready simultaneously
  1178  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  1179  		t.Fatal(err)
  1180  	}
  1181  
  1182  	// Use scale subresource to scale the deployment up to 3
  1183  	testScalingUsingScaleSubresource(t, tester, 3)
  1184  	// Use the scale subresource to scale the deployment down to 0
  1185  	testScalingUsingScaleSubresource(t, tester, 0)
  1186  }
  1187  
  1188  // This test verifies that the Deployment does orphan a ReplicaSet when the ReplicaSet's
  1189  // .Labels field is changed to no longer match the Deployment's selector. It also partially
  1190  // verifies that collision avoidance mechanism is triggered when a Deployment's new ReplicaSet
  1191  // is orphaned, even without PodTemplateSpec change. Refer comment below for more info:
  1192  // https://github.com/kubernetes/kubernetes/pull/59212#discussion_r166465113
  1193  func TestReplicaSetOrphaningAndAdoptionWhenLabelsChange(t *testing.T) {
  1194  	_, ctx := ktesting.NewTestContext(t)
  1195  	ctx, cancel := context.WithCancel(ctx)
  1196  	defer cancel()
  1197  
  1198  	closeFn, rm, dc, informers, c := dcSetup(ctx, t)
  1199  	defer closeFn()
  1200  
  1201  	name := "test-replicaset-orphaning-and-adoption-when-labels-change"
  1202  	ns := framework.CreateNamespaceOrDie(c, name, t)
  1203  	defer framework.DeleteNamespaceOrDie(c, ns, t)
  1204  
  1205  	deploymentName := "deployment"
  1206  	replicas := int32(1)
  1207  	tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  1208  	var err error
  1209  	tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  1210  	if err != nil {
  1211  		t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  1212  	}
  1213  
  1214  	// Start informer and controllers
  1215  	stopControllers := runControllersAndInformers(t, rm, dc, informers)
  1216  	defer stopControllers()
  1217  
  1218  	// Wait for the Deployment to be updated to revision 1
  1219  	if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  1220  		t.Fatal(err)
  1221  	}
  1222  
  1223  	// Ensure the deployment completes while marking its pods as ready simultaneously
  1224  	if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  1225  		t.Fatal(err)
  1226  	}
  1227  
  1228  	// Orphaning: deployment should remove OwnerReference from a RS when the RS's labels change to not match its labels
  1229  
  1230  	// Get replicaset of the deployment
  1231  	rs, err := testutil.GetNewReplicaSet(tester.deployment, c)
  1232  	if err != nil {
  1233  		t.Fatalf("failed to get replicaset of deployment %q: %v", deploymentName, err)
  1234  	}
  1235  	if rs == nil {
  1236  		t.Fatalf("unable to find replicaset of deployment %q", deploymentName)
  1237  	}
  1238  
  1239  	// Verify controllerRef of the replicaset is not nil and pointing to the deployment
  1240  	controllerRef := metav1.GetControllerOf(rs)
  1241  	if controllerRef == nil {
  1242  		t.Fatalf("controllerRef of replicaset %q is nil", rs.Name)
  1243  	}
  1244  	if controllerRef.UID != tester.deployment.UID {
  1245  		t.Fatalf("controllerRef of replicaset %q has a different UID: Expected %v, got %v", rs.Name, tester.deployment.UID, controllerRef.UID)
  1246  	}
  1247  
  1248  	// Change the replicaset's labels to not match the deployment's labels
  1249  	labelMap := map[string]string{"new-name": "new-test"}
  1250  	rs, err = tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  1251  		update.Labels = labelMap
  1252  	})
  1253  	if err != nil {
  1254  		t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  1255  	}
  1256  
  1257  	// Wait for the controllerRef of the replicaset to become nil
  1258  	rsClient := tester.c.AppsV1().ReplicaSets(ns.Name)
  1259  	if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1260  		rs, err = rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1261  		if err != nil {
  1262  			return false, err
  1263  		}
  1264  		return metav1.GetControllerOf(rs) == nil, nil
  1265  	}); err != nil {
  1266  		t.Fatalf("failed to wait for controllerRef of replicaset %q to become nil: %v", rs.Name, err)
  1267  	}
  1268  
  1269  	// Wait for the deployment to create a new replicaset
  1270  	// This will trigger collision avoidance due to deterministic nature of replicaset name
  1271  	// i.e., the new replicaset will have a name with different hash to preserve name uniqueness
  1272  	var newRS *apps.ReplicaSet
  1273  	if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1274  		newRS, err = testutil.GetNewReplicaSet(tester.deployment, c)
  1275  		if err != nil {
  1276  			return false, fmt.Errorf("failed to get new replicaset of deployment %q after orphaning: %v", deploymentName, err)
  1277  		}
  1278  		return newRS != nil, nil
  1279  	}); err != nil {
  1280  		t.Fatalf("failed to wait for deployment %q to create a new replicaset after orphaning: %v", deploymentName, err)
  1281  	}
  1282  	if newRS.UID == rs.UID {
  1283  		t.Fatalf("expect deployment %q to create a new replicaset different from the orphaned one, but it isn't", deploymentName)
  1284  	}
  1285  
  1286  	// Adoption: deployment should add controllerRef to a RS when the RS's labels change to match its labels
  1287  
  1288  	// Change the old replicaset's labels to match the deployment's labels
  1289  	rs, err = tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  1290  		update.Labels = testLabels()
  1291  	})
  1292  	if err != nil {
  1293  		t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  1294  	}
  1295  
  1296  	// Wait for the deployment to adopt the old replicaset
  1297  	if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1298  		rs, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1299  		if err != nil {
  1300  			return false, err
  1301  		}
  1302  		controllerRef = metav1.GetControllerOf(rs)
  1303  		return controllerRef != nil && controllerRef.UID == tester.deployment.UID, nil
  1304  	}); err != nil {
  1305  		t.Fatalf("failed waiting for replicaset adoption by deployment %q to complete: %v", deploymentName, err)
  1306  	}
  1307  }
  1308  

View as plain text