thousand_deployments_retry_test.go

Documentation: sigs.k8s.io/cli-utils/test/stress

     1  // Copyright 2022 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package stress
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"time"
    10  
    11  	. "github.com/onsi/ginkgo/v2"
    12  	. "github.com/onsi/gomega"
    13  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    14  	"k8s.io/klog/v2"
    15  	"sigs.k8s.io/cli-utils/pkg/apply"
    16  	"sigs.k8s.io/cli-utils/pkg/apply/event"
    17  	"sigs.k8s.io/cli-utils/pkg/common"
    18  	"sigs.k8s.io/cli-utils/pkg/inventory"
    19  	"sigs.k8s.io/cli-utils/test/e2e/e2eutil"
    20  	"sigs.k8s.io/cli-utils/test/e2e/invconfig"
    21  	"sigs.k8s.io/controller-runtime/pkg/client"
    22  )
    23  
    24  // thousandDeploymentsRetryTest tests one pre-existing namespace with 1,000
    25  // Deployments in it. The wait timeout is set too short to confirm
    26  // reconciliation, but the apply/destroy is retried until success.
    27  //
    28  // The Deployments themselves are easy to get status on, but with the retrieval
    29  // of generated resource status (ReplicaSets & Pods), this becomes expensive.
    30  func thousandDeploymentsRetryTest(ctx context.Context, c client.Client, invConfig invconfig.InventoryConfig, inventoryName, namespaceName string) {
    31  	By("Apply LOTS of resources")
    32  	applier := invConfig.ApplierFactoryFunc()
    33  	inventoryID := fmt.Sprintf("%s-%s", inventoryName, namespaceName)
    34  
    35  	inventoryInfo := invconfig.CreateInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID)
    36  
    37  	resources := []*unstructured.Unstructured{}
    38  
    39  	deploymentObjTemplate := e2eutil.ManifestToUnstructured([]byte(deploymentYaml))
    40  	deploymentObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})
    41  
    42  	objectCount := 1000
    43  
    44  	for i := 1; i <= objectCount; i++ {
    45  		deploymentObj := deploymentObjTemplate.DeepCopy()
    46  		deploymentObj.SetNamespace(namespaceName)
    47  
    48  		// change name & selector labels to avoid overlap between deployments
    49  		name := fmt.Sprintf("nginx-%d", i)
    50  		deploymentObj.SetName(name)
    51  		err := unstructured.SetNestedField(deploymentObj.Object, name, "spec", "selector", "matchLabels", "app")
    52  		Expect(err).ToNot(HaveOccurred())
    53  		err = unstructured.SetNestedField(deploymentObj.Object, name, "spec", "template", "metadata", "labels", "app")
    54  		Expect(err).ToNot(HaveOccurred())
    55  
    56  		resources = append(resources, deploymentObj)
    57  	}
    58  
    59  	defer func() {
    60  		By("Cleanup Deployments")
    61  		e2eutil.DeleteAllUnstructuredIfExists(ctx, c, deploymentObjTemplate)
    62  	}()
    63  
    64  	startTotal := time.Now()
    65  
    66  	var applierEvents []event.Event
    67  
    68  	maxAttempts := 15
    69  	reconcileTimeout := 2 * time.Minute
    70  
    71  	for attempt := 1; attempt <= maxAttempts; attempt++ {
    72  		start := time.Now()
    73  
    74  		applierEvents = e2eutil.RunCollect(applier.Run(ctx, inventoryInfo, resources, apply.ApplierOptions{
    75  			// SSA reduces GET+PATCH to just PATCH, which is faster
    76  			ServerSideOptions: common.ServerSideOptions{
    77  				ServerSideApply: true,
    78  				ForceConflicts:  true,
    79  				FieldManager:    "cli-utils.kubernetes.io",
    80  			},
    81  			ReconcileTimeout: reconcileTimeout,
    82  			EmitStatusEvents: false,
    83  		}))
    84  
    85  		duration := time.Since(start)
    86  		klog.Infof("Applier.Run execution time (attempt: %d): %v", attempt, duration)
    87  
    88  		e2eutil.ExpectNoEventErrors(applierEvents)
    89  
    90  		// Retry if ReconcileTimeout
    91  		retry := false
    92  		for _, e := range applierEvents {
    93  			if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout {
    94  				retry = true
    95  			}
    96  		}
    97  		if !retry {
    98  			break
    99  		}
   100  	}
   101  
   102  	durationTotal := time.Since(startTotal)
   103  	klog.Infof("Applier.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal)
   104  
   105  	e2eutil.ExpectNoReconcileTimeouts(applierEvents)
   106  
   107  	By("Verify inventory created")
   108  	invConfig.InvSizeVerifyFunc(ctx, c, inventoryName, namespaceName, inventoryID, len(resources), len(resources))
   109  
   110  	By(fmt.Sprintf("Verify %d Deployments created", objectCount))
   111  	e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, objectCount)
   112  
   113  	By("Destroy LOTS of resources")
   114  	destroyer := invConfig.DestroyerFactoryFunc()
   115  
   116  	startTotal = time.Now()
   117  
   118  	var destroyerEvents []event.Event
   119  
   120  	for attempt := 1; attempt <= maxAttempts; attempt++ {
   121  		start := time.Now()
   122  
   123  		destroyerEvents = e2eutil.RunCollect(destroyer.Run(ctx, inventoryInfo, apply.DestroyerOptions{
   124  			InventoryPolicy: inventory.PolicyAdoptIfNoInventory,
   125  			DeleteTimeout:   reconcileTimeout,
   126  		}))
   127  
   128  		duration := time.Since(start)
   129  		klog.Infof("Destroyer.Run execution time (attempt: %d): %v", attempt, duration)
   130  
   131  		e2eutil.ExpectNoEventErrors(destroyerEvents)
   132  
   133  		// Retry if ReconcileTimeout
   134  		retry := false
   135  		for _, e := range applierEvents {
   136  			if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout {
   137  				retry = true
   138  			}
   139  		}
   140  		if !retry {
   141  			break
   142  		}
   143  	}
   144  
   145  	durationTotal = time.Since(startTotal)
   146  	klog.Infof("Destroyer.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal)
   147  
   148  	e2eutil.ExpectNoReconcileTimeouts(applierEvents)
   149  
   150  	By("Verify inventory deleted")
   151  	invConfig.InvNotExistsFunc(ctx, c, inventoryName, namespaceName, inventoryID)
   152  
   153  	By(fmt.Sprintf("Verify %d Deployments deleted", objectCount))
   154  	e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, 0)
   155  }
   156
View as plain text