objectrestarter_test.go

Documentation: edge-infra.dev/pkg/k8s/runtime/objectrestarter

     1  package objectrestarter
     2  
     3  import (
     4  	"context"
     5  	"os"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/stretchr/testify/suite"
    10  	appsv1 "k8s.io/api/apps/v1"
    11  	corev1 "k8s.io/api/core/v1"
    12  	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    13  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    14  	"k8s.io/apimachinery/pkg/runtime/schema"
    15  	"sigs.k8s.io/cli-utils/pkg/object"
    16  	"sigs.k8s.io/controller-runtime/pkg/client"
    17  
    18  	"edge-infra.dev/pkg/k8s/runtime/sap"
    19  	"edge-infra.dev/test/framework"
    20  	"edge-infra.dev/test/framework/integration"
    21  	"edge-infra.dev/test/framework/k8s"
    22  	"edge-infra.dev/test/framework/k8s/envtest"
    23  )
    24  
    25  func TestMain(m *testing.M) {
    26  	framework.HandleFlags()
    27  	os.Exit(m.Run())
    28  }
    29  
    30  type Suite struct {
    31  	*framework.Framework
    32  	*k8s.K8s
    33  	ctx          context.Context
    34  	timeout      time.Duration
    35  	tick         time.Duration
    36  	workloadName string
    37  }
    38  
    39  // Test setup function which runs the rest of the tests via Suite
    40  func TestObjectRestarter(t *testing.T) {
    41  	testEnv := envtest.Setup(envtest.WithoutCRDs())
    42  
    43  	k := k8s.New(testEnv.Config)
    44  	f := framework.New("objectrestarter").Register(k)
    45  
    46  	s := &Suite{
    47  		Framework:    f,
    48  		K8s:          k,
    49  		ctx:          context.Background(),
    50  		timeout:      k8s.Timeouts.DefaultTimeout,
    51  		tick:         k8s.Timeouts.Tick,
    52  		workloadName: "redis-master",
    53  	}
    54  
    55  	suite.Run(t, s)
    56  
    57  	t.Cleanup(func() {
    58  		f.NoError(testEnv.Stop())
    59  	})
    60  }
    61  
    62  func (s *Suite) TestObjectRestarter_Integration() {
    63  	integration.SkipIfNot(s.Framework)
    64  
    65  	_, deployment := s.testWorkload()
    66  
    67  	if err := s.Client.Create(s.ctx, deployment); err != nil {
    68  		s.FailNow("failed to create deployment", err)
    69  	}
    70  
    71  	s.Log("waiting on deployment to become ready so it can be restarted")
    72  	// TODO(aw185176): is this correct wrt client options? handle in framework/k8s sweep
    73  	mgr, err := sap.NewResourceManagerFromConfig(
    74  		s.RESTConfig(),
    75  		client.Options{},
    76  		k8s.FieldManagerOwner(s.Framework),
    77  	)
    78  	if err != nil {
    79  		s.FailNow("failed to create resource manager", err)
    80  	}
    81  	err = mgr.WaitForSet(s.ctx, []object.ObjMetadata{
    82  		{
    83  			Name:      deployment.Name,
    84  			Namespace: deployment.Namespace,
    85  			// for some reason deployment.GroupVersionKind().GroupKind() doesn't work
    86  			GroupKind: schema.GroupKind{
    87  				Group: "apps",
    88  				Kind:  "Deployment",
    89  			},
    90  		},
    91  	}, sap.WaitOptions{Timeout: s.timeout})
    92  	s.NoError(err, "deployment never became ready")
    93  
    94  	s.restart(deployment)
    95  	s.checkAnnotation(deployment)
    96  
    97  	// this is actually quite paranoid as we are verifying that the built-in K8s
    98  	// mechanism works, but it is useful to confirm that we are using the
    99  	// mechanism correctly both now and over time as K8s itself changes.
   100  	s.Log("waiting for deployment to restart")
   101  	d := &appsv1.Deployment{}
   102  	s.Eventually(func() bool {
   103  		_ = s.Client.Get(s.ctx, client.ObjectKeyFromObject(deployment), d)
   104  		return deployCondition(
   105  			d.Status.Conditions,
   106  			appsv1.DeploymentProgressing,
   107  			corev1.ConditionTrue,
   108  		)
   109  	}, s.timeout, s.tick, "deployment never began progressing again")
   110  }
   111  
   112  func (s *Suite) TestObjectRestarter_Unit() {
   113  	// we don't want to run this for integration tests because it does the same
   114  	// thing and we don't want to create additional namespaces during integration
   115  	// tests
   116  	integration.SkipIf(s.Framework)
   117  
   118  	namespace, deployment := s.testWorkload()
   119  
   120  	if err := s.Client.Create(s.ctx, namespace); err != nil {
   121  		s.FailNow("failed to create namespace", err)
   122  	}
   123  
   124  	if err := s.Client.Create(s.ctx, deployment); err != nil {
   125  		s.FailNow("failed to create deployment", err)
   126  	}
   127  
   128  	s.restart(deployment)
   129  	s.checkAnnotation(deployment)
   130  }
   131  
   132  func (s *Suite) TestIsObjectRestartable() {
   133  	s.True(IsObjectRestartable(&appsv1.Deployment{}))
   134  	s.True(IsObjectRestartable(&appsv1.StatefulSet{}))
   135  	s.True(IsObjectRestartable(&appsv1.DaemonSet{}))
   136  
   137  	workload := &unstructured.Unstructured{}
   138  	workload.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment"))
   139  	s.True(IsObjectRestartable(workload))
   140  }
   141  
   142  //
   143  // test helpers
   144  //
   145  
   146  func (s *Suite) checkAnnotation(deployment *appsv1.Deployment) {
   147  	s.Log("fetching deployment to confirm it was changed")
   148  	d := &appsv1.Deployment{}
   149  	s.NoError(s.Client.Get(s.ctx, client.ObjectKeyFromObject(deployment), d))
   150  	s.True(
   151  		v1.HasAnnotation(d.Spec.Template.ObjectMeta, restartAnnotation),
   152  		"deployment was not annotated",
   153  	)
   154  }
   155  
   156  func (s *Suite) restart(deployment *appsv1.Deployment) {
   157  	s.Log("patching deployment with restart annotation")
   158  	s.NoError(Restart(s.ctx, s.Client, deployment), "failed to patch deployment")
   159  }
   160  
   161  func (s *Suite) testWorkload() (namespace *corev1.Namespace, deployment *appsv1.Deployment) {
   162  	n := &corev1.Namespace{
   163  		ObjectMeta: v1.ObjectMeta{
   164  			Name: "obj-restarter-rest-deployment",
   165  		},
   166  	}
   167  
   168  	d := &appsv1.Deployment{
   169  		ObjectMeta: v1.ObjectMeta{
   170  			Name: "redis-master",
   171  		},
   172  		Spec: appsv1.DeploymentSpec{
   173  			Selector: &v1.LabelSelector{
   174  				MatchLabels: map[string]string{
   175  					"app": "redis",
   176  				},
   177  			},
   178  			Template: corev1.PodTemplateSpec{
   179  				ObjectMeta: v1.ObjectMeta{
   180  					Labels: map[string]string{
   181  						"app": "redis",
   182  					},
   183  				},
   184  				Spec: corev1.PodSpec{
   185  					Containers: []corev1.Container{
   186  						{
   187  							Name:  "master",
   188  							Image: "redis:6",
   189  						},
   190  					},
   191  				},
   192  			},
   193  		},
   194  	}
   195  
   196  	if integration.IsIntegrationTest() {
   197  		d.Namespace = s.Namespace
   198  	} else {
   199  		d.Namespace = n.Name
   200  	}
   201  
   202  	return n, d
   203  }
   204  
   205  // this is janky as hell because i couldn't figure out how to get the kstatus
   206  // generic status readers stuff working
   207  func deployCondition(cc []appsv1.DeploymentCondition, t appsv1.DeploymentConditionType, s corev1.ConditionStatus) bool {
   208  	for _, c := range cc {
   209  		if c.Type == t && c.Status == s {
   210  			return true
   211  		}
   212  	}
   213  	return false
   214  }
   215
View as plain text