...

Source file src/edge-infra.dev/test/e2e/linkerd/workloadinjection/workloadinjection_test.go

Documentation: edge-infra.dev/test/e2e/linkerd/workloadinjection

     1  package linkerd
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/stretchr/testify/assert"
    11  	"github.com/stretchr/testify/require"
    12  	"gotest.tools/v3/poll"
    13  	corev1 "k8s.io/api/core/v1"
    14  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    15  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    16  	"k8s.io/apimachinery/pkg/runtime"
    17  	"k8s.io/apimachinery/pkg/types"
    18  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    19  	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
    20  	ctrl "sigs.k8s.io/controller-runtime"
    21  	"sigs.k8s.io/controller-runtime/pkg/client"
    22  
    23  	"edge-infra.dev/pkg/edge/linkerd"
    24  	l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1"
    25  	"edge-infra.dev/pkg/k8s/runtime/conditions"
    26  	"edge-infra.dev/pkg/k8s/runtime/controller"
    27  	"edge-infra.dev/pkg/lib/fog"
    28  	"edge-infra.dev/test/f2"
    29  	"edge-infra.dev/test/f2/x/ktest"
    30  	"edge-infra.dev/test/f2/x/ktest/envtest"
    31  )
    32  
    33  var f f2.Framework
    34  
    35  func TestMain(m *testing.M) {
    36  	ctrl.SetLogger(fog.New())
    37  	f = f2.New(context.Background(),
    38  		f2.WithExtensions(
    39  			ktest.New(
    40  				ktest.SkipNamespaceCreation(),
    41  				ktest.WithCtrlManager(createManager),
    42  				ktest.WithEnvtestOptions(envtest.WithoutCRDs()),
    43  			),
    44  		),
    45  	).
    46  		Setup(func(ctx f2.Context) (f2.Context, error) {
    47  			k, err := ktest.FromContext(ctx)
    48  			if err != nil {
    49  				return ctx, err
    50  			}
    51  			// delete all linkerd workloadinjections
    52  			err = k.Client.DeleteAllOf(ctx, &l5dv1alpha1.LinkerdWorkloadInjection{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: new(int64)}})
    53  			if err != nil {
    54  				return ctx, err
    55  			}
    56  			l5d := &l5dv1alpha1.Linkerd{}
    57  			err = k.Client.Get(ctx, types.NamespacedName{Name: l5dv1alpha1.Name}, l5d)
    58  			if err != nil {
    59  				return ctx, err
    60  			}
    61  			// disable LinkerdWorkloadInjection/linkerd
    62  			l5d.Annotations[l5dv1alpha1.WorkloadInjectionDisabledAnnotation] = "true"
    63  			return ctx, k.Client.Update(ctx, l5d)
    64  		}).
    65  		Teardown(func(ctx f2.Context) (f2.Context, error) {
    66  			k, err := ktest.FromContext(ctx)
    67  			if err != nil {
    68  				return ctx, err
    69  			}
    70  			l5d := &l5dv1alpha1.Linkerd{}
    71  			err = k.Client.Get(ctx, types.NamespacedName{Name: l5dv1alpha1.Name}, l5d)
    72  			if err != nil {
    73  				return ctx, err
    74  			}
    75  			// re-enable LinkerdWorkloadInjection/linkerd
    76  			delete(l5d.Annotations, l5dv1alpha1.WorkloadInjectionDisabledAnnotation)
    77  			return ctx, k.Client.Update(ctx, l5d)
    78  		}).
    79  		WithLabel("dsds", "true")
    80  	os.Exit(f.Run(m))
    81  }
    82  
    83  func TestSpecificNamespaceWorkloadinjection(t *testing.T) {
    84  	const specificNamespaceWLI = "specific-namespace"
    85  	var k *ktest.K8s
    86  	var beforeRestart time.Time
    87  	feature := f2.NewFeature("workloadinjection").
    88  		Setup("create workloadinjection", func(ctx f2.Context, t *testing.T) f2.Context {
    89  			k = ktest.FromContextT(ctx, t)
    90  			beforeRestart = time.Now()
    91  			require.NoError(t, k.Client.Create(ctx, &l5dv1alpha1.LinkerdWorkloadInjection{
    92  				ObjectMeta: metav1.ObjectMeta{Name: specificNamespaceWLI},
    93  				Spec: l5dv1alpha1.LinkerdWorkloadInjectionSpec{
    94  					Namespaces: []string{"corednsctl"},
    95  					Force:      true,
    96  				},
    97  			}))
    98  			return ctx
    99  		}).
   100  		Test("workloadinjection completed", func(ctx f2.Context, t *testing.T) f2.Context {
   101  			k = ktest.FromContextT(ctx, t)
   102  			wli := l5dv1alpha1.LinkerdWorkloadInjection{}
   103  			require.Eventually(t, func() bool {
   104  				if err := k.Client.Get(ctx, types.NamespacedName{Name: specificNamespaceWLI}, &wli); err != nil {
   105  					return false
   106  				}
   107  				return wli.IsCompleted() && conditions.IsReady(&wli) && wli.Status.Inventory != nil && len(wli.Status.Inventory.Entries) == 1
   108  			}, ktest.Timeout*3, ktest.Tick, "Wait for linkerd reinjection job to complete")
   109  			assert.Empty(t, wli.Status.FailedInventory)
   110  			return ctx
   111  		}).
   112  		Test("specific namespace is restarted", func(ctx f2.Context, t *testing.T) f2.Context {
   113  			k = ktest.FromContextT(ctx, t)
   114  			// check pods restarted
   115  			k.WaitOn(t, func(_ poll.LogT) poll.Result {
   116  				podList := &corev1.PodList{}
   117  				require.NoError(t, k.Client.List(ctx, podList, client.InNamespace("corednsctl")))
   118  				for _, pod := range podList.Items {
   119  					// ignore these pods
   120  					if !pod.DeletionTimestamp.IsZero() {
   121  						continue
   122  					}
   123  					if pod.Status.Phase == corev1.PodRunning && !pod.CreationTimestamp.After(beforeRestart) {
   124  						return poll.Continue("pod still running: %s", pod.Name)
   125  					}
   126  				}
   127  				return poll.Success()
   128  			}, poll.WithDelay(ktest.Tick), poll.WithTimeout(ktest.Timeout*5))
   129  			return ctx
   130  		}).
   131  		Teardown("delete created workloadinjection", func(ctx f2.Context, t *testing.T) f2.Context {
   132  			deleteWLI(ctx, t, specificNamespaceWLI)
   133  			return ctx
   134  		}).
   135  		Feature()
   136  	f.Test(t, feature)
   137  }
   138  
   139  func TestAllNamespacesWorkloadinjection(t *testing.T) {
   140  	const allNamespacesWLI = "all-namespaces"
   141  	var k *ktest.K8s
   142  	var beforeRestart time.Time
   143  	feature := f2.NewFeature("workloadinjection").
   144  		Setup("create workloadinjection", func(ctx f2.Context, t *testing.T) f2.Context {
   145  			k = ktest.FromContextT(ctx, t)
   146  			beforeRestart = time.Now()
   147  			require.NoError(t, k.Client.Create(ctx, &l5dv1alpha1.LinkerdWorkloadInjection{
   148  				ObjectMeta: metav1.ObjectMeta{Name: allNamespacesWLI},
   149  				Spec: l5dv1alpha1.LinkerdWorkloadInjectionSpec{
   150  					Namespaces: []string{},
   151  					Force:      true,
   152  				},
   153  			}))
   154  			return ctx
   155  		}).
   156  		Test("workloadinjection completed", func(ctx f2.Context, t *testing.T) f2.Context {
   157  			k = ktest.FromContextT(ctx, t)
   158  			wli := l5dv1alpha1.LinkerdWorkloadInjection{}
   159  			require.Eventually(t, func() bool {
   160  				if err := k.Client.Get(ctx, types.NamespacedName{Name: allNamespacesWLI}, &wli); err != nil {
   161  					return false
   162  				}
   163  				return wli.IsCompleted() && conditions.IsReady(&wli) && wli.Status.Inventory != nil
   164  			}, ktest.Timeout*5, ktest.Tick, "Wait for linkerd reinjection job to complete")
   165  			assert.True(t, len(wli.Status.Inventory.Entries) > 1)
   166  			assert.Empty(t, wli.Status.FailedInventory)
   167  			return ctx
   168  		}).
   169  		Test("all namespaces are restarted", func(ctx f2.Context, t *testing.T) f2.Context {
   170  			k = ktest.FromContextT(ctx, t)
   171  			// check pods restarted
   172  			k.WaitOn(t, func(_ poll.LogT) poll.Result {
   173  				podList := &corev1.PodList{}
   174  				require.NoError(t, k.Client.List(ctx, podList))
   175  				stalePods := []string{}
   176  				for _, pod := range podList.Items {
   177  					// ignore these pods
   178  					if pod.Namespace == "fluent-operator" || pod.Spec.HostNetwork || !isLinkerdEnabled(pod) || !pod.DeletionTimestamp.IsZero() {
   179  						continue
   180  					}
   181  					if pod.Status.Phase == corev1.PodRunning && !pod.CreationTimestamp.After(beforeRestart) {
   182  						stalePods = append(stalePods, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name))
   183  					}
   184  				}
   185  				if len(stalePods) == 0 {
   186  					return poll.Success()
   187  				}
   188  				return poll.Continue("pods still running: %v", stalePods)
   189  			}, poll.WithDelay(ktest.Tick), poll.WithTimeout(ktest.Timeout*5))
   190  			return ctx
   191  		}).
   192  		Teardown("delete created workloadinjection", func(ctx f2.Context, t *testing.T) f2.Context {
   193  			deleteWLI(ctx, t, allNamespacesWLI)
   194  			return ctx
   195  		}).
   196  		Feature()
   197  	f.Test(t, feature)
   198  }
   199  
   200  func deleteWLI(ctx f2.Context, t *testing.T, name string) {
   201  	k := ktest.FromContextT(ctx, t)
   202  	require.NoError(t, k.Client.Delete(ctx, &l5dv1alpha1.LinkerdWorkloadInjection{ObjectMeta: metav1.ObjectMeta{Name: name}}))
   203  	require.Eventually(t, func() bool {
   204  		err := k.Client.Get(ctx, types.NamespacedName{Name: name}, &l5dv1alpha1.LinkerdWorkloadInjection{})
   205  		return apierrors.IsNotFound(err)
   206  	}, ktest.Timeout*5, ktest.Tick, "wait for workloadinjection to delete")
   207  }
   208  
   209  func createManager(opts ...controller.Option) (ctrl.Manager, error) {
   210  	mgrCfg, mgrOpts := controller.ProcessOptions(opts...)
   211  	mgrOpts.Scheme = createScheme()
   212  	return ctrl.NewManager(mgrCfg, mgrOpts)
   213  }
   214  
   215  func createScheme() *runtime.Scheme {
   216  	scheme := runtime.NewScheme()
   217  	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
   218  	utilruntime.Must(l5dv1alpha1.AddToScheme(scheme))
   219  	return scheme
   220  }
   221  
   222  func isLinkerdEnabled(pod corev1.Pod) bool {
   223  	if inject, found := pod.GetAnnotations()[linkerd.InjectionAnnotation]; found && inject == "enabled" {
   224  		return true
   225  	}
   226  	return false
   227  }
   228  

View as plain text