package linkerd import ( "fmt" "testing" "time" "cuelang.org/go/pkg/strings" goVersion "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gotest.tools/v3/assert/cmp" "gotest.tools/v3/poll" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "edge-infra.dev/pkg/edge/linkerd" "edge-infra.dev/pkg/edge/linkerd/certs/trustanchor" l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1" "edge-infra.dev/test/f2" "edge-infra.dev/test/f2/x/ktest" "edge-infra.dev/test/f2/x/ktest/kustomization" l5dhelm "edge-infra.dev/third_party/k8s/linkerd/helm" ) func TestManualTrustAnchorRotation(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd initialTrustAnchorSecret *corev1.Secret injected []string ) manualRotation := f2.NewFeature("manual trust anchor rotation"). Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) nodes := &corev1.NodeList{} require.NoError(t, k.Client.List(ctx, nodes)) initialTrustAnchorSecret = &corev1.Secret{} require.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), initialTrustAnchorSecret)) return ctx }). Test("manual rotation", func(ctx f2.Context, t *testing.T) f2.Context { l5d = &l5dv1alpha1.Linkerd{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "linkerd.edge.ncr.com"}, l5d)) patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf( `{"metadata":{"annotations":{"%s":"%s"}}}`, trustanchor.ManualTrustAnchorRotation, "true"), )) assert.NoError(t, k.Client.Patch(ctx, l5d, patch)) assert.Eventually(t, func() bool { return trustanchor.IsRotated(ctx, k.Client) }, ktest.Timeout*2, ktest.Tick, "Wait for linkerd trust anchor secret to be rotated") return ctx }). Test("check manual rotation annotation", func(ctx f2.Context, t *testing.T) f2.Context { l5d = &l5dv1alpha1.Linkerd{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "linkerd.edge.ncr.com"}, l5d)) assert.Eventually(t, func() bool { return !trustanchor.HasManualRotationAnnotation(l5d) }, ktest.Timeout*2, ktest.Tick, "wait for manual rotation annotation to be removed") return ctx }). Test("trust anchor check", func(ctx f2.Context, t *testing.T) f2.Context { currentTrustAnchorSecret := &corev1.Secret{} assert.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), currentTrustAnchorSecret)) assert.NotEqual(t, currentTrustAnchorSecret.Data["tls.crt"], initialTrustAnchorSecret.Data["tls.crt"]) caBundleOK, err := trustanchor.CheckCaBundle(ctx, k.Client) assert.NoError(t, err) assert.True(t, caBundleOK) return ctx }). Test("verify linkerd CR status", func(ctx f2.Context, t *testing.T) f2.Context { l5d := &l5dv1alpha1.Linkerd{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "linkerd.edge.ncr.com"}, l5d)) assert.True(t, isReady(l5d.Status.Conditions)) assert.NotNil(t, l5d.Status.Version) injected = l5d.Status.InjectedNamespaces return ctx }). Test("verify linkerdworkloadinjection CR status", func(ctx f2.Context, t *testing.T) f2.Context { workloadinjection := &l5dv1alpha1.LinkerdWorkloadInjection{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.Namespace, Namespace: linkerd.Namespace}, workloadinjection)) assert.True(t, isReady(workloadinjection.Status.Conditions)) return ctx }). Test("verify injected workloads are meshed successfully", func(ctx f2.Context, t *testing.T) f2.Context { podList := &corev1.PodList{} require.NoError(t, k.Client.List(ctx, podList)) for _, pod := range podList.Items { if pod.Status.Phase == corev1.PodRunning && isInjected(injected, pod.Namespace) && !pod.Spec.HostNetwork && !isLinkerdDisabled(pod) { assert.True(t, isMeshed(pod) && proxyReady(pod), pod.Name) } } return ctx }). Feature() f.Test(t, manualRotation) } // TestAutomaticTrustAnchorRotation is an e2e test that'll need to be run against a cluster at an older // version of linkerd to what is latest for us. If it isn't, this test will be skipped. func TestAutomaticTrustAnchorRotation(t *testing.T) { const ( l5dObjName = l5dv1alpha1.Name ) var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd initialTrustAnchorSecret *corev1.Secret injected []string latestL5dVersion = l5dhelm.Version ) automaticRotation := f2.NewFeature("automatic trust anchor rotation"). Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) nodes := &corev1.NodeList{} require.NoError(t, k.Client.List(ctx, nodes)) initialTrustAnchorSecret = &corev1.Secret{} require.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), initialTrustAnchorSecret)) return ctx }). Setup("linkerd version check", func(ctx f2.Context, t *testing.T) f2.Context { l5d = &l5dv1alpha1.Linkerd{} require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: l5dObjName}, l5d)) versionCheckSuccess, err := isLinkerdUpgradable(l5d, latestL5dVersion) require.NoError(t, err) if !versionCheckSuccess { t.SkipNow() } return ctx }). Setup("linkerd upgrade", func(ctx f2.Context, t *testing.T) f2.Context { crdManifests, err := kustomization.ProcessManifests(ctx.RunID, linkerdCRDManifests, linkerd.Namespace) require.NoError(t, err) for _, manifest := range crdManifests { require.NoError(t, k.Client.Create(ctx, manifest)) } controllerManifests, err := kustomization.ProcessManifests(ctx.RunID, linkerdControllerManifests, linkerd.Namespace) require.NoError(t, err) for _, manifest := range controllerManifests { require.NoError(t, k.Client.Create(ctx, manifest)) } return ctx }). Setup("wait for linkerd upgrade", func(ctx f2.Context, t *testing.T) f2.Context { l5d = &l5dv1alpha1.Linkerd{} require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: l5dObjName}, l5d)) k.WaitOn(t, k.Check(l5d, func(_ client.Object) cmp.Result { require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: l5dObjName}, l5d)) if l5d.Status.Version == latestL5dVersion { return cmp.ResultSuccess } return cmp.ResultFailure("linkerd not upgraded") }), poll.WithTimeout(time.Minute*2)) return ctx }). Test("verify linkerd CR status", func(ctx f2.Context, t *testing.T) f2.Context { l5d := &l5dv1alpha1.Linkerd{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "linkerd.edge.ncr.com"}, l5d)) assert.True(t, isReady(l5d.Status.Conditions)) assert.NotNil(t, l5d.Status.Version) injected = l5d.Status.InjectedNamespaces return ctx }). Test("verify linkerdworkloadinjection CR status", func(ctx f2.Context, t *testing.T) f2.Context { workloadinjection := &l5dv1alpha1.LinkerdWorkloadInjection{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.Namespace, Namespace: linkerd.Namespace}, workloadinjection)) assert.True(t, isReady(workloadinjection.Status.Conditions)) return ctx }). Test("verify injected workloads are meshed successfully", func(ctx f2.Context, t *testing.T) f2.Context { podList := &corev1.PodList{} require.NoError(t, k.Client.List(ctx, podList)) for _, pod := range podList.Items { if pod.Status.Phase == corev1.PodRunning && isInjected(injected, pod.Namespace) && !pod.Spec.HostNetwork && !isLinkerdDisabled(pod) { assert.True(t, isMeshed(pod) && proxyReady(pod), pod.Name) } } return ctx }).Feature() f.Test(t, automaticRotation) } func isLinkerdUpgradable(l5d *l5dv1alpha1.Linkerd, latestL5dVersion string) (bool, error) { currentVersion, err := goVersion.NewVersion(strings.TrimPrefix(l5d.Status.Version, "edge-")) if err != nil { return false, err } latestVersion, err := goVersion.NewVersion(strings.TrimPrefix(latestL5dVersion, "edge-")) if err != nil { return false, err } return currentVersion.LessThan(latestVersion), nil }