package linkerd import ( "context" "embed" "fmt" "io/fs" "os" "slices" "strconv" "testing" certmgr "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" serverv1beta1 "github.com/linkerd/linkerd2/controller/gen/apis/server/v1beta1" serverauthv1beta1 "github.com/linkerd/linkerd2/controller/gen/apis/serverauthorization/v1beta1" "github.com/linkerd/linkerd2/pkg/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1" "edge-infra.dev/pkg/lib/fog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "edge-infra.dev/pkg/edge/linkerd" "edge-infra.dev/pkg/k8s/runtime/controller" "edge-infra.dev/test/f2" "edge-infra.dev/test/f2/x/ktest" ) //go:embed manifests var manifests embed.FS var linkerdCRDManifests []byte var linkerdControllerManifests []byte var f f2.Framework var ( linkerdCRName = "linkerd.edge.ncr.com" expectedSecrets = []string{ "edge-docker-pull-secret", "linkerd-identity-issuer", "linkerd-policy-validator-k8s-tls", "linkerd-proxy-injector-k8s-tls", "linkerd-sp-validator-k8s-tls", "linkerd-trust-anchor", } ) func TestMain(m *testing.M) { ctrl.SetLogger(fog.New()) f = f2.New(context.Background(), f2.WithExtensions( ktest.New( ktest.SkipNamespaceCreation(), ktest.WithCtrlManager(createManager), ), ), ). Setup(func(ctx f2.Context) (f2.Context, error) { // Load linkerd manifests var err error linkerdCRDManifests, err = fs.ReadFile(manifests, "manifests/linkerd-crds_manifests.yaml") if err != nil { return ctx, err } linkerdControllerManifests, err = fs.ReadFile(manifests, "manifests/linkerd-controller-generic_manifests.yaml") if err != nil { return ctx, err } return ctx, nil }). WithLabel("dsds", "true") os.Exit(f.Run(m)) } func TestLinkerdHealthcheck(t *testing.T) { var ( k *ktest.K8s numNodes int thickPOS bool certDuration int renewBefore int err error injected []string ) healthcheck := f2.NewFeature("linkerd healthcheck"). Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) nodes := &corev1.NodeList{} require.NoError(t, k.Client.List(ctx, nodes)) numNodes = len(nodes.Items) cm := &corev1.ConfigMap{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "topology-info", Namespace: "kube-public"}, cm)) thickPOS, err = strconv.ParseBool(cm.Data["thick_pos"]) certDuration, err = strconv.Atoi(cm.Data["linkerd_identity_issuer_cert_duration"]) require.NoError(t, err) renewBefore, err = strconv.Atoi(cm.Data["linkerd_identity_issuer_cert_renew_before"]) require.NoError(t, err) return ctx }). Test("verify linkerdctl pod status", func(ctx f2.Context, t *testing.T) f2.Context { pods := corev1.PodList{} assert.NoError(t, k.Client.List(ctx, &pods, client.MatchingFields{"metadata.namespace": "linkerdctl"})) assert.Equal(t, 1, len(pods.Items)) assert.Equal(t, corev1.PodRunning, pods.Items[0].Status.Phase) return ctx }). Test("verify linkerdctl manifests exist", func(ctx f2.Context, t *testing.T) f2.Context { deploymentList := &appsv1.DeploymentList{} assert.NoError(t, k.Client.List(ctx, deploymentList, client.MatchingFields{"metadata.namespace": linkerd.Namespace})) for _, deployment := range deploymentList.Items { assert.True(t, isDeploymentReady(deployment, thickPOS, numNodes)) } daemonsetList := &appsv1.DaemonSetList{} assert.NoError(t, k.Client.List(ctx, daemonsetList, client.MatchingFields{"metadata.namespace": linkerd.Namespace})) for _, daemonset := range daemonsetList.Items { assert.True(t, isDaemonsetReady(daemonset, thickPOS, numNodes)) } return ctx }). Test("verify certmanager resources exist", func(ctx f2.Context, t *testing.T) f2.Context { issuer := &certmgr.Issuer{} require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, issuer)) assert.Equal(t, linkerd.TrustAnchorName, issuer.Spec.IssuerConfig.CA.SecretName) cert := &certmgr.Certificate{} require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.IssuerName, Namespace: linkerd.Namespace}, cert)) assert.Equal(t, linkerd.TrustAnchorName, cert.Spec.IssuerRef.Name) assert.Equal(t, certmgr.IssuerKind, cert.Spec.IssuerRef.Kind) assert.Equal(t, certDuration, int(cert.Spec.Duration.Hours())) assert.Equal(t, renewBefore, int(cert.Spec.RenewBefore.Hours())) return ctx }). Test("verify required secrets exist", func(ctx f2.Context, t *testing.T) f2.Context { secretList := &corev1.SecretList{} require.NoError(t, k.Client.List(ctx, secretList, client.MatchingFields{"metadata.namespace": linkerd.Namespace})) secretNames := []string{} for _, secret := range secretList.Items { secretNames = append(secretNames, secret.Name) } assert.ElementsMatch(t, expectedSecrets, secretNames) return ctx }). Test("verify linkerd CR status", func(ctx f2.Context, t *testing.T) f2.Context { l5d := &l5dv1alpha1.Linkerd{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerdCRName}, l5d)) assert.True(t, isReady(l5d.Status.Conditions)) assert.NotNil(t, l5d.Status.Version) injected = l5d.Status.InjectedNamespaces return ctx }). Test("verify linkerdworkloadinjection CR status", func(ctx f2.Context, t *testing.T) f2.Context { workloadinjection := &l5dv1alpha1.LinkerdWorkloadInjection{} assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.Namespace, Namespace: linkerd.Namespace}, workloadinjection)) assert.True(t, isReady(workloadinjection.Status.Conditions)) return ctx }). Test("verify injected workloads are meshed successfully", func(ctx f2.Context, t *testing.T) f2.Context { podList := &corev1.PodList{} require.NoError(t, k.Client.List(ctx, podList)) for _, pod := range podList.Items { if pod.Status.Phase == corev1.PodRunning && isInjected(injected, pod.Namespace) && !pod.Spec.HostNetwork && !isLinkerdDisabled(pod) { assert.True(t, isMeshed(pod) && proxyReady(pod), pod.Name) } } return ctx }). Test("verify disabled namespaces are not meshed", func(ctx f2.Context, t *testing.T) f2.Context { podList := &corev1.PodList{} require.NoError(t, k.Client.List(ctx, podList)) l5d := &l5dv1alpha1.Linkerd{} require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerdCRName}, l5d)) excluded := []string{} for keys := range l5d.ExcludedNamespacesMap() { excluded = append(excluded, keys) } for _, pod := range podList.Items { if slices.Contains(excluded, pod.Namespace) && pod.Namespace != linkerd.Namespace { assert.True(t, pod.Status.Phase == corev1.PodRunning, fmt.Sprintf("%s/%s should be running", pod.Namespace, pod.Name)) assert.False(t, isInjected(injected, pod.Namespace), fmt.Sprintf("%s/%s should not be marked for injection", pod.Namespace, pod.Name)) assert.False(t, isMeshed(pod), fmt.Sprintf("%s/%s should not be meshed", pod.Namespace, pod.Name)) } } return ctx }).Feature() f.Test(t, healthcheck) } func isLinkerdDisabled(pod corev1.Pod) bool { if inject, found := pod.GetAnnotations()[linkerd.InjectionAnnotation]; found && inject == "disabled" { return true } return false } func isReady(conditions []metav1.Condition) bool { for _, condition := range conditions { if condition.Type == "Ready" && condition.Status == metav1.ConditionTrue { return true } } return false } func isDeploymentReady(deployment appsv1.Deployment, thickPOS bool, numNodes int) bool { if thickPOS { return deployment.Status.ReadyReplicas == int32(0) } return deployment.Status.ReadyReplicas == int32(numNodes) /* #nosec G115 */ } func isDaemonsetReady(daemonset appsv1.DaemonSet, thickPOS bool, numNodes int) bool { if thickPOS { return daemonset.Status.NumberReady == int32(numNodes) /* #nosec G115 */ } return daemonset.Status.NumberReady == int32(0) } func isInjected(injected []string, namespace string) bool { for _, i := range injected { if i == namespace { return true } } return false } func isMeshed(pod corev1.Pod) bool { for _, initContainer := range pod.Spec.InitContainers { if initContainer.Name == k8s.ProxyContainerName { return true } } return false } func proxyReady(pod corev1.Pod) bool { for _, initContainerStatus := range pod.Status.InitContainerStatuses { if initContainerStatus.Name == k8s.ProxyContainerName { return initContainerStatus.Ready } } return false } func createManager(opts ...controller.Option) (ctrl.Manager, error) { mgrCfg, mgrOpts := controller.ProcessOptions(opts...) mgrOpts.Scheme = createScheme() return ctrl.NewManager(mgrCfg, mgrOpts) } func createScheme() *runtime.Scheme { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(l5dv1alpha1.AddToScheme(scheme)) utilruntime.Must(certmgr.AddToScheme(scheme)) utilruntime.Must(serverv1beta1.AddToScheme(scheme)) utilruntime.Must(serverauthv1beta1.AddToScheme(scheme)) return scheme }