package integration import ( "context" "fmt" "os" "reflect" "strconv" "testing" "time" certmgr "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gotest.tools/v3/fs" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "edge-infra.dev/pkg/edge/component/build" "edge-infra.dev/pkg/edge/constants/api/workload" "edge-infra.dev/pkg/edge/info" linkerd "edge-infra.dev/pkg/edge/linkerd" "edge-infra.dev/pkg/edge/linkerd/certs/trustanchor" l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1" "edge-infra.dev/pkg/edge/linkerd/k8s/controllers" "edge-infra.dev/pkg/sds/ien/topology" "edge-infra.dev/test/f2" "edge-infra.dev/test/f2/x/ktest" ) var f f2.Framework var ( numTestNodes = 3 l5dHeadlessServices = []string{ "linkerd-dst-headless", "linkerd-identity-headless", "linkerd-policy", } testNS1Name = "test-ns-1" testNS2Name = "test-ns-2" testNamespace1 = corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: testNS1Name, Labels: map[string]string{workload.Label: "test-ns-1"}, }, } testNamespace2 = corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: testNS2Name, Labels: map[string]string{workload.Label: "test-ns-2"}, }, } edgeInfo = corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", APIVersion: metav1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Name: info.EdgeConfigMapName, Namespace: metav1.NamespacePublic, }, Data: map[string]string{ info.Banner: "test-banner", info.ProjectID: "test-project", info.StoreName: "test-store", info.ClusterEdgeID: "test-cluster-edge-id", info.ClusterType: "test-cluster-type", info.K8sClusterLocation: "test-location", info.FleetType: "test-fleet", info.ForemanProjectID: "test-top-level-project-id", info.BannerID: "test-banner-id", info.EdgeAPIEndpoint: "https:/test/api/v2", }, } expectedInventoryID1 = "test-ns-1_app1_apps_Deployment" expectedInventoryID2 = "test-ns-2_app2_apps_Deployment" expectedInventoryVersion = "v1" expectedL5dDaemonSets = []client.Object{ &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.Identity, Namespace: linkerd.Namespace, }, }, &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.Destination, Namespace: linkerd.Namespace, }, }, &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.ProxyInjector, Namespace: linkerd.Namespace, }, }, } readyPodStatus = corev1.PodStatus{ Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{ { Type: corev1.PodReady, Status: corev1.ConditionTrue, }, }, } ) func TestMain(m *testing.M) { f = f2.New(context.Background(), f2.WithExtensions( ktest.New( ktest.WithCtrlManager(controllers.CreateControllerManager), ktest.WithGracefulTimeout("5m"), ktest.WithCertManager(), ), )). Setup(func(ctx f2.Context) (f2.Context, error) { k, err := ktest.FromContext(ctx) if err != nil { return ctx, err } // Override timeouts if we aren't using a live cluster if !*k.Env.UseExistingCluster { k.Timeout = 5 * time.Second k.Tick = 10 * time.Millisecond } return ctx, nil }). Teardown() os.Exit(f.Run(m)) } func TestLinkerdController(t *testing.T) { var ( k *ktest.K8s dirPath string ) // setup test directory for l5d manifests.yaml dir := fs.NewDir(t, "/etc/l5d") defer dir.Remove() dirPath = dir.Path() t.Setenv("L5D_DIR", dirPath) feature := f2.NewFeature("Linkerd Controller"). Setup("Register Controllers", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) registry := build.DefaultPublicContainerRegistry l5dcfg := controllers.Config{Registry: ®istry, L5dDirPath: &dirPath} _, err := controllers.RegisterControllers(l5dcfg, k.Manager) require.NoError(t, err) return ctx }). Setup("create namespaces, deployments, pods, linkerd and edge-info config", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, edgeInfo.DeepCopy())) require.NoError(t, k.Client.Create(ctx, testNamespace1.DeepCopy())) require.NoError(t, k.Client.Create(ctx, testNamespace2.DeepCopy())) testDeployment1 := createDeployment(testNS1Name, "app1") require.NoError(t, k.Client.Create(ctx, testDeployment1.DeepCopy())) testDeployment2 := createDeployment(testNS2Name, "app2") require.NoError(t, k.Client.Create(ctx, testDeployment2.DeepCopy())) k.WaitOn(t, k.ObjsExist([]client.Object{testDeployment1, testDeployment2})) testPod1 := createPod(testNS1Name, "app1-abcde", false, ownerReference(testDeployment1)) require.NoError(t, k.Client.Create(ctx, testPod1)) testPod1.Status = readyPodStatus require.NoError(t, k.Client.Status().Update(ctx, testPod1)) testPod2 := createPod(testNS2Name, "app2-fghij", false, ownerReference(testDeployment2)) require.NoError(t, k.Client.Create(ctx, testPod2)) testPod2.Status = readyPodStatus require.NoError(t, k.Client.Status().Update(ctx, testPod2)) require.NoError(t, k.Client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: linkerd.Namespace}})) require.NoError(t, k.Client.Create(ctx, identityTrustRootsConfigMap())) l5d := l5d() require.NoError(t, k.Client.Create(ctx, l5d)) return ctx }). Setup("create topology-info ConfigMap with thick-pos enabled", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, topologyInfoConfigMap(true))) return ctx }). Setup("create nodes to test the correct number of Linkerd control-plane pods are created", func(ctx f2.Context, t *testing.T) f2.Context { for nodeNum := 0; nodeNum < numTestNodes; nodeNum++ { require.NoError(t, k.Client.Create(ctx, node(strconv.Itoa(nodeNum)))) } return ctx }). Test("trust anchor secret created", func(ctx f2.Context, t *testing.T) f2.Context { secret := &corev1.Secret{} assert.Eventually(t, func() bool { err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, secret) return !errors.IsNotFound(err) }, ktest.Timeout, ktest.Tick, "trust anchor secret not created") assert.Contains(t, secret.Data, corev1.TLSCertKey) assert.Contains(t, secret.Data, corev1.TLSPrivateKeyKey) return ctx }). Test("trust anchor issuer created", func(ctx f2.Context, t *testing.T) f2.Context { issuer := &certmgr.Issuer{} assert.Eventually(t, func() bool { err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, issuer) return !errors.IsNotFound(err) }, ktest.Timeout, ktest.Tick, "trust anchor issuer not created") assert.Equal(t, linkerd.TrustAnchorName, issuer.Spec.IssuerConfig.CA.SecretName) return ctx }). Test("certificate created", func(ctx f2.Context, t *testing.T) f2.Context { cert := &certmgr.Certificate{} assert.Eventually(t, func() bool { err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.IssuerName, Namespace: linkerd.Namespace}, cert) return !errors.IsNotFound(err) }, ktest.Timeout, ktest.Tick, "certificate never created") assert.Equal(t, linkerd.TrustAnchorName, cert.Spec.IssuerRef.Name) assert.Equal(t, certmgr.IssuerKind, cert.Spec.IssuerRef.Kind) assert.Equal(t, linkerd.DefaultThickPosIdentityIssuerCertificateDurationHours, uint(cert.Spec.Duration.Hours())) assert.Equal(t, linkerd.DefaultThickPosIdentityIssuerCertificateRenewBeforeHours, uint(cert.Spec.RenewBefore.Hours())) return ctx }). Test("thick-pos configured daemonsets", func(ctx f2.Context, t *testing.T) f2.Context { k.WaitOn(t, k.ObjsExist(expectedL5dDaemonSets)) return ctx }). Test("thick-pos configured services", func(ctx f2.Context, t *testing.T) f2.Context { assert.Eventually(t, func() bool { return servicesAreThickPosConfigured(ctx, k.Client) }, ktest.Timeout, ktest.Tick, "Linkerd services were never thick-pos configured") return ctx }). Feature() f.Test(t, feature) } func l5d() *l5dv1alpha1.Linkerd { return &l5dv1alpha1.Linkerd{ ObjectMeta: metav1.ObjectMeta{ Name: l5dv1alpha1.Name, }, Spec: l5dv1alpha1.LinkerdSpec{ // TODO: integration test needs prometheus operator to be installed, should // probably verify that install is successful with/without monitoring enabled // and that the monitoring objects end up on inventory of reconciled L5d resource Monitoring: l5dv1alpha1.Monitoring{Enabled: false}, Injection: l5dv1alpha1.Injection{Enabled: true}, }, } } func l5dreinjectionjob(spec l5dv1alpha1.LinkerdWorkloadInjectionSpec) *l5dv1alpha1.LinkerdWorkloadInjection { return &l5dv1alpha1.LinkerdWorkloadInjection{ ObjectMeta: metav1.ObjectMeta{ Name: "linkerd", }, Spec: spec, } } func node(nodeName string) *corev1.Node { return &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("test-node-%v", nodeName), }, Status: corev1.NodeStatus{ NodeInfo: corev1.NodeSystemInfo{ KubeletVersion: "1.28.9", }, }, } } func identityTrustRootsConfigMap() *corev1.ConfigMap { return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.LinkerdIdentityConfigMap, Namespace: linkerd.Namespace, }, Data: map[string]string{ trustanchor.CaBundleKey: "", }, } } func topologyInfoConfigMap(thickPos bool) *corev1.ConfigMap { info := topology.New() info.ThickPOS = thickPos if thickPos { info.LinkerdIdentityCertDuration = int(linkerd.DefaultThickPosIdentityIssuerCertificateDurationHours) info.LinkerdIdentityCertRenewBefore = int(linkerd.DefaultThickPosIdentityIssuerCertificateRenewBeforeHours) } return info.ToConfigMap() } // Check headless Linkerd services are no longer headless and use local internal traffic policy func servicesAreThickPosConfigured(ctx context.Context, k8sClient client.Client) bool { for _, serviceName := range l5dHeadlessServices { service := &corev1.Service{} if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: linkerd.Namespace, Name: serviceName}, service); err != nil { return false } if service.Spec.ClusterIP == "None" { return false } if *service.Spec.InternalTrafficPolicy != corev1.ServiceInternalTrafficPolicyLocal { return false } } return true } func createDeployment(namespace, name string) *appsv1.Deployment { replica := int32(1) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: name, }, Spec: appsv1.DeploymentSpec{ Replicas: &replica, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "app", }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "app": "app", }, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name: "nginx", Image: "nginx", }, }, }, }, }, } } func createPod(namespace, name string, host bool, reference []metav1.OwnerReference) *corev1.Pod { pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: name, Labels: map[string]string{ "app": "app", }, OwnerReferences: reference, }, Spec: corev1.PodSpec{ HostNetwork: host, Containers: []corev1.Container{ { Name: "nginx", Image: "nginx", }, }, }, } return pod } func ownerReference(obj *appsv1.Deployment) []metav1.OwnerReference { return []metav1.OwnerReference{ *metav1.NewControllerRef(obj, appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(appsv1.Deployment{}).Name())), } }