1 package linkerd
2
3 import (
4 "context"
5 "embed"
6 "fmt"
7 "io/fs"
8 "os"
9 "slices"
10 "strconv"
11 "testing"
12
13 certmgr "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
14 serverv1beta1 "github.com/linkerd/linkerd2/controller/gen/apis/server/v1beta1"
15 serverauthv1beta1 "github.com/linkerd/linkerd2/controller/gen/apis/serverauthorization/v1beta1"
16 "github.com/linkerd/linkerd2/pkg/k8s"
17 "github.com/stretchr/testify/assert"
18 "github.com/stretchr/testify/require"
19 appsv1 "k8s.io/api/apps/v1"
20 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21 "k8s.io/apimachinery/pkg/types"
22 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
23 clientgoscheme "k8s.io/client-go/kubernetes/scheme"
24 ctrl "sigs.k8s.io/controller-runtime"
25 "sigs.k8s.io/controller-runtime/pkg/client"
26
27 l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1"
28 "edge-infra.dev/pkg/lib/fog"
29
30 corev1 "k8s.io/api/core/v1"
31 "k8s.io/apimachinery/pkg/runtime"
32
33 "edge-infra.dev/pkg/edge/linkerd"
34 "edge-infra.dev/pkg/k8s/runtime/controller"
35 "edge-infra.dev/test/f2"
36 "edge-infra.dev/test/f2/x/ktest"
37 )
38
39
40 var manifests embed.FS
41
42 var linkerdCRDManifests []byte
43 var linkerdControllerManifests []byte
44
45 var f f2.Framework
46
47 var (
48 linkerdCRName = "linkerd.edge.ncr.com"
49 expectedSecrets = []string{
50 "edge-docker-pull-secret",
51 "linkerd-identity-issuer",
52 "linkerd-policy-validator-k8s-tls",
53 "linkerd-proxy-injector-k8s-tls",
54 "linkerd-sp-validator-k8s-tls",
55 "linkerd-trust-anchor",
56 }
57 )
58
59 func TestMain(m *testing.M) {
60 ctrl.SetLogger(fog.New())
61 f = f2.New(context.Background(),
62 f2.WithExtensions(
63 ktest.New(
64 ktest.SkipNamespaceCreation(),
65 ktest.WithCtrlManager(createManager),
66 ),
67 ),
68 ).
69 Setup(func(ctx f2.Context) (f2.Context, error) {
70
71 var err error
72 linkerdCRDManifests, err = fs.ReadFile(manifests, "manifests/linkerd-crds_manifests.yaml")
73 if err != nil {
74 return ctx, err
75 }
76 linkerdControllerManifests, err = fs.ReadFile(manifests, "manifests/linkerd-controller-generic_manifests.yaml")
77 if err != nil {
78 return ctx, err
79 }
80 return ctx, nil
81 }).
82 WithLabel("dsds", "true")
83 os.Exit(f.Run(m))
84 }
85
86 func TestLinkerdHealthcheck(t *testing.T) {
87 var (
88 k *ktest.K8s
89 numNodes int
90 thickPOS bool
91 certDuration int
92 renewBefore int
93 err error
94 injected []string
95 )
96 healthcheck := f2.NewFeature("linkerd healthcheck").
97 Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context {
98 k = ktest.FromContextT(ctx, t)
99
100 nodes := &corev1.NodeList{}
101 require.NoError(t, k.Client.List(ctx, nodes))
102 numNodes = len(nodes.Items)
103
104 cm := &corev1.ConfigMap{}
105 assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: "topology-info", Namespace: "kube-public"}, cm))
106 thickPOS, err = strconv.ParseBool(cm.Data["thick_pos"])
107
108 certDuration, err = strconv.Atoi(cm.Data["linkerd_identity_issuer_cert_duration"])
109 require.NoError(t, err)
110 renewBefore, err = strconv.Atoi(cm.Data["linkerd_identity_issuer_cert_renew_before"])
111 require.NoError(t, err)
112 return ctx
113 }).
114 Test("verify linkerdctl pod status", func(ctx f2.Context, t *testing.T) f2.Context {
115 pods := corev1.PodList{}
116 assert.NoError(t, k.Client.List(ctx, &pods, client.MatchingFields{"metadata.namespace": "linkerdctl"}))
117 assert.Equal(t, 1, len(pods.Items))
118 assert.Equal(t, corev1.PodRunning, pods.Items[0].Status.Phase)
119 return ctx
120 }).
121 Test("verify linkerdctl manifests exist", func(ctx f2.Context, t *testing.T) f2.Context {
122 deploymentList := &appsv1.DeploymentList{}
123 assert.NoError(t, k.Client.List(ctx, deploymentList, client.MatchingFields{"metadata.namespace": linkerd.Namespace}))
124 for _, deployment := range deploymentList.Items {
125 assert.True(t, isDeploymentReady(deployment, thickPOS, numNodes))
126 }
127 daemonsetList := &appsv1.DaemonSetList{}
128 assert.NoError(t, k.Client.List(ctx, daemonsetList, client.MatchingFields{"metadata.namespace": linkerd.Namespace}))
129 for _, daemonset := range daemonsetList.Items {
130 assert.True(t, isDaemonsetReady(daemonset, thickPOS, numNodes))
131 }
132 return ctx
133 }).
134 Test("verify certmanager resources exist", func(ctx f2.Context, t *testing.T) f2.Context {
135 issuer := &certmgr.Issuer{}
136 require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, issuer))
137 assert.Equal(t, linkerd.TrustAnchorName, issuer.Spec.IssuerConfig.CA.SecretName)
138
139 cert := &certmgr.Certificate{}
140 require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.IssuerName, Namespace: linkerd.Namespace}, cert))
141 assert.Equal(t, linkerd.TrustAnchorName, cert.Spec.IssuerRef.Name)
142 assert.Equal(t, certmgr.IssuerKind, cert.Spec.IssuerRef.Kind)
143 assert.Equal(t, certDuration, int(cert.Spec.Duration.Hours()))
144 assert.Equal(t, renewBefore, int(cert.Spec.RenewBefore.Hours()))
145 return ctx
146 }).
147 Test("verify required secrets exist", func(ctx f2.Context, t *testing.T) f2.Context {
148 secretList := &corev1.SecretList{}
149 require.NoError(t, k.Client.List(ctx, secretList, client.MatchingFields{"metadata.namespace": linkerd.Namespace}))
150
151 secretNames := []string{}
152 for _, secret := range secretList.Items {
153 secretNames = append(secretNames, secret.Name)
154 }
155 assert.ElementsMatch(t, expectedSecrets, secretNames)
156 return ctx
157 }).
158 Test("verify linkerd CR status", func(ctx f2.Context, t *testing.T) f2.Context {
159 l5d := &l5dv1alpha1.Linkerd{}
160 assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerdCRName}, l5d))
161 assert.True(t, isReady(l5d.Status.Conditions))
162 assert.NotNil(t, l5d.Status.Version)
163 injected = l5d.Status.InjectedNamespaces
164 return ctx
165 }).
166 Test("verify linkerdworkloadinjection CR status", func(ctx f2.Context, t *testing.T) f2.Context {
167 workloadinjection := &l5dv1alpha1.LinkerdWorkloadInjection{}
168 assert.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.Namespace, Namespace: linkerd.Namespace}, workloadinjection))
169 assert.True(t, isReady(workloadinjection.Status.Conditions))
170 return ctx
171 }).
172 Test("verify injected workloads are meshed successfully", func(ctx f2.Context, t *testing.T) f2.Context {
173 podList := &corev1.PodList{}
174 require.NoError(t, k.Client.List(ctx, podList))
175 for _, pod := range podList.Items {
176 if pod.Status.Phase == corev1.PodRunning && isInjected(injected, pod.Namespace) && !pod.Spec.HostNetwork && !isLinkerdDisabled(pod) {
177 assert.True(t, isMeshed(pod) && proxyReady(pod), pod.Name)
178 }
179 }
180 return ctx
181 }).
182 Test("verify disabled namespaces are not meshed", func(ctx f2.Context, t *testing.T) f2.Context {
183 podList := &corev1.PodList{}
184 require.NoError(t, k.Client.List(ctx, podList))
185 l5d := &l5dv1alpha1.Linkerd{}
186 require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerdCRName}, l5d))
187 excluded := []string{}
188 for keys := range l5d.ExcludedNamespacesMap() {
189 excluded = append(excluded, keys)
190 }
191 for _, pod := range podList.Items {
192 if slices.Contains(excluded, pod.Namespace) && pod.Namespace != linkerd.Namespace {
193 assert.True(t, pod.Status.Phase == corev1.PodRunning, fmt.Sprintf("%s/%s should be running", pod.Namespace, pod.Name))
194 assert.False(t, isInjected(injected, pod.Namespace), fmt.Sprintf("%s/%s should not be marked for injection", pod.Namespace, pod.Name))
195 assert.False(t, isMeshed(pod), fmt.Sprintf("%s/%s should not be meshed", pod.Namespace, pod.Name))
196 }
197 }
198 return ctx
199 }).Feature()
200 f.Test(t, healthcheck)
201 }
202
203 func isLinkerdDisabled(pod corev1.Pod) bool {
204 if inject, found := pod.GetAnnotations()[linkerd.InjectionAnnotation]; found && inject == "disabled" {
205 return true
206 }
207 return false
208 }
209
210 func isReady(conditions []metav1.Condition) bool {
211 for _, condition := range conditions {
212 if condition.Type == "Ready" && condition.Status == metav1.ConditionTrue {
213 return true
214 }
215 }
216 return false
217 }
218
219 func isDeploymentReady(deployment appsv1.Deployment, thickPOS bool, numNodes int) bool {
220 if thickPOS {
221 return deployment.Status.ReadyReplicas == int32(0)
222 }
223 return deployment.Status.ReadyReplicas == int32(numNodes)
224 }
225
226 func isDaemonsetReady(daemonset appsv1.DaemonSet, thickPOS bool, numNodes int) bool {
227 if thickPOS {
228 return daemonset.Status.NumberReady == int32(numNodes)
229 }
230 return daemonset.Status.NumberReady == int32(0)
231 }
232
233 func isInjected(injected []string, namespace string) bool {
234 for _, i := range injected {
235 if i == namespace {
236 return true
237 }
238 }
239 return false
240 }
241
242 func isMeshed(pod corev1.Pod) bool {
243 for _, initContainer := range pod.Spec.InitContainers {
244 if initContainer.Name == k8s.ProxyContainerName {
245 return true
246 }
247 }
248 return false
249 }
250
251 func proxyReady(pod corev1.Pod) bool {
252 for _, initContainerStatus := range pod.Status.InitContainerStatuses {
253 if initContainerStatus.Name == k8s.ProxyContainerName {
254 return initContainerStatus.Ready
255 }
256 }
257 return false
258 }
259
260 func createManager(opts ...controller.Option) (ctrl.Manager, error) {
261 mgrCfg, mgrOpts := controller.ProcessOptions(opts...)
262 mgrOpts.Scheme = createScheme()
263 return ctrl.NewManager(mgrCfg, mgrOpts)
264 }
265
266 func createScheme() *runtime.Scheme {
267 scheme := runtime.NewScheme()
268 utilruntime.Must(clientgoscheme.AddToScheme(scheme))
269 utilruntime.Must(l5dv1alpha1.AddToScheme(scheme))
270 utilruntime.Must(certmgr.AddToScheme(scheme))
271 utilruntime.Must(serverv1beta1.AddToScheme(scheme))
272 utilruntime.Must(serverauthv1beta1.AddToScheme(scheme))
273 return scheme
274 }
275
View as plain text