1 package integration
2
3 import (
4 "context"
5 "fmt"
6 "os"
7 "reflect"
8 "strconv"
9 "testing"
10 "time"
11
12 certmgr "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
13 "github.com/stretchr/testify/assert"
14 "github.com/stretchr/testify/require"
15 "gotest.tools/v3/fs"
16 appsv1 "k8s.io/api/apps/v1"
17 corev1 "k8s.io/api/core/v1"
18 "k8s.io/apimachinery/pkg/api/errors"
19 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20 "k8s.io/apimachinery/pkg/types"
21 "sigs.k8s.io/controller-runtime/pkg/client"
22
23 "edge-infra.dev/pkg/edge/component/build"
24 "edge-infra.dev/pkg/edge/constants/api/workload"
25 "edge-infra.dev/pkg/edge/info"
26 linkerd "edge-infra.dev/pkg/edge/linkerd"
27 "edge-infra.dev/pkg/edge/linkerd/certs/trustanchor"
28 l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1"
29 "edge-infra.dev/pkg/edge/linkerd/k8s/controllers"
30 "edge-infra.dev/pkg/sds/ien/topology"
31 "edge-infra.dev/test/f2"
32 "edge-infra.dev/test/f2/x/ktest"
33 )
34
35 var f f2.Framework
36
37 var (
38 numTestNodes = 3
39
40 l5dHeadlessServices = []string{
41 "linkerd-dst-headless",
42 "linkerd-identity-headless",
43 "linkerd-policy",
44 }
45 testNS1Name = "test-ns-1"
46 testNS2Name = "test-ns-2"
47 testNamespace1 = corev1.Namespace{
48 ObjectMeta: metav1.ObjectMeta{
49 Name: testNS1Name,
50 Labels: map[string]string{workload.Label: "test-ns-1"},
51 },
52 }
53 testNamespace2 = corev1.Namespace{
54 ObjectMeta: metav1.ObjectMeta{
55 Name: testNS2Name,
56 Labels: map[string]string{workload.Label: "test-ns-2"},
57 },
58 }
59 edgeInfo = corev1.ConfigMap{
60 TypeMeta: metav1.TypeMeta{
61 Kind: "ConfigMap",
62 APIVersion: metav1.SchemeGroupVersion.String(),
63 },
64 ObjectMeta: metav1.ObjectMeta{
65 Name: info.EdgeConfigMapName,
66 Namespace: metav1.NamespacePublic,
67 },
68 Data: map[string]string{
69 info.Banner: "test-banner",
70 info.ProjectID: "test-project",
71 info.StoreName: "test-store",
72 info.ClusterEdgeID: "test-cluster-edge-id",
73 info.ClusterType: "test-cluster-type",
74 info.K8sClusterLocation: "test-location",
75 info.FleetType: "test-fleet",
76 info.ForemanProjectID: "test-top-level-project-id",
77 info.BannerID: "test-banner-id",
78 info.EdgeAPIEndpoint: "https:/test/api/v2",
79 },
80 }
81 expectedInventoryID1 = "test-ns-1_app1_apps_Deployment"
82 expectedInventoryID2 = "test-ns-2_app2_apps_Deployment"
83 expectedInventoryVersion = "v1"
84
85 expectedL5dDaemonSets = []client.Object{
86 &appsv1.DaemonSet{
87 ObjectMeta: metav1.ObjectMeta{
88 Name: linkerd.Identity,
89 Namespace: linkerd.Namespace,
90 },
91 },
92 &appsv1.DaemonSet{
93 ObjectMeta: metav1.ObjectMeta{
94 Name: linkerd.Destination,
95 Namespace: linkerd.Namespace,
96 },
97 },
98 &appsv1.DaemonSet{
99 ObjectMeta: metav1.ObjectMeta{
100 Name: linkerd.ProxyInjector,
101 Namespace: linkerd.Namespace,
102 },
103 },
104 }
105 readyPodStatus = corev1.PodStatus{
106 Phase: corev1.PodRunning,
107 Conditions: []corev1.PodCondition{
108 {
109 Type: corev1.PodReady,
110 Status: corev1.ConditionTrue,
111 },
112 },
113 }
114 )
115
116 func TestMain(m *testing.M) {
117 f = f2.New(context.Background(),
118 f2.WithExtensions(
119 ktest.New(
120 ktest.WithCtrlManager(controllers.CreateControllerManager),
121 ktest.WithGracefulTimeout("5m"),
122 ktest.WithCertManager(),
123 ),
124 )).
125 Setup(func(ctx f2.Context) (f2.Context, error) {
126 k, err := ktest.FromContext(ctx)
127 if err != nil {
128 return ctx, err
129 }
130
131
132 if !*k.Env.UseExistingCluster {
133 k.Timeout = 5 * time.Second
134 k.Tick = 10 * time.Millisecond
135 }
136
137 return ctx, nil
138 }).
139 Teardown()
140 os.Exit(f.Run(m))
141 }
142
143 func TestLinkerdController(t *testing.T) {
144 var (
145 k *ktest.K8s
146 dirPath string
147 )
148
149
150 dir := fs.NewDir(t, "/etc/l5d")
151 defer dir.Remove()
152 dirPath = dir.Path()
153 t.Setenv("L5D_DIR", dirPath)
154
155 feature := f2.NewFeature("Linkerd Controller").
156 Setup("Register Controllers", func(ctx f2.Context, t *testing.T) f2.Context {
157 k = ktest.FromContextT(ctx, t)
158 registry := build.DefaultPublicContainerRegistry
159 l5dcfg := controllers.Config{Registry: ®istry, L5dDirPath: &dirPath}
160 _, err := controllers.RegisterControllers(l5dcfg, k.Manager)
161 require.NoError(t, err)
162 return ctx
163 }).
164 Setup("create namespaces, deployments, pods, linkerd and edge-info config", func(ctx f2.Context, t *testing.T) f2.Context {
165 require.NoError(t, k.Client.Create(ctx, edgeInfo.DeepCopy()))
166
167 require.NoError(t, k.Client.Create(ctx, testNamespace1.DeepCopy()))
168 require.NoError(t, k.Client.Create(ctx, testNamespace2.DeepCopy()))
169
170 testDeployment1 := createDeployment(testNS1Name, "app1")
171 require.NoError(t, k.Client.Create(ctx, testDeployment1.DeepCopy()))
172
173 testDeployment2 := createDeployment(testNS2Name, "app2")
174 require.NoError(t, k.Client.Create(ctx, testDeployment2.DeepCopy()))
175
176 k.WaitOn(t, k.ObjsExist([]client.Object{testDeployment1, testDeployment2}))
177
178 testPod1 := createPod(testNS1Name, "app1-abcde", false, ownerReference(testDeployment1))
179 require.NoError(t, k.Client.Create(ctx, testPod1))
180 testPod1.Status = readyPodStatus
181 require.NoError(t, k.Client.Status().Update(ctx, testPod1))
182
183 testPod2 := createPod(testNS2Name, "app2-fghij", false, ownerReference(testDeployment2))
184 require.NoError(t, k.Client.Create(ctx, testPod2))
185 testPod2.Status = readyPodStatus
186 require.NoError(t, k.Client.Status().Update(ctx, testPod2))
187
188 require.NoError(t, k.Client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: linkerd.Namespace}}))
189 require.NoError(t, k.Client.Create(ctx, identityTrustRootsConfigMap()))
190
191 l5d := l5d()
192 require.NoError(t, k.Client.Create(ctx, l5d))
193 return ctx
194 }).
195 Setup("create topology-info ConfigMap with thick-pos enabled", func(ctx f2.Context, t *testing.T) f2.Context {
196 require.NoError(t, k.Client.Create(ctx, topologyInfoConfigMap(true)))
197 return ctx
198 }).
199 Setup("create nodes to test the correct number of Linkerd control-plane pods are created", func(ctx f2.Context, t *testing.T) f2.Context {
200 for nodeNum := 0; nodeNum < numTestNodes; nodeNum++ {
201 require.NoError(t, k.Client.Create(ctx, node(strconv.Itoa(nodeNum))))
202 }
203 return ctx
204 }).
205 Test("trust anchor secret created", func(ctx f2.Context, t *testing.T) f2.Context {
206 secret := &corev1.Secret{}
207 assert.Eventually(t, func() bool {
208 err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, secret)
209 return !errors.IsNotFound(err)
210 }, ktest.Timeout, ktest.Tick, "trust anchor secret not created")
211 assert.Contains(t, secret.Data, corev1.TLSCertKey)
212 assert.Contains(t, secret.Data, corev1.TLSPrivateKeyKey)
213 return ctx
214 }).
215 Test("trust anchor issuer created", func(ctx f2.Context, t *testing.T) f2.Context {
216 issuer := &certmgr.Issuer{}
217 assert.Eventually(t, func() bool {
218 err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}, issuer)
219 return !errors.IsNotFound(err)
220 }, ktest.Timeout, ktest.Tick, "trust anchor issuer not created")
221 assert.Equal(t, linkerd.TrustAnchorName, issuer.Spec.IssuerConfig.CA.SecretName)
222 return ctx
223 }).
224 Test("certificate created", func(ctx f2.Context, t *testing.T) f2.Context {
225 cert := &certmgr.Certificate{}
226 assert.Eventually(t, func() bool {
227 err := k.Client.Get(ctx, types.NamespacedName{Name: linkerd.IssuerName, Namespace: linkerd.Namespace}, cert)
228 return !errors.IsNotFound(err)
229 }, ktest.Timeout, ktest.Tick, "certificate never created")
230 assert.Equal(t, linkerd.TrustAnchorName, cert.Spec.IssuerRef.Name)
231 assert.Equal(t, certmgr.IssuerKind, cert.Spec.IssuerRef.Kind)
232 assert.Equal(t, linkerd.DefaultThickPosIdentityIssuerCertificateDurationHours, uint(cert.Spec.Duration.Hours()))
233 assert.Equal(t, linkerd.DefaultThickPosIdentityIssuerCertificateRenewBeforeHours, uint(cert.Spec.RenewBefore.Hours()))
234 return ctx
235 }).
236 Test("thick-pos configured daemonsets", func(ctx f2.Context, t *testing.T) f2.Context {
237 k.WaitOn(t, k.ObjsExist(expectedL5dDaemonSets))
238 return ctx
239 }).
240 Test("thick-pos configured services", func(ctx f2.Context, t *testing.T) f2.Context {
241 assert.Eventually(t, func() bool {
242 return servicesAreThickPosConfigured(ctx, k.Client)
243 }, ktest.Timeout, ktest.Tick, "Linkerd services were never thick-pos configured")
244 return ctx
245 }).
246 Feature()
247
248 f.Test(t, feature)
249 }
250
251 func l5d() *l5dv1alpha1.Linkerd {
252 return &l5dv1alpha1.Linkerd{
253 ObjectMeta: metav1.ObjectMeta{
254 Name: l5dv1alpha1.Name,
255 },
256 Spec: l5dv1alpha1.LinkerdSpec{
257
258
259
260 Monitoring: l5dv1alpha1.Monitoring{Enabled: false},
261 Injection: l5dv1alpha1.Injection{Enabled: true},
262 },
263 }
264 }
265
266 func l5dreinjectionjob(spec l5dv1alpha1.LinkerdWorkloadInjectionSpec) *l5dv1alpha1.LinkerdWorkloadInjection {
267 return &l5dv1alpha1.LinkerdWorkloadInjection{
268 ObjectMeta: metav1.ObjectMeta{
269 Name: "linkerd",
270 },
271 Spec: spec,
272 }
273 }
274
275 func node(nodeName string) *corev1.Node {
276 return &corev1.Node{
277 ObjectMeta: metav1.ObjectMeta{
278 Name: fmt.Sprintf("test-node-%v", nodeName),
279 },
280 Status: corev1.NodeStatus{
281 NodeInfo: corev1.NodeSystemInfo{
282 KubeletVersion: "1.28.9",
283 },
284 },
285 }
286 }
287
288 func identityTrustRootsConfigMap() *corev1.ConfigMap {
289 return &corev1.ConfigMap{
290 ObjectMeta: metav1.ObjectMeta{
291 Name: linkerd.LinkerdIdentityConfigMap,
292 Namespace: linkerd.Namespace,
293 },
294 Data: map[string]string{
295 trustanchor.CaBundleKey: "",
296 },
297 }
298 }
299
300 func topologyInfoConfigMap(thickPos bool) *corev1.ConfigMap {
301 info := topology.New()
302 info.ThickPOS = thickPos
303 if thickPos {
304 info.LinkerdIdentityCertDuration = int(linkerd.DefaultThickPosIdentityIssuerCertificateDurationHours)
305 info.LinkerdIdentityCertRenewBefore = int(linkerd.DefaultThickPosIdentityIssuerCertificateRenewBeforeHours)
306 }
307 return info.ToConfigMap()
308 }
309
310
311 func servicesAreThickPosConfigured(ctx context.Context, k8sClient client.Client) bool {
312 for _, serviceName := range l5dHeadlessServices {
313 service := &corev1.Service{}
314 if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: linkerd.Namespace, Name: serviceName}, service); err != nil {
315 return false
316 }
317 if service.Spec.ClusterIP == "None" {
318 return false
319 }
320 if *service.Spec.InternalTrafficPolicy != corev1.ServiceInternalTrafficPolicyLocal {
321 return false
322 }
323 }
324
325 return true
326 }
327
328 func createDeployment(namespace, name string) *appsv1.Deployment {
329 replica := int32(1)
330 return &appsv1.Deployment{
331 TypeMeta: metav1.TypeMeta{
332 Kind: "Deployment",
333 APIVersion: "apps/v1",
334 },
335 ObjectMeta: metav1.ObjectMeta{
336 Namespace: namespace,
337 Name: name,
338 },
339 Spec: appsv1.DeploymentSpec{
340 Replicas: &replica,
341 Selector: &metav1.LabelSelector{
342 MatchLabels: map[string]string{
343 "app": "app",
344 },
345 },
346 Template: corev1.PodTemplateSpec{
347 ObjectMeta: metav1.ObjectMeta{
348 Labels: map[string]string{
349 "app": "app",
350 },
351 },
352 Spec: corev1.PodSpec{
353 Containers: []corev1.Container{
354 {
355 Name: "nginx",
356 Image: "nginx",
357 },
358 },
359 },
360 },
361 },
362 }
363 }
364
365 func createPod(namespace, name string, host bool, reference []metav1.OwnerReference) *corev1.Pod {
366 pod := &corev1.Pod{
367 TypeMeta: metav1.TypeMeta{
368 APIVersion: "v1",
369 Kind: "Pod",
370 },
371 ObjectMeta: metav1.ObjectMeta{
372 Namespace: namespace,
373 Name: name,
374 Labels: map[string]string{
375 "app": "app",
376 },
377 OwnerReferences: reference,
378 },
379 Spec: corev1.PodSpec{
380 HostNetwork: host,
381 Containers: []corev1.Container{
382 {
383 Name: "nginx",
384 Image: "nginx",
385 },
386 },
387 },
388 }
389
390 return pod
391 }
392
393 func ownerReference(obj *appsv1.Deployment) []metav1.OwnerReference {
394 return []metav1.OwnerReference{
395 *metav1.NewControllerRef(obj,
396 appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(appsv1.Deployment{}).Name())),
397 }
398 }
399
View as plain text