1 package couchctl
2
3 import (
4 "fmt"
5 "testing"
6
7 persistenceApi "edge-infra.dev/pkg/edge/apis/persistence/v1alpha1"
8 "edge-infra.dev/pkg/edge/constants"
9 "edge-infra.dev/pkg/edge/constants/api/cluster"
10 "edge-infra.dev/pkg/edge/constants/api/fleet"
11 "edge-infra.dev/pkg/edge/controllers/envctl/pkg/nameutils"
12 dsapi "edge-infra.dev/pkg/edge/datasync/apis/v1alpha1"
13 "edge-infra.dev/pkg/edge/datasync/couchdb"
14 "edge-infra.dev/pkg/k8s/testing/kmp"
15 v1ien "edge-infra.dev/pkg/sds/ien/k8s/apis/v1"
16 nodemeta "edge-infra.dev/pkg/sds/ien/node"
17 "edge-infra.dev/test/f2"
18 "edge-infra.dev/test/f2/integration"
19 "edge-infra.dev/test/f2/x/ktest"
20
21 "github.com/stretchr/testify/require"
22 "gotest.tools/v3/poll"
23 appsv1 "k8s.io/api/apps/v1"
24 corev1 "k8s.io/api/core/v1"
25 "k8s.io/apimachinery/pkg/api/resource"
26 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27 "sigs.k8s.io/controller-runtime/pkg/client"
28 )
29
30 func TestPersistenceController(t *testing.T) {
31 nodes := &corev1.NodeList{}
32 leaderNode := &corev1.Node{}
33 fin := f2.NewFeature("PersistenceController").
34 WithLabel(_fleetType, fleet.Store).
35 WithLabel(_clusterType, cluster.Generic, cluster.DSDS).
36 Setup("Node Exists For Persistence", func(ctx f2.Context, t *testing.T) f2.Context {
37 k := ktest.FromContextT(ctx, t)
38 k.WaitOn(t, func(_ poll.LogT) poll.Result {
39 if err := k.Client.List(ctx, nodes); err != nil {
40 return poll.Error(err)
41 }
42 if len(nodes.Items) > 0 {
43 return poll.Success()
44 }
45 return poll.Continue("No K8s nodes found for Persistence Controller")
46 })
47
48 require.NoError(t, client.IgnoreAlreadyExists(k.Client.Create(ctx, couchDBPersistence)))
49 return ctx
50 }).
51 Test("CouchDBPersistence Ready", func(ctx f2.Context, t *testing.T) f2.Context {
52 k := ktest.FromContextT(ctx, t)
53 k.WaitOn(t, k.Check(couchDBPersistence, kmp.IsReady()))
54 return ctx
55 }).
56 Test("Leader Node Elected", func(ctx f2.Context, t *testing.T) f2.Context {
57 k := ktest.FromContextT(ctx, t)
58 k.WaitOn(t, func(_ poll.LogT) poll.Result {
59 leaderNodes := &corev1.NodeList{}
60 if err := k.Client.List(ctx, leaderNodes, client.MatchingLabels{
61 couchdb.NodeLeaderLabel: couchdb.LabelValueTrue,
62 }); err != nil {
63 return poll.Error(err)
64 }
65 if len(leaderNodes.Items) == 1 &&
66 leaderNodes.Items[0].Labels[nodemeta.ClassLabel] == string(v1ien.Server) {
67 leaderNode = &leaderNodes.Items[0]
68 return poll.Success()
69 }
70 return poll.Continue("expected a leader node, node(s) found: %d", len(leaderNodes.Items))
71 })
72 return ctx
73 }).
74 Test("CouchDBPersistence Servers Created", func(ctx f2.Context, t *testing.T) f2.Context {
75 k := ktest.FromContextT(ctx, t)
76 for _, ni := range nodeInfo(nodes) {
77 su := LaneSubstitution(ni, nil, couchCtlConfig.ReplicationDB(), string(leaderNode.UID))
78 for _, server := range couchDBPersistence.Spec.Servers {
79
80 _server := server
81 _server.Name = fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName)
82 k.WaitOn(t, k.ObjExists(&_server))
83 require.Equal(t, _server.Annotations[couchdb.StatefulSetLabel], su.CouchDBStatefulSet)
84 require.Equal(t, _server.Labels[nodemeta.LaneLabel], su.LaneNumber)
85 require.Equal(t, _server.Spec.Type, su.ServerType)
86 }
87 }
88 return ctx
89 }).
90 Test("CouchDBPersistence Databases Created", func(ctx f2.Context, t *testing.T) f2.Context {
91 k := ktest.FromContextT(ctx, t)
92 for _, ni := range nodeInfo(nodes) {
93 su := LaneSubstitution(ni, nil, couchCtlConfig.ReplicationDB(), string(leaderNode.UID))
94 for _, database := range couchDBPersistence.Spec.Databases {
95
96 _database := database
97 _database.Name = fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName)
98 k.WaitOn(t, k.ObjExists(&_database))
99 if couchCtlConfig.IsDSDS() {
100 require.NotEmpty(t, _database.Labels[couchdb.NodeUIDLabel])
101 }
102 require.Equal(t, _database.Spec.Name, fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName))
103 require.Equal(t, _database.Spec.Security.Admins.Names[0], fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName))
104 require.Equal(t, _database.Spec.ServerRef.Name, fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName))
105 }
106 }
107 return ctx
108 }).
109 Test("CouchDBPersistence Users Created", func(ctx f2.Context, t *testing.T) f2.Context {
110 k := ktest.FromContextT(ctx, t)
111 for _, ni := range nodeInfo(nodes) {
112 su := LaneSubstitution(ni, nil, couchCtlConfig.ReplicationDB(), string(leaderNode.UID))
113 for _, user := range couchDBPersistence.Spec.Users {
114
115 _user := user
116 _user.Name = fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName)
117 k.WaitOn(t, k.ObjExists(&_user))
118 if couchCtlConfig.IsDSDS() {
119 require.NotEmpty(t, _user.Labels[couchdb.NodeUIDLabel])
120 }
121 require.Equal(t, _user.Spec.User.Name, fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName))
122 require.Equal(t, _user.Spec.Provider.Name, fmt.Sprintf("%s-provider", couchDBPersistence.Name))
123 }
124 }
125 return ctx
126 }).
127 Test("CouchDBPersistence Replications Created", func(ctx f2.Context, t *testing.T) f2.Context {
128 k := ktest.FromContextT(ctx, t)
129 for _, ni := range nodeInfo(nodes) {
130 su := LaneSubstitution(ni, nil, couchCtlConfig.ReplicationDB(), string(leaderNode.UID))
131 for _, replication := range couchDBPersistence.Spec.Replications {
132
133 _replication := replication
134 _replication.Name = fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName)
135 k.WaitOn(t, k.ObjExists(&_replication))
136 if couchCtlConfig.IsDSDS() {
137 require.NotEmpty(t, _replication.Labels[couchdb.NodeUIDLabel])
138 }
139
140 require.Equal(t, _replication.Spec.Datasets[0].Provider.Name, "replication-user")
141 require.Equal(t, _replication.Spec.Source.Name, su.ReplicationSecret)
142 require.Equal(t, _replication.Spec.Target.Name, fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.ServerName))
143 }
144 }
145 return ctx
146 }).
147 Test("CouchDBPersistence StatefulSets Created", func(ctx f2.Context, t *testing.T) f2.Context {
148 k := ktest.FromContextT(ctx, t)
149 for _, ni := range nodeInfo(nodes) {
150 su := LaneSubstitution(ni, nil, couchCtlConfig.ReplicationDB(), string(leaderNode.UID))
151 for _, sts := range couchDBPersistence.Spec.StatefulSets {
152
153 sts := sts
154 sts.Name = fmt.Sprintf("%s-%s", couchDBPersistence.Name, su.CouchDBStatefulSet)
155 k.WaitOn(t, k.ObjExists(&sts))
156 require.Equal(t, sts.Spec.Template.Spec.Volumes[0].Name, "config")
157 require.Equal(t, sts.Spec.Template.Spec.Volumes[0].ConfigMap.Name, su.CouchDBStatefulSet)
158 require.Equal(t, sts.Spec.Selector.MatchLabels[persistenceApi.InstanceLabel], sts.Name)
159 require.Equal(t, sts.Spec.Template.ObjectMeta.Labels[persistenceApi.InstanceLabel], sts.Name)
160 require.Nil(t, sts.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
161 require.NotNil(t, sts.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
162 if couchCtlConfig.IsDSDS() {
163 require.NotEmpty(t, sts.Labels[couchdb.NodeUIDLabel])
164 }
165 if su.ServerType == dsapi.Store {
166 require.Equal(t, sts.Spec.Template.ObjectMeta.Labels[couchdb.NodeLeaderLabel], couchdb.LabelValueTrue)
167 }
168 }
169 }
170 return ctx
171 }).
172 Feature()
173 f.Test(t, fin)
174 }
175
176 func newCouchDBPersistence(name string) *dsapi.CouchDBPersistence {
177 persistName := fmt.Sprintf("pst-%s", name)
178 server := dsapi.NewStoreCouchDBServer()
179
180 server.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
181 server.Annotations[couchdb.StatefulSetLabel] = string(CouchDBStatefulSet)
182 server.Spec.Type = dsapi.ServerType(ServerType)
183
184 server.Spec.URI = fmt.Sprintf("%s-0.data-sync-couchdb.data-sync-couchdb.svc.cluster.local", CouchDBStatefulSet)
185 if integration.IsL1() {
186 server.Spec.URI = defaultHost
187 }
188
189 db := newCouchDBDatabase(persistName, server)
190
191 db.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
192 db.Spec.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
193 db.Spec.Security.Admins.Names[0] = fmt.Sprintf("%s-%s", persistName, string(ServerName))
194 db.Spec.ServerRef.Name = server.Name
195
196 user := newCouchDBUser(persistName, persistName, server)
197
198 user.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
199 user.Spec.User.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
200
201 repl := newCouchDBReplicationSet(persistName, server)
202
203 repl.Name = fmt.Sprintf("%s-%s", persistName, string(ServerName))
204 repl.Spec.Datasets[0].Name = string(ReplicationDB)
205
206 repl.Spec.Source.Name = string(ReplicationSecret)
207 repl.Spec.Source.Namespace = string(ReplicationSecretNS)
208 repl.Spec.Target.Name = server.Name
209
210
211 stsName := fmt.Sprintf("%s-%s", persistName, string(CouchDBStatefulSet))
212 configMapName := string(CouchDBStatefulSet)
213 sts := newStatefulSet(stsName, server, configMapName)
214
215 return &dsapi.CouchDBPersistence{
216 TypeMeta: metav1.TypeMeta{
217 APIVersion: dsapi.GroupVersion.String(),
218 Kind: "CouchDBPersistence",
219 },
220 ObjectMeta: metav1.ObjectMeta{
221 Name: persistName,
222 Namespace: server.Namespace,
223 },
224 Spec: dsapi.CouchDBPersistenceSpec{
225 Servers: []dsapi.CouchDBServer{
226 *server,
227 },
228 Databases: []dsapi.CouchDBDatabase{
229 *db,
230 },
231 Users: []dsapi.CouchDBUser{
232 *user,
233 },
234 Replications: []dsapi.CouchDBReplicationSet{
235 *repl,
236 },
237 StatefulSets: []appsv1.StatefulSet{
238 *sts,
239 },
240 },
241 }
242 }
243
244 func nodeInfo(nodes *corev1.NodeList) []*nameutils.NodeInfo {
245 if couchCtlConfig.IsDSDS() {
246 var info []*nameutils.NodeInfo
247 for i := range nodes.Items {
248 node := nodes.Items[i]
249 ni, _ := nameutils.GetNodeInfo(node, LaneNumberSubstitutionMaxLength)
250 info = append(info, ni)
251 }
252 return info
253 }
254 return []*nameutils.NodeInfo{
255 {
256 Hostname: "",
257 Lane: "",
258 Class: v1ien.Server,
259 Role: v1ien.ControlPlane,
260 },
261 }
262 }
263
264 func newStatefulSet(name string, server *dsapi.CouchDBServer, configMapMount string) *appsv1.StatefulSet {
265 return &appsv1.StatefulSet{
266 TypeMeta: metav1.TypeMeta{
267 APIVersion: appsv1.SchemeGroupVersion.String(),
268 Kind: "StatefulSet",
269 },
270 ObjectMeta: metav1.ObjectMeta{
271 Name: name,
272 Namespace: server.Namespace,
273 Labels: map[string]string{
274 "platform.edge.ncr.com/component": "data-sync-couchdb",
275 },
276 },
277 Spec: appsv1.StatefulSetSpec{
278 PodManagementPolicy: appsv1.ParallelPodManagement,
279 ServiceName: server.Namespace,
280 Selector: &metav1.LabelSelector{
281 MatchLabels: map[string]string{
282 "platform.edge.ncr.com/component": "data-sync-couchdb",
283 },
284 },
285 Template: corev1.PodTemplateSpec{
286 ObjectMeta: metav1.ObjectMeta{
287 Labels: map[string]string{
288 "platform.edge.ncr.com/component": "data-sync-couchdb",
289 },
290 },
291 Spec: corev1.PodSpec{
292 ServiceAccountName: server.Namespace,
293 InitContainers: []corev1.Container{
294 {
295 Name: "init-copy",
296 Image: "us-east1-docker.pkg.dev/ret-edge-pltf-infra/thirdparty/index.docker.io/library/busybox:latest",
297
298 Command: []string{"sh", "-c", "cp /tmp/chart.ini /default.d; ls -lrt /default.d;"},
299 VolumeMounts: []corev1.VolumeMount{
300 {
301 Name: "config",
302 MountPath: "/tmp/",
303 },
304 {
305 Name: "config-storage",
306 MountPath: "/default.d",
307 },
308 },
309 ImagePullPolicy: corev1.PullIfNotPresent,
310 },
311 },
312 Containers: []corev1.Container{
313 {
314 Name: "couchdb",
315 Image: "us-east1-docker.pkg.dev/ret-edge-pltf-infra/thirdparty/couchdb:3.3.1",
316 Ports: []corev1.ContainerPort{
317 {
318 Name: "couchdb",
319 ContainerPort: int32(5984),
320 },
321 {
322 Name: "epmd",
323 ContainerPort: int32(4369),
324 },
325 {
326 ContainerPort: int32(9100),
327 },
328 },
329 Env: []corev1.EnvVar{
330 {
331 Name: "COUCHDB_USER",
332 ValueFrom: &corev1.EnvVarSource{
333 SecretKeyRef: &corev1.SecretKeySelector{
334 LocalObjectReference: corev1.LocalObjectReference{
335 Name: couchdb.StoreSecretName,
336 },
337 Key: couchdb.SecretUsername,
338 },
339 },
340 },
341 {
342 Name: "COUCHDB_PASSWORD",
343 ValueFrom: &corev1.EnvVarSource{
344 SecretKeyRef: &corev1.SecretKeySelector{
345 LocalObjectReference: corev1.LocalObjectReference{
346 Name: couchdb.StoreSecretName,
347 },
348 Key: couchdb.SecretPassword,
349 },
350 },
351 },
352 {
353 Name: "COUCHDB_SECRET",
354 ValueFrom: &corev1.EnvVarSource{
355 SecretKeyRef: &corev1.SecretKeySelector{
356 LocalObjectReference: corev1.LocalObjectReference{
357 Name: couchdb.StoreSecretName,
358 },
359 Key: couchdb.SecretCookieName,
360 },
361 },
362 },
363 {
364 Name: "couch_namespace",
365 ValueFrom: &corev1.EnvVarSource{
366 FieldRef: &corev1.ObjectFieldSelector{
367 FieldPath: "metadata.namespace",
368 },
369 },
370 },
371 {
372 Name: "ERL_FLAGS",
373 Value: "-name couchdb -setcookie $(COUCHDB_SECRET)",
374 },
375 },
376 Resources: corev1.ResourceRequirements{
377 Limits: map[corev1.ResourceName]resource.Quantity{
378 corev1.ResourceCPU: resource.MustParse("4"),
379 corev1.ResourceMemory: resource.MustParse("4Gi"),
380 },
381 Requests: map[corev1.ResourceName]resource.Quantity{
382 corev1.ResourceCPU: resource.MustParse("50m"),
383 corev1.ResourceMemory: resource.MustParse("150Mi"),
384 },
385 },
386 VolumeMounts: []corev1.VolumeMount{
387 {
388 Name: "config-storage",
389 MountPath: "/opt/couchdb/etc/default.d",
390 },
391 {
392 Name: "database-storage",
393 MountPath: "/opt/couchdb/data",
394 },
395 },
396 LivenessProbe: &corev1.Probe{
397 ProbeHandler: corev1.ProbeHandler{
398 Exec: &corev1.ExecAction{
399 Command: []string{"curl", "$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up"},
400 },
401 },
402 FailureThreshold: 3,
403 PeriodSeconds: 10,
404 SuccessThreshold: 1,
405 TimeoutSeconds: 5,
406 },
407 ReadinessProbe: &corev1.Probe{
408 ProbeHandler: corev1.ProbeHandler{
409 Exec: &corev1.ExecAction{
410 Command: []string{"curl", "$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up"},
411 },
412 },
413 FailureThreshold: 3,
414 InitialDelaySeconds: 0,
415 PeriodSeconds: 10,
416 SuccessThreshold: 1,
417 TimeoutSeconds: 1,
418 },
419 },
420 },
421 Volumes: []corev1.Volume{
422 {
423 Name: "config",
424 VolumeSource: corev1.VolumeSource{
425 ConfigMap: &corev1.ConfigMapVolumeSource{
426 LocalObjectReference: corev1.LocalObjectReference{
427 Name: configMapMount,
428 },
429 Items: []corev1.KeyToPath{{
430 Key: IniFileKey,
431 Path: "chart.ini",
432 }},
433 },
434 },
435 },
436 {
437 Name: "config-storage",
438 VolumeSource: corev1.VolumeSource{
439 EmptyDir: &corev1.EmptyDirVolumeSource{},
440 },
441 },
442 {
443 Name: couchdb.StoreSecretName,
444 VolumeSource: corev1.VolumeSource{
445 Secret: &corev1.SecretVolumeSource{
446 SecretName: couchdb.StoreSecretName,
447 },
448 },
449 },
450 },
451 ImagePullSecrets: []corev1.LocalObjectReference{
452 {
453 Name: constants.EdgeDockerSecret,
454 },
455 },
456 },
457 },
458 VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
459 {
460 ObjectMeta: metav1.ObjectMeta{
461 Name: "database-storage",
462 Labels: map[string]string{
463 "platform.edge.ncr.com/component": "data-sync-couchdb",
464 },
465 },
466 Spec: corev1.PersistentVolumeClaimSpec{
467 Resources: corev1.VolumeResourceRequirements{
468 Requests: corev1.ResourceList{
469 corev1.ResourceStorage: resource.MustParse("10Gi"),
470 },
471 },
472 AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
473 },
474 },
475 },
476 },
477 }
478 }
479
View as plain text