package envctl import ( "context" "strings" "edge-infra.dev/pkg/edge/apis/meta" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" persistenceApi "edge-infra.dev/pkg/edge/apis/persistence/v1alpha1" "edge-infra.dev/pkg/k8s/runtime/inventory" nodemeta "edge-infra.dev/pkg/sds/ien/node" ) type testNode struct { Name string Labels map[string]string } type testCasePersistence struct { Name string NodeFilter []corev1.NodeSelectorTerm NumNodes int StartingStateFulSetNodes []string } func (s *Suite) TestPersistence() { var err error nodes := []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, } tests := []testCasePersistence{ { //Create 1 peristence no node filter then delete the persistence Name: "test-1", NodeFilter: nil, NumNodes: 2, StartingStateFulSetNodes: []string{}, }, { //Create 1 persistence with node filter then delete the persistence Name: "test-2", NodeFilter: []corev1.NodeSelectorTerm{ {MatchExpressions: []corev1.NodeSelectorRequirement{ {Key: nodemeta.LaneLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{"lane-1"}}, }}, }, NumNodes: 1, StartingStateFulSetNodes: []string{"worker-2"}, }, { //create 1 peristence with nodes filter no matches then delete the persistence Name: "test-3", NodeFilter: []corev1.NodeSelectorTerm{ {MatchExpressions: []corev1.NodeSelectorRequirement{ {Key: nodemeta.LaneLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{"lane-does-not-exist"}}, }}, }, NumNodes: 0, StartingStateFulSetNodes: []string{"worker-1", "worker-2"}, }, } //run test cases for _, terst := range tests { s.updateNodes(nodes) for _, startSS := range terst.StartingStateFulSetNodes { node := &corev1.Node{} s.Eventually(func() bool { err = s.Client.Get(context.TODO(), types.NamespacedName{Name: startSS}, node) return err == nil }, s.timeout, s.tick, "expected node not created") statefulSet := &v1.StatefulSet{} err := yaml.Unmarshal([]byte(ss), statefulSet) s.Assert().NoError(err) ss := statefulSet.DeepCopy() ss.Name = instanceName(*node, terst.Name, map[string]struct{}{}) ss.Namespace = "default" err = s.Client.Create(context.TODO(), ss) s.Assert().NoError(err) s.Eventually(func() bool { err = s.Client.Get(context.TODO(), types.NamespacedName{Name: ss.Name, Namespace: ss.Namespace}, statefulSet) return err == nil }, s.timeout, s.tick, "expected ss not created") } //use node list to verify the correct ss have been created nodeList := &corev1.NodeList{} filter := convertNodeSelectorToLabelSelector(terst.NodeFilter, s.Log) s.Eventually(func() bool { err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter}) return err == nil && len(nodeList.Items) == terst.NumNodes }, s.timeout, s.tick, "expected node not created") s.Assert().NoError(err) persistence := s.createPersistence(terst.Name, "data-sync-couchdb", terst.NodeFilter) err = s.Client.Create(context.TODO(), persistence) s.Assert().NoError(err) pers := &persistenceApi.Persistence{} s.Eventually(func() bool { err = s.Client.Get(context.TODO(), types.NamespacedName{ Namespace: "default", Name: terst.Name, }, pers) return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodes) }, s.timeout, s.tick, "status not set on persistence") ssl := &v1.StatefulSetList{} s.Eventually(func() bool { err = s.Client.List(context.TODO(), ssl) return err == nil && len(ssl.Items) == len(nodeList.Items) }, s.timeout, s.tick, "statedulsets not created for all nodes") s.verifyStatesulSets(nodeList, ssl, terst.Name, "data-sync-couchdb") //verify delete err = s.Client.Delete(context.TODO(), pers) s.Assert().NoError(err) //replicate owner ref delete https://book.kubebuilder.io/reference/envtest.html?highlight=testing#testing-considerations err = s.Client.List(context.TODO(), ssl) s.Assert().NoError(err) for _, deleteSS := range ssl.Items { ssToDelete := deleteSS err = s.Client.Delete(context.TODO(), &ssToDelete) s.Assert().NoError(err) } s.Eventually(func() bool { err = s.Client.List(context.TODO(), ssl) return err == nil && len(ssl.Items) == 0 }, s.timeout, s.tick, "statedulsets cleaned up for all nodes") } } type testCasePersistenceScaling struct { Name string NodeFilter []corev1.NodeSelectorTerm StartNodes []testNode NumNodesStart int EndNodes []testNode NumNodesEnd int } func (s *Suite) TestPersistenceScaling() { var err error tests := []testCasePersistenceScaling{ { //change labels Name: "test-1", NodeFilter: nil, StartNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesStart: 2, EndNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}}, }, NumNodesEnd: 1, }, { //add node with matching label Name: "test-2", NodeFilter: nil, StartNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesStart: 1, EndNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesEnd: 2, }, { //add node with non matching label Name: "test-3", NodeFilter: nil, StartNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesStart: 1, EndNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}}, }, NumNodesEnd: 1, }, { //remove node with matching label Name: "test-4", NodeFilter: nil, StartNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesStart: 2, EndNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesEnd: 1, }, { //remove node with non matching label Name: "test-5", NodeFilter: nil, StartNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, {Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}}, }, NumNodesStart: 1, EndNodes: []testNode{ {Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}}, }, NumNodesEnd: 1, }, } for _, terst := range tests { s.updateNodes(terst.StartNodes) //use node list to verify the correct ss have been created nodeList := &corev1.NodeList{} filter := convertNodeSelectorToLabelSelector(terst.NodeFilter, s.Log) s.Eventually(func() bool { err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter}) return err == nil && len(nodeList.Items) == terst.NumNodesStart }, s.timeout, s.tick, "expected node not created") s.Assert().NoError(err) persistence := s.createPersistence(terst.Name, "", terst.NodeFilter) err = s.Client.Create(context.TODO(), persistence) s.Assert().NoError(err) pers := &persistenceApi.Persistence{} s.Eventually(func() bool { err = s.Client.Get(context.TODO(), types.NamespacedName{ Namespace: "default", Name: terst.Name, }, pers) return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodesStart) }, s.timeout, s.tick, "status not set on persistence") ssl := &v1.StatefulSetList{} s.Eventually(func() bool { err = s.Client.List(context.TODO(), ssl) return err == nil && len(ssl.Items) == len(nodeList.Items) }, s.timeout, s.tick, "statedulsets not created for all nodes") s.verifyStatesulSets(nodeList, ssl, terst.Name, "") s.updateNodes(terst.EndNodes) //use node list to verify the correct ss have been updated nodeList = &corev1.NodeList{} s.Eventually(func() bool { err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter}) return err == nil && len(nodeList.Items) == terst.NumNodesEnd }, s.timeout, s.tick, "expected node not created") s.Assert().NoError(err) pers = &persistenceApi.Persistence{} s.Eventually(func() bool { err = s.Client.Get(context.TODO(), types.NamespacedName{ Namespace: "default", Name: terst.Name, }, pers) return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodesEnd) }, s.timeout, s.tick, "status not set on persistence") ssl = &v1.StatefulSetList{} s.Eventually(func() bool { err = s.Client.List(context.TODO(), ssl) return err == nil && len(ssl.Items) == len(nodeList.Items) }, s.timeout, s.tick, "statedulsets not created for all nodes") s.verifyStatesulSets(nodeList, ssl, terst.Name, "") //verify delete err = s.Client.Delete(context.TODO(), pers) s.Assert().NoError(err) //replicate owner ref delete https://book.kubebuilder.io/reference/envtest.html?highlight=testing#testing-considerations err = s.Client.List(context.TODO(), ssl) s.Assert().NoError(err) for _, deleteSS := range ssl.Items { ssToDelete := deleteSS err = s.Client.Delete(context.TODO(), &ssToDelete) s.Assert().NoError(err) } s.Eventually(func() bool { err = s.Client.List(context.TODO(), ssl) return err == nil && len(ssl.Items) == 0 }, s.timeout, s.tick, "statedulsets cleaned up for all nodes") //remove nodes for next test s.updateNodes([]testNode{}) //use node list to verify the correct ss have been updated nodeList = &corev1.NodeList{} s.Eventually(func() bool { err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter}) return err == nil && len(nodeList.Items) == 0 }, s.timeout, s.tick, "expected node not created") s.Assert().NoError(err) } } // createPersistence helper to create the persistence func (s *Suite) createPersistence(name, nameSubstitution string, selectors []corev1.NodeSelectorTerm) *persistenceApi.Persistence { statefulSet := &v1.StatefulSet{} err := yaml.Unmarshal([]byte(ss), statefulSet) s.Assert().NoError(err) statefulSet.ObjectMeta.Name = name var nameSub *string if len(nameSubstitution) > 0 { nameSub = &nameSubstitution } return &persistenceApi.Persistence{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, Spec: persistenceApi.PersistenceSpec{ NodeSelectorTerms: selectors, StatefulSet: *statefulSet, NameSubstitution: nameSub, }, } } // updateNodes helper to update nodes on the cluster based on test node struct func (s *Suite) updateNodes(nodes []testNode) { nodesOnCluster := &corev1.NodeList{} err := s.Client.List(context.TODO(), nodesOnCluster) s.Assert().NoError(err) //delete nodes that don't match for _, nodeOnCluster := range nodesOnCluster.Items { nodeShouldExist := false nodeOnCluster := nodeOnCluster for _, node := range nodes { if node.Name == nodeOnCluster.Name { nodeShouldExist = true } } if !nodeShouldExist { err = s.Client.Delete(context.TODO(), &nodeOnCluster) s.Assert().NoError(err) } } //add missing nodes or update e for _, node := range nodes { nodeAlreadyExists := false for _, nodeOnCluster := range nodesOnCluster.Items { nodeOnCluster := nodeOnCluster if node.Name == nodeOnCluster.Name { nodeAlreadyExists = true nodeOnCluster.Labels = node.Labels err = s.Client.Update(context.TODO(), &nodeOnCluster) s.Assert().NoError(err) } } if !nodeAlreadyExists { newNode := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{Name: node.Name, Labels: node.Labels}, } err = s.Client.Create(context.TODO(), newNode) s.Assert().NoError(err) } } } // verifyInventory helper to verify inventory is what we expect func verifyInventory(i *inventory.ResourceInventory, numNodes int) bool { // if no nodes match inventory will not be set if numNodes == 0 && i == nil { return true } else if i != nil && len(i.Entries) == numNodes { //if inventory size matches the number of nodes return true } return false } // verifyStatesulSets helper to verify statefulSets func (s *Suite) verifyStatesulSets(nodeList *corev1.NodeList, ssl *v1.StatefulSetList, name, nameSubstitute string) { s.Assert().Equal(len(nodeList.Items), len(ssl.Items)) for _, node := range nodeList.Items { matchingSS := false for _, ssInstance := range ssl.Items { if strings.Contains(ssInstance.Name, meta.Hash(string(node.UID))) { //verify everything set instanceName := instanceName(node, name, map[string]struct{}{}) s.Assert().Equal(instanceName, ssInstance.GetName()) s.Assert().Equal(instanceName, ssInstance.Spec.Template.ObjectMeta.Labels[persistenceApi.InstanceLabel]) s.Assert().Equal(instanceName, ssInstance.Spec.Selector.MatchLabels[persistenceApi.InstanceLabel]) if len(nameSubstitute) > 0 { s.Assert().Equal(instanceName, ssInstance.Spec.Template.Spec.Volumes[0].ConfigMap.Name) } //todo verify node affinity matchingSS = true } } s.Assert().True(matchingSS) } } // ss stateful set string yaml to marshal into a go type persistence for testing var ss = `apiVersion: apps/v1 kind: StatefulSet metadata: name: data-sync-couchdb namespace: default labels: platform.edge.ncr.com/component: data-sync-couchdb spec: replicas: 1 selector: matchLabels: platform.edge.ncr.com/component: data-sync-couchdb template: metadata: labels: platform.edge.ncr.com/component: data-sync-couchdb spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "node.ncr.com/lane" operator: In values: - "lane-4" serviceAccountName: data-sync-couchdb initContainers: - name: init-copy image: us-east1-docker.pkg.dev/ret-edge-pltf-infra/thirdparty/index.docker.io/library/busybox:latest command: - 'sh' - '-c' - 'cp /tmp/chart.ini /default.d; cp /tmp/prometheus.ini /default.d; ls -lrt /default.d;' volumeMounts: - name: config mountPath: /tmp/ - name: config-storage mountPath: /default.d imagePullPolicy: IfNotPresent containers: - name: couchdb image: us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/couchdb@sha256:658ef40b47f068cdbc8e8a069d18e72df1a37c38f26890e7e2543decc24246fe ports: - name: couchdb containerPort: 5984 - name: epmd containerPort: 4369 - containerPort: 9100 env: - name: COUCHDB_USER valueFrom: secretKeyRef: name: couchdb-local-creds key: username - name: COUCHDB_PASSWORD valueFrom: secretKeyRef: name: couchdb-local-creds key: password - name: COUCHDB_SECRET valueFrom: secretKeyRef: name: couchdb-local-creds key: cookieAuthSecret - name: couch_namespace valueFrom: fieldRef: fieldPath: metadata.namespace - name: ERL_FLAGS value: " -name couchdb -setcookie monster " resources: {} volumeMounts: - name: config-storage mountPath: /opt/couchdb/etc/default.d - name: database-storage mountPath: /opt/couchdb/data - name: couchdb-local-creds readOnly: true mountPath: /opt/secrets/data-sync-couchdb-creds livenessProbe: exec: command: - curl - $COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up failureThreshold: 3 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 readinessProbe: exec: command: - curl - $COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up failureThreshold: 3 initialDelaySeconds: 0 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 imagePullPolicy: IfNotPresent volumes: - name: config configMap: name: data-sync-couchdb items: - key: inifile path: chart.ini - name: config-storage emptyDir: {} - name: couchdb-local-creds secret: secretName: couchdb-local-creds imagePullSecrets: - name: edge-docker-pull-secret volumeClaimTemplates: - metadata: name: database-storage labels: platform.edge.ncr.com/component: data-sync-couchdb spec: resources: requests: storage: "10Gi" accessModes: - "ReadWriteOnce" serviceName: data-sync-couchdb podManagementPolicy: Parallel`