Source file src/edge-infra.dev/pkg/edge/controllers/envctl/persistence_controller_test.go

Documentation: edge-infra.dev/pkg/edge/controllers/envctl

     1  package envctl
     2  
     3  import (
     4  	"context"
     5  	"strings"
     6  
     7  	"edge-infra.dev/pkg/edge/apis/meta"
     8  
     9  	v1 "k8s.io/api/apps/v1"
    10  	corev1 "k8s.io/api/core/v1"
    11  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    12  	"k8s.io/apimachinery/pkg/types"
    13  	"sigs.k8s.io/controller-runtime/pkg/client"
    14  	"sigs.k8s.io/yaml"
    15  
    16  	persistenceApi "edge-infra.dev/pkg/edge/apis/persistence/v1alpha1"
    17  	"edge-infra.dev/pkg/k8s/runtime/inventory"
    18  	nodemeta "edge-infra.dev/pkg/sds/ien/node"
    19  )
    20  
    21  type testNode struct {
    22  	Name   string
    23  	Labels map[string]string
    24  }
    25  
    26  type testCasePersistence struct {
    27  	Name                     string
    28  	NodeFilter               []corev1.NodeSelectorTerm
    29  	NumNodes                 int
    30  	StartingStateFulSetNodes []string
    31  }
    32  
    33  func (s *Suite) TestPersistence() {
    34  	var err error
    35  	nodes := []testNode{
    36  		{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
    37  		{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
    38  	}
    39  	tests := []testCasePersistence{
    40  		{ //Create 1 peristence no node filter then delete the persistence
    41  			Name:                     "test-1",
    42  			NodeFilter:               nil,
    43  			NumNodes:                 2,
    44  			StartingStateFulSetNodes: []string{},
    45  		},
    46  		{ //Create 1 persistence with node filter then delete the persistence
    47  			Name: "test-2",
    48  			NodeFilter: []corev1.NodeSelectorTerm{
    49  				{MatchExpressions: []corev1.NodeSelectorRequirement{
    50  					{Key: nodemeta.LaneLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{"lane-1"}},
    51  				}},
    52  			},
    53  			NumNodes:                 1,
    54  			StartingStateFulSetNodes: []string{"worker-2"},
    55  		},
    56  		{ //create 1 peristence with nodes filter no matches then delete the persistence
    57  			Name: "test-3",
    58  			NodeFilter: []corev1.NodeSelectorTerm{
    59  				{MatchExpressions: []corev1.NodeSelectorRequirement{
    60  					{Key: nodemeta.LaneLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{"lane-does-not-exist"}},
    61  				}},
    62  			},
    63  			NumNodes:                 0,
    64  			StartingStateFulSetNodes: []string{"worker-1", "worker-2"},
    65  		},
    66  	}
    67  
    68  	//run test cases
    69  	for _, terst := range tests {
    70  		s.updateNodes(nodes)
    71  
    72  		for _, startSS := range terst.StartingStateFulSetNodes {
    73  			node := &corev1.Node{}
    74  			s.Eventually(func() bool {
    75  				err = s.Client.Get(context.TODO(), types.NamespacedName{Name: startSS}, node)
    76  				return err == nil
    77  			}, s.timeout, s.tick, "expected node not created")
    78  			statefulSet := &v1.StatefulSet{}
    79  			err := yaml.Unmarshal([]byte(ss), statefulSet)
    80  			s.Assert().NoError(err)
    81  			ss := statefulSet.DeepCopy()
    82  			ss.Name = instanceName(*node, terst.Name, map[string]struct{}{})
    83  			ss.Namespace = "default"
    84  			err = s.Client.Create(context.TODO(), ss)
    85  			s.Assert().NoError(err)
    86  			s.Eventually(func() bool {
    87  				err = s.Client.Get(context.TODO(), types.NamespacedName{Name: ss.Name, Namespace: ss.Namespace}, statefulSet)
    88  				return err == nil
    89  			}, s.timeout, s.tick, "expected ss not created")
    90  		}
    91  
    92  		//use node list to verify the correct ss have been created
    93  		nodeList := &corev1.NodeList{}
    94  		filter := convertNodeSelectorToLabelSelector(terst.NodeFilter, s.Log)
    95  		s.Eventually(func() bool {
    96  			err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter})
    97  			return err == nil && len(nodeList.Items) == terst.NumNodes
    98  		}, s.timeout, s.tick, "expected node not created")
    99  		s.Assert().NoError(err)
   100  
   101  		persistence := s.createPersistence(terst.Name, "data-sync-couchdb", terst.NodeFilter)
   102  		err = s.Client.Create(context.TODO(), persistence)
   103  		s.Assert().NoError(err)
   104  
   105  		pers := &persistenceApi.Persistence{}
   106  		s.Eventually(func() bool {
   107  			err = s.Client.Get(context.TODO(), types.NamespacedName{
   108  				Namespace: "default",
   109  				Name:      terst.Name,
   110  			}, pers)
   111  			return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodes)
   112  		}, s.timeout, s.tick, "status not set on persistence")
   113  
   114  		ssl := &v1.StatefulSetList{}
   115  		s.Eventually(func() bool {
   116  			err = s.Client.List(context.TODO(), ssl)
   117  			return err == nil && len(ssl.Items) == len(nodeList.Items)
   118  		}, s.timeout, s.tick, "statedulsets not created for all nodes")
   119  
   120  		s.verifyStatesulSets(nodeList, ssl, terst.Name, "data-sync-couchdb")
   121  
   122  		//verify delete
   123  		err = s.Client.Delete(context.TODO(), pers)
   124  		s.Assert().NoError(err)
   125  		//replicate owner ref delete https://book.kubebuilder.io/reference/envtest.html?highlight=testing#testing-considerations
   126  		err = s.Client.List(context.TODO(), ssl)
   127  		s.Assert().NoError(err)
   128  		for _, deleteSS := range ssl.Items {
   129  			ssToDelete := deleteSS
   130  			err = s.Client.Delete(context.TODO(), &ssToDelete)
   131  			s.Assert().NoError(err)
   132  		}
   133  		s.Eventually(func() bool {
   134  			err = s.Client.List(context.TODO(), ssl)
   135  			return err == nil && len(ssl.Items) == 0
   136  		}, s.timeout, s.tick, "statedulsets cleaned up for all nodes")
   137  	}
   138  }
   139  
   140  type testCasePersistenceScaling struct {
   141  	Name          string
   142  	NodeFilter    []corev1.NodeSelectorTerm
   143  	StartNodes    []testNode
   144  	NumNodesStart int
   145  	EndNodes      []testNode
   146  	NumNodesEnd   int
   147  }
   148  
   149  func (s *Suite) TestPersistenceScaling() {
   150  	var err error
   151  	tests := []testCasePersistenceScaling{
   152  		{ //change labels
   153  			Name:       "test-1",
   154  			NodeFilter: nil,
   155  			StartNodes: []testNode{
   156  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   157  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   158  			},
   159  			NumNodesStart: 2,
   160  			EndNodes: []testNode{
   161  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   162  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}},
   163  			},
   164  			NumNodesEnd: 1,
   165  		},
   166  		{ //add node with matching label
   167  			Name:       "test-2",
   168  			NodeFilter: nil,
   169  			StartNodes: []testNode{
   170  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   171  			},
   172  			NumNodesStart: 1,
   173  			EndNodes: []testNode{
   174  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   175  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   176  			},
   177  			NumNodesEnd: 2,
   178  		},
   179  		{ //add node with non matching label
   180  			Name:       "test-3",
   181  			NodeFilter: nil,
   182  			StartNodes: []testNode{
   183  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   184  			},
   185  			NumNodesStart: 1,
   186  			EndNodes: []testNode{
   187  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   188  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}},
   189  			},
   190  			NumNodesEnd: 1,
   191  		},
   192  		{ //remove node with matching label
   193  			Name:       "test-4",
   194  			NodeFilter: nil,
   195  			StartNodes: []testNode{
   196  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   197  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   198  			},
   199  			NumNodesStart: 2,
   200  			EndNodes: []testNode{
   201  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   202  			},
   203  			NumNodesEnd: 1,
   204  		},
   205  		{ //remove node with non matching label
   206  			Name:       "test-5",
   207  			NodeFilter: nil,
   208  			StartNodes: []testNode{
   209  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   210  				{Name: "worker-2", Labels: map[string]string{nodemeta.LaneLabel: "lane-2", nodemeta.ClassLabel: "no-match"}},
   211  			},
   212  			NumNodesStart: 1,
   213  			EndNodes: []testNode{
   214  				{Name: "worker-1", Labels: map[string]string{nodemeta.LaneLabel: "lane-1", nodemeta.ClassLabel: persistenceApi.TouchpointLabel}},
   215  			},
   216  			NumNodesEnd: 1,
   217  		},
   218  	}
   219  
   220  	for _, terst := range tests {
   221  		s.updateNodes(terst.StartNodes)
   222  
   223  		//use node list to verify the correct ss have been created
   224  		nodeList := &corev1.NodeList{}
   225  		filter := convertNodeSelectorToLabelSelector(terst.NodeFilter, s.Log)
   226  		s.Eventually(func() bool {
   227  			err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter})
   228  			return err == nil && len(nodeList.Items) == terst.NumNodesStart
   229  		}, s.timeout, s.tick, "expected node not created")
   230  		s.Assert().NoError(err)
   231  
   232  		persistence := s.createPersistence(terst.Name, "", terst.NodeFilter)
   233  		err = s.Client.Create(context.TODO(), persistence)
   234  		s.Assert().NoError(err)
   235  
   236  		pers := &persistenceApi.Persistence{}
   237  		s.Eventually(func() bool {
   238  			err = s.Client.Get(context.TODO(), types.NamespacedName{
   239  				Namespace: "default",
   240  				Name:      terst.Name,
   241  			}, pers)
   242  			return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodesStart)
   243  		}, s.timeout, s.tick, "status not set on persistence")
   244  
   245  		ssl := &v1.StatefulSetList{}
   246  		s.Eventually(func() bool {
   247  			err = s.Client.List(context.TODO(), ssl)
   248  			return err == nil && len(ssl.Items) == len(nodeList.Items)
   249  		}, s.timeout, s.tick, "statedulsets not created for all nodes")
   250  
   251  		s.verifyStatesulSets(nodeList, ssl, terst.Name, "")
   252  
   253  		s.updateNodes(terst.EndNodes)
   254  
   255  		//use node list to verify the correct ss have been updated
   256  		nodeList = &corev1.NodeList{}
   257  		s.Eventually(func() bool {
   258  			err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter})
   259  			return err == nil && len(nodeList.Items) == terst.NumNodesEnd
   260  		}, s.timeout, s.tick, "expected node not created")
   261  		s.Assert().NoError(err)
   262  
   263  		pers = &persistenceApi.Persistence{}
   264  		s.Eventually(func() bool {
   265  			err = s.Client.Get(context.TODO(), types.NamespacedName{
   266  				Namespace: "default",
   267  				Name:      terst.Name,
   268  			}, pers)
   269  			return err == nil && verifyInventory(pers.Status.Inventory, terst.NumNodesEnd)
   270  		}, s.timeout, s.tick, "status not set on persistence")
   271  		ssl = &v1.StatefulSetList{}
   272  		s.Eventually(func() bool {
   273  			err = s.Client.List(context.TODO(), ssl)
   274  			return err == nil && len(ssl.Items) == len(nodeList.Items)
   275  		}, s.timeout, s.tick, "statedulsets not created for all nodes")
   276  
   277  		s.verifyStatesulSets(nodeList, ssl, terst.Name, "")
   278  
   279  		//verify delete
   280  		err = s.Client.Delete(context.TODO(), pers)
   281  		s.Assert().NoError(err)
   282  		//replicate owner ref delete https://book.kubebuilder.io/reference/envtest.html?highlight=testing#testing-considerations
   283  		err = s.Client.List(context.TODO(), ssl)
   284  		s.Assert().NoError(err)
   285  		for _, deleteSS := range ssl.Items {
   286  			ssToDelete := deleteSS
   287  			err = s.Client.Delete(context.TODO(), &ssToDelete)
   288  			s.Assert().NoError(err)
   289  		}
   290  		s.Eventually(func() bool {
   291  			err = s.Client.List(context.TODO(), ssl)
   292  			return err == nil && len(ssl.Items) == 0
   293  		}, s.timeout, s.tick, "statedulsets cleaned up for all nodes")
   294  
   295  		//remove nodes for next test
   296  		s.updateNodes([]testNode{})
   297  		//use node list to verify the correct ss have been updated
   298  		nodeList = &corev1.NodeList{}
   299  		s.Eventually(func() bool {
   300  			err = s.Client.List(context.TODO(), nodeList, &client.ListOptions{LabelSelector: filter})
   301  			return err == nil && len(nodeList.Items) == 0
   302  		}, s.timeout, s.tick, "expected node not created")
   303  		s.Assert().NoError(err)
   304  	}
   305  }
   306  
   307  // createPersistence helper to create the persistence
   308  func (s *Suite) createPersistence(name, nameSubstitution string, selectors []corev1.NodeSelectorTerm) *persistenceApi.Persistence {
   309  	statefulSet := &v1.StatefulSet{}
   310  	err := yaml.Unmarshal([]byte(ss), statefulSet)
   311  	s.Assert().NoError(err)
   312  	statefulSet.ObjectMeta.Name = name
   313  	var nameSub *string
   314  	if len(nameSubstitution) > 0 {
   315  		nameSub = &nameSubstitution
   316  	}
   317  	return &persistenceApi.Persistence{
   318  		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"},
   319  		Spec: persistenceApi.PersistenceSpec{
   320  			NodeSelectorTerms: selectors,
   321  			StatefulSet:       *statefulSet,
   322  			NameSubstitution:  nameSub,
   323  		},
   324  	}
   325  }
   326  
   327  // updateNodes helper to update nodes on the cluster based on test node struct
   328  func (s *Suite) updateNodes(nodes []testNode) {
   329  	nodesOnCluster := &corev1.NodeList{}
   330  	err := s.Client.List(context.TODO(), nodesOnCluster)
   331  	s.Assert().NoError(err)
   332  	//delete nodes that don't match
   333  	for _, nodeOnCluster := range nodesOnCluster.Items {
   334  		nodeShouldExist := false
   335  		nodeOnCluster := nodeOnCluster
   336  		for _, node := range nodes {
   337  			if node.Name == nodeOnCluster.Name {
   338  				nodeShouldExist = true
   339  			}
   340  		}
   341  		if !nodeShouldExist {
   342  			err = s.Client.Delete(context.TODO(), &nodeOnCluster)
   343  			s.Assert().NoError(err)
   344  		}
   345  	}
   346  	//add missing nodes or update e
   347  	for _, node := range nodes {
   348  		nodeAlreadyExists := false
   349  		for _, nodeOnCluster := range nodesOnCluster.Items {
   350  			nodeOnCluster := nodeOnCluster
   351  			if node.Name == nodeOnCluster.Name {
   352  				nodeAlreadyExists = true
   353  				nodeOnCluster.Labels = node.Labels
   354  				err = s.Client.Update(context.TODO(), &nodeOnCluster)
   355  				s.Assert().NoError(err)
   356  			}
   357  		}
   358  		if !nodeAlreadyExists {
   359  			newNode := &corev1.Node{
   360  				ObjectMeta: metav1.ObjectMeta{Name: node.Name, Labels: node.Labels},
   361  			}
   362  			err = s.Client.Create(context.TODO(), newNode)
   363  			s.Assert().NoError(err)
   364  		}
   365  	}
   366  }
   367  
   368  // verifyInventory helper to verify inventory is what we expect
   369  func verifyInventory(i *inventory.ResourceInventory, numNodes int) bool {
   370  	// if no nodes match inventory will not be set
   371  	if numNodes == 0 && i == nil {
   372  		return true
   373  	} else if i != nil && len(i.Entries) == numNodes { //if inventory size matches the number of nodes
   374  		return true
   375  	}
   376  	return false
   377  }
   378  
   379  // verifyStatesulSets helper to verify statefulSets
   380  func (s *Suite) verifyStatesulSets(nodeList *corev1.NodeList, ssl *v1.StatefulSetList, name, nameSubstitute string) {
   381  	s.Assert().Equal(len(nodeList.Items), len(ssl.Items))
   382  
   383  	for _, node := range nodeList.Items {
   384  		matchingSS := false
   385  		for _, ssInstance := range ssl.Items {
   386  			if strings.Contains(ssInstance.Name, meta.Hash(string(node.UID))) {
   387  				//verify everything set
   388  				instanceName := instanceName(node, name, map[string]struct{}{})
   389  				s.Assert().Equal(instanceName, ssInstance.GetName())
   390  				s.Assert().Equal(instanceName, ssInstance.Spec.Template.ObjectMeta.Labels[persistenceApi.InstanceLabel])
   391  				s.Assert().Equal(instanceName, ssInstance.Spec.Selector.MatchLabels[persistenceApi.InstanceLabel])
   392  				if len(nameSubstitute) > 0 {
   393  					s.Assert().Equal(instanceName, ssInstance.Spec.Template.Spec.Volumes[0].ConfigMap.Name)
   394  				}
   395  
   396  				//todo verify node affinity
   397  				matchingSS = true
   398  			}
   399  		}
   400  		s.Assert().True(matchingSS)
   401  	}
   402  }
   403  
   404  // ss stateful set string yaml to marshal into a go type persistence for testing
   405  var ss = `apiVersion: apps/v1
   406  kind: StatefulSet
   407  metadata:
   408    name: data-sync-couchdb
   409    namespace: default
   410    labels:
   411      platform.edge.ncr.com/component: data-sync-couchdb
   412  spec:
   413    replicas: 1
   414    selector:
   415      matchLabels:
   416        platform.edge.ncr.com/component: data-sync-couchdb
   417    template:
   418      metadata:
   419        labels:
   420          platform.edge.ncr.com/component: data-sync-couchdb
   421      spec:
   422        affinity:
   423          nodeAffinity:
   424            requiredDuringSchedulingIgnoredDuringExecution:
   425              nodeSelectorTerms:
   426                - matchExpressions:
   427                    - key: "node.ncr.com/lane"
   428                      operator: In
   429                      values:
   430                        - "lane-4"
   431        serviceAccountName: data-sync-couchdb
   432        initContainers:
   433        - name: init-copy
   434          image: us-east1-docker.pkg.dev/ret-edge-pltf-infra/thirdparty/index.docker.io/library/busybox:latest
   435          command:
   436          - 'sh'
   437          - '-c'
   438          - 'cp /tmp/chart.ini /default.d; cp /tmp/prometheus.ini /default.d; ls -lrt /default.d;'
   439          volumeMounts:
   440          - name: config
   441            mountPath: /tmp/
   442          - name: config-storage
   443            mountPath: /default.d
   444          imagePullPolicy: IfNotPresent
   445        containers:
   446        - name: couchdb
   447          image: us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/couchdb@sha256:658ef40b47f068cdbc8e8a069d18e72df1a37c38f26890e7e2543decc24246fe
   448          ports:
   449          - name: couchdb
   450            containerPort: 5984
   451          - name: epmd
   452            containerPort: 4369
   453          - containerPort: 9100
   454          env:
   455          - name: COUCHDB_USER
   456            valueFrom:
   457              secretKeyRef:
   458                name: couchdb-local-creds
   459                key: username
   460          - name: COUCHDB_PASSWORD
   461            valueFrom:
   462              secretKeyRef:
   463                name: couchdb-local-creds
   464                key: password
   465          - name: COUCHDB_SECRET
   466            valueFrom:
   467              secretKeyRef:
   468                name: couchdb-local-creds
   469                key: cookieAuthSecret
   470          - name: couch_namespace
   471            valueFrom:
   472              fieldRef:
   473                fieldPath: metadata.namespace
   474          - name: ERL_FLAGS
   475            value: " -name couchdb  -setcookie monster "
   476          resources: {}
   477          volumeMounts:
   478          - name: config-storage
   479            mountPath: /opt/couchdb/etc/default.d
   480          - name: database-storage
   481            mountPath: /opt/couchdb/data
   482          - name: couchdb-local-creds
   483            readOnly: true
   484            mountPath: /opt/secrets/data-sync-couchdb-creds
   485          livenessProbe:
   486            exec:
   487              command:
   488              - curl
   489              - $COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up
   490            failureThreshold: 3
   491            periodSeconds: 10
   492            successThreshold: 1
   493            timeoutSeconds: 1
   494          readinessProbe:
   495            exec:
   496              command:
   497              - curl
   498              - $COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984/_up
   499            failureThreshold: 3
   500            initialDelaySeconds: 0
   501            periodSeconds: 10
   502            successThreshold: 1
   503            timeoutSeconds: 1
   504          imagePullPolicy: IfNotPresent
   505        volumes:
   506        - name: config
   507          configMap:
   508            name: data-sync-couchdb
   509            items:
   510            - key: inifile
   511              path: chart.ini
   512        - name: config-storage
   513          emptyDir: {}
   514        - name: couchdb-local-creds
   515          secret:
   516            secretName: couchdb-local-creds
   517        imagePullSecrets:
   518        - name: edge-docker-pull-secret
   519    volumeClaimTemplates:
   520    - metadata:
   521        name: database-storage
   522        labels:
   523          platform.edge.ncr.com/component: data-sync-couchdb
   524      spec:
   525        resources:
   526          requests:
   527            storage: "10Gi"
   528        accessModes:
   529        - "ReadWriteOnce"
   530    serviceName: data-sync-couchdb
   531    podManagementPolicy: Parallel`
   532  

View as plain text