package clusterctl import ( "fmt" containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1" "github.com/google/uuid" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gkeClusterApi "edge-infra.dev/pkg/edge/apis/gkecluster/v1alpha1" "edge-infra.dev/pkg/edge/constants/api/fleet" "edge-infra.dev/pkg/edge/k8objectsutils" "edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta" "edge-infra.dev/pkg/lib/gcp/iam" edgeUUID "edge-infra.dev/pkg/lib/uuid" "edge-infra.dev/test/framework/integration" ) func (s *Suite) TestGKEClusterReconciler() { name := uuid.New().String() gkeCluster := gkeClusterApi.New(s.ProjectID, s.Banner.Name, s.Organization, name, s.Location, s.NodeVersion, s.NumNodes, fleet.Store, name) key := gkeCluster.ContainerClusterKey() s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}})) s.NoError(s.Client.Create(s.ctx, gkeCluster)) // check for ContainerCluster creation s.Eventually(func() bool { err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected GKECluster was never created") // check for ContainerCluster creation cluster := &containerAPI.ContainerCluster{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, cluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected ContainerCluster was never created") s.Equal(meta.DeletionPolicyAbandon, cluster.Annotations[meta.DeletionPolicyAnnotation]) // check for ContainerNodePool creation nodePool := &containerAPI.ContainerNodePool{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, nodePool) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected ContainerNodePool was never created") s.Equal(meta.DeletionPolicyAbandon, nodePool.Annotations[meta.DeletionPolicyAnnotation]) s.Equal(clusterConfigs["store"].MachineType, *nodePool.Spec.NodeConfig.MachineType) if !integration.IsIntegrationTest() { cluster.Status = containerAPI.ContainerClusterStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}} s.NoError(s.Client.Update(s.ctx, cluster)) nodePool.Status = containerAPI.ContainerNodePoolStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}} s.NoError(s.Client.Update(s.ctx, nodePool)) } s.Eventually(func() bool { s.NoError(s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)) return gkeCluster.Status.Inventory != nil && len(gkeCluster.Status.Inventory.Entries) == 2 }, s.timeout, s.tick, "expected GKECluster inventory was never created") s.NoError(s.Client.Delete(s.ctx, gkeCluster)) } func (s *Suite) TestGKEClusterImmutableFields() { integration.SkipIf(s.Framework) name := uuid.New().String() gkeCluster := gkeClusterApi.New(s.ProjectID, s.Banner.Name, s.Organization, name, s.Location, s.NodeVersion, s.NumNodes, fleet.Store, name) key := gkeCluster.ContainerClusterKey() cc := k8objectsutils.BuildContainerCluster(gkeCluster, key) desc := "tester desc" cc.Spec.Description = &desc cc.Spec.ReleaseChannel = &containerAPI.ClusterReleaseChannel{Channel: "BETA"} s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}})) s.NoError(s.Client.Create(s.ctx, cc)) cluster := &containerAPI.ContainerCluster{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, cluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected ContainerCluster was never created") s.Equal(cluster.Spec.ReleaseChannel.Channel, "BETA") s.Equal(*cluster.Spec.Description, "tester desc") s.NoError(s.Client.Create(s.ctx, gkeCluster)) // check for ContainerCluster creation s.Eventually(func() bool { err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected GKECluster was never created") // check for ContainerCluster creation s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, cluster) if err != nil { return false } if cluster.Spec.ReleaseChannel.Channel != "STABLE" { return false } if cluster.Spec.Description == nil || *cluster.Spec.Description != "tester desc" { return false } return true }, s.timeout, s.tick, "expected ContainerCluster was never created") } func (s *Suite) TestGKEClusterReconciler_FleetClusterInfra() { name := uuid.New().String() // create a cluster-infra cluster gkeCluster := gkeClusterApi.New(s.ProjectID, s.Banner.Name, s.Organization, name, s.Location, s.NodeVersion, s.NumNodes, fleet.Cluster, name) s.testGKEClusterReconciler(gkeCluster) } func (s *Suite) TestGKEClusterReconciler_TenantClusterInfra() { name := uuid.New().String() // create a cluster-infra cluster gkeCluster := gkeClusterApi.New("tenant-project", // not a top level project s.Banner.Name, s.Organization, name, s.Location, s.NodeVersion, s.NumNodes, fleet.Cluster, name) s.testGKEClusterReconciler(gkeCluster) } func (s *Suite) testGKEClusterReconciler(gkeCluster *gkeClusterApi.GKECluster) { key := gkeCluster.ContainerClusterKey() s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}})) s.NoError(s.Client.Create(s.ctx, gkeCluster)) // check for ContainerCluster creation s.Eventually(func() bool { err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected GKECluster was never created") // check for ContainerCluster creation cluster := &containerAPI.ContainerCluster{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, cluster) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected ContainerCluster was never created") // verify new ContainerCluster s.NotNil(cluster.Spec.NetworkRef, "expected networkRef to be set on new GKE ContainerClusters") s.NotNil(cluster.Spec.SubnetworkRef, "expected subnetworkRef to be set on new GKE ContainerClusters") s.NotNil(cluster.Spec.IpAllocationPolicy, "expected ipAllocationPolicy to be set on new GKE ContainerClusters") s.Equal(meta.DeletionPolicyAbandon, cluster.Annotations[meta.DeletionPolicyAnnotation]) // check masterAuthorizedNetwork setting s.NotEmpty(cluster.Spec.MasterAuthorizedNetworksConfig.CidrBlocks, "expected a MasterAuthorizedNetworksConfig block") // check for ContainerNodePool creation nodePool := &containerAPI.ContainerNodePool{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, key, nodePool) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected ContainerNodePool was never created") s.Equal(meta.DeletionPolicyAbandon, nodePool.Annotations[meta.DeletionPolicyAnnotation]) if !integration.IsIntegrationTest() { cluster.Status = containerAPI.ContainerClusterStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}} s.NoError(s.Client.Update(s.ctx, cluster)) nodePool.Status = containerAPI.ContainerNodePoolStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}} s.NoError(s.Client.Update(s.ctx, nodePool)) } clusterClient, err := s.ClusterClient(*cluster, client.Options{Scheme: s.Scheme}) s.NoError(err) hash := edgeUUID.FromUUID(gkeCluster.ObjectMeta.Name).Hash() clusterCtlSAName := fmt.Sprintf("cctl-%s", hash) syncedObjectCtlSAName := fmt.Sprintf("soctl-%s", hash) // check clusterctl service account created clusterctlSA := &corev1.ServiceAccount{} s.Eventually(func() bool { err := clusterClient.Get(s.ctx, client.ObjectKey{Namespace: "clusterctl", Name: "clusterctl"}, clusterctlSA) return !errors.IsNotFound(err) && iam.SvcAccountEmail(clusterCtlSAName, gkeCluster.Spec.ProjectID) == clusterctlSA.Annotations["iam.gke.io/gcp-service-account"] }, s.timeout, s.tick, "expected clusterctl service account to be created") // check synced object service account created syncedobjectctlSA := &corev1.ServiceAccount{} s.Eventually(func() bool { err := clusterClient.Get(s.ctx, client.ObjectKey{Namespace: "syncedobjectctl", Name: "syncedobjectctl"}, syncedobjectctlSA) return !errors.IsNotFound(err) && iam.SvcAccountEmail(syncedObjectCtlSAName, gkeCluster.Spec.ProjectID) == syncedobjectctlSA.Annotations["iam.gke.io/gcp-service-account"] }, s.timeout, s.tick, "expected syncedobject service account to be created") s.NoError(s.Client.Delete(s.ctx, gkeCluster)) }