package dennis import ( "context" "encoding/json" "fmt" "os" "testing" "time" compute "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/compute/v1beta1" dns "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/dns/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1" "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta" "edge-infra.dev/pkg/k8s/runtime/controller" "edge-infra.dev/test" "edge-infra.dev/test/framework" "edge-infra.dev/test/framework/gcp" "edge-infra.dev/test/framework/integration" "edge-infra.dev/test/framework/k8s" "edge-infra.dev/test/framework/k8s/envtest" ) func TestMain(m *testing.M) { // register framework flags + parse them, must be done before actual test // execution framework.HandleFlags() os.Exit(m.Run()) } type Suite struct { *framework.Framework *k8s.K8s ctx context.Context timeout time.Duration tick time.Duration projectID string dnsProjectID string domain string zone string } func TestDNSRecordController(t *testing.T) { testEnv := envtest.Setup() mgr, _, err := create(controller.WithCfg(testEnv.Config), controller.WithMetricsAddress("0")) test.NoError(err) k := k8s.New(testEnv.Config, k8s.WithCtrlManager(mgr), k8s.WithKonfigKonnector()) f := framework.New("dennis"). Component("dennis"). Register(k) s := &Suite{ Framework: f, K8s: k, ctx: context.Background(), domain: "my-domain.edge-infra.dev", projectID: "ret-edge-foo-fighters", dnsProjectID: "ret-edge-dennys", zone: "edge-infra", } if integration.IsIntegrationTest() { s.projectID = gcp.GCloud.ProjectID s.dnsProjectID = gcp.GCloud.ProjectID s.timeout = k8s.Timeouts.DefaultTimeout s.tick = k8s.Timeouts.Tick } else { s.timeout = time.Second * 1 s.tick = time.Millisecond * 10 } suite.Run(t, s) t.Cleanup(func() { f.NoError(testEnv.Stop()) }) } func (s *Suite) TestDNSRecordCreation_SameNamespace() { addy := s.createAddy("same-namespace", true) s.NoError(s.Client.Create(s.ctx, addy)) d := &dns.DNSRecordSet{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.Equal(s.domain, d.Spec.Name) s.Equal(s.zone, d.Spec.ManagedZoneRef.Name) id, err := meta.GetProjectAnnotation(d.ObjectMeta) s.NoError(err) s.Equal(s.dnsProjectID, id) s.Equal("", d.Spec.ManagedZoneRef.Namespace) } func (s *Suite) TestDNSRecordCreation_DifferentNamespace() { zoneNS := "edge-infra-namespace" zone := fmt.Sprintf("%s/%s", zoneNS, s.zone) addy := s.createAddy("diff-namespace", true) addy.ObjectMeta.Annotations[ManagedZoneAnnotation] = zone s.NoError(s.Client.Create(s.ctx, addy)) d := &dns.DNSRecordSet{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.Equal(s.domain, d.Spec.Name) s.Equal(s.zone, d.Spec.ManagedZoneRef.Name) id, err := meta.GetProjectAnnotation(d.ObjectMeta) s.NoError(err) s.Equal(s.dnsProjectID, id) s.Equal(zoneNS, d.Spec.ManagedZoneRef.Namespace) } func (s *Suite) TestDNSRecordNotCreated() { addy := s.createAddy("no-record", false) s.NoError(s.Client.Create(s.ctx, addy)) s.Never(func() bool { d := &dns.DNSRecordSet{} err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "DNSRecordSet created when it shouldn't have been") } func (s *Suite) TestDNSRecordSetGarbageCollection() { integration.SkipIfNot(s.Framework) addy := s.createAddy("to-be-deleted", true) s.NoError(s.Client.Create(s.ctx, addy)) s.Eventually(func() bool { d := &dns.DNSRecordSet{} err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.NoError(s.Client.Delete(s.ctx, addy)) s.Eventually(func() bool { d := &dns.DNSRecordSet{} err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return errors.IsNotFound(err) }, s.timeout, s.tick, "DNSRecordSet was never garbage collected") } func (s *Suite) TestDNSRecordSSA() { addy := s.createAddy("server-side-apply", true) s.NoError(s.Client.Create(s.ctx, addy)) d := &dns.DNSRecordSet{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.Equal(s.domain, d.Spec.Name) s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: addy.ObjectMeta.Namespace, }, addy) return !errors.IsNotFound(err) }, s.timeout, s.tick, "could not retrieve latest computeaddress") testChangeDomain := "test.change.domain" metav1.SetMetaDataAnnotation(&addy.ObjectMeta, NameAnnotation, testChangeDomain) s.NoError(s.Client.Update(s.ctx, addy)) s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) && d.ObjectMeta.Generation != 1 }, s.timeout, s.tick, "expected DNSRecordSet was never updated") s.Equal(testChangeDomain, d.Spec.Name) } func (s *Suite) TestDNSRecordMigrateToNewProject() { addy := s.createAddy("migrate-projects", true) s.Require().NoError(s.Client.Create(s.ctx, addy)) d := &dns.DNSRecordSet{} s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.Equal(s.domain, d.Spec.Name) s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: addy.ObjectMeta.Namespace, }, addy) return !errors.IsNotFound(err) }, s.timeout, s.tick, "could not retrieve latest computeaddress") testDuplicateToNewProject := "ret-edge-prod-dennys" recordConfigs := []RecordConfig{ { Name: s.domain, ManagedZone: s.zone, DNSProjectID: testDuplicateToNewProject, }, } recordConfigAnnoVal, _ := json.Marshal(recordConfigs) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, RecordConfigsAnnotation, string(recordConfigAnnoVal)) s.Require().NoError(s.Client.Update(s.ctx, addy)) outName := addy.ObjectMeta.Name + genSuffix(0) s.Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: outName, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") s.Equal(testDuplicateToNewProject, d.ObjectMeta.Annotations[meta.ProjectAnnotation]) } func (s *Suite) TestDNSRecordFromRecordConfigs() { addy := s.createAddy("from-record-configs", false) id := "uniq." // at least one part of project, zone, or domain must be unique recordConfigs := []RecordConfig{ { Name: s.domain, ManagedZone: s.zone, DNSProjectID: s.dnsProjectID, }, { Name: id + s.domain, ManagedZone: s.zone, DNSProjectID: s.dnsProjectID, }, { Name: s.domain, ManagedZone: id + s.zone, DNSProjectID: s.dnsProjectID, }, { Name: s.domain, ManagedZone: s.zone, DNSProjectID: id + s.dnsProjectID, }, } recordConfigAnnoVal, err := json.Marshal(recordConfigs) s.Require().NoError(err) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, RecordConfigsAnnotation, string(recordConfigAnnoVal)) s.Require().NoError(s.Client.Create(s.ctx, addy)) for i := 0; i < 4; i++ { drsName := fmt.Sprintf("%s%s", addy.ObjectMeta.Name, genSuffix(i)) d0 := &dns.DNSRecordSet{} s.Require().Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: drsName, Namespace: s.Namespace, }, d0) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") } } func (s *Suite) TestDNSRecordPreventDuplicates() { addy := s.createAddy("prevent-duplicates", true) recordConfigs := []RecordConfig{ { Name: s.domain, ManagedZone: s.zone, DNSProjectID: s.dnsProjectID, }, } recordConfigAnnoVal, err := json.Marshal(recordConfigs) s.NoError(err) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, RecordConfigsAnnotation, string(recordConfigAnnoVal)) s.NoError(s.Client.Create(s.ctx, addy)) // created by RecordConfig outName0 := addy.ObjectMeta.Name + genSuffix(0) d0 := &dns.DNSRecordSet{} s.Never(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: outName0, Namespace: s.Namespace, }, d0) return !errors.IsNotFound(err) }, s.timeout, s.tick, "DNSRecordSet created when it shouldn't have been") // created by singular annotations d := &dns.DNSRecordSet{} s.Never(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: addy.ObjectMeta.Name, Namespace: s.Namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "DNSRecordSet created when it shouldn't have been") } func (s *Suite) TestDNSRecordConfigFromYaml() { computeAddressYaml := []byte(fmt.Sprintf(` apiVersion: compute.cnrm.cloud.google.com/v1beta1 kind: ComputeAddress metadata: name: edge-ingress-ip namespace: %s annotations: dns.edge.ncr.com/record-configs: | [ { "name":"dev0.edge-preprod.dev.", "managed-zone":"infra/edge-preprod-dns-zone", "dns-project-id":"ret-edge-pltf-infra" }, { "name":"dev0.edge-preprod.dev.", "managed-zone":"preprod-infra/edge-preprod-dns-managed-zone", "dns-project-id":"ret-edge-pltf-preprod-infra" } ] spec: location: global resourceID: edge-ingress-ip `, s.Namespace)) addy := &compute.ComputeAddress{} s.Require().NoError( yaml.Unmarshal(computeAddressYaml, addy), "failed to unmarshal computeaddress from json document string", ) ip := "192.158.1.20" addy.Spec = compute.ComputeAddressSpec{Address: &ip} addy.Status = compute.ComputeAddressStatus{ Conditions: []v1alpha1.Condition{ {Type: "Ready", Status: v1.ConditionTrue, Reason: "I said so"}, }, } s.Require().NoError(s.Client.Create(s.ctx, addy)) // created by RecordConfig outName0 := addy.ObjectMeta.Name + genSuffix(0) outNamespace0 := addy.ObjectMeta.Namespace d0 := &dns.DNSRecordSet{} s.Require().Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: outName0, Namespace: outNamespace0, }, d0) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") } func (s *Suite) TestDeletionPolicyAnnoIsCopied() { tname := "deletion-policy-is-copied" dname := fmt.Sprintf("%s.%s", tname, s.domain) addy := s.createAddy(tname, false) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, meta.DeletionPolicyAnnotation, meta.DeletionPolicyAbandon) recordConfigs := []RecordConfig{ { Name: dname, ManagedZone: s.zone, DNSProjectID: s.dnsProjectID, }, } recordConfigAnnoVal, err := json.Marshal(recordConfigs) s.NoError(err) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, RecordConfigsAnnotation, string(recordConfigAnnoVal)) s.NoError(s.Client.Create(s.ctx, addy)) // expect deletion-policy to be copied from parent ComputeAddress d := s.eventuallyGetRecordSet(addy.ObjectMeta.Name+genSuffix(0), s.Namespace) s.Require().Equal(meta.DeletionPolicyAbandon, d.Annotations[meta.DeletionPolicyAnnotation]) } func (s *Suite) TestDeletionPolicyAnnoEmpty() { tname := "deletion-policy-is-empty" dname := fmt.Sprintf("%s.%s", tname, s.domain) addy := s.createAddy(tname, false) recordConfigs := []RecordConfig{ { Name: dname, ManagedZone: s.zone, DNSProjectID: s.dnsProjectID, }, } recordConfigAnnoVal, err := json.Marshal(recordConfigs) s.NoError(err) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, RecordConfigsAnnotation, string(recordConfigAnnoVal)) s.NoError(s.Client.Create(s.ctx, addy)) // expect deletion-policy to be empty if not set on parent ComputeAddress d := s.eventuallyGetRecordSet(addy.ObjectMeta.Name+genSuffix(0), s.Namespace) s.Require().Equal("", d.Annotations[meta.DeletionPolicyAnnotation]) } func (s *Suite) eventuallyGetRecordSet(name, namespace string) *dns.DNSRecordSet { d := &dns.DNSRecordSet{} s.Require().Eventually(func() bool { err := s.Client.Get(s.ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, d) return !errors.IsNotFound(err) }, s.timeout, s.tick, "expected DNSRecordSet was never created") return d } func (s *Suite) createAddy(name string, addAnnos bool) *compute.ComputeAddress { addy := &compute.ComputeAddress{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: s.Namespace, }, Spec: compute.ComputeAddressSpec{Location: "global"}, } meta.SetProjectAnnotation(&addy.ObjectMeta, s.projectID) if addAnnos { metav1.SetMetaDataAnnotation(&addy.ObjectMeta, NameAnnotation, s.domain) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, ManagedZoneAnnotation, s.zone) metav1.SetMetaDataAnnotation(&addy.ObjectMeta, DNSProjectAnnotation, s.dnsProjectID) } if !integration.IsIntegrationTest() { ip := "192.158.1.20" addy.Spec = compute.ComputeAddressSpec{Address: &ip} addy.Status = compute.ComputeAddressStatus{ Conditions: []v1alpha1.Condition{ {Type: "Ready", Status: v1.ConditionTrue, Reason: "I said so"}, }, } } return addy }