package vpnconfig import ( "context" "encoding/base64" "encoding/json" "fmt" "net/netip" "os" "strings" "testing" iamAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/iam/v1beta1" emissaryv3alpha1 "github.com/emissary-ingress/emissary/v3/pkg/api/getambassador.io/v3alpha1" goext "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kruntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "edge-infra.dev/pkg/edge/api/mocks" v1cluster "edge-infra.dev/pkg/edge/apis/cluster/v1alpha1" v1alpha1syncedobject "edge-infra.dev/pkg/edge/apis/syncedobject/apis/v1alpha1" "edge-infra.dev/pkg/sds/ingress/emissary" "edge-infra.dev/pkg/sds/remoteaccess/constants" v1vpnconfig "edge-infra.dev/pkg/sds/remoteaccess/k8s/apis/vpnconfigs/v1" "edge-infra.dev/pkg/sds/remoteaccess/wireguard/vpn" "edge-infra.dev/test/f2" ) var f f2.Framework var ( vpnCIDR = "172.16.16.0/28" testVPNCIDRConfigMap = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: constants.VPNConfigMapName}, Data: map[string]string{constants.VPNConfigMapCIDRKey: vpnCIDR}, } outOfSubnetIPAddress = "179.16.17.3" inSubnetIPAddress = "172.16.16.12" relayIPAddress = "172.16.16.0" vpnSubnet netip.Prefix ) func init() { vpnSubnet, _ = netip.ParsePrefix(vpnCIDR) } var ( isEnabled = true isDisabled = false projectID = "ret-edge-b79we3ikmc7j9mihuwst2" clusterAName = "cluster-a" clusterBName = "cluster-b" clusterCName = "cluster-c" clusterDName = "cluster-d" clusterEName = "cluster-e" testClusterA = createCluster(clusterAName, "us-east1-c") testClusterB = createCluster(clusterBName, "us-east1-c") testClusterC = createCluster(clusterCName, "us-east1-c") testClusterD = createCluster(clusterDName, "us-east1-d") testClusterE = createCluster(clusterEName, "us-east1-d") testVPNConfigNoIPAddress = createVPNConfig(clusterAName, "", isEnabled) testVPNConfigOutOfSubnetIPAddress = createVPNConfig(clusterBName, outOfSubnetIPAddress, isEnabled) testVPNConfig = createVPNConfig(clusterCName, inSubnetIPAddress, isEnabled) testVPNConfigUnavailableIPAddress = createVPNConfig(clusterDName, relayIPAddress, isEnabled) testVPNConfigDisabled = createVPNConfig(clusterEName, "", isDisabled) testWireguardImageCM = createConfigMap("wireguard-image") testNginxImageCM = createConfigMap("nginx-image") loadBalancerIPAddress = "34.123.45.67" testWireguardRelayService = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: constants.RelayName}, Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{ {IP: loadBalancerIPAddress}, }}, }, } emissaryHostname = "project.edge-preprod.dev" testEmissaryHost = &emissaryv3alpha1.Host{ ObjectMeta: metav1.ObjectMeta{ Namespace: emissary.IngressNamespace, Name: emissary.RemoteAccessHostName, }, Spec: &emissaryv3alpha1.HostSpec{ Hostname: emissaryHostname, }, } testEmissaryDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: emissary.IngressName, Namespace: emissary.IngressNamespace, Labels: map[string]string{constants.K8sNameLabel: emissary.IngressName}, }, } emissaryPodName = emissary.IngressName + "-weaw4-as0cm" testEmissaryPod = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: emissaryPodName, Namespace: emissary.IngressNamespace, Labels: map[string]string{constants.K8sNameLabel: emissary.IngressName}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{testEmissaryContainer}, }, } testEmissaryPodWithWgContainer = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: emissaryPodName, Namespace: emissary.IngressNamespace, Labels: map[string]string{constants.K8sNameLabel: emissary.IngressName}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{testEmissaryContainer, testWireguardContainer}, }, } testEmissaryContainer = corev1.Container{Name: emissary.IngressName} testWireguardContainer = corev1.Container{Name: constants.WireguardContainerName} restartAnnotation = "kubectl.kubernetes.io/restartedAt" ) func TestMain(m *testing.M) { f = f2.New(context.Background(), f2.WithExtensions()). Setup(). Teardown() os.Exit(f.Run(m)) } func TestVPNConfig(t *testing.T) { var ( vpnConfig *v1vpnconfig.VPNConfig vpnConfigWithNoIPAddress *v1vpnconfig.VPNConfig vpnConfigOutOfSubnetIPAddress *v1vpnconfig.VPNConfig vpnConfigWithUnavailableIPAddress *v1vpnconfig.VPNConfig sm *mocks.MockSecretManagerService ) feature := f2.NewFeature("VPNConfig"). Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context { vpnConfig = getVPNConfig() vpnConfigWithNoIPAddress = getVPNConfigWithNoIPAddress() vpnConfigOutOfSubnetIPAddress = getVPNConfigWitOutOfSubnetIPAddress() vpnConfigWithUnavailableIPAddress = getVPNConfigWithUnavailableIPAddress() sm = createSecretManagerServiceMock(t) return ctx }). Test("CIDR ConfigMap not existing returns an error", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(vpnConfig, testClusterC, testWireguardRelayService, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.True(t, kerrors.IsNotFound(Update(ctx, k, sm, vpn, vpnConfig, testClusterC))) return ctx }). Test("VPNConfig without an IP address has a valid IP set", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigWithNoIPAddress, testClusterA, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigWithNoIPAddress, testClusterA)) assert.NotEmpty(t, vpnConfigWithNoIPAddress.IP()) ipAddress, err := netip.ParseAddr(vpnConfigWithNoIPAddress.IP().String()) assert.NoError(t, err) assert.True(t, vpnSubnet.Contains(ipAddress)) return ctx }). Test("VPNConfig without an out of subnet IP address has a valid IP reassigned", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigOutOfSubnetIPAddress, testClusterB, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigOutOfSubnetIPAddress, testClusterB)) assert.NotEqual(t, outOfSubnetIPAddress, vpnConfigOutOfSubnetIPAddress.IP()) ipAddress, err := netip.ParseAddr(vpnConfigOutOfSubnetIPAddress.IP().String()) assert.NoError(t, err) assert.True(t, vpnSubnet.Contains(ipAddress)) return ctx }). Test("VPNConfig with a valid IP address is not changed", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) updatedVPNConfig := &v1vpnconfig.VPNConfig{} require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig)) assert.Equal(t, inSubnetIPAddress, updatedVPNConfig.IP().String()) return ctx }). Test("maximum number of stores are updated, each has a unique IP address", func(ctx f2.Context, t *testing.T) f2.Context { var ( vpnConfigs, k = generateFakeClientWithVPNConfigs(14) ) sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(14) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) ipAddresses := []string{} for _, vpnConfig := range vpnConfigs { assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC)) updatedVPNConfig := &v1vpnconfig.VPNConfig{} require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig)) ipAddress := updatedVPNConfig.IP() assert.NotContains(t, ipAddresses, ipAddress) ipAddresses = append(ipAddresses, ipAddress.String()) } return ctx }). Test("IP addresses cannot be assigned to VPNConfigs when IP address pool is full", func(ctx f2.Context, t *testing.T) f2.Context { var ( vpnConfigs, k = generateFakeClientWithVPNConfigs(15) ) sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(14) expectedError := vpn.ErrNoIPAddressesAvailable vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) // no error for first 14 stores for idx, vpnConfig := range vpnConfigs { if idx == 14 { break } assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC)) updatedVPNConfig := &v1vpnconfig.VPNConfig{} require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig)) } // 15th store should not be given an IP address vpnConfig := vpnConfigs[14] assert.Equal(t, expectedError, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC)) return ctx }). Test("VPNConfig with relay IP address has new address assigned", func(ctx f2.Context, t *testing.T) f2.Context { k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigWithUnavailableIPAddress, testClusterD, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigWithUnavailableIPAddress, testClusterD)) assert.NotEqual(t, t, vpnConfigWithUnavailableIPAddress.IP(), vpn.Relay().IPAddress) return ctx }).Feature() f.Test(t, feature) } func TestRelayWireguard(t *testing.T) { var ( vpnConfig *v1vpnconfig.VPNConfig sm *mocks.MockSecretManagerService ) feature := f2.NewFeature("RelayWireguard"). Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context { vpnConfig = getVPNConfig() sm = createSecretManagerServiceMock(t) return ctx }). Test("Wireguard relay secret is created", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) secret := &corev1.Secret{} assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: constants.VPNNamespace, Name: constants.RelayName}, secret)) return ctx }).Feature() f.Test(t, feature) } func TestClientWireguard(t *testing.T) { var ( vpnConfig *v1vpnconfig.VPNConfig sm *mocks.MockSecretManagerService ) feature := f2.NewFeature("ClientWireguard"). Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context { vpnConfig = getVPNConfig() sm = createSecretManagerServiceMock(t) return ctx }). Test("Wireguard client secret is created", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) secret := &corev1.Secret{} assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: emissary.IngressNamespace, Name: constants.ClientName}, secret)) return ctx }). Test("Emissary deployment is restarted successfully", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testEmissaryDeployment, testEmissaryPod, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) // check Emissary was restarted deployment := &appsv1.Deployment{} assert.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(testEmissaryDeployment), deployment)) assert.Contains(t, deployment.Spec.Template.ObjectMeta.Annotations, restartAnnotation) k = getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testEmissaryDeployment, testEmissaryPodWithWgContainer, testWireguardImageCM, testNginxImageCM) require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) // check Emissary was not restarted deployment = &appsv1.Deployment{} assert.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(testEmissaryDeployment), deployment)) assert.NotContains(t, deployment.Spec.Template.ObjectMeta.Annotations, restartAnnotation) return ctx }).Feature() f.Test(t, feature) } func TestStoreWireguard(t *testing.T) { var ( vpnConfig *v1vpnconfig.VPNConfig vpnConfigOutOfSubnetIPAddress *v1vpnconfig.VPNConfig disabledVPNConfig *v1vpnconfig.VPNConfig sm *mocks.MockSecretManagerService ) feature := f2.NewFeature("StoreWireguard"). Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context { vpnConfig = getVPNConfig() disabledVPNConfig = getDisabledVPNConfig() vpnConfigOutOfSubnetIPAddress = getVPNConfigWitOutOfSubnetIPAddress() sm = createSecretManagerServiceMock(t) return ctx }). Test("Store deployment external secret synced object is created", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) // get the created external secret synced object key := client.ObjectKey{ Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName), } syncedObject := &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) // check the synced object is a valid external secret object, _ := base64.StdEncoding.DecodeString(syncedObject.Spec.Object) externalSecret := &goext.ExternalSecret{} assert.NoError(t, json.Unmarshal(object, externalSecret)) // check external secret references correct secret assert.Equal(t, externalSecret.Spec.Target.Name, constants.StoreName) return ctx }). Test("Multiple stores are configured", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, vpnConfigOutOfSubnetIPAddress, testClusterC, testClusterB, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigOutOfSubnetIPAddress, testClusterB)) // check the external secret synced objects are created for both stores key := client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)} syncedObject := &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) key = client.ObjectKey{Namespace: clusterBName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterBName)} syncedObject = &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) return ctx }). Test("Store can be removed", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) sm.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) // add store assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) // check objects were created key := client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)} syncedObject := &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) // remove store assert.NoError(t, Remove(ctx, k, sm, vpn, vpnConfig, testClusterC)) // check objects no longer exist key = client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)} syncedObject = &v1alpha1syncedobject.SyncedObject{} err = k.Get(ctx, key, syncedObject) assert.True(t, kerrors.IsNotFound(err)) return ctx }). Test("Stores can be disabled", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, disabledVPNConfig, testClusterC, testClusterE, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) assert.NoError(t, Update(ctx, k, sm, vpn, disabledVPNConfig, testClusterE)) // check the external secret synced object exists for the enabled cluster key := client.ObjectKey{Namespace: vpnConfig.ClusterEdgeID(), Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, vpnConfig.ClusterEdgeID())} syncedObject := &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) // check the external secret synced object exists for the enabled cluster, secret manager mock ensures secret data is empty key = client.ObjectKey{Namespace: disabledVPNConfig.ClusterEdgeID(), Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, disabledVPNConfig.ClusterEdgeID())} syncedObject = &v1alpha1syncedobject.SyncedObject{} assert.NoError(t, k.Get(ctx, key, syncedObject)) // check the relay configuration secret only have 2 peers (client and the enabled store) relaySecret := &corev1.Secret{} assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: constants.VPNNamespace, Name: constants.RelayName}, relaySecret)) relaySecretLines := getLinesFromWireguardSecretData(relaySecret) assert.Len(t, relaySecretLines, 18) assert.Equal(t, "[Peer]", relaySecretLines[8]) assert.Equal(t, fmt.Sprintf("AllowedIPs = %s/32", vpn.Client().GetIPAddress()), relaySecretLines[10]) assert.Equal(t, "[Peer]", relaySecretLines[13]) assert.Equal(t, fmt.Sprintf("AllowedIPs = %s/32", vpnConfig.IP()), relaySecretLines[15]) // check the client configuration secret only has one allowed IP address (the enabled store) clientSecret := &corev1.Secret{} assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: emissary.IngressNamespace, Name: constants.ClientName}, clientSecret)) clientSecretLines := getLinesFromWireguardSecretData(clientSecret) allowedIPsLine := clientSecretLines[9] expectedAllowedIPsLine := fmt.Sprintf("AllowedIPs = %s/32", vpnConfig.IP()) assert.Equal(t, expectedAllowedIPsLine, allowedIPsLine) return ctx }). Feature() f.Test(t, feature) } func TestStoreEmissaryMappings(t *testing.T) { var ( vpnConfig *v1vpnconfig.VPNConfig sm *mocks.MockSecretManagerService ) feature := f2.NewFeature("StoreEmissaryMappings"). Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context { vpnConfig = getVPNConfig() sm = createSecretManagerServiceMock(t) return ctx }). Test("Emissary mapping is created", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())} mapping := &emissaryv3alpha1.Mapping{} assert.NoError(t, k.Get(ctx, key, mapping)) expectedPrefix := fmt.Sprintf("/remoteaccess/%s/.*", clusterCName) expectedRegexPattern := fmt.Sprintf("/remoteaccess/%s/(.*)", clusterCName) expectedRegexSubstitution := "/\\1" assert.Equal(t, emissaryHostname, mapping.Spec.Hostname) assert.Equal(t, vpnConfig.IP().String(), mapping.Spec.Service) assert.Equal(t, expectedPrefix, mapping.Spec.Prefix) assert.True(t, *mapping.Spec.PrefixRegex) assert.Equal(t, expectedRegexPattern, mapping.Spec.RegexRewrite.Pattern) assert.Equal(t, expectedRegexSubstitution, mapping.Spec.RegexRewrite.Substitution) return ctx }). Test("Emissary mapping is deleted", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) sm.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).Return(nil) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())} mapping := &emissaryv3alpha1.Mapping{} assert.NoError(t, k.Get(ctx, key, mapping)) assert.NoError(t, Remove(ctx, k, sm, vpn, vpnConfig, testClusterC)) err = k.Get(ctx, key, mapping) assert.True(t, kerrors.IsNotFound(err)) return ctx }). Test("Emissary mapping is updated", func(ctx f2.Context, t *testing.T) f2.Context { sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2) k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM) vpn, err := vpn.New() require.NoError(t, err) require.NoError(t, vpn.UpdateRelay(ctx, k)) require.NoError(t, vpn.UpdateClient(ctx, k)) assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())} mapping := &emissaryv3alpha1.Mapping{} assert.NoError(t, k.Get(ctx, key, mapping)) assert.Equal(t, vpnConfig.IP().String(), mapping.Spec.Service) // fake the IP address changing to trigger another update updatedIPAddress := "172.16.16.12" vpnConfig.Status.IP = updatedIPAddress assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC)) assert.NoError(t, k.Get(ctx, key, mapping)) assert.Equal(t, updatedIPAddress, mapping.Spec.Service) return ctx }). Feature() f.Test(t, feature) } func getVPNConfigWithNoIPAddress() *v1vpnconfig.VPNConfig { return testVPNConfigNoIPAddress.DeepCopy() } func getVPNConfigWitOutOfSubnetIPAddress() *v1vpnconfig.VPNConfig { return testVPNConfigOutOfSubnetIPAddress.DeepCopy() } func getVPNConfig() *v1vpnconfig.VPNConfig { return testVPNConfig.DeepCopy() } func getVPNConfigWithUnavailableIPAddress() *v1vpnconfig.VPNConfig { return testVPNConfigUnavailableIPAddress.DeepCopy() } func getDisabledVPNConfig() *v1vpnconfig.VPNConfig { return testVPNConfigDisabled.DeepCopy() } // Creates a fake K8s client with given objects present func getFakeClientWithObjects(objects ...client.Object) client.Client { return fake.NewClientBuilder().WithScheme(createScheme()).WithObjects(objects...).Build() } // Creates schema for the fake client func createScheme() *kruntime.Scheme { scheme := kruntime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(v1vpnconfig.AddToScheme(scheme)) utilruntime.Must(v1cluster.AddToScheme(scheme)) utilruntime.Must(v1alpha1syncedobject.AddToScheme(scheme)) utilruntime.Must(emissaryv3alpha1.AddToScheme(scheme)) utilruntime.Must(iamAPI.AddToScheme(scheme)) return scheme } func createSecretManagerServiceMock(t *testing.T) *mocks.MockSecretManagerService { mockCtrl := gomock.NewController(t) return mocks.NewMockSecretManagerService(mockCtrl) } func generateFakeClientWithVPNConfigs(numVPNConfigs int) ([]client.Object, client.Client) { vpnConfigs := []client.Object{} clusters := []client.Object{} for idx := 0; idx < numVPNConfigs; idx++ { // test subnet has 16 addresses with 2 reserved for relay and client vpnConfig := getVPNConfigWithNoIPAddress() clusterName := fmt.Sprintf("%s-%d", vpnConfig.GetName(), idx) vpnConfig.ObjectMeta.Name = clusterName vpnConfigs = append(vpnConfigs, vpnConfig) cluster := testClusterA.DeepCopy() cluster.ObjectMeta.Name = clusterName clusters = append(clusters, cluster) } return vpnConfigs, fake.NewClientBuilder().WithScheme(createScheme()). WithObjects(testVPNCIDRConfigMap, testClusterA, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM). WithObjects(vpnConfigs...). WithObjects(clusters...). Build() } func getLinesFromWireguardSecretData(secret *corev1.Secret) []string { secretData := secret.StringData[constants.WireguardSecretField] return strings.Split(secretData, "\n") } func createCluster(name, location string) *v1cluster.Cluster { return &v1cluster.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: v1cluster.ClusterSpec{ Banner: "dev0-zynstra", Fleet: "store", Location: location, Name: "4c4d-30-05-22", Organization: "edge-dev0-retail-gmi062", ProjectID: projectID, Type: "sds", }, } } func createVPNConfig(name, ip string, enabled bool) *v1vpnconfig.VPNConfig { return &v1vpnconfig.VPNConfig{ ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: name}, Spec: v1vpnconfig.VPNConfigSpec{ Enabled: enabled, }, Status: &v1vpnconfig.VPNConfigStatus{ IP: ip, }, } } func createConfigMap(name string) *corev1.ConfigMap { return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: name}, Data: map[string]string{ "image": "test-image", }, } }