...

Source file src/edge-infra.dev/pkg/sds/remoteaccess/k8s/controllers/wireguardctl/vpnconfig/vpnconfig_test.go

Documentation: edge-infra.dev/pkg/sds/remoteaccess/k8s/controllers/wireguardctl/vpnconfig

     1  package vpnconfig
     2  
     3  import (
     4  	"context"
     5  	"encoding/base64"
     6  	"encoding/json"
     7  	"fmt"
     8  	"net/netip"
     9  	"os"
    10  	"strings"
    11  	"testing"
    12  
    13  	iamAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/iam/v1beta1"
    14  	emissaryv3alpha1 "github.com/emissary-ingress/emissary/v3/pkg/api/getambassador.io/v3alpha1"
    15  	goext "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
    16  	"github.com/golang/mock/gomock"
    17  	"github.com/stretchr/testify/assert"
    18  	"github.com/stretchr/testify/require"
    19  	appsv1 "k8s.io/api/apps/v1"
    20  	corev1 "k8s.io/api/core/v1"
    21  	kerrors "k8s.io/apimachinery/pkg/api/errors"
    22  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    23  	kruntime "k8s.io/apimachinery/pkg/runtime"
    24  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    25  	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
    26  	"sigs.k8s.io/controller-runtime/pkg/client"
    27  	"sigs.k8s.io/controller-runtime/pkg/client/fake"
    28  
    29  	"edge-infra.dev/pkg/edge/api/mocks"
    30  	v1cluster "edge-infra.dev/pkg/edge/apis/cluster/v1alpha1"
    31  	v1alpha1syncedobject "edge-infra.dev/pkg/edge/apis/syncedobject/apis/v1alpha1"
    32  	"edge-infra.dev/pkg/sds/ingress/emissary"
    33  	"edge-infra.dev/pkg/sds/remoteaccess/constants"
    34  	v1vpnconfig "edge-infra.dev/pkg/sds/remoteaccess/k8s/apis/vpnconfigs/v1"
    35  	"edge-infra.dev/pkg/sds/remoteaccess/wireguard/vpn"
    36  	"edge-infra.dev/test/f2"
    37  )
    38  
    39  var f f2.Framework
    40  
    41  var (
    42  	vpnCIDR = "172.16.16.0/28"
    43  
    44  	testVPNCIDRConfigMap = &corev1.ConfigMap{
    45  		ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: constants.VPNConfigMapName},
    46  		Data:       map[string]string{constants.VPNConfigMapCIDRKey: vpnCIDR},
    47  	}
    48  
    49  	outOfSubnetIPAddress = "179.16.17.3"
    50  	inSubnetIPAddress    = "172.16.16.12"
    51  	relayIPAddress       = "172.16.16.0"
    52  
    53  	vpnSubnet netip.Prefix
    54  )
    55  
    56  func init() {
    57  	vpnSubnet, _ = netip.ParsePrefix(vpnCIDR)
    58  }
    59  
    60  var (
    61  	isEnabled  = true
    62  	isDisabled = false
    63  
    64  	projectID = "ret-edge-b79we3ikmc7j9mihuwst2"
    65  
    66  	clusterAName = "cluster-a"
    67  	clusterBName = "cluster-b"
    68  	clusterCName = "cluster-c"
    69  	clusterDName = "cluster-d"
    70  	clusterEName = "cluster-e"
    71  	testClusterA = createCluster(clusterAName, "us-east1-c")
    72  	testClusterB = createCluster(clusterBName, "us-east1-c")
    73  	testClusterC = createCluster(clusterCName, "us-east1-c")
    74  	testClusterD = createCluster(clusterDName, "us-east1-d")
    75  	testClusterE = createCluster(clusterEName, "us-east1-d")
    76  
    77  	testVPNConfigNoIPAddress          = createVPNConfig(clusterAName, "", isEnabled)
    78  	testVPNConfigOutOfSubnetIPAddress = createVPNConfig(clusterBName, outOfSubnetIPAddress, isEnabled)
    79  	testVPNConfig                     = createVPNConfig(clusterCName, inSubnetIPAddress, isEnabled)
    80  	testVPNConfigUnavailableIPAddress = createVPNConfig(clusterDName, relayIPAddress, isEnabled)
    81  	testVPNConfigDisabled             = createVPNConfig(clusterEName, "", isDisabled)
    82  
    83  	testWireguardImageCM = createConfigMap("wireguard-image")
    84  	testNginxImageCM     = createConfigMap("nginx-image")
    85  
    86  	loadBalancerIPAddress     = "34.123.45.67"
    87  	testWireguardRelayService = &corev1.Service{
    88  		ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: constants.RelayName},
    89  		Status: corev1.ServiceStatus{
    90  			LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{
    91  				{IP: loadBalancerIPAddress},
    92  			}},
    93  		},
    94  	}
    95  
    96  	emissaryHostname = "project.edge-preprod.dev"
    97  	testEmissaryHost = &emissaryv3alpha1.Host{
    98  		ObjectMeta: metav1.ObjectMeta{
    99  			Namespace: emissary.IngressNamespace,
   100  			Name:      emissary.RemoteAccessHostName,
   101  		},
   102  		Spec: &emissaryv3alpha1.HostSpec{
   103  			Hostname: emissaryHostname,
   104  		},
   105  	}
   106  
   107  	testEmissaryDeployment = &appsv1.Deployment{
   108  		ObjectMeta: metav1.ObjectMeta{
   109  			Name:      emissary.IngressName,
   110  			Namespace: emissary.IngressNamespace,
   111  			Labels:    map[string]string{constants.K8sNameLabel: emissary.IngressName},
   112  		},
   113  	}
   114  
   115  	emissaryPodName = emissary.IngressName + "-weaw4-as0cm"
   116  	testEmissaryPod = &corev1.Pod{
   117  		ObjectMeta: metav1.ObjectMeta{
   118  			Name:      emissaryPodName,
   119  			Namespace: emissary.IngressNamespace,
   120  			Labels:    map[string]string{constants.K8sNameLabel: emissary.IngressName},
   121  		},
   122  		Spec: corev1.PodSpec{
   123  			Containers: []corev1.Container{testEmissaryContainer},
   124  		},
   125  	}
   126  	testEmissaryPodWithWgContainer = &corev1.Pod{
   127  		ObjectMeta: metav1.ObjectMeta{
   128  			Name:      emissaryPodName,
   129  			Namespace: emissary.IngressNamespace,
   130  			Labels:    map[string]string{constants.K8sNameLabel: emissary.IngressName},
   131  		},
   132  		Spec: corev1.PodSpec{
   133  			Containers: []corev1.Container{testEmissaryContainer, testWireguardContainer},
   134  		},
   135  	}
   136  	testEmissaryContainer  = corev1.Container{Name: emissary.IngressName}
   137  	testWireguardContainer = corev1.Container{Name: constants.WireguardContainerName}
   138  
   139  	restartAnnotation = "kubectl.kubernetes.io/restartedAt"
   140  )
   141  
   142  func TestMain(m *testing.M) {
   143  	f = f2.New(context.Background(), f2.WithExtensions()).
   144  		Setup().
   145  		Teardown()
   146  	os.Exit(f.Run(m))
   147  }
   148  
   149  func TestVPNConfig(t *testing.T) {
   150  	var (
   151  		vpnConfig                         *v1vpnconfig.VPNConfig
   152  		vpnConfigWithNoIPAddress          *v1vpnconfig.VPNConfig
   153  		vpnConfigOutOfSubnetIPAddress     *v1vpnconfig.VPNConfig
   154  		vpnConfigWithUnavailableIPAddress *v1vpnconfig.VPNConfig
   155  		sm                                *mocks.MockSecretManagerService
   156  	)
   157  	feature := f2.NewFeature("VPNConfig").
   158  		Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context {
   159  			vpnConfig = getVPNConfig()
   160  			vpnConfigWithNoIPAddress = getVPNConfigWithNoIPAddress()
   161  			vpnConfigOutOfSubnetIPAddress = getVPNConfigWitOutOfSubnetIPAddress()
   162  			vpnConfigWithUnavailableIPAddress = getVPNConfigWithUnavailableIPAddress()
   163  			sm = createSecretManagerServiceMock(t)
   164  
   165  			return ctx
   166  		}).
   167  		Test("CIDR ConfigMap not existing returns an error", func(ctx f2.Context, t *testing.T) f2.Context {
   168  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   169  
   170  			k := getFakeClientWithObjects(vpnConfig, testClusterC, testWireguardRelayService, testWireguardImageCM, testNginxImageCM)
   171  
   172  			vpn, err := vpn.New()
   173  			require.NoError(t, err)
   174  
   175  			require.NoError(t, err)
   176  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   177  			require.NoError(t, vpn.UpdateClient(ctx, k))
   178  			assert.True(t, kerrors.IsNotFound(Update(ctx, k, sm, vpn, vpnConfig, testClusterC)))
   179  
   180  			return ctx
   181  		}).
   182  		Test("VPNConfig without an IP address has a valid IP set", func(ctx f2.Context, t *testing.T) f2.Context {
   183  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   184  
   185  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigWithNoIPAddress, testClusterA, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   186  			vpn, err := vpn.New()
   187  			require.NoError(t, err)
   188  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   189  			require.NoError(t, vpn.UpdateClient(ctx, k))
   190  
   191  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigWithNoIPAddress, testClusterA))
   192  			assert.NotEmpty(t, vpnConfigWithNoIPAddress.IP())
   193  
   194  			ipAddress, err := netip.ParseAddr(vpnConfigWithNoIPAddress.IP().String())
   195  			assert.NoError(t, err)
   196  			assert.True(t, vpnSubnet.Contains(ipAddress))
   197  
   198  			return ctx
   199  		}).
   200  		Test("VPNConfig without an out of subnet IP address has a valid IP reassigned", func(ctx f2.Context, t *testing.T) f2.Context {
   201  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   202  
   203  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigOutOfSubnetIPAddress, testClusterB, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   204  			vpn, err := vpn.New()
   205  			require.NoError(t, err)
   206  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   207  			require.NoError(t, vpn.UpdateClient(ctx, k))
   208  
   209  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigOutOfSubnetIPAddress, testClusterB))
   210  			assert.NotEqual(t, outOfSubnetIPAddress, vpnConfigOutOfSubnetIPAddress.IP())
   211  
   212  			ipAddress, err := netip.ParseAddr(vpnConfigOutOfSubnetIPAddress.IP().String())
   213  			assert.NoError(t, err)
   214  			assert.True(t, vpnSubnet.Contains(ipAddress))
   215  
   216  			return ctx
   217  		}).
   218  		Test("VPNConfig with a valid IP address is not changed", func(ctx f2.Context, t *testing.T) f2.Context {
   219  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   220  
   221  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   222  			vpn, err := vpn.New()
   223  			require.NoError(t, err)
   224  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   225  			require.NoError(t, vpn.UpdateClient(ctx, k))
   226  
   227  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   228  
   229  			updatedVPNConfig := &v1vpnconfig.VPNConfig{}
   230  			require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig))
   231  
   232  			assert.Equal(t, inSubnetIPAddress, updatedVPNConfig.IP().String())
   233  
   234  			return ctx
   235  		}).
   236  		Test("maximum number of stores are updated, each has a unique IP address", func(ctx f2.Context, t *testing.T) f2.Context {
   237  			var (
   238  				vpnConfigs, k = generateFakeClientWithVPNConfigs(14)
   239  			)
   240  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(14)
   241  
   242  			vpn, err := vpn.New()
   243  			require.NoError(t, err)
   244  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   245  			require.NoError(t, vpn.UpdateClient(ctx, k))
   246  
   247  			ipAddresses := []string{}
   248  			for _, vpnConfig := range vpnConfigs {
   249  				assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC))
   250  				updatedVPNConfig := &v1vpnconfig.VPNConfig{}
   251  				require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig))
   252  
   253  				ipAddress := updatedVPNConfig.IP()
   254  				assert.NotContains(t, ipAddresses, ipAddress)
   255  				ipAddresses = append(ipAddresses, ipAddress.String())
   256  			}
   257  
   258  			return ctx
   259  		}).
   260  		Test("IP addresses cannot be assigned to VPNConfigs when IP address pool is full", func(ctx f2.Context, t *testing.T) f2.Context {
   261  			var (
   262  				vpnConfigs, k = generateFakeClientWithVPNConfigs(15)
   263  			)
   264  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(14)
   265  			expectedError := vpn.ErrNoIPAddressesAvailable
   266  
   267  			vpn, err := vpn.New()
   268  			require.NoError(t, err)
   269  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   270  			require.NoError(t, vpn.UpdateClient(ctx, k))
   271  
   272  			// no error for first 14 stores
   273  			for idx, vpnConfig := range vpnConfigs {
   274  				if idx == 14 {
   275  					break
   276  				}
   277  				assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC))
   278  				updatedVPNConfig := &v1vpnconfig.VPNConfig{}
   279  				require.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(vpnConfig), updatedVPNConfig))
   280  			}
   281  
   282  			// 15th store should not be given an IP address
   283  			vpnConfig := vpnConfigs[14]
   284  			assert.Equal(t, expectedError, Update(ctx, k, sm, vpn, vpnConfig.(*v1vpnconfig.VPNConfig), testClusterC))
   285  
   286  			return ctx
   287  		}).
   288  		Test("VPNConfig with relay IP address has new address assigned", func(ctx f2.Context, t *testing.T) f2.Context {
   289  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfigWithUnavailableIPAddress, testClusterD, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   290  			vpn, err := vpn.New()
   291  			require.NoError(t, err)
   292  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   293  			require.NoError(t, vpn.UpdateClient(ctx, k))
   294  
   295  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigWithUnavailableIPAddress, testClusterD))
   296  
   297  			assert.NotEqual(t, t, vpnConfigWithUnavailableIPAddress.IP(), vpn.Relay().IPAddress)
   298  
   299  			return ctx
   300  		}).Feature()
   301  
   302  	f.Test(t, feature)
   303  }
   304  
   305  func TestRelayWireguard(t *testing.T) {
   306  	var (
   307  		vpnConfig *v1vpnconfig.VPNConfig
   308  		sm        *mocks.MockSecretManagerService
   309  	)
   310  	feature := f2.NewFeature("RelayWireguard").
   311  		Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context {
   312  			vpnConfig = getVPNConfig()
   313  			sm = createSecretManagerServiceMock(t)
   314  			return ctx
   315  		}).
   316  		Test("Wireguard relay secret is created", func(ctx f2.Context, t *testing.T) f2.Context {
   317  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   318  
   319  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   320  			vpn, err := vpn.New()
   321  			require.NoError(t, err)
   322  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   323  			require.NoError(t, vpn.UpdateClient(ctx, k))
   324  
   325  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   326  
   327  			secret := &corev1.Secret{}
   328  			assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: constants.VPNNamespace, Name: constants.RelayName}, secret))
   329  			return ctx
   330  		}).Feature()
   331  
   332  	f.Test(t, feature)
   333  }
   334  
   335  func TestClientWireguard(t *testing.T) {
   336  	var (
   337  		vpnConfig *v1vpnconfig.VPNConfig
   338  		sm        *mocks.MockSecretManagerService
   339  	)
   340  	feature := f2.NewFeature("ClientWireguard").
   341  		Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context {
   342  			vpnConfig = getVPNConfig()
   343  			sm = createSecretManagerServiceMock(t)
   344  			return ctx
   345  		}).
   346  		Test("Wireguard client secret is created", func(ctx f2.Context, t *testing.T) f2.Context {
   347  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   348  
   349  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   350  			vpn, err := vpn.New()
   351  			require.NoError(t, err)
   352  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   353  			require.NoError(t, vpn.UpdateClient(ctx, k))
   354  
   355  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   356  
   357  			secret := &corev1.Secret{}
   358  			assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: emissary.IngressNamespace, Name: constants.ClientName}, secret))
   359  			return ctx
   360  		}).
   361  		Test("Emissary deployment is restarted successfully", func(ctx f2.Context, t *testing.T) f2.Context {
   362  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
   363  
   364  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testEmissaryDeployment, testEmissaryPod, testWireguardImageCM, testNginxImageCM)
   365  			vpn, err := vpn.New()
   366  			require.NoError(t, err)
   367  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   368  			require.NoError(t, vpn.UpdateClient(ctx, k))
   369  
   370  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   371  
   372  			// check Emissary was restarted
   373  			deployment := &appsv1.Deployment{}
   374  			assert.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(testEmissaryDeployment), deployment))
   375  			assert.Contains(t, deployment.Spec.Template.ObjectMeta.Annotations, restartAnnotation)
   376  
   377  			k = getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testEmissaryDeployment, testEmissaryPodWithWgContainer, testWireguardImageCM, testNginxImageCM)
   378  			require.NoError(t, err)
   379  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   380  			require.NoError(t, vpn.UpdateClient(ctx, k))
   381  
   382  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   383  
   384  			// check Emissary was not restarted
   385  			deployment = &appsv1.Deployment{}
   386  			assert.NoError(t, k.Get(ctx, client.ObjectKeyFromObject(testEmissaryDeployment), deployment))
   387  			assert.NotContains(t, deployment.Spec.Template.ObjectMeta.Annotations, restartAnnotation)
   388  			return ctx
   389  		}).Feature()
   390  
   391  	f.Test(t, feature)
   392  }
   393  
   394  func TestStoreWireguard(t *testing.T) {
   395  	var (
   396  		vpnConfig                     *v1vpnconfig.VPNConfig
   397  		vpnConfigOutOfSubnetIPAddress *v1vpnconfig.VPNConfig
   398  		disabledVPNConfig             *v1vpnconfig.VPNConfig
   399  		sm                            *mocks.MockSecretManagerService
   400  	)
   401  	feature := f2.NewFeature("StoreWireguard").
   402  		Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context {
   403  			vpnConfig = getVPNConfig()
   404  			disabledVPNConfig = getDisabledVPNConfig()
   405  			vpnConfigOutOfSubnetIPAddress = getVPNConfigWitOutOfSubnetIPAddress()
   406  			sm = createSecretManagerServiceMock(t)
   407  
   408  			return ctx
   409  		}).
   410  		Test("Store deployment external secret synced object is created", func(ctx f2.Context, t *testing.T) f2.Context {
   411  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   412  
   413  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   414  			vpn, err := vpn.New()
   415  			require.NoError(t, err)
   416  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   417  			require.NoError(t, vpn.UpdateClient(ctx, k))
   418  
   419  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   420  
   421  			// get the created external secret synced object
   422  			key := client.ObjectKey{
   423  				Namespace: clusterCName,
   424  				Name:      fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName),
   425  			}
   426  			syncedObject := &v1alpha1syncedobject.SyncedObject{}
   427  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   428  
   429  			// check the synced object is a valid external secret
   430  			object, _ := base64.StdEncoding.DecodeString(syncedObject.Spec.Object)
   431  			externalSecret := &goext.ExternalSecret{}
   432  			assert.NoError(t, json.Unmarshal(object, externalSecret))
   433  
   434  			// check external secret references correct secret
   435  			assert.Equal(t, externalSecret.Spec.Target.Name, constants.StoreName)
   436  
   437  			return ctx
   438  		}).
   439  		Test("Multiple stores are configured", func(ctx f2.Context, t *testing.T) f2.Context {
   440  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
   441  
   442  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, vpnConfigOutOfSubnetIPAddress, testClusterC, testClusterB, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   443  			vpn, err := vpn.New()
   444  			require.NoError(t, err)
   445  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   446  			require.NoError(t, vpn.UpdateClient(ctx, k))
   447  
   448  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   449  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfigOutOfSubnetIPAddress, testClusterB))
   450  
   451  			// check the external secret synced objects are created for both stores
   452  			key := client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)}
   453  			syncedObject := &v1alpha1syncedobject.SyncedObject{}
   454  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   455  			key = client.ObjectKey{Namespace: clusterBName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterBName)}
   456  			syncedObject = &v1alpha1syncedobject.SyncedObject{}
   457  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   458  
   459  			return ctx
   460  		}).
   461  		Test("Store can be removed", func(ctx f2.Context, t *testing.T) f2.Context {
   462  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   463  			sm.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).Return(nil)
   464  
   465  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   466  			vpn, err := vpn.New()
   467  			require.NoError(t, err)
   468  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   469  			require.NoError(t, vpn.UpdateClient(ctx, k))
   470  
   471  			// add store
   472  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   473  
   474  			// check objects were created
   475  			key := client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)}
   476  			syncedObject := &v1alpha1syncedobject.SyncedObject{}
   477  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   478  
   479  			// remove store
   480  			assert.NoError(t, Remove(ctx, k, sm, vpn, vpnConfig, testClusterC))
   481  
   482  			// check objects no longer exist
   483  			key = client.ObjectKey{Namespace: clusterCName, Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, clusterCName)}
   484  			syncedObject = &v1alpha1syncedobject.SyncedObject{}
   485  			err = k.Get(ctx, key, syncedObject)
   486  			assert.True(t, kerrors.IsNotFound(err))
   487  
   488  			return ctx
   489  		}).
   490  		Test("Stores can be disabled", func(ctx f2.Context, t *testing.T) f2.Context {
   491  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
   492  
   493  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, disabledVPNConfig, testClusterC, testClusterE, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   494  			vpn, err := vpn.New()
   495  			require.NoError(t, err)
   496  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   497  			require.NoError(t, vpn.UpdateClient(ctx, k))
   498  
   499  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   500  			assert.NoError(t, Update(ctx, k, sm, vpn, disabledVPNConfig, testClusterE))
   501  
   502  			// check the external secret synced object exists for the enabled cluster
   503  			key := client.ObjectKey{Namespace: vpnConfig.ClusterEdgeID(), Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, vpnConfig.ClusterEdgeID())}
   504  			syncedObject := &v1alpha1syncedobject.SyncedObject{}
   505  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   506  
   507  			// check the external secret synced object exists for the enabled cluster, secret manager mock ensures secret data is empty
   508  			key = client.ObjectKey{Namespace: disabledVPNConfig.ClusterEdgeID(), Name: fmt.Sprintf("%s-externalsecret-%s", constants.StoreName, disabledVPNConfig.ClusterEdgeID())}
   509  			syncedObject = &v1alpha1syncedobject.SyncedObject{}
   510  			assert.NoError(t, k.Get(ctx, key, syncedObject))
   511  
   512  			// check the relay configuration secret only have 2 peers (client and the enabled store)
   513  			relaySecret := &corev1.Secret{}
   514  			assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: constants.VPNNamespace, Name: constants.RelayName}, relaySecret))
   515  			relaySecretLines := getLinesFromWireguardSecretData(relaySecret)
   516  			assert.Len(t, relaySecretLines, 18)
   517  			assert.Equal(t, "[Peer]", relaySecretLines[8])
   518  			assert.Equal(t, fmt.Sprintf("AllowedIPs = %s/32", vpn.Client().GetIPAddress()), relaySecretLines[10])
   519  			assert.Equal(t, "[Peer]", relaySecretLines[13])
   520  			assert.Equal(t, fmt.Sprintf("AllowedIPs = %s/32", vpnConfig.IP()), relaySecretLines[15])
   521  
   522  			// check the client configuration secret only has one allowed IP address (the enabled store)
   523  			clientSecret := &corev1.Secret{}
   524  			assert.NoError(t, k.Get(ctx, client.ObjectKey{Namespace: emissary.IngressNamespace, Name: constants.ClientName}, clientSecret))
   525  			clientSecretLines := getLinesFromWireguardSecretData(clientSecret)
   526  			allowedIPsLine := clientSecretLines[9]
   527  			expectedAllowedIPsLine := fmt.Sprintf("AllowedIPs = %s/32", vpnConfig.IP())
   528  			assert.Equal(t, expectedAllowedIPsLine, allowedIPsLine)
   529  
   530  			return ctx
   531  		}).
   532  		Feature()
   533  
   534  	f.Test(t, feature)
   535  }
   536  
   537  func TestStoreEmissaryMappings(t *testing.T) {
   538  	var (
   539  		vpnConfig *v1vpnconfig.VPNConfig
   540  		sm        *mocks.MockSecretManagerService
   541  	)
   542  	feature := f2.NewFeature("StoreEmissaryMappings").
   543  		Setup("Create mock secret manager and vpn config", func(ctx f2.Context, t *testing.T) f2.Context {
   544  			vpnConfig = getVPNConfig()
   545  			sm = createSecretManagerServiceMock(t)
   546  
   547  			return ctx
   548  		}).
   549  		Test("Emissary mapping is created", func(ctx f2.Context, t *testing.T) f2.Context {
   550  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   551  
   552  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   553  			vpn, err := vpn.New()
   554  			require.NoError(t, err)
   555  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   556  			require.NoError(t, vpn.UpdateClient(ctx, k))
   557  
   558  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   559  
   560  			key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())}
   561  			mapping := &emissaryv3alpha1.Mapping{}
   562  			assert.NoError(t, k.Get(ctx, key, mapping))
   563  
   564  			expectedPrefix := fmt.Sprintf("/remoteaccess/%s/.*", clusterCName)
   565  			expectedRegexPattern := fmt.Sprintf("/remoteaccess/%s/(.*)", clusterCName)
   566  			expectedRegexSubstitution := "/\\1"
   567  
   568  			assert.Equal(t, emissaryHostname, mapping.Spec.Hostname)
   569  			assert.Equal(t, vpnConfig.IP().String(), mapping.Spec.Service)
   570  			assert.Equal(t, expectedPrefix, mapping.Spec.Prefix)
   571  			assert.True(t, *mapping.Spec.PrefixRegex)
   572  			assert.Equal(t, expectedRegexPattern, mapping.Spec.RegexRewrite.Pattern)
   573  			assert.Equal(t, expectedRegexSubstitution, mapping.Spec.RegexRewrite.Substitution)
   574  
   575  			return ctx
   576  		}).
   577  		Test("Emissary mapping is deleted", func(ctx f2.Context, t *testing.T) f2.Context {
   578  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
   579  			sm.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).Return(nil)
   580  
   581  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   582  			vpn, err := vpn.New()
   583  			require.NoError(t, err)
   584  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   585  			require.NoError(t, vpn.UpdateClient(ctx, k))
   586  
   587  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   588  
   589  			key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())}
   590  			mapping := &emissaryv3alpha1.Mapping{}
   591  			assert.NoError(t, k.Get(ctx, key, mapping))
   592  
   593  			assert.NoError(t, Remove(ctx, k, sm, vpn, vpnConfig, testClusterC))
   594  
   595  			err = k.Get(ctx, key, mapping)
   596  			assert.True(t, kerrors.IsNotFound(err))
   597  
   598  			return ctx
   599  		}).
   600  		Test("Emissary mapping is updated", func(ctx f2.Context, t *testing.T) f2.Context {
   601  			sm.EXPECT().AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
   602  
   603  			k := getFakeClientWithObjects(testVPNCIDRConfigMap, vpnConfig, testClusterC, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM)
   604  			vpn, err := vpn.New()
   605  			require.NoError(t, err)
   606  			require.NoError(t, vpn.UpdateRelay(ctx, k))
   607  			require.NoError(t, vpn.UpdateClient(ctx, k))
   608  
   609  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   610  
   611  			key := client.ObjectKey{Namespace: emissary.IngressNamespace, Name: fmt.Sprintf("%s-%s", constants.StoreName, vpnConfig.ClusterEdgeID())}
   612  			mapping := &emissaryv3alpha1.Mapping{}
   613  			assert.NoError(t, k.Get(ctx, key, mapping))
   614  			assert.Equal(t, vpnConfig.IP().String(), mapping.Spec.Service)
   615  
   616  			// fake the IP address changing to trigger another update
   617  			updatedIPAddress := "172.16.16.12"
   618  			vpnConfig.Status.IP = updatedIPAddress
   619  
   620  			assert.NoError(t, Update(ctx, k, sm, vpn, vpnConfig, testClusterC))
   621  
   622  			assert.NoError(t, k.Get(ctx, key, mapping))
   623  			assert.Equal(t, updatedIPAddress, mapping.Spec.Service)
   624  
   625  			return ctx
   626  		}).
   627  		Feature()
   628  
   629  	f.Test(t, feature)
   630  }
   631  
   632  func getVPNConfigWithNoIPAddress() *v1vpnconfig.VPNConfig {
   633  	return testVPNConfigNoIPAddress.DeepCopy()
   634  }
   635  
   636  func getVPNConfigWitOutOfSubnetIPAddress() *v1vpnconfig.VPNConfig {
   637  	return testVPNConfigOutOfSubnetIPAddress.DeepCopy()
   638  }
   639  
   640  func getVPNConfig() *v1vpnconfig.VPNConfig {
   641  	return testVPNConfig.DeepCopy()
   642  }
   643  
   644  func getVPNConfigWithUnavailableIPAddress() *v1vpnconfig.VPNConfig {
   645  	return testVPNConfigUnavailableIPAddress.DeepCopy()
   646  }
   647  
   648  func getDisabledVPNConfig() *v1vpnconfig.VPNConfig {
   649  	return testVPNConfigDisabled.DeepCopy()
   650  }
   651  
   652  // Creates a fake K8s client with given objects present
   653  func getFakeClientWithObjects(objects ...client.Object) client.Client {
   654  	return fake.NewClientBuilder().WithScheme(createScheme()).WithObjects(objects...).Build()
   655  }
   656  
   657  // Creates schema for the fake client
   658  func createScheme() *kruntime.Scheme {
   659  	scheme := kruntime.NewScheme()
   660  	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
   661  	utilruntime.Must(v1vpnconfig.AddToScheme(scheme))
   662  	utilruntime.Must(v1cluster.AddToScheme(scheme))
   663  	utilruntime.Must(v1alpha1syncedobject.AddToScheme(scheme))
   664  	utilruntime.Must(emissaryv3alpha1.AddToScheme(scheme))
   665  	utilruntime.Must(iamAPI.AddToScheme(scheme))
   666  	return scheme
   667  }
   668  
   669  func createSecretManagerServiceMock(t *testing.T) *mocks.MockSecretManagerService {
   670  	mockCtrl := gomock.NewController(t)
   671  	return mocks.NewMockSecretManagerService(mockCtrl)
   672  }
   673  
   674  func generateFakeClientWithVPNConfigs(numVPNConfigs int) ([]client.Object, client.Client) {
   675  	vpnConfigs := []client.Object{}
   676  	clusters := []client.Object{}
   677  	for idx := 0; idx < numVPNConfigs; idx++ { // test subnet has 16 addresses with 2 reserved for relay and client
   678  		vpnConfig := getVPNConfigWithNoIPAddress()
   679  		clusterName := fmt.Sprintf("%s-%d", vpnConfig.GetName(), idx)
   680  
   681  		vpnConfig.ObjectMeta.Name = clusterName
   682  		vpnConfigs = append(vpnConfigs, vpnConfig)
   683  
   684  		cluster := testClusterA.DeepCopy()
   685  		cluster.ObjectMeta.Name = clusterName
   686  		clusters = append(clusters, cluster)
   687  	}
   688  
   689  	return vpnConfigs, fake.NewClientBuilder().WithScheme(createScheme()).
   690  		WithObjects(testVPNCIDRConfigMap, testClusterA, testWireguardRelayService, testEmissaryHost, testWireguardImageCM, testNginxImageCM).
   691  		WithObjects(vpnConfigs...).
   692  		WithObjects(clusters...).
   693  		Build()
   694  }
   695  
   696  func getLinesFromWireguardSecretData(secret *corev1.Secret) []string {
   697  	secretData := secret.StringData[constants.WireguardSecretField]
   698  	return strings.Split(secretData, "\n")
   699  }
   700  
   701  func createCluster(name, location string) *v1cluster.Cluster {
   702  	return &v1cluster.Cluster{
   703  		ObjectMeta: metav1.ObjectMeta{Name: name},
   704  		Spec: v1cluster.ClusterSpec{
   705  			Banner:       "dev0-zynstra",
   706  			Fleet:        "store",
   707  			Location:     location,
   708  			Name:         "4c4d-30-05-22",
   709  			Organization: "edge-dev0-retail-gmi062",
   710  			ProjectID:    projectID,
   711  			Type:         "sds",
   712  		},
   713  	}
   714  }
   715  
   716  func createVPNConfig(name, ip string, enabled bool) *v1vpnconfig.VPNConfig {
   717  	return &v1vpnconfig.VPNConfig{
   718  		ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: name},
   719  		Spec: v1vpnconfig.VPNConfigSpec{
   720  			Enabled: enabled,
   721  		},
   722  		Status: &v1vpnconfig.VPNConfigStatus{
   723  			IP: ip,
   724  		},
   725  	}
   726  }
   727  
   728  func createConfigMap(name string) *corev1.ConfigMap {
   729  	return &corev1.ConfigMap{
   730  		ObjectMeta: metav1.ObjectMeta{Namespace: constants.VPNNamespace, Name: name},
   731  		Data: map[string]string{
   732  			"image": "test-image",
   733  		},
   734  	}
   735  }
   736  

View as plain text