Source file src/github.com/linkerd/linkerd2/test/integration/multicluster/install_test.go

Documentation: github.com/linkerd/linkerd2/test/integration/multicluster

     1  package multiclustertest
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/ecdsa"
     6  	"crypto/elliptic"
     7  	"crypto/rand"
     8  	"crypto/x509"
     9  	"crypto/x509/pkix"
    10  	"encoding/pem"
    11  	"fmt"
    12  	"math/big"
    13  	"os"
    14  	"testing"
    15  	"time"
    16  
    17  	mcHealthcheck "github.com/linkerd/linkerd2/multicluster/cmd"
    18  	"github.com/linkerd/linkerd2/pkg/healthcheck"
    19  	"github.com/linkerd/linkerd2/pkg/version"
    20  	"github.com/linkerd/linkerd2/testutil"
    21  )
    22  
    23  //////////////////////
    24  ///   TEST SETUP   ///
    25  //////////////////////
    26  
    27  var (
    28  	TestHelper     *testutil.TestHelper
    29  	contexts       map[string]string
    30  	testDataDiffer testutil.TestDataDiffer
    31  )
    32  
    33  type (
    34  	multiclusterCerts struct {
    35  		ca         []byte
    36  		issuerCert []byte
    37  		issuerKey  []byte
    38  	}
    39  )
    40  
    41  func TestMain(m *testing.M) {
    42  	TestHelper = testutil.NewTestHelper()
    43  	os.Exit(m.Run())
    44  }
    45  
    46  // TestInstall will install the linkerd control plane to be used in the rest of
    47  // the deep suite tests.
    48  func TestInstall(t *testing.T) {
    49  	// Create temporary directory to create shared trust anchor and issuer
    50  	// certificates
    51  	tmpDir, err := os.MkdirTemp("", "multicluster-certs")
    52  	if err != nil {
    53  		testutil.AnnotatedFatal(t, "failed to create temp dir", err)
    54  	}
    55  
    56  	defer os.RemoveAll(tmpDir)
    57  
    58  	// Generate CA certificate
    59  	certs, err := createMulticlusterCertificates()
    60  	if err != nil {
    61  		testutil.AnnotatedFatal(t, "failed to create multicluster certificates", err)
    62  	}
    63  
    64  	// First, write CA to file
    65  	rootPath := fmt.Sprintf("%s/%s", tmpDir, "ca.crt")
    66  	// Write file with numberic mode 0400 -- u=r
    67  	if err = os.WriteFile(rootPath, certs.ca, 0400); err != nil {
    68  		testutil.AnnotatedFatal(t, "failed to create CA certificate", err)
    69  	}
    70  
    71  	// Second, write issuer key and cert to files
    72  	issuerCertPath := fmt.Sprintf("%s/%s", tmpDir, "issuer.crt")
    73  	issuerKeyPath := fmt.Sprintf("%s/%s", tmpDir, "issuer.key")
    74  
    75  	if err = os.WriteFile(issuerCertPath, certs.issuerCert, 0400); err != nil {
    76  		testutil.AnnotatedFatal(t, "failed to create issuer certificate", err)
    77  	}
    78  
    79  	if err = os.WriteFile(issuerKeyPath, certs.issuerKey, 0400); err != nil {
    80  		testutil.AnnotatedFatal(t, "failed to create issuer key", err)
    81  	}
    82  
    83  	// Install CRDs
    84  	cmd := []string{
    85  		"install",
    86  		"--crds",
    87  		"--controller-log-level", "debug",
    88  		"--set", fmt.Sprintf("proxy.image.version=%s", TestHelper.GetVersion()),
    89  		"--set", "heartbeatSchedule=1 2 3 4 5",
    90  		"--identity-trust-anchors-file", rootPath,
    91  		"--identity-issuer-certificate-file", issuerCertPath,
    92  		"--identity-issuer-key-file", issuerKeyPath,
    93  	}
    94  
    95  	// Global state to keep track of clusters
    96  	contexts = TestHelper.GetMulticlusterContexts()
    97  	for _, ctx := range contexts {
    98  		// Pipe cmd & args to `linkerd`
    99  		cmd := append([]string{"--context=" + ctx}, cmd...)
   100  		out, err := TestHelper.LinkerdRun(cmd...)
   101  		if err != nil {
   102  			testutil.AnnotatedFatal(t, "'linkerd install' command failed", err)
   103  		}
   104  		// Apply manifest from stdin
   105  		out, err = TestHelper.KubectlApplyWithContext(out, ctx, "-f", "-")
   106  		if err != nil {
   107  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   108  				"'kubectl apply' command failed\n%s", out)
   109  		}
   110  	}
   111  
   112  	// Install control-plane
   113  	cmd = []string{
   114  		"install",
   115  		"--controller-log-level", "debug",
   116  		"--set", "proxyInit.image.name=ghcr.io/linkerd/proxy-init",
   117  		"--set", fmt.Sprintf("proxyInit.image.version=%s", version.ProxyInitVersion),
   118  		"--set", fmt.Sprintf("proxy.image.version=%s", TestHelper.GetVersion()),
   119  		"--set", "heartbeatSchedule=1 2 3 4 5",
   120  		"--identity-trust-anchors-file", rootPath,
   121  		"--identity-issuer-certificate-file", issuerCertPath,
   122  		"--identity-issuer-key-file", issuerKeyPath,
   123  	}
   124  
   125  	// Global state to keep track of clusters
   126  	contexts = TestHelper.GetMulticlusterContexts()
   127  	for _, ctx := range contexts {
   128  		// Pipe cmd & args to `linkerd`
   129  		cmd := append([]string{"--context=" + ctx}, cmd...)
   130  		out, err := TestHelper.LinkerdRun(cmd...)
   131  		if err != nil {
   132  			testutil.AnnotatedFatal(t, "'linkerd install' command failed", err)
   133  		}
   134  		// Apply manifest from stdin
   135  		out, err = TestHelper.KubectlApplyWithContext(out, ctx, "-f", "-")
   136  		if err != nil {
   137  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   138  				"'kubectl apply' command failed\n%s", out)
   139  		}
   140  	}
   141  
   142  }
   143  
   144  func TestInstallMulticluster(t *testing.T) {
   145  	for k, ctx := range contexts {
   146  		var out string
   147  		var err error
   148  		// Source context should be installed without a gateway
   149  		if k == testutil.SourceContextKey {
   150  			out, err = TestHelper.LinkerdRun("--context="+ctx, "multicluster", "install", "--gateway=false")
   151  		} else {
   152  			out, err = TestHelper.LinkerdRun("--context="+ctx, "multicluster", "install")
   153  		}
   154  
   155  		if err != nil {
   156  			testutil.AnnotatedFatal(t, "'linkerd multicluster install' command failed", err)
   157  		}
   158  
   159  		out, err = TestHelper.KubectlApplyWithContext(out, ctx, "-f", "-")
   160  		if err != nil {
   161  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   162  				"'kubectl apply' command failed\n%s", out)
   163  		}
   164  	}
   165  
   166  	// Wait for gateways to come up in target cluster
   167  	TestHelper.WaitRolloutWithContext(t, testutil.MulticlusterDeployReplicas, contexts[testutil.TargetContextKey])
   168  
   169  	TestHelper.AddInstalledExtension("multicluster")
   170  }
   171  
   172  func TestMulticlusterResourcesPostInstall(t *testing.T) {
   173  	multiclusterSvcs := []testutil.Service{
   174  		{Namespace: "linkerd-multicluster", Name: "linkerd-gateway"},
   175  	}
   176  
   177  	TestHelper.SwitchContext(contexts[testutil.TargetContextKey])
   178  	testutil.TestResourcesPostInstall(TestHelper.GetMulticlusterNamespace(), multiclusterSvcs, testutil.MulticlusterDeployReplicas, TestHelper, t)
   179  }
   180  
   181  func TestLinkClusters(t *testing.T) {
   182  	// Get the control plane node IP, this is used to communicate with the
   183  	// API Server address.
   184  	// k3s runs an API server on the control plane node, the docker
   185  	// container IP suffices for a connection between containers to happen
   186  	// since they run on a shared network.
   187  	lbCmd := []string{
   188  		"get", "node",
   189  		"-n", " -l=node-role.kubernetes.io/control-plane=true",
   190  		"-o", "go-template={{ (index (index .items 0).status.addresses 0).address }}",
   191  	}
   192  
   193  	// Link target cluster to source
   194  	// * source cluster should support headless services
   195  	linkName := "target"
   196  	lbIP, err := TestHelper.KubectlWithContext("", contexts[testutil.TargetContextKey], lbCmd...)
   197  	if err != nil {
   198  		testutil.AnnotatedFatalf(t, "'kubectl get' command failed",
   199  			"'kubectl get' command failed\n%s", lbIP)
   200  	}
   201  
   202  	linkCmd := []string{
   203  		"--context=" + contexts[testutil.TargetContextKey],
   204  		"--cluster-name", linkName,
   205  		"--api-server-address", fmt.Sprintf("https://%s:6443", lbIP),
   206  		"--set", "enableHeadlessServices=true",
   207  		"multicluster", "link",
   208  		"--log-format", "json",
   209  		"--log-level", "debug",
   210  	}
   211  
   212  	out, err := TestHelper.LinkerdRun(linkCmd...)
   213  	if err != nil {
   214  		testutil.AnnotatedFatalf(t, "'linkerd multicluster link' command failed", "'linkerd multicluster link' command failed: %s\n%s", out, err)
   215  	}
   216  
   217  	out, err = TestHelper.KubectlApplyWithContext(out, contexts[testutil.SourceContextKey], "-f", "-")
   218  	if err != nil {
   219  		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   220  			"'kubectl apply' command failed\n%s", out)
   221  	}
   222  
   223  	// Link source cluster to target
   224  	// * source cluster does not have a gateway, so the link will reflect that
   225  	linkName = "source"
   226  	lbIP, err = TestHelper.KubectlWithContext("", contexts[testutil.SourceContextKey], lbCmd...)
   227  	if err != nil {
   228  		testutil.AnnotatedFatalf(t, "'kubectl get' command failed",
   229  			"'kubectl get' command failed\n%s", lbIP)
   230  	}
   231  
   232  	linkCmd = []string{
   233  		"--context=" + contexts[testutil.SourceContextKey],
   234  		"--cluster-name", linkName, "--gateway=false",
   235  		"--api-server-address", fmt.Sprintf("https://%s:6443", lbIP),
   236  		"multicluster", "link",
   237  		"--log-format", "json",
   238  		"--log-level", "debug",
   239  	}
   240  
   241  	out, err = TestHelper.LinkerdRun(linkCmd...)
   242  	if err != nil {
   243  		testutil.AnnotatedFatalf(t, "'linkerd multicluster link' command failed", "'linkerd multicluster link' command failed: %s\n%s", out, err)
   244  	}
   245  
   246  	out, err = TestHelper.KubectlApplyWithContext(out, contexts[testutil.TargetContextKey], "-f", "-")
   247  	if err != nil {
   248  		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   249  			"'kubectl apply' command failed\n%s", out)
   250  	}
   251  
   252  }
   253  
   254  func TestCheckMulticluster(t *testing.T) {
   255  	// Run `linkerd check` for both clusters, expect multicluster checks to be
   256  	// run and pass successfully
   257  	for _, ctx := range contexts {
   258  		// First, switch context to make sure we check pods in the cluster we're
   259  		// supposed to be checking them in. This will rebuild the clientset
   260  		if err := TestHelper.SwitchContext(ctx); err != nil {
   261  			testutil.AnnotatedFatalf(t, "failed to rebuild helper clientset with new context", "failed to rebuild helper clientset with new context [%s]: %v", ctx, err)
   262  		}
   263  
   264  		err := TestHelper.TestCheckWith([]healthcheck.CategoryID{mcHealthcheck.LinkerdMulticlusterExtensionCheck}, "--context", ctx)
   265  		if err != nil {
   266  			t.Fatalf("'linkerd check' command failed: %s", err)
   267  		}
   268  	}
   269  
   270  	// Check resources after link were created successfully in source cluster (e.g.
   271  	// secrets)
   272  	t.Run("Outputs resources that allow service-mirror controllers to connect to target cluster", func(t *testing.T) {
   273  		if err := TestHelper.SwitchContext(contexts[testutil.TargetContextKey]); err != nil {
   274  			testutil.AnnotatedFatalf(t,
   275  				"failed to rebuild helper clientset with new context",
   276  				"failed to rebuild helper clientset with new context [%s]: %v",
   277  				contexts[testutil.TargetContextKey], err)
   278  		}
   279  		name := "foo"
   280  		out, err := TestHelper.LinkerdRun("mc", "allow", "--service-account-name", name)
   281  		if err != nil {
   282  			testutil.AnnotatedFatalf(t,
   283  				"failed to execute 'mc allow' command",
   284  				"failed to execute 'mc allow' command %s\n%s",
   285  				err.Error(), out)
   286  		}
   287  		params := map[string]string{
   288  			"Version":     TestHelper.GetVersion(),
   289  			"AccountName": name,
   290  		}
   291  		if err = testDataDiffer.DiffTestYAMLTemplate("allow.golden", out, params); err != nil {
   292  			testutil.AnnotatedFatalf(t,
   293  				"received unexpected output",
   294  				"received unexpected output\n%s",
   295  				err.Error())
   296  		}
   297  	})
   298  }
   299  
   300  //////////////////////
   301  ///   CERT UTILS   ///
   302  //////////////////////
   303  
   304  func createMulticlusterCertificates() (multiclusterCerts, error) {
   305  	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
   306  	if err != nil {
   307  		return multiclusterCerts{}, err
   308  	}
   309  
   310  	var serialNumber int64 = 1
   311  	caTemplate := createCertificateTemplate("root.linkerd.cluster.local", big.NewInt(serialNumber))
   312  	caTemplate.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
   313  	// Create self-signed CA. Pass in its own pub key and its own private key
   314  	caDerBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
   315  	if err != nil {
   316  		return multiclusterCerts{}, err
   317  	}
   318  
   319  	// Increment serial number to generate next certificate (issuer)
   320  	serialNumber++
   321  	issuerDerBytes, issuerECKey, err := createIssuerCertificate(serialNumber, &caTemplate, caKey)
   322  	if err != nil {
   323  		return multiclusterCerts{}, err
   324  	}
   325  
   326  	// Convert keypairs to DER encoding. We don't care about the CA key so we
   327  	// only encode (to export) the issuer keys
   328  	issuerDerKey, err := x509.MarshalECPrivateKey(issuerECKey)
   329  	if err != nil {
   330  		return multiclusterCerts{}, err
   331  	}
   332  
   333  	// Finally, get strings from der blocks
   334  	// we don't care about CA's private key, it won't be written to a file
   335  	ca, _, err := tryDerToPem(caDerBytes, []byte{})
   336  	if err != nil {
   337  		return multiclusterCerts{}, err
   338  	}
   339  
   340  	issuer, issuerKey, err := tryDerToPem(issuerDerBytes, issuerDerKey)
   341  	if err != nil {
   342  		return multiclusterCerts{}, err
   343  	}
   344  
   345  	return multiclusterCerts{
   346  		ca:         ca,
   347  		issuerCert: issuer,
   348  		issuerKey:  issuerKey,
   349  	}, nil
   350  }
   351  
   352  // createCertificateTemplate will bootstrap a certificate based on the arguments
   353  // passed in, with a validty of 24h
   354  func createCertificateTemplate(subjectCommonName string, serialNumber *big.Int) x509.Certificate {
   355  	return x509.Certificate{
   356  		SerialNumber:          serialNumber,
   357  		Subject:               pkix.Name{CommonName: subjectCommonName},
   358  		NotBefore:             time.Now(),
   359  		NotAfter:              time.Now().Add(time.Hour * 24),
   360  		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
   361  		BasicConstraintsValid: true,
   362  		MaxPathLen:            0,
   363  		IsCA:                  true,
   364  	}
   365  }
   366  
   367  // createIssuerCertificate accepts a serial number, a CA template and the CA's
   368  // key and it creates and signs an intermediate certificate. The function
   369  // returns the certificate in DER encoding along with its keypair
   370  func createIssuerCertificate(serialNumber int64, caTemplate *x509.Certificate, caKey *ecdsa.PrivateKey) ([]byte, *ecdsa.PrivateKey, error) {
   371  	// Generate keypair first
   372  	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
   373  	if err != nil {
   374  		return []byte{}, nil, err
   375  	}
   376  
   377  	// Create issuer template
   378  	template := createCertificateTemplate("identity.linkerd.cluster.local", big.NewInt(serialNumber))
   379  	template.KeyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign | x509.KeyUsageCRLSign
   380  
   381  	// Create issuer certificate signed by CA, we pass in parent template and
   382  	// parent key
   383  	derBytes, err := x509.CreateCertificate(rand.Reader, &template, caTemplate, &key.PublicKey, caKey)
   384  	if err != nil {
   385  		return []byte{}, nil, err
   386  	}
   387  
   388  	return derBytes, key, nil
   389  }
   390  
   391  // tryDerToPem converts a DER encoded byte block and a DER encoded ECDSA keypair
   392  // to a PEM encoded block
   393  func tryDerToPem(derBlock []byte, key []byte) ([]byte, []byte, error) {
   394  	certOut := &bytes.Buffer{}
   395  	certPemBlock := pem.Block{Type: "CERTIFICATE", Bytes: derBlock}
   396  	if err := pem.Encode(certOut, &certPemBlock); err != nil {
   397  		return []byte{}, []byte{}, err
   398  	}
   399  
   400  	if len(key) == 0 {
   401  		return certOut.Bytes(), []byte{}, nil
   402  	}
   403  
   404  	keyOut := &bytes.Buffer{}
   405  	keyPemBlock := pem.Block{Type: "EC PRIVATE KEY", Bytes: key}
   406  	if err := pem.Encode(keyOut, &keyPemBlock); err != nil {
   407  		return []byte{}, []byte{}, err
   408  	}
   409  
   410  	return certOut.Bytes(), keyOut.Bytes(), nil
   411  }
   412  

View as plain text