Source file src/github.com/linkerd/linkerd2/test/integration/upgrade-edge/upgrade_edge_test.go

Documentation: github.com/linkerd/linkerd2/test/integration/upgrade-edge

     1  package edgeupgradetest
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"os"
     8  	"runtime"
     9  	"strings"
    10  	"testing"
    11  
    12  	"github.com/go-test/deep"
    13  	"github.com/linkerd/linkerd2/pkg/flags"
    14  	"github.com/linkerd/linkerd2/pkg/k8s"
    15  	"github.com/linkerd/linkerd2/pkg/tree"
    16  	"github.com/linkerd/linkerd2/testutil"
    17  )
    18  
    19  var (
    20  	TestHelper *testutil.TestHelper
    21  
    22  	configMapUID string
    23  
    24  	linkerdSvcEdge = []testutil.Service{
    25  		{Namespace: "linkerd", Name: "linkerd-dst"},
    26  		{Namespace: "linkerd", Name: "linkerd-identity"},
    27  
    28  		{Namespace: "linkerd", Name: "linkerd-dst-headless"},
    29  		{Namespace: "linkerd", Name: "linkerd-identity-headless"},
    30  	}
    31  
    32  	// skippedInboundPorts lists some ports to be marked as skipped, which will
    33  	// be verified in test/integration/inject
    34  	skippedInboundPorts    = "1234,5678"
    35  	skippedOutboundPorts   = "1234,5678"
    36  	linkerdBaseEdgeVersion string
    37  )
    38  
    39  func TestMain(m *testing.M) {
    40  	TestHelper = testutil.NewTestHelper()
    41  	os.Exit(m.Run())
    42  }
    43  
    44  func TestInstallResourcesPreUpgrade(t *testing.T) {
    45  	versions, err := TestHelper.GetReleaseChannelVersions()
    46  	if err != nil {
    47  		testutil.AnnotatedFatal(t, "failed to get the latest release channels versions", err)
    48  	}
    49  	linkerdBaseEdgeVersion = versions["edge"]
    50  
    51  	tmpDir, err := os.MkdirTemp("", "upgrade-cli")
    52  	if err != nil {
    53  		testutil.AnnotatedFatal(t, "failed to create temp dir", err)
    54  	}
    55  	defer os.RemoveAll(tmpDir)
    56  
    57  	cliPath := fmt.Sprintf("%s/linkerd2-cli-%s-%s-%s", tmpDir, linkerdBaseEdgeVersion, runtime.GOOS, runtime.GOARCH)
    58  	if err := TestHelper.DownloadCLIBinary(cliPath, linkerdBaseEdgeVersion); err != nil {
    59  		testutil.AnnotatedFatal(t, "failed to fetch cli executable", err)
    60  	}
    61  
    62  	// Nest all pre-upgrade tests here so they can install and check resources
    63  	// using the latest edge CLI
    64  	t.Run(fmt.Sprintf("installing Linkerd %s control plane", linkerdBaseEdgeVersion), func(t *testing.T) {
    65  		out, err := TestHelper.CmdRun(cliPath, "install", "--crds")
    66  		if err != nil {
    67  			testutil.AnnotatedFatalf(t, "'linkerd install --crds' command failed", "'linkerd install --crds' command failed:\n%v", err)
    68  		}
    69  
    70  		out, err = TestHelper.KubectlApply(out, "")
    71  		if err != nil {
    72  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
    73  				"'kubectl apply' command failed\n%s", out)
    74  		}
    75  
    76  		waitArgs := []string{
    77  			"wait", "--for", "condition=established", "--timeout=60s", "crd",
    78  			"authorizationpolicies.policy.linkerd.io",
    79  			"meshtlsauthentications.policy.linkerd.io",
    80  			"networkauthentications.policy.linkerd.io",
    81  			"servers.policy.linkerd.io",
    82  			"serverauthorizations.policy.linkerd.io",
    83  		}
    84  		if _, err := TestHelper.Kubectl("", waitArgs...); err != nil {
    85  			testutil.AnnotatedFatalf(t, "'kubectl wait crd' command failed", "'kubectl wait crd' command failed:\n%v", err)
    86  		}
    87  
    88  		out, err = TestHelper.CmdRun(cliPath,
    89  			"install",
    90  			"--controller-log-level=debug",
    91  			"--set=proxyInit.ignoreInboundPorts=1234\\,5678",
    92  		)
    93  		if err != nil {
    94  			testutil.AnnotatedFatalf(t, "'linkerd install' command failed", "'linkerd install' command failed:\n%v", err)
    95  		}
    96  
    97  		out, err = TestHelper.KubectlApply(out, "")
    98  		if err != nil {
    99  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   100  				"'kubectl apply' command failed\n%s", out)
   101  		}
   102  
   103  		TestHelper.WaitRollout(t, testutil.LinkerdDeployReplicasEdge)
   104  	})
   105  
   106  	// TestInstallViz will install the viz extension to be used by the rest of the
   107  	// tests in the viz suite
   108  	t.Run(fmt.Sprintf("installing Linkerd %s viz extension", linkerdBaseEdgeVersion), func(t *testing.T) {
   109  		out, err := TestHelper.CmdRun(cliPath, "viz", "install", "--set", fmt.Sprintf("namespace=%s", TestHelper.GetVizNamespace()))
   110  		if err != nil {
   111  			testutil.AnnotatedFatal(t, "'linkerd viz install' command failed", err)
   112  		}
   113  
   114  		out, err = TestHelper.KubectlApplyWithArgs(out)
   115  		if err != nil {
   116  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   117  				"'kubectl apply' command failed\n%s", out)
   118  		}
   119  
   120  		TestHelper.WaitRollout(t, testutil.LinkerdVizDeployReplicas)
   121  		TestHelper.AddInstalledExtension("viz")
   122  
   123  	})
   124  
   125  	// Check client and server versions are what we expect them to be
   126  	t.Run(fmt.Sprintf("check version is %s pre-upgrade", linkerdBaseEdgeVersion), func(t *testing.T) {
   127  		out, err := TestHelper.CmdRun(cliPath, "version")
   128  		if err != nil {
   129  			testutil.AnnotatedFatalf(t, "'linkerd version' command failed", "'linkerd version' command failed\n%s", err.Error())
   130  		}
   131  
   132  		if !strings.Contains(out, fmt.Sprintf("Client version: %s", linkerdBaseEdgeVersion)) {
   133  			testutil.AnnotatedFatalf(t, "'linkerd version' command failed", "'linkerd version' command failed\nexpected client version: %s, got: %s", linkerdBaseEdgeVersion, out)
   134  		}
   135  		if !strings.Contains(out, fmt.Sprintf("Server version: %s", linkerdBaseEdgeVersion)) {
   136  			testutil.AnnotatedFatalf(t, "'linkerd version' command failed", "'linkerd version' command failed\nexpected server version: %s, got: %s", linkerdBaseEdgeVersion, out)
   137  		}
   138  	})
   139  }
   140  
   141  func TestUpgradeTestAppWorksBeforeUpgrade(t *testing.T) {
   142  	ctx := context.Background()
   143  	testAppNamespace := "upgrade-test"
   144  
   145  	// create namespace, and install test app
   146  	if err := TestHelper.CreateDataPlaneNamespaceIfNotExists(ctx, testAppNamespace, map[string]string{k8s.ProxyInjectAnnotation: "enabled"}); err != nil {
   147  		testutil.AnnotatedFatalf(t, "failed to create namespace", "failed to create namespace %s: %s", testAppNamespace, err)
   148  	}
   149  
   150  	if _, err := TestHelper.Kubectl("", "apply", "-f", "./testdata/emoji.yaml", "-n", testAppNamespace); err != nil {
   151  		testutil.AnnotatedFatalf(t, "'kubectl apply' failed", "'kubectl apply' failed: %s", err)
   152  	}
   153  
   154  	// make sure app is running
   155  	for _, deploy := range []string{"emoji", "voting", "web"} {
   156  		if err := TestHelper.CheckPods(ctx, testAppNamespace, deploy, 1); err != nil {
   157  			var rce *testutil.RestartCountError
   158  			if errors.As(err, &rce) {
   159  				testutil.AnnotatedWarn(t, "CheckPods timed-out", rce)
   160  			} else {
   161  				testutil.AnnotatedError(t, "CheckPods timed-out", err)
   162  			}
   163  		}
   164  	}
   165  
   166  	if err := testutil.ExerciseTestAppEndpoint("/api/list", testAppNamespace, TestHelper); err != nil {
   167  		testutil.AnnotatedFatalf(t, "error exercising test app endpoint before upgrade",
   168  			"error exercising test app endpoint before upgrade %s", err)
   169  	}
   170  }
   171  
   172  func TestRetrieveUidPreUpgrade(t *testing.T) {
   173  	var err error
   174  	configMapUID, err = TestHelper.KubernetesHelper.GetConfigUID(context.Background(), TestHelper.GetLinkerdNamespace())
   175  	if err != nil || configMapUID == "" {
   176  		testutil.AnnotatedFatalf(t, "error retrieving linkerd-config's uid",
   177  			"error retrieving linkerd-config's uid: %s", err)
   178  	}
   179  }
   180  
   181  func TestUpgradeCli(t *testing.T) {
   182  	cmd := "upgrade"
   183  	args := []string{
   184  		"--controller-log-level", "debug",
   185  		"--set", "proxyInit.ignoreInboundPorts=1234\\,5678",
   186  		"--set", "heartbeatSchedule=1 2 3 4 5",
   187  		"--set", "proxyInit.ignoreOutboundPorts=1234\\,5678",
   188  	}
   189  
   190  	// Upgrade CRDs.
   191  	exec := append([]string{cmd}, append(args, "--crds")...)
   192  	out, err := TestHelper.LinkerdRun(exec...)
   193  	if err != nil {
   194  		testutil.AnnotatedFatal(t, "'linkerd upgrade --crds' command failed", err)
   195  	}
   196  	cmdOut, err := TestHelper.KubectlApply(out, "")
   197  	if err != nil {
   198  		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   199  			"'kubectl apply' command failed\n%s", cmdOut)
   200  	}
   201  
   202  	// Upgrade control plane.
   203  	exec = append([]string{cmd}, args...)
   204  	out, err = TestHelper.LinkerdRun(exec...)
   205  	if err != nil {
   206  		testutil.AnnotatedFatal(t, "'linkerd upgrade' command failed", err)
   207  	}
   208  
   209  	// Limit the pruning only to known resources
   210  	// that we intend to be delete in this stage to prevent it
   211  	// from deleting other resources that have the
   212  	// label
   213  	cmdOut, err = TestHelper.KubectlApplyWithArgs(out, []string{
   214  		"--prune",
   215  		"-l", "linkerd.io/control-plane-ns=linkerd",
   216  		"--prune-allowlist", "apps/v1/deployment",
   217  		"--prune-allowlist", "core/v1/service",
   218  		"--prune-allowlist", "core/v1/configmap",
   219  	}...)
   220  	if err != nil {
   221  		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   222  			"'kubectl apply' command failed\n%s", cmdOut)
   223  	}
   224  
   225  	TestHelper.WaitRollout(t, testutil.LinkerdDeployReplicasEdge)
   226  
   227  	// It is necessary to clone LinkerdVizDeployReplicas so that we do not
   228  	// mutate its original value.
   229  	expectedDeployments := make(map[string]testutil.DeploySpec)
   230  	for k, v := range testutil.LinkerdVizDeployReplicas {
   231  		expectedDeployments[k] = v
   232  	}
   233  
   234  	// Install Linkerd Viz Extension
   235  	vizCmd := []string{
   236  		"viz",
   237  		"install",
   238  		"--set", fmt.Sprintf("namespace=%s", TestHelper.GetVizNamespace()),
   239  	}
   240  	out, err = TestHelper.LinkerdRun(vizCmd...)
   241  	if err != nil {
   242  		testutil.AnnotatedFatal(t, "'linkerd viz install' command failed", err)
   243  	}
   244  
   245  	out, err = TestHelper.KubectlApplyWithArgs(out, []string{
   246  		"--prune",
   247  		"-l", "linkerd.io/extension=viz",
   248  	}...)
   249  	if err != nil {
   250  		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   251  			"'kubectl apply' command failed\n%s", out)
   252  	}
   253  
   254  	TestHelper.WaitRollout(t, expectedDeployments)
   255  
   256  }
   257  
   258  func TestControlPlaneResourcesPostInstall(t *testing.T) {
   259  	expectedDeployments := testutil.LinkerdDeployReplicasEdge
   260  	expectedServices := linkerdSvcEdge
   261  	vizServices := []testutil.Service{
   262  		{Namespace: "linkerd-viz", Name: "web"},
   263  		{Namespace: "linkerd-viz", Name: "tap"},
   264  		{Namespace: "linkerd-viz", Name: "prometheus"},
   265  	}
   266  	expectedServices = append(expectedServices, vizServices...)
   267  	expectedDeployments["prometheus"] = testutil.DeploySpec{Namespace: "linkerd-viz", Replicas: 1}
   268  
   269  	testutil.TestResourcesPostInstall(TestHelper.GetLinkerdNamespace(), expectedServices, expectedDeployments, TestHelper, t)
   270  }
   271  
   272  func TestRetrieveUidPostUpgrade(t *testing.T) {
   273  	newConfigMapUID, err := TestHelper.KubernetesHelper.GetConfigUID(context.Background(), TestHelper.GetLinkerdNamespace())
   274  	if err != nil || newConfigMapUID == "" {
   275  		testutil.AnnotatedFatalf(t, "error retrieving linkerd-config's uid",
   276  			"error retrieving linkerd-config's uid: %s", err)
   277  	}
   278  	if configMapUID != newConfigMapUID {
   279  		testutil.AnnotatedFatalf(t, "linkerd-config's uid after upgrade doesn't match its value before the upgrade",
   280  			"linkerd-config's uid after upgrade [%s] doesn't match its value before the upgrade [%s]",
   281  			newConfigMapUID, configMapUID,
   282  		)
   283  	}
   284  }
   285  
   286  func TestOverridesSecret(t *testing.T) {
   287  	configOverridesSecret, err := TestHelper.KubernetesHelper.GetSecret(context.Background(), TestHelper.GetLinkerdNamespace(), "linkerd-config-overrides")
   288  	if err != nil {
   289  		testutil.AnnotatedFatalf(t, "could not retrieve linkerd-config-overrides",
   290  			"could not retrieve linkerd-config-overrides\n%s", err)
   291  	}
   292  
   293  	overrides := configOverridesSecret.Data["linkerd-config-overrides"]
   294  	overridesTree, err := tree.BytesToTree(overrides)
   295  	if err != nil {
   296  		testutil.AnnotatedFatalf(t, "could not retrieve linkerd-config-overrides",
   297  			"could not retrieve linkerd-config-overrides\n%s", err)
   298  	}
   299  
   300  	// Check for fields that were added during install
   301  	testCases := []struct {
   302  		path  []string
   303  		value string
   304  	}{
   305  		{
   306  			[]string{"controllerLogLevel"},
   307  			"debug",
   308  		},
   309  		{
   310  			[]string{"proxyInit", "ignoreInboundPorts"},
   311  			skippedInboundPorts,
   312  		},
   313  		{
   314  			[]string{"proxyInit", "ignoreOutboundPorts"},
   315  			skippedOutboundPorts,
   316  		},
   317  	}
   318  
   319  	for _, tc := range testCases {
   320  		tc := tc // pin
   321  		t.Run(fmt.Sprintf("%s: %s", strings.Join(tc.path, "/"), tc.value), func(t *testing.T) {
   322  			finalValue, err := overridesTree.GetString(tc.path...)
   323  			if err != nil {
   324  				testutil.AnnotatedFatalf(t, "could not perform tree.GetString",
   325  					"could not perform tree.GetString\n%s", err)
   326  			}
   327  
   328  			if tc.value != finalValue {
   329  				testutil.AnnotatedFatalf(t, fmt.Sprintf("Values at path %s do not match", strings.Join(tc.path, "/")),
   330  					"Expected value at [%s] to be [%s] but received [%s]",
   331  					strings.Join(tc.path, "/"), tc.value, finalValue)
   332  			}
   333  		})
   334  	}
   335  
   336  	extractValue := func(t *testing.T, path ...string) string {
   337  		val, err := overridesTree.GetString(path...)
   338  		if err != nil {
   339  			testutil.AnnotatedFatalf(t, "error calling overridesTree.GetString()",
   340  				"error calling overridesTree.GetString(): %s", err)
   341  			return ""
   342  
   343  		}
   344  		return val
   345  	}
   346  
   347  	t.Run("Check if any unknown fields snuck in", func(t *testing.T) {
   348  		knownKeys := tree.Tree{
   349  			"controllerLogLevel": "debug",
   350  			"heartbeatSchedule":  "1 2 3 4 5",
   351  			"identity": tree.Tree{
   352  				"issuer": tree.Tree{
   353  					"tls": tree.Tree{
   354  						"crtPEM": extractValue(t, "identity", "issuer", "tls", "crtPEM"),
   355  						"keyPEM": extractValue(t, "identity", "issuer", "tls", "keyPEM"),
   356  					},
   357  				},
   358  			},
   359  			"identityTrustAnchorsPEM": extractValue(t, "identityTrustAnchorsPEM"),
   360  			"proxyInit": tree.Tree{
   361  				"ignoreInboundPorts":  skippedInboundPorts,
   362  				"ignoreOutboundPorts": skippedOutboundPorts,
   363  			},
   364  		}
   365  
   366  		if reg := os.Getenv(flags.EnvOverrideDockerRegistry); reg != "" {
   367  			knownKeys["controllerImage"] = reg + "/controller"
   368  			knownKeys["debugContainer"] = tree.Tree{
   369  				"image": tree.Tree{
   370  					"name": reg + "/debug",
   371  				},
   372  			}
   373  			knownKeys["policyController"] = tree.Tree{
   374  				"image": tree.Tree{
   375  					"name": reg + "/policy-controller",
   376  				},
   377  			}
   378  			knownKeys["proxy"] = tree.Tree{
   379  				"image": tree.Tree{
   380  					"name": reg + "/proxy",
   381  				},
   382  			}
   383  			knownKeys["proxyInit"].(tree.Tree)["image"] = tree.Tree{
   384  				"name": reg + "/proxy-init",
   385  			}
   386  
   387  		}
   388  
   389  		// Check if the keys in overridesTree match with knownKeys
   390  		if diff := deep.Equal(overridesTree.String(), knownKeys.String()); diff != nil {
   391  			testutil.AnnotatedFatalf(t, "Overrides and knownKeys are different", "%+v", diff)
   392  		}
   393  	})
   394  }
   395  
   396  func TestVersionPostInstall(t *testing.T) {
   397  	err := TestHelper.CheckVersion(TestHelper.GetVersion())
   398  	if err != nil {
   399  		testutil.AnnotatedFatalf(t, "Version command failed",
   400  			"Version command failed\n%s", err.Error())
   401  	}
   402  }
   403  
   404  func TestCheckProxyPostUpgrade(t *testing.T) {
   405  	if err := TestHelper.TestCheckProxy(TestHelper.GetVersion(), TestHelper.GetLinkerdNamespace()); err != nil {
   406  		t.Fatalf("'linkerd check --proxy' command failed: %s", err)
   407  	}
   408  }
   409  
   410  func TestUpgradeTestAppWorksAfterUpgrade(t *testing.T) {
   411  	testAppNamespace := "upgrade-test"
   412  
   413  	// Restart pods after upgrade to make sure they're re-injected with the
   414  	// latest proxy
   415  	if _, err := TestHelper.Kubectl("", "rollout", "restart", "deploy", "-n", testAppNamespace); err != nil {
   416  		testutil.AnnotatedFatalf(t, "'kubectl rollout' failed", "'kubectl rollout' failed: %s", err)
   417  	}
   418  
   419  	// make sure app is running before proceeding
   420  	ctx := context.Background()
   421  	for _, deploy := range []string{"emoji", "voting", "web"} {
   422  		if err := TestHelper.CheckPods(ctx, testAppNamespace, deploy, 1); err != nil {
   423  			var rce *testutil.RestartCountError
   424  			if errors.As(err, &rce) {
   425  				testutil.AnnotatedWarn(t, "CheckPods timed-out", rce)
   426  			} else {
   427  				testutil.AnnotatedError(t, "CheckPods timed-out", err)
   428  			}
   429  		}
   430  	}
   431  
   432  	if err := testutil.ExerciseTestAppEndpoint("/api/vote?choice=:policeman:", testAppNamespace, TestHelper); err != nil {
   433  		testutil.AnnotatedFatalf(t, "error exercising test app endpoint after upgrade",
   434  			"error exercising test app endpoint after upgrade %s", err)
   435  	}
   436  }
   437  

View as plain text