upgrade.go

Documentation: github.com/linkerd/linkerd2/cli/cmd

     1  package cmd
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  	"os"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/linkerd/linkerd2/cli/flag"
    13  	l5dcharts "github.com/linkerd/linkerd2/pkg/charts/linkerd2"
    14  	"github.com/linkerd/linkerd2/pkg/config"
    15  	flagspkg "github.com/linkerd/linkerd2/pkg/flags"
    16  	"github.com/linkerd/linkerd2/pkg/healthcheck"
    17  	"github.com/linkerd/linkerd2/pkg/k8s"
    18  	"github.com/linkerd/linkerd2/pkg/tls"
    19  	"github.com/spf13/cobra"
    20  	"github.com/spf13/pflag"
    21  	valuespkg "helm.sh/helm/v3/pkg/cli/values"
    22  	corev1 "k8s.io/api/core/v1"
    23  	kerrors "k8s.io/apimachinery/pkg/api/errors"
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  	"sigs.k8s.io/yaml"
    26  )
    27  
    28  const (
    29  	failMessage            = "For troubleshooting help, visit: https://linkerd.io/upgrade/#troubleshooting\n"
    30  	trustRootChangeMessage = "Rotating the trust anchors will affect existing proxies\nSee https://linkerd.io/2/tasks/rotating_identity_certificates/ for more information"
    31  )
    32  
    33  var (
    34  	manifests string
    35  	force     bool
    36  )
    37  
    38  /* The upgrade commands all follow the same flow:
    39   * 1. Load default values from the Linkerd2 chart
    40   * 2. Update the values with stored overrides
    41   * 3. Apply flags to further modify the values
    42   * 4. Render the chart using those values
    43   *
    44   * The individual commands (upgrade, upgrade config, and upgrade control-plane)
    45   * differ in which flags are available to each, what pre-check validations
    46   * are done, and which subset of the chart is rendered.
    47   */
    48  func newCmdUpgrade() *cobra.Command {
    49  	values, err := l5dcharts.NewValues()
    50  	if err != nil {
    51  		fmt.Fprint(os.Stderr, err.Error())
    52  		os.Exit(1)
    53  	}
    54  
    55  	var crds bool
    56  	var options valuespkg.Options
    57  	installUpgradeFlags, installUpgradeFlagSet, err := makeInstallUpgradeFlags(values)
    58  	if err != nil {
    59  		fmt.Fprint(os.Stderr, err.Error())
    60  		os.Exit(1)
    61  	}
    62  	proxyFlags, proxyFlagSet := makeProxyFlags(values)
    63  	flags := flattenFlags(installUpgradeFlags, proxyFlags)
    64  
    65  	upgradeFlagSet := makeUpgradeFlags()
    66  
    67  	cmd := &cobra.Command{
    68  		Use:   "upgrade [flags]",
    69  		Args:  cobra.NoArgs,
    70  		Short: "Output Kubernetes configs to upgrade an existing Linkerd control plane",
    71  		Long: `Output Kubernetes configs to upgrade an existing Linkerd control plane.
    72  
    73  Note that the default flag values for this command come from the Linkerd control
    74  plane. The default values displayed in the Flags section below only apply to the
    75  install command.
    76  
    77  The upgrade can be configured by using the --set, --values, --set-string and --set-file flags.
    78  A full list of configurable values can be found at https://www.github.com/linkerd/linkerd2/tree/main/charts/linkerd2/README.md
    79  `,
    80  
    81  		Example: `  # Upgrade CRDs first
    82    linkerd upgrade --crds | kubectl apply -f -
    83  
    84    # Then upgrade the control plane
    85    linkerd upgrade | kubectl apply -f -
    86  
    87    # And lastly, remove linkerd resources that no longer exist in the current version
    88    linkerd prune | kubectl delete -f -`,
    89  		RunE: func(cmd *cobra.Command, args []string) error {
    90  			if crds {
    91  				// The CRD chart is not configurable.
    92  				// TODO(ver): Error if values have been configured?
    93  				if _, err := upgradeCRDs(options).WriteTo(os.Stdout); err != nil {
    94  					fmt.Fprintln(os.Stderr, err.Error())
    95  					os.Exit(1)
    96  				}
    97  				return nil
    98  			}
    99  
   100  			k, err := k8s.NewAPI(kubeconfigPath, kubeContext, impersonate, impersonateGroup, 0)
   101  			if err != nil {
   102  				return fmt.Errorf("failed to create a kubernetes client: %w", err)
   103  			}
   104  
   105  			if err = upgradeControlPlaneRunE(cmd.Context(), k, flags, options, manifests); err != nil {
   106  				fmt.Fprintln(os.Stderr, err.Error())
   107  				os.Exit(1)
   108  			}
   109  			return nil
   110  		},
   111  	}
   112  
   113  	cmd.Flags().AddFlagSet(installUpgradeFlagSet)
   114  	cmd.Flags().AddFlagSet(proxyFlagSet)
   115  	cmd.PersistentFlags().AddFlagSet(upgradeFlagSet)
   116  	flagspkg.AddValueOptionsFlags(cmd.Flags(), &options)
   117  	cmd.Flags().BoolVar(&crds, "crds", false, "Upgrade Linkerd CRDs")
   118  
   119  	return cmd
   120  }
   121  
   122  // makeConfigClient is used to re-initialize the Kubernetes client in order
   123  // to fetch existing configuration. It accepts two arguments: a Kubernetes
   124  // client, and a path to a manifest file. If the manifest path is empty, the
   125  // client will not be re-initialized. When non-empty, the client will be
   126  // replaced by a fake Kubernetes client that will hold the values parsed from
   127  // the manifest.
   128  func makeConfigClient(k *k8s.KubernetesAPI, localManifestPath string) (*k8s.KubernetesAPI, error) {
   129  	if localManifestPath == "" {
   130  		return k, nil
   131  	}
   132  
   133  	// We need a Kubernetes client to fetch configs and issuer secrets.
   134  	readers, err := read(localManifestPath)
   135  	if err != nil {
   136  		return nil, fmt.Errorf("failed to parse manifests from %s: %w", localManifestPath, err)
   137  	}
   138  
   139  	k, err = k8s.NewFakeAPIFromManifests(readers)
   140  	if err != nil {
   141  		return nil, fmt.Errorf("failed to parse Kubernetes objects from manifest %s: %w", localManifestPath, err)
   142  	}
   143  	return k, nil
   144  }
   145  
   146  // makeUpgradeFlags returns a FlagSet of flags that are only accessible at upgrade-time
   147  // and not at install-time.  These flags do not configure the Values used to
   148  // render the chart but instead modify the behavior of the upgrade command itself.
   149  // They are not persisted in any way.
   150  func makeUpgradeFlags() *pflag.FlagSet {
   151  	upgradeFlags := pflag.NewFlagSet("upgrade-only", pflag.ExitOnError)
   152  
   153  	upgradeFlags.StringVar(
   154  		&manifests, "from-manifests", "",
   155  		"Read config from a Linkerd install YAML rather than from Kubernetes",
   156  	)
   157  	upgradeFlags.BoolVar(
   158  		&force, "force", false,
   159  		"Force upgrade operation even when issuer certificate does not work with the trust anchors of all proxies",
   160  	)
   161  	return upgradeFlags
   162  }
   163  
   164  func upgradeControlPlaneRunE(ctx context.Context, k *k8s.KubernetesAPI, flags []flag.Flag, options valuespkg.Options, localManifestPath string) error {
   165  
   166  	crds := bytes.Buffer{}
   167  	err := renderCRDs(&crds, options)
   168  	if err != nil {
   169  		return err
   170  	}
   171  
   172  	err = healthcheck.CheckCustomResourceDefinitions(ctx, k, crds.String())
   173  	if err != nil {
   174  		return fmt.Errorf("Linkerd CRDs must be installed first. Run linkerd upgrade with the --crds flag:\n%w", err)
   175  	}
   176  
   177  	// Re-initialize client if a local manifest path is used
   178  	k, err = makeConfigClient(k, localManifestPath)
   179  	if err != nil {
   180  		return err
   181  	}
   182  
   183  	buf, err := upgradeControlPlane(ctx, k, flags, options)
   184  	if err != nil {
   185  		return err
   186  	}
   187  
   188  	for _, flag := range flags {
   189  		if flag.Name() == "identity-trust-anchors-file" && flag.IsSet() {
   190  			fmt.Fprintf(os.Stderr, "\n%s %s\n\n", warnStatus, trustRootChangeMessage)
   191  		}
   192  	}
   193  
   194  	_, err = buf.WriteTo(os.Stdout)
   195  	return err
   196  }
   197  
   198  func upgradeCRDs(options valuespkg.Options) *bytes.Buffer {
   199  	var buf bytes.Buffer
   200  	if err := renderCRDs(&buf, options); err != nil {
   201  		upgradeErrorf("Could not render upgrade configuration: %s", err)
   202  	}
   203  	return &buf
   204  }
   205  
   206  func upgradeControlPlane(ctx context.Context, k *k8s.KubernetesAPI, flags []flag.Flag, options valuespkg.Options) (*bytes.Buffer, error) {
   207  	values, err := loadStoredValues(ctx, k)
   208  	if err != nil {
   209  		return nil, fmt.Errorf("failed to load stored values: %w", err)
   210  	}
   211  
   212  	if values == nil {
   213  		return nil, errors.New(
   214  			`Could not find the linkerd-config-overrides secret.
   215  			If Linkerd was installed with Helm, please use Helm to perform upgrades`)
   216  	}
   217  
   218  	err = flag.ApplySetFlags(values, flags)
   219  	if err != nil {
   220  		return nil, err
   221  	}
   222  
   223  	if values.Identity.Issuer.Scheme == string(corev1.SecretTypeTLS) {
   224  		for _, flag := range flags {
   225  			if (flag.Name() == "identity-issuer-certificate-file" || flag.Name() == "identity-issuer-key-file") && flag.IsSet() {
   226  				return nil, errors.New("cannot update issuer certificates if you are using external cert management solution")
   227  			}
   228  		}
   229  	}
   230  
   231  	err = validateValues(ctx, k, values)
   232  	if err != nil {
   233  		return nil, err
   234  	}
   235  	if !force && values.Identity.Issuer.Scheme == k8s.IdentityIssuerSchemeLinkerd {
   236  		err = ensureIssuerCertWorksWithAllProxies(ctx, k, values)
   237  		if err != nil {
   238  			return nil, err
   239  		}
   240  	}
   241  
   242  	// Create values override
   243  	valuesOverrides, err := options.MergeValues(nil)
   244  	if err != nil {
   245  		return nil, err
   246  	}
   247  	if !isRunAsRoot(valuesOverrides) {
   248  		err = healthcheck.CheckNodesHaveNonDockerRuntime(ctx, k)
   249  		if err != nil {
   250  			return nil, err
   251  		}
   252  	}
   253  
   254  	var buf bytes.Buffer
   255  	if err = renderControlPlane(&buf, values, valuesOverrides); err != nil {
   256  		upgradeErrorf("Could not render upgrade configuration: %s", err)
   257  	}
   258  	return &buf, nil
   259  }
   260  
   261  func loadStoredValues(ctx context.Context, k *k8s.KubernetesAPI) (*l5dcharts.Values, error) {
   262  	// Load the default values from the chart.
   263  	values, err := l5dcharts.NewValues()
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  
   268  	// Load the stored overrides from the linkerd-config-overrides secret.
   269  	secret, err := k.CoreV1().Secrets(controlPlaneNamespace).Get(ctx, "linkerd-config-overrides", metav1.GetOptions{})
   270  	if kerrors.IsNotFound(err) {
   271  		return nil, nil
   272  	}
   273  	if err != nil {
   274  		return nil, err
   275  	}
   276  
   277  	bytes, ok := secret.Data["linkerd-config-overrides"]
   278  	if !ok {
   279  		return nil, errors.New("secret/linkerd-config-overrides is missing linkerd-config-overrides data")
   280  	}
   281  
   282  	bytes, err = config.RemoveGlobalFieldIfPresent(bytes)
   283  	if err != nil {
   284  		return nil, err
   285  	}
   286  
   287  	// Unmarshal the overrides directly onto the values.  This has the effect
   288  	// of merging the two with the overrides taking priority.
   289  	err = yaml.Unmarshal(bytes, values)
   290  	if err != nil {
   291  		return nil, err
   292  	}
   293  
   294  	return values, nil
   295  }
   296  
   297  // upgradeErrorf prints the error message and quits the upgrade process
   298  func upgradeErrorf(format string, a ...interface{}) {
   299  	template := fmt.Sprintf("%s %s\n%s\n", failStatus, format, failMessage)
   300  	fmt.Fprintf(os.Stderr, template, a...)
   301  	os.Exit(1)
   302  }
   303  
   304  func ensureIssuerCertWorksWithAllProxies(ctx context.Context, k *k8s.KubernetesAPI, values *l5dcharts.Values) error {
   305  	cred, err := tls.ValidateAndCreateCreds(
   306  		values.Identity.Issuer.TLS.CrtPEM,
   307  		values.Identity.Issuer.TLS.KeyPEM,
   308  	)
   309  	if err != nil {
   310  		return err
   311  	}
   312  
   313  	meshedPods, err := healthcheck.GetMeshedPodsIdentityData(ctx, k, "")
   314  	var problematicPods []string
   315  	if err != nil {
   316  		return err
   317  	}
   318  	for _, pod := range meshedPods {
   319  		// Skip control plane pods since they load their trust anchors from the linkerd-identity-trust-anchors configmap.
   320  		if pod.Namespace == controlPlaneNamespace {
   321  			continue
   322  		}
   323  		anchors, err := tls.DecodePEMCertPool(pod.Anchors)
   324  
   325  		if anchors != nil {
   326  			err = cred.Verify(anchors, "", time.Time{})
   327  		}
   328  
   329  		if err != nil {
   330  			problematicPods = append(problematicPods, fmt.Sprintf("* %s/%s", pod.Namespace, pod.Name))
   331  		}
   332  	}
   333  
   334  	if len(problematicPods) > 0 {
   335  		errorMessageHeader := "You are attempting to use an issuer certificate which does not validate against the trust anchors of the following pods:"
   336  		errorMessageFooter := "These pods do not have the current trust bundle and must be restarted.  Use the --force flag to proceed anyway (this will likely prevent those pods from sending or receiving traffic)."
   337  		return fmt.Errorf("%s\n\t%s\n%s", errorMessageHeader, strings.Join(problematicPods, "\n\t"), errorMessageFooter)
   338  	}
   339  	return nil
   340  }
   341
View as plain text