utils.go

Documentation: k8s.io/kubernetes/test/e2e_node/remote

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package remote
    18  
    19  import (
    20  	"fmt"
    21  	"path/filepath"
    22  
    23  	"k8s.io/klog/v2"
    24  
    25  	"k8s.io/kubernetes/test/e2e_node/builder"
    26  )
    27  
    28  // utils.go contains functions used across test suites.
    29  
    30  const (
    31  	cniVersion       = "v1.4.0"
    32  	cniDirectory     = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin".
    33  	cniConfDirectory = "cni/net.d"
    34  
    35  	ecrCredentialProviderVersion = "v1.27.1"
    36  )
    37  
    38  const cniConfig = `{
    39    "name": "mynet",
    40    "type": "bridge",
    41    "bridge": "mynet0",
    42    "isDefaultGateway": true,
    43    "forceAddress": false,
    44    "ipMasq": true,
    45    "hairpinMode": true,
    46    "ipam": {
    47      "type": "host-local",
    48      "subnet": "10.10.0.0/16"
    49    }
    50  }
    51  `
    52  
    53  const credentialGCPProviderConfig = `kind: CredentialProviderConfig
    54  apiVersion: kubelet.config.k8s.io/v1
    55  providers:
    56    - name: gcp-credential-provider
    57      apiVersion: credentialprovider.kubelet.k8s.io/v1
    58      matchImages:
    59      - "gcr.io"
    60      - "*.gcr.io"
    61      - "container.cloud.google.com"
    62      - "*.pkg.dev"
    63      defaultCacheDuration: 1m`
    64  
    65  const credentialAWSProviderConfig = `kind: CredentialProviderConfig
    66  apiVersion: kubelet.config.k8s.io/v1
    67  providers:
    68  - name: ecr-credential-provider
    69    apiVersion: credentialprovider.kubelet.k8s.io/v1
    70    matchImages:
    71    - "*.dkr.ecr.*.amazonaws.com"
    72    - "*.dkr.ecr.*.amazonaws.com.cn"
    73    - "*.dkr.ecr-fips.*.amazonaws.com"
    74    - "*.dkr.ecr.us-iso-east-1.c2s.ic.gov"
    75    - "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov"
    76    defaultCacheDuration: 12h`
    77  
    78  func getCNIURL() string {
    79  	cniArch := "amd64"
    80  	if builder.IsTargetArchArm64() {
    81  		cniArch = "arm64"
    82  	}
    83  	cniURL := fmt.Sprintf("https://storage.googleapis.com/k8s-artifacts-cni/release/%s/cni-plugins-linux-%s-%s.tgz", cniVersion, cniArch, cniVersion)
    84  	return cniURL
    85  
    86  }
    87  
    88  // Install the cni plugin and add basic bridge configuration to the
    89  // configuration directory.
    90  func setupCNI(host, workspace string) error {
    91  	klog.V(2).Infof("Install CNI on %q", host)
    92  	cniPath := filepath.Join(workspace, cniDirectory)
    93  	klog.V(2).Infof("Install CNI on path %q", cniPath)
    94  	cmd := getSSHCommand(" ; ",
    95  		fmt.Sprintf("mkdir -p %s", cniPath),
    96  		fmt.Sprintf("curl -s -L %s | tar -xz -C %s", getCNIURL(), cniPath),
    97  	)
    98  	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
    99  		return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output)
   100  	}
   101  
   102  	// The added CNI network config is not needed for kubenet. It is only
   103  	// used when testing the CNI network plugin, but is added in both cases
   104  	// for consistency and simplicity.
   105  	klog.V(2).Infof("Adding CNI configuration on %q", host)
   106  	cniConfigPath := filepath.Join(workspace, cniConfDirectory)
   107  	cmd = getSSHCommand(" ; ",
   108  		fmt.Sprintf("mkdir -p %s", cniConfigPath),
   109  		fmt.Sprintf("echo %s > %s", quote(cniConfig), filepath.Join(cniConfigPath, "mynet.conf")),
   110  	)
   111  	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
   112  		return fmt.Errorf("failed to write cni configuration on %q: %v output: %q", host, err, output)
   113  	}
   114  	return nil
   115  }
   116  
   117  func getECRCredentialProviderURL() string {
   118  	arch := "amd64"
   119  	if builder.IsTargetArchArm64() {
   120  		arch = "arm64"
   121  	}
   122  	return fmt.Sprintf("https://artifacts.k8s.io/binaries/cloud-provider-aws/%s/linux/%s/ecr-credential-provider-linux-%s", ecrCredentialProviderVersion, arch, arch)
   123  }
   124  
   125  func setupECRCredentialProvider(host, workspace string) error {
   126  	klog.V(2).Infof("Install ecr-credential-provider on %q at %q", host, workspace)
   127  	cmd := getSSHCommand(" ; ",
   128  		fmt.Sprintf("curl -s -Lo %s/ecr-credential-provider %s", workspace, getECRCredentialProviderURL()),
   129  		fmt.Sprintf("chmod a+x %s/ecr-credential-provider", workspace),
   130  	)
   131  	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
   132  		return fmt.Errorf("failed to install ecr-credential-provider plugin on %q: %v output: %q", host, err, output)
   133  	}
   134  	return nil
   135  }
   136  
   137  func configureCredentialProvider(host, workspace string) error {
   138  	klog.V(2).Infof("Configuring kubelet credential provider on %q", host)
   139  
   140  	credentialProviderConfig := credentialGCPProviderConfig
   141  	if GetSSHUser() == "ec2-user" {
   142  		credentialProviderConfig = credentialAWSProviderConfig
   143  		err := setupECRCredentialProvider(host, workspace)
   144  		if err != nil {
   145  			return err
   146  		}
   147  	}
   148  
   149  	cmd := getSSHCommand(" ; ",
   150  		fmt.Sprintf("echo %s > %s", quote(credentialProviderConfig), filepath.Join(workspace, "credential-provider.yaml")),
   151  	)
   152  	if output, err := SSH(host, "sh", "-c", cmd); err != nil {
   153  		return fmt.Errorf("failed to write credential provider configuration on %q: %v output: %q", host, err, output)
   154  	}
   155  
   156  	return nil
   157  }
   158  
   159  // configureFirewall configures iptable firewall rules.
   160  func configureFirewall(host string) error {
   161  	klog.V(2).Infof("Configure iptables firewall rules on %q", host)
   162  
   163  	// Since the goal is to enable connectivity without taking into account current rule,
   164  	// we can just prepend the accept rules directly without any check
   165  	cmd := getSSHCommand("&&",
   166  		"iptables -I INPUT 1 -w -p tcp -j ACCEPT",
   167  		"iptables -I INPUT 1 -w -p udp -j ACCEPT",
   168  		"iptables -I INPUT 1 -w -p icmp -j ACCEPT",
   169  		"iptables -I FORWARD 1 -w -p tcp -j ACCEPT",
   170  		"iptables -I FORWARD 1 -w -p udp -j ACCEPT",
   171  		"iptables -I FORWARD 1 -w -p icmp -j ACCEPT",
   172  	)
   173  	output, err := SSH(host, "sh", "-c", cmd)
   174  	if err != nil {
   175  		return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output)
   176  	}
   177  	return nil
   178  }
   179  
   180  // cleanupNodeProcesses kills all running node processes may conflict with the test.
   181  func cleanupNodeProcesses(host string) {
   182  	klog.V(2).Infof("Killing any existing node processes on %q", host)
   183  	cmd := getSSHCommand(" ; ",
   184  		"pkill kubelet",
   185  		"pkill kube-apiserver",
   186  		"pkill etcd",
   187  		"pkill e2e_node.test",
   188  	)
   189  	// No need to log an error if pkill fails since pkill will fail if the commands are not running.
   190  	// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
   191  	// logs about failing to bind the required ports.
   192  	SSH(host, "sh", "-c", cmd)
   193  }
   194  
   195  // Quotes a shell literal so it can be nested within another shell scope.
   196  func quote(s string) string {
   197  	return fmt.Sprintf("'\"'\"'%s'\"'\"'", s)
   198  }
   199
View as plain text