node_e2e.go

Documentation: k8s.io/kubernetes/test/e2e_node/remote

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package remote
    18  
    19  import (
    20  	"fmt"
    21  	"os"
    22  	"os/exec"
    23  	"path/filepath"
    24  	"regexp"
    25  	"strings"
    26  	"time"
    27  
    28  	"k8s.io/klog/v2"
    29  
    30  	"k8s.io/kubernetes/test/e2e_node/builder"
    31  	"k8s.io/kubernetes/test/e2e_node/system"
    32  	"k8s.io/kubernetes/test/utils"
    33  )
    34  
    35  // NodeE2ERemote contains the specific functions in the node e2e test suite.
    36  type NodeE2ERemote struct{}
    37  
    38  // init initializes the node e2e test suite.
    39  func init() {
    40  	RegisterTestSuite("default", &NodeE2ERemote{})
    41  }
    42  
    43  // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests
    44  func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
    45  	// Build the executables
    46  	if err := builder.BuildGo(); err != nil {
    47  		return fmt.Errorf("failed to build the dependencies: %w", err)
    48  	}
    49  
    50  	// Make sure we can find the newly built binaries
    51  	buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
    52  	if err != nil {
    53  		return fmt.Errorf("failed to locate kubernetes build output directory: %w", err)
    54  	}
    55  
    56  	rootDir, err := utils.GetK8sRootDir()
    57  	if err != nil {
    58  		return fmt.Errorf("failed to locate kubernetes root directory: %w", err)
    59  	}
    60  
    61  	// Copy binaries
    62  	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"}
    63  	for _, bin := range requiredBins {
    64  		source := filepath.Join(buildOutputDir, bin)
    65  		klog.V(2).Infof("Copying binaries from %s", source)
    66  		if _, err := os.Stat(source); err != nil {
    67  			return fmt.Errorf("failed to locate test binary %s: %w", bin, err)
    68  		}
    69  		out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput()
    70  		if err != nil {
    71  			return fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out)
    72  		}
    73  	}
    74  
    75  	if systemSpecName != "" {
    76  		// Copy system spec file
    77  		source := filepath.Join(rootDir, system.SystemSpecPath, systemSpecName+".yaml")
    78  		if _, err := os.Stat(source); err != nil {
    79  			return fmt.Errorf("failed to locate system spec %q: %w", source, err)
    80  		}
    81  		out, err := exec.Command("cp", source, tardir).CombinedOutput()
    82  		if err != nil {
    83  			return fmt.Errorf("failed to copy system spec %q: %v, output: %q", source, err, out)
    84  		}
    85  	}
    86  
    87  	return nil
    88  }
    89  
    90  // prependMemcgNotificationFlag prepends the flag for enabling memcg
    91  // notification to args and returns the result.
    92  func prependMemcgNotificationFlag(args string) string {
    93  	return "--kubelet-flags=--kernel-memcg-notification=true " + args
    94  }
    95  
    96  // prependCredentialProviderFlag prepends the flags for enabling
    97  // a credential provider plugin.
    98  func prependCredentialProviderFlag(args, workspace string) string {
    99  	credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml")
   100  	featureGateFlag := "--kubelet-flags=--feature-gates=DisableKubeletCloudCredentialProviders=true"
   101  	configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig)
   102  	binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace)
   103  	return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args)
   104  }
   105  
   106  // osSpecificActions takes OS specific actions required for the node tests
   107  func osSpecificActions(args, host, workspace string) (string, error) {
   108  	output, err := getOSDistribution(host)
   109  	if err != nil {
   110  		return "", fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
   111  	}
   112  	switch {
   113  	case strings.Contains(output, "fedora"), strings.Contains(output, "rhcos"),
   114  		strings.Contains(output, "centos"), strings.Contains(output, "rhel"):
   115  		return args, setKubeletSELinuxLabels(host, workspace)
   116  	case strings.Contains(output, "gci"), strings.Contains(output, "cos"):
   117  		args = prependMemcgNotificationFlag(args)
   118  		return prependCredentialProviderFlag(args, workspace), nil
   119  	case strings.Contains(output, "ubuntu"):
   120  		args = prependCredentialProviderFlag(args, workspace)
   121  		return prependMemcgNotificationFlag(args), nil
   122  	case strings.Contains(output, "amzn"):
   123  		args = prependCredentialProviderFlag(args, workspace)
   124  		return prependMemcgNotificationFlag(args), nil
   125  	}
   126  	return args, nil
   127  }
   128  
   129  // setKubeletSELinuxLabels set the appropriate SELinux labels for the
   130  // kubelet on Fedora CoreOS distribution
   131  func setKubeletSELinuxLabels(host, workspace string) error {
   132  	cmd := getSSHCommand(" && ",
   133  		fmt.Sprintf("/usr/bin/chcon -u system_u -r object_r -t kubelet_exec_t %s", filepath.Join(workspace, "kubelet")),
   134  		fmt.Sprintf("/usr/bin/chcon -u system_u -r object_r -t bin_t %s", filepath.Join(workspace, "e2e_node.test")),
   135  		fmt.Sprintf("/usr/bin/chcon -u system_u -r object_r -t bin_t %s", filepath.Join(workspace, "ginkgo")),
   136  		fmt.Sprintf("/usr/bin/chcon -u system_u -r object_r -t bin_t %s", filepath.Join(workspace, "mounter")),
   137  		fmt.Sprintf("/usr/bin/chcon -R -u system_u -r object_r -t bin_t %s", filepath.Join(workspace, "cni", "bin/")),
   138  	)
   139  	output, err := SSH(host, "sh", "-c", cmd)
   140  	if err != nil {
   141  		return fmt.Errorf("Unable to apply SELinux labels. Err: %v, Output:\n%s", err, output)
   142  	}
   143  	return nil
   144  }
   145  
   146  func getOSDistribution(host string) (string, error) {
   147  	output, err := SSH(host, "cat", "/etc/os-release")
   148  	if err != nil {
   149  		return "", fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
   150  	}
   151  
   152  	var re = regexp.MustCompile(`(?m)^ID="?(\w+)"?`)
   153  	subMatch := re.FindStringSubmatch(output)
   154  	if len(subMatch) > 0 {
   155  		return subMatch[1], nil
   156  	}
   157  
   158  	return "", fmt.Errorf("Unable to parse os-release for the host, %s", host)
   159  }
   160  
   161  // RunTest runs test on the node.
   162  func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, extraEnvs, runtimeConfig string, timeout time.Duration) (string, error) {
   163  	// Install the cni plugins and add a basic CNI configuration.
   164  	// TODO(random-liu): Do this in cloud init after we remove containervm test.
   165  	if err := setupCNI(host, workspace); err != nil {
   166  		return "", err
   167  	}
   168  
   169  	// Configure iptables firewall rules
   170  	if err := configureFirewall(host); err != nil {
   171  		return "", err
   172  	}
   173  
   174  	// Install the kubelet credential provider plugin
   175  	if err := configureCredentialProvider(host, workspace); err != nil {
   176  		return "", err
   177  	}
   178  
   179  	// Kill any running node processes
   180  	cleanupNodeProcesses(host)
   181  
   182  	testArgs, err := osSpecificActions(testArgs, host, workspace)
   183  	if err != nil {
   184  		return "", err
   185  	}
   186  
   187  	systemSpecFile := ""
   188  	if systemSpecName != "" {
   189  		systemSpecFile = systemSpecName + ".yaml"
   190  	}
   191  
   192  	outputGinkgoFile := filepath.Join(results, fmt.Sprintf("%s-ginkgo.log", host))
   193  
   194  	// Run the tests
   195  	klog.V(2).Infof("Starting tests on %q", host)
   196  	cmd := getSSHCommand(" && ",
   197  		fmt.Sprintf("cd %s", workspace),
   198  		// Note, we need to have set -o pipefail here to ensure we return the appriorate exit code from ginkgo; not tee
   199  		fmt.Sprintf("set -o pipefail; timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s 2>&1 | tee -i %s",
   200  			timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs, outputGinkgoFile),
   201  	)
   202  	return SSH(host, "/bin/bash", "-c", cmd)
   203  }
   204
View as plain text