node_conformance.go

Documentation: k8s.io/kubernetes/test/e2e_node/remote

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package remote
    18  
    19  import (
    20  	"crypto/rand"
    21  	"encoding/base64"
    22  	"fmt"
    23  	"math"
    24  	"os"
    25  	"os/exec"
    26  	"path/filepath"
    27  	"runtime"
    28  	"strings"
    29  	"time"
    30  
    31  	"k8s.io/klog/v2"
    32  
    33  	"k8s.io/kubernetes/test/e2e_node/builder"
    34  	"k8s.io/kubernetes/test/utils"
    35  )
    36  
    37  // ConformanceRemote contains the specific functions in the node conformance test suite.
    38  type ConformanceRemote struct{}
    39  
    40  func init() {
    41  	RegisterTestSuite("conformance", &ConformanceRemote{})
    42  }
    43  
    44  // getConformanceDirectory gets node conformance test build directory.
    45  func getConformanceDirectory() (string, error) {
    46  	k8sRoot, err := utils.GetK8sRootDir()
    47  	if err != nil {
    48  		return "", err
    49  	}
    50  	return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil
    51  }
    52  
    53  // commandToString is a helper function which formats command to string.
    54  func commandToString(c *exec.Cmd) string {
    55  	return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ")
    56  }
    57  
    58  // Image path constants.
    59  const (
    60  	conformanceRegistry         = "registry.k8s.io"
    61  	conformanceArch             = runtime.GOARCH
    62  	conformanceTarfile          = "node_conformance.tar"
    63  	conformanceTestBinary       = "e2e_node.test"
    64  	conformanceImageLoadTimeout = time.Duration(30) * time.Second
    65  )
    66  
    67  // timestamp is used as an unique id of current test.
    68  var timestamp = getTimestamp()
    69  
    70  // getConformanceTestImageName returns name of the conformance test image given the system spec name.
    71  func getConformanceTestImageName(systemSpecName string) string {
    72  	if systemSpecName == "" {
    73  		return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp)
    74  	}
    75  	return fmt.Sprintf("%s/node-test-%s-%s:%s", conformanceRegistry, systemSpecName, conformanceArch, timestamp)
    76  }
    77  
    78  // buildConformanceTest builds node conformance test image tarball into binDir.
    79  func buildConformanceTest(binDir, systemSpecName string) error {
    80  	// Get node conformance directory.
    81  	conformancePath, err := getConformanceDirectory()
    82  	if err != nil {
    83  		return fmt.Errorf("failed to get node conformance directory: %w", err)
    84  	}
    85  	// Build docker image.
    86  	cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir,
    87  		"REGISTRY="+conformanceRegistry,
    88  		"ARCH="+conformanceArch,
    89  		"VERSION="+timestamp,
    90  		"SYSTEM_SPEC_NAME="+systemSpecName)
    91  	if output, err := cmd.CombinedOutput(); err != nil {
    92  		return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q",
    93  			commandToString(cmd), err, output)
    94  	}
    95  	// Save docker image into tar file.
    96  	cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceTestImageName(systemSpecName))
    97  	if output, err := cmd.CombinedOutput(); err != nil {
    98  		return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q",
    99  			commandToString(cmd), err, output)
   100  	}
   101  	return nil
   102  }
   103  
   104  // SetupTestPackage sets up the test package with binaries k8s required for node conformance test
   105  func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) error {
   106  	// Build the executables
   107  	if err := builder.BuildGo(); err != nil {
   108  		return fmt.Errorf("failed to build the dependencies: %w", err)
   109  	}
   110  
   111  	// Make sure we can find the newly built binaries
   112  	buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
   113  	if err != nil {
   114  		return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
   115  	}
   116  
   117  	// Build node conformance tarball.
   118  	if err := buildConformanceTest(buildOutputDir, systemSpecName); err != nil {
   119  		return fmt.Errorf("failed to build node conformance test: %w", err)
   120  	}
   121  
   122  	// Copy files
   123  	requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile}
   124  	for _, file := range requiredFiles {
   125  		source := filepath.Join(buildOutputDir, file)
   126  		if _, err := os.Stat(source); err != nil {
   127  			return fmt.Errorf("failed to locate test file %s: %w", file, err)
   128  		}
   129  		output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput()
   130  		if err != nil {
   131  			return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output)
   132  		}
   133  	}
   134  
   135  	return nil
   136  }
   137  
   138  // loadConformanceImage loads node conformance image from tar file.
   139  func loadConformanceImage(host, workspace string) error {
   140  	klog.Info("Loading conformance image from tarfile")
   141  	tarfile := filepath.Join(workspace, conformanceTarfile)
   142  	if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(),
   143  		"docker", "load", "-i", tarfile); err != nil {
   144  		return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q",
   145  			tarfile, err, output)
   146  	}
   147  	return nil
   148  }
   149  
   150  // kubeletLauncherLog is the log of kubelet launcher.
   151  const kubeletLauncherLog = "kubelet-launcher.log"
   152  
   153  // kubeletPodPath is a fixed known pod specification path. We can not use the random pod
   154  // manifest directory generated in e2e_node.test because we need to mount the directory into
   155  // the conformance test container, it's easier if it's a known directory.
   156  // TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script.
   157  var kubeletPodPath = "conformance-pod-manifest-" + timestamp
   158  
   159  // getPodPath returns pod manifest full path.
   160  func getPodPath(workspace string) string {
   161  	return filepath.Join(workspace, kubeletPodPath)
   162  }
   163  
   164  // isSystemd returns whether the node is a systemd node.
   165  func isSystemd(host string) (bool, error) {
   166  	// Returns "systemd" if /run/systemd/system is found, empty string otherwise.
   167  	output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true")
   168  	if err != nil {
   169  		return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output)
   170  	}
   171  	return strings.TrimSpace(output) != "", nil
   172  }
   173  
   174  // launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode.
   175  // This is a temporary solution, we should change node e2e to use the same node bootstrap
   176  // with cluster e2e and launch kubelet outside of the test for both regular node e2e and
   177  // node conformance test.
   178  // TODO(random-liu): Switch to use standard node bootstrap script.
   179  func launchKubelet(host, workspace, results, testArgs, bearerToken string) error {
   180  	podManifestPath := getPodPath(workspace)
   181  	if output, err := SSH(host, "mkdir", podManifestPath); err != nil {
   182  		return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q",
   183  			podManifestPath, err, output)
   184  	}
   185  	startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --node-name=%s"+
   186  		" --bearer-token=%s"+
   187  		" --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1",
   188  		conformanceTestBinary, host, bearerToken, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog))
   189  	var cmd []string
   190  	systemd, err := isSystemd(host)
   191  	if err != nil {
   192  		return fmt.Errorf("failed to check systemd: %w", err)
   193  	}
   194  	if systemd {
   195  		cmd = []string{
   196  			"systemd-run", "sh", "-c", getSSHCommand(" && ",
   197  				// Switch to workspace.
   198  				fmt.Sprintf("cd %s", workspace),
   199  				// Launch kubelet by running e2e_node.test in run-kubelet-mode.
   200  				startKubeletCmd,
   201  			),
   202  		}
   203  	} else {
   204  		cmd = []string{
   205  			"sh", "-c", getSSHCommand(" && ",
   206  				// Switch to workspace.
   207  				fmt.Sprintf("cd %s", workspace),
   208  				// Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup.
   209  				fmt.Sprintf("(nohup %s &)", startKubeletCmd),
   210  			),
   211  		}
   212  	}
   213  	klog.V(2).Infof("Launch kubelet with command: %v", cmd)
   214  	output, err := SSH(host, cmd...)
   215  	if err != nil {
   216  		return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q",
   217  			cmd, err, output)
   218  	}
   219  	klog.Info("Successfully launch kubelet")
   220  	return nil
   221  }
   222  
   223  // kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet.
   224  const kubeletStopGracePeriod = 10 * time.Second
   225  
   226  // stopKubelet stops kubelet launcher and kubelet gracefully.
   227  func stopKubelet(host, workspace string) error {
   228  	klog.Info("Gracefully stop kubelet launcher")
   229  	if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil {
   230  		return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q",
   231  			err, output)
   232  	}
   233  	klog.Info("Wait for kubelet launcher to stop")
   234  	stopped := false
   235  	for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) {
   236  		// Check whether the process is still running.
   237  		output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true")
   238  		if err != nil {
   239  			return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q",
   240  				err, output)
   241  		}
   242  		// Kubelet is stopped
   243  		if strings.TrimSpace(output) == "" {
   244  			stopped = true
   245  			break
   246  		}
   247  	}
   248  	if !stopped {
   249  		klog.Info("Forcibly stop kubelet")
   250  		if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil {
   251  			return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q",
   252  				err, output)
   253  		}
   254  	}
   255  	klog.Info("Successfully stop kubelet")
   256  	// Clean up the pod manifest path
   257  	podManifestPath := getPodPath(workspace)
   258  	if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil {
   259  		return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q",
   260  			podManifestPath, err, output)
   261  	}
   262  	return nil
   263  }
   264  
   265  // RunTest runs test on the node.
   266  func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName, extraEnvs, _ string, timeout time.Duration) (string, error) {
   267  	// Install the cni plugins and add a basic CNI configuration.
   268  	if err := setupCNI(host, workspace); err != nil {
   269  		return "", err
   270  	}
   271  
   272  	// Configure iptables firewall rules.
   273  	if err := configureFirewall(host); err != nil {
   274  		return "", err
   275  	}
   276  
   277  	// Kill any running node processes.
   278  	cleanupNodeProcesses(host)
   279  
   280  	// Load node conformance image.
   281  	if err := loadConformanceImage(host, workspace); err != nil {
   282  		return "", err
   283  	}
   284  
   285  	bearerToken, err := generateSecureToken(16)
   286  	if err != nil {
   287  		return "", err
   288  	}
   289  
   290  	// Launch kubelet.
   291  	if err := launchKubelet(host, workspace, results, testArgs, bearerToken); err != nil {
   292  		return "", err
   293  	}
   294  	// Stop kubelet.
   295  	defer func() {
   296  		if err := stopKubelet(host, workspace); err != nil {
   297  			// Only log an error if failed to stop kubelet because it is not critical.
   298  			klog.Errorf("failed to stop kubelet: %v", err)
   299  		}
   300  	}()
   301  
   302  	// Run the tests
   303  	klog.V(2).Infof("Starting tests on %q", host)
   304  	podManifestPath := getPodPath(workspace)
   305  	cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s -e TEST_ARGS=--bearer-token=%s %s'",
   306  		timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, bearerToken, getConformanceTestImageName(systemSpecName))
   307  	return SSH(host, "sh", "-c", cmd)
   308  }
   309  
   310  // generateSecureToken returns a string of length tokenLen, consisting
   311  // of random bytes encoded as base64 for use as a Bearer Token during
   312  // communication with an APIServer
   313  func generateSecureToken(tokenLen int) (string, error) {
   314  	// Number of bytes to be tokenLen when base64 encoded.
   315  	tokenSize := math.Ceil(float64(tokenLen) * 6 / 8)
   316  	rawToken := make([]byte, int(tokenSize))
   317  	if _, err := rand.Read(rawToken); err != nil {
   318  		return "", err
   319  	}
   320  	encoded := base64.RawURLEncoding.EncodeToString(rawToken)
   321  	token := encoded[:tokenLen]
   322  	return token, nil
   323  }
   324
View as plain text