services.go

Documentation: k8s.io/kubernetes/test/e2e_node/services

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package services
    18  
    19  import (
    20  	"fmt"
    21  	"os"
    22  	"os/exec"
    23  	"path"
    24  	"testing"
    25  
    26  	"k8s.io/klog/v2"
    27  
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  )
    30  
    31  // E2EServices starts and stops e2e services in a separate process. The test
    32  // uses it to start and stop all e2e services.
    33  type E2EServices struct {
    34  	// monitorParent determines whether the sub-processes should watch and die with the current
    35  	// process.
    36  	rmDirs        []string
    37  	monitorParent bool
    38  	services      *server
    39  	kubelet       *server
    40  	logs          logFiles
    41  }
    42  
    43  // NewE2EServices returns a new E2EServices instance.
    44  func NewE2EServices(monitorParent bool) *E2EServices {
    45  	return &E2EServices{
    46  		monitorParent: monitorParent,
    47  		// Special log files that need to be collected for additional debugging.
    48  		logs: getLogFiles(),
    49  	}
    50  }
    51  
    52  // Start starts the e2e services in another process by calling back into the
    53  // test binary.  Returns when all e2e services are ready or an error.
    54  //
    55  // We want to statically link e2e services into the test binary, but we don't
    56  // want their glog output to pollute the test result. So we run the binary in
    57  // run-services-mode to start e2e services in another process.
    58  // The function starts 2 processes:
    59  // * internal e2e services: services which statically linked in the test binary - apiserver, etcd and
    60  // namespace controller.
    61  // * kubelet: kubelet binary is outside. (We plan to move main kubelet start logic out when we have
    62  // standard kubelet launcher)
    63  func (e *E2EServices) Start(featureGates map[string]bool) error {
    64  	var err error
    65  	if e.services, err = e.startInternalServices(); err != nil {
    66  		return fmt.Errorf("failed to start internal services: %w", err)
    67  	}
    68  	klog.Infof("Node services started.")
    69  	// running the kubelet depends on whether we are running conformance test-suite
    70  	if framework.TestContext.NodeConformance {
    71  		klog.Info("nothing to do in node-e2e-services, running conformance suite")
    72  	} else {
    73  		// Start kubelet
    74  		e.kubelet, err = e.startKubelet(featureGates)
    75  		if err != nil {
    76  			return fmt.Errorf("failed to start kubelet: %w", err)
    77  		}
    78  		klog.Infof("Kubelet started.")
    79  	}
    80  	return nil
    81  }
    82  
    83  // Stop stops the e2e services.
    84  func (e *E2EServices) Stop() {
    85  	defer func() {
    86  		if !framework.TestContext.NodeConformance {
    87  			// Collect log files.
    88  			e.collectLogFiles()
    89  		}
    90  	}()
    91  	if e.services != nil {
    92  		if err := e.services.kill(); err != nil {
    93  			klog.Errorf("Failed to stop services: %v", err)
    94  		}
    95  	}
    96  	if e.kubelet != nil {
    97  		if err := e.kubelet.kill(); err != nil {
    98  			klog.Errorf("Failed to kill kubelet: %v", err)
    99  		}
   100  		// Stop the kubelet systemd unit which will delete the kubelet transient unit.
   101  		if err := e.kubelet.stopUnit(); err != nil {
   102  			klog.Errorf("Failed to stop kubelet systemd unit: %v", err)
   103  		}
   104  	}
   105  	for _, d := range e.rmDirs {
   106  		err := os.RemoveAll(d)
   107  		if err != nil {
   108  			klog.Errorf("Failed to delete directory %s: %v", d, err)
   109  		}
   110  	}
   111  }
   112  
   113  // RunE2EServices actually start the e2e services. This function is used to
   114  // start e2e services in current process. This is only used in run-services-mode.
   115  func RunE2EServices(t *testing.T) {
   116  	e := newE2EServices()
   117  	if err := e.run(t); err != nil {
   118  		klog.Fatalf("Failed to run e2e services: %v", err)
   119  	}
   120  }
   121  
   122  const (
   123  	// services.log is the combined log of all services
   124  	servicesLogFile = "services.log"
   125  	// LogVerbosityLevel is consistent with the level used in a cluster e2e test.
   126  	LogVerbosityLevel = "4"
   127  )
   128  
   129  // startInternalServices starts the internal services in a separate process.
   130  func (e *E2EServices) startInternalServices() (*server, error) {
   131  	testBin, err := os.Executable()
   132  	if err != nil {
   133  		return nil, fmt.Errorf("can't get current binary: %w", err)
   134  	}
   135  	// Pass all flags into the child process, so that it will see the same flag set.
   136  	startCmd := exec.Command(testBin,
   137  		append(
   138  			[]string{"--run-services-mode", fmt.Sprintf("--bearer-token=%s", framework.TestContext.BearerToken)},
   139  			os.Args[1:]...,
   140  		)...)
   141  	server := newServer("services", startCmd, nil, nil, getServicesHealthCheckURLs(), servicesLogFile, e.monitorParent, false, "")
   142  	return server, server.start()
   143  }
   144  
   145  // collectLogFiles collects logs of interest either via journalctl or by creating sym
   146  // links. Since we scp files from the remote directory, symlinks will be
   147  // treated as normal files and file contents will be copied over.
   148  func (e *E2EServices) collectLogFiles() {
   149  	// Nothing to do if report dir is not specified.
   150  	if framework.TestContext.ReportDir == "" {
   151  		return
   152  	}
   153  	klog.Info("Fetching log files...")
   154  	journaldFound := isJournaldAvailable()
   155  	for targetFileName, log := range e.logs {
   156  		targetLink := path.Join(framework.TestContext.ReportDir, targetFileName)
   157  		if journaldFound {
   158  			// Skip log files that do not have an equivalent in journald-based machines.
   159  			if len(log.JournalctlCommand) == 0 {
   160  				continue
   161  			}
   162  			klog.Infof("Get log file %q with journalctl command %v.", targetFileName, log.JournalctlCommand)
   163  			out, err := exec.Command("journalctl", log.JournalctlCommand...).CombinedOutput()
   164  			if err != nil {
   165  				klog.Errorf("failed to get %q from journald: %v, %v", targetFileName, string(out), err)
   166  			} else {
   167  				if err = os.WriteFile(targetLink, out, 0644); err != nil {
   168  					klog.Errorf("failed to write logs to %q: %v", targetLink, err)
   169  				}
   170  			}
   171  			continue
   172  		}
   173  		for _, file := range log.Files {
   174  			if _, err := os.Stat(file); err != nil {
   175  				// Expected file not found on this distro.
   176  				continue
   177  			}
   178  			if err := copyLogFile(file, targetLink); err != nil {
   179  				klog.Error(err)
   180  			} else {
   181  				break
   182  			}
   183  		}
   184  	}
   185  }
   186  
   187  // isJournaldAvailable returns whether the system executing the tests uses
   188  // journald.
   189  func isJournaldAvailable() bool {
   190  	_, err := exec.LookPath("journalctl")
   191  	return err == nil
   192  }
   193  
   194  func copyLogFile(src, target string) error {
   195  	// If not a journald based distro, then just symlink files.
   196  	if out, err := exec.Command("cp", src, target).CombinedOutput(); err != nil {
   197  		return fmt.Errorf("failed to copy %q to %q: %v, %v", src, target, out, err)
   198  	}
   199  	if out, err := exec.Command("chmod", "a+r", target).CombinedOutput(); err != nil {
   200  		return fmt.Errorf("failed to make log file %q world readable: %v, %v", target, out, err)
   201  	}
   202  	return nil
   203  }
   204
View as plain text