run_remote_suite.go

Documentation: k8s.io/kubernetes/test/e2e_node/remote

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package remote
    18  
    19  import (
    20  	"flag"
    21  	"fmt"
    22  	"log"
    23  	"math/rand"
    24  	"os"
    25  	"os/exec"
    26  	"os/signal"
    27  	"strings"
    28  	"sync"
    29  	"time"
    30  
    31  	"k8s.io/klog/v2"
    32  )
    33  
    34  var mode = flag.String("mode", "gce", "Mode to operate in. One of gce|ssh. Defaults to gce")
    35  var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
    36  var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
    37  var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
    38  var imageConfigDir = flag.String("image-config-dir", "", "(optional) path to image config files")
    39  var images = flag.String("images", "", "images to test")
    40  var hosts = flag.String("hosts", "", "hosts to test")
    41  var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
    42  var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created")
    43  var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
    44  var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
    45  var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
    46  var (
    47  	arc Archive
    48  )
    49  
    50  // Archive contains path info in the archive.
    51  type Archive struct {
    52  	sync.Once
    53  	path string
    54  	err  error
    55  }
    56  
    57  func getFlag(name string) string {
    58  	lookup := flag.Lookup(name)
    59  	if lookup == nil {
    60  		return ""
    61  	}
    62  	return lookup.Value.String()
    63  }
    64  
    65  func RunRemoteTestSuite(testSuite TestSuite) {
    66  	// Listen for SIGINT and ignore the first one. In case SIGINT is sent to this
    67  	// process and all its children, we ignore it here, while our children ssh connections
    68  	// are stopped. This allows us to gather artifacts and print out test state before
    69  	// being killed.
    70  	c := make(chan os.Signal, 2)
    71  	signal.Notify(c, os.Interrupt)
    72  	go func() {
    73  		<-c
    74  		fmt.Printf("Received SIGINT. Will exit on next SIGINT.\n")
    75  		<-c
    76  		fmt.Printf("Received another SIGINT. Will exit.\n")
    77  		os.Exit(1)
    78  	}()
    79  
    80  	rand.Seed(time.Now().UnixNano())
    81  	if *buildOnly {
    82  		// Build the archive and exit
    83  		CreateTestArchive(testSuite,
    84  			getFlag("system-spec-name"),
    85  			getFlag("kubelet-config-file"))
    86  		return
    87  	}
    88  
    89  	// Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh
    90  	allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags)
    91  	fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags)
    92  
    93  	var runner Runner
    94  	cfg := Config{
    95  		InstanceNamePrefix: *instanceNamePrefix,
    96  		ImageConfigFile:    *imageConfigFile,
    97  		ImageConfigDir:     *imageConfigDir,
    98  		Images:             splitCommaList(*images),
    99  		Hosts:              parseHostsList(*hosts),
   100  		GinkgoFlags:        allGinkgoFlags,
   101  		DeleteInstances:    *deleteInstances,
   102  		Cleanup:            *cleanup,
   103  		TestArgs:           *testArgs,
   104  		ExtraEnvs:          getFlag("extra-envs"),
   105  		RuntimeConfig:      getFlag("runtime-config"),
   106  		SystemSpecName:     getFlag("system-spec-name"),
   107  	}
   108  
   109  	var sshRunner Runner
   110  
   111  	if *mode == "ssh" {
   112  		runner = NewSSHRunner(cfg)
   113  	} else {
   114  		getRunner, err := GetRunner(*mode)
   115  		if err != nil {
   116  			klog.Fatalf("getting runner mode %q : %v", *mode, err)
   117  		}
   118  		runner = getRunner(cfg)
   119  		sshRunner = NewSSHRunner(cfg)
   120  	}
   121  
   122  	if err := runner.Validate(); err != nil {
   123  		klog.Fatalf("validating remote config, %s", err)
   124  	}
   125  
   126  	// Setup coloring
   127  	stat, _ := os.Stdout.Stat()
   128  	useColor := (stat.Mode() & os.ModeCharDevice) != 0
   129  	blue := ""
   130  	noColour := ""
   131  	if useColor {
   132  		blue = "\033[0;34m"
   133  		noColour = "\033[0m"
   134  	}
   135  
   136  	results := make(chan *TestResult)
   137  
   138  	path, err := arc.getArchive(testSuite)
   139  	if err != nil {
   140  		log.Fatalf("unable to create test archive: %s", err)
   141  	}
   142  	defer arc.deleteArchive()
   143  
   144  	running := runner.StartTests(testSuite, path, results)
   145  	// You can potentially run SSH based tests while running image based test as well.  The GCE provider does this, see
   146  	// test-e2e-node.sh.
   147  	if sshRunner != nil && len(cfg.Hosts) > 0 {
   148  		running += sshRunner.StartTests(testSuite, path, results)
   149  	}
   150  
   151  	// Wait for all tests to complete and emit the results
   152  	errCount := 0
   153  	exitOk := true
   154  	for i := 0; i < running; i++ {
   155  		tr := <-results
   156  		host := tr.Host
   157  		fmt.Println() // Print an empty line
   158  		fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
   159  		fmt.Printf("%s>                              START TEST                                >%s\n", blue, noColour)
   160  		fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
   161  		fmt.Printf("Start Test Suite on Host %s\n", host)
   162  		fmt.Printf("%s\n", tr.Output)
   163  		if tr.Err != nil {
   164  			errCount++
   165  			fmt.Printf("Failure Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n%v\n", host, tr.Err)
   166  		} else {
   167  			fmt.Printf("Success Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n", host)
   168  		}
   169  		exitOk = exitOk && tr.ExitOK
   170  		fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
   171  		fmt.Printf("%s<                              FINISH TEST                               <%s\n", blue, noColour)
   172  		fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
   173  		fmt.Println() // Print an empty line
   174  	}
   175  	// Set the exit code if there were failures
   176  	if !exitOk {
   177  		fmt.Printf("Failure: %d errors encountered.\n", errCount)
   178  		callGubernator(*gubernator)
   179  		arc.deleteArchive()
   180  		os.Exit(1)
   181  	}
   182  	callGubernator(*gubernator)
   183  }
   184  
   185  func splitCommaList(s string) []string {
   186  	if len(s) == 0 {
   187  		return nil
   188  	}
   189  	return strings.Split(s, ",")
   190  }
   191  
   192  func callGubernator(gubernator bool) {
   193  	if gubernator {
   194  		fmt.Println("Running gubernator.sh")
   195  		output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output()
   196  
   197  		if err != nil {
   198  			fmt.Println("gubernator.sh Failed")
   199  			fmt.Println(err)
   200  			return
   201  		}
   202  		fmt.Printf("%s", output)
   203  	}
   204  	return
   205  }
   206  
   207  func (a *Archive) getArchive(suite TestSuite) (string, error) {
   208  	a.Do(func() {
   209  		a.path, a.err = CreateTestArchive(suite,
   210  			getFlag("system-spec-name"),
   211  			getFlag("kubelet-config-file"))
   212  	})
   213  	return a.path, a.err
   214  }
   215  
   216  func (a *Archive) deleteArchive() {
   217  	path, err := a.getArchive(nil)
   218  	if err != nil {
   219  		return
   220  	}
   221  	os.Remove(path)
   222  }
   223  
   224  // parseHostsList splits a host list of the form a=1.2.3.4,b=5.6.7.8 into the list of hosts [a,b] while registering the
   225  // given addresses
   226  func parseHostsList(hostList string) []string {
   227  	if len(hostList) == 0 {
   228  		return nil
   229  	}
   230  	hosts := strings.Split(hostList, ",")
   231  	var hostsOnly []string
   232  	for _, host := range hosts {
   233  		segs := strings.Split(host, "=")
   234  		if len(segs) == 2 {
   235  			AddHostnameIP(segs[0], segs[1])
   236  		} else if len(segs) > 2 {
   237  			klog.Fatalf("invalid format of host %q", hostList)
   238  		}
   239  		hostsOnly = append(hostsOnly, segs[0])
   240  	}
   241  	return hostsOnly
   242  }
   243
View as plain text