common.go

Documentation: sigs.k8s.io/release-utils/util

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package util
    18  
    19  import (
    20  	"bufio"
    21  	"errors"
    22  	"fmt"
    23  	"io"
    24  	"os"
    25  	"os/signal"
    26  	"path/filepath"
    27  	"regexp"
    28  	"strings"
    29  	"syscall"
    30  
    31  	"github.com/blang/semver/v4"
    32  	"github.com/sirupsen/logrus"
    33  
    34  	"sigs.k8s.io/release-utils/command"
    35  )
    36  
    37  const (
    38  	TagPrefix = "v"
    39  )
    40  
    41  var (
    42  	regexpCRLF       = regexp.MustCompile(`\015$`)
    43  	regexpCtrlChar   = regexp.MustCompile(`\x1B[\[(](\d{1,2}(;\d{1,2})?)?[mKB]`)
    44  	regexpOauthToken = regexp.MustCompile(`[a-f0-9]{40}:x-oauth-basic`)
    45  	regexpGitToken   = regexp.MustCompile(`git:[a-f0-9]{35,40}@github\.com`)
    46  )
    47  
    48  // UserInputError a custom error to handle more user input info
    49  type UserInputError struct {
    50  	ErrorString string
    51  	isCtrlC     bool
    52  }
    53  
    54  // Error return the error string
    55  func (e UserInputError) Error() string {
    56  	return e.ErrorString
    57  }
    58  
    59  // IsCtrlC return true if the user has hit Ctrl+C
    60  func (e UserInputError) IsCtrlC() bool {
    61  	return e.isCtrlC
    62  }
    63  
    64  // NewUserInputError creates a new UserInputError
    65  func NewUserInputError(message string, ctrlC bool) UserInputError {
    66  	return UserInputError{
    67  		ErrorString: message,
    68  		isCtrlC:     ctrlC,
    69  	}
    70  }
    71  
    72  // PackagesAvailable takes a slice of packages and determines if they are installed
    73  // on the host OS. Replaces common::check_packages.
    74  func PackagesAvailable(packages ...string) (bool, error) {
    75  	type packageVerifier struct {
    76  		cmd  string
    77  		args []string
    78  	}
    79  	type packageChecker struct {
    80  		manager  string
    81  		verifier *packageVerifier
    82  	}
    83  	var checker *packageChecker
    84  
    85  	for _, x := range []struct {
    86  		possiblePackageManagers []string
    87  		verifierCmd             string
    88  		verifierArgs            []string
    89  	}{
    90  		{ // Debian, Ubuntu and similar
    91  			[]string{"apt"},
    92  			"dpkg",
    93  			[]string{"-l"},
    94  		},
    95  		{ // Fedora, openSUSE and similar
    96  			[]string{"dnf", "yum", "zypper"},
    97  			"rpm",
    98  			[]string{"--quiet", "-q"},
    99  		},
   100  		{ // ArchLinux and similar
   101  			[]string{"yay", "pacaur", "pacman"},
   102  			"pacman",
   103  			[]string{"-Qs"},
   104  		},
   105  	} {
   106  		// Find a working package verifier
   107  		if !command.Available(x.verifierCmd) {
   108  			logrus.Debugf("Skipping not available package verifier %s",
   109  				x.verifierCmd)
   110  			continue
   111  		}
   112  
   113  		// Find a working package manager
   114  		packageManager := ""
   115  		for _, mgr := range x.possiblePackageManagers {
   116  			if command.Available(mgr) {
   117  				packageManager = mgr
   118  				break
   119  			}
   120  			logrus.Debugf("Skipping not available package manager %s", mgr)
   121  		}
   122  		if packageManager == "" {
   123  			return false, fmt.Errorf(
   124  				"unable to find working package manager for verifier `%s`",
   125  				x.verifierCmd,
   126  			)
   127  		}
   128  
   129  		checker = &packageChecker{
   130  			manager:  packageManager,
   131  			verifier: &packageVerifier{x.verifierCmd, x.verifierArgs},
   132  		}
   133  		break
   134  	}
   135  	if checker == nil {
   136  		return false, errors.New("unable to find working package manager")
   137  	}
   138  	logrus.Infof("Assuming %q as package manager", checker.manager)
   139  
   140  	missingPkgs := []string{}
   141  	for _, pkg := range packages {
   142  		logrus.Infof("Checking if %q has been installed", pkg)
   143  
   144  		args := checker.verifier.args
   145  		args = append(args, pkg)
   146  		if err := command.New(checker.verifier.cmd, args...).
   147  			RunSilentSuccess(); err != nil {
   148  			logrus.Infof("Adding %s to missing packages", pkg)
   149  			missingPkgs = append(missingPkgs, pkg)
   150  		}
   151  	}
   152  
   153  	if len(missingPkgs) > 0 {
   154  		logrus.Warnf("The following packages are not installed via %s: %s",
   155  			checker.manager, strings.Join(missingPkgs, ", "))
   156  
   157  		// TODO: `install` might not be the install command for every package
   158  		// manager
   159  		logrus.Infof("Install them with: sudo %s install %s",
   160  			checker.manager, strings.Join(missingPkgs, " "))
   161  		return false, nil
   162  	}
   163  
   164  	return true, nil
   165  }
   166  
   167  /*
   168  #############################################################################
   169  # Simple yes/no prompt
   170  #
   171  # @optparam default -n(default)/-y/-e (default to n, y or make (e)xplicit)
   172  # @param message
   173  common::askyorn () {
   174    local yorn
   175    local def=n
   176    local msg="y/N"
   177  
   178    case $1 in
   179    -y) # yes default
   180        def="y" msg="Y/n"
   181        shift
   182        ;;
   183    -e) # Explicit
   184        def="" msg="y/n"
   185        shift
   186        ;;
   187    -n) shift
   188        ;;
   189    esac
   190  
   191    while [[ $yorn != [yYnN] ]]; do
   192      logecho -n "$*? ($msg): "
   193      read yorn
   194      : ${yorn:=$def}
   195    done
   196  
   197    # Final test to set return code
   198    [[ $yorn == [yY] ]]
   199  }
   200  */
   201  
   202  // readInput prints a question and then reads an answer from the user
   203  //
   204  // If the user presses Ctrl+C instead of answering, this funtcion will
   205  // return an error crafted with UserInputError. This error can be queried
   206  // to find out if the user canceled the input using its method IsCtrlC:
   207  //
   208  //	if err.(util.UserInputError).IsCtrlC() {}
   209  //
   210  // Note that in case of cancelling input, the user will still have to press
   211  // enter to finish the scan.
   212  func readInput(question string) (string, error) {
   213  	fmt.Print(question)
   214  
   215  	// Trap Ctrl+C if a user wishes to cancel the input
   216  	inputChannel := make(chan string, 1)
   217  	signalChannel := make(chan os.Signal, 1)
   218  	signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)
   219  	defer func() {
   220  		signal.Stop(signalChannel)
   221  		close(signalChannel)
   222  	}()
   223  	go func() {
   224  		scanner := bufio.NewScanner(os.Stdin)
   225  		scanner.Scan()
   226  		response := scanner.Text()
   227  		inputChannel <- response
   228  		close(inputChannel)
   229  	}()
   230  
   231  	select {
   232  	case <-signalChannel:
   233  		return "", NewUserInputError("Input canceled", true)
   234  	case response := <-inputChannel:
   235  		return response, nil
   236  	}
   237  }
   238  
   239  // Ask asks the user a question, expecting a known response expectedResponse
   240  //
   241  // You may specify a single response as a string or a series
   242  // of valid/invalid responses with an optional default.
   243  //
   244  // To specify the valid responses, either pass a string or craft a series
   245  // of answers using the following format:
   246  //
   247  //	"|successAnswers|nonSuccessAnswers|defaultAnswer"
   248  //
   249  // The successAnswers and nonSuccessAnswers can be either a string or a
   250  // series os responses like:
   251  //
   252  //	"|opt1a:opt1b|opt2a:opt2b|defaultAnswer"
   253  //
   254  // This example will accept opt1a and opt1b as successful answers, opt2a and
   255  // opt2b as unsuccessful answers and in case of an empty answer, it will
   256  // return "defaultAnswer" as success.
   257  //
   258  // To consider the default as a success, simply list them with the rest of the
   259  // non successful answers.
   260  func Ask(question, expectedResponse string, retries int) (answer string, success bool, err error) {
   261  	attempts := 1
   262  
   263  	if retries < 0 {
   264  		fmt.Printf("Retries was set to a number less than zero (%d). Please specify a positive number of retries or zero, if you want to ask unconditionally.\n", retries)
   265  	}
   266  
   267  	const (
   268  		partsSeparator string = "|"
   269  		optsSeparator  string = ":"
   270  	)
   271  
   272  	successAnswers := make([]string, 0)
   273  	nonSuccessAnswers := make([]string, 0)
   274  	defaultAnswer := ""
   275  
   276  	// Check out if string has several options
   277  	if strings.Contains(expectedResponse, partsSeparator) {
   278  		parts := strings.Split(expectedResponse, partsSeparator)
   279  		if len(parts) > 3 {
   280  			return "", false, errors.New("answer spec malformed")
   281  		}
   282  		// The first part has the answers to consider a success
   283  		if strings.Contains(expectedResponse, parts[0]) {
   284  			successAnswers = strings.Split(parts[0], optsSeparator)
   285  		}
   286  		// If there is a second part, its non success, but expected responses
   287  		if len(parts) >= 2 {
   288  			if strings.Contains(parts[1], optsSeparator) {
   289  				nonSuccessAnswers = strings.Split(parts[1], optsSeparator)
   290  			} else {
   291  				nonSuccessAnswers = append(nonSuccessAnswers, parts[1])
   292  			}
   293  		}
   294  		// If we have a fourth part, its the default answer
   295  		if len(parts) == 3 {
   296  			defaultAnswer = parts[2]
   297  		}
   298  	}
   299  
   300  	for attempts <= retries {
   301  		// Read the input from the user
   302  		answer, err = readInput(fmt.Sprintf("%s (%d/%d) \n", question, attempts, retries))
   303  		if err != nil {
   304  			return answer, false, err
   305  		}
   306  
   307  		// if we have multiple options, use those and ignore the expected string
   308  		if len(successAnswers) > 0 {
   309  			// check the right answers
   310  			for _, testResponse := range successAnswers {
   311  				if answer == testResponse {
   312  					return answer, true, nil
   313  				}
   314  			}
   315  
   316  			// if we have wrong, but accepted answers, try those
   317  			for _, testResponse := range nonSuccessAnswers {
   318  				if answer == testResponse {
   319  					return answer, false, nil
   320  				}
   321  
   322  				// If answer is the default, and it is a nonSuccess, return it
   323  				if answer == "" && defaultAnswer == testResponse {
   324  					return defaultAnswer, false, nil
   325  				}
   326  			}
   327  		} else if answer == expectedResponse {
   328  			return answer, true, nil
   329  		}
   330  
   331  		if answer == "" && defaultAnswer != "" {
   332  			return defaultAnswer, true, nil
   333  		}
   334  
   335  		fmt.Printf("Expected '%s', but got '%s'\n", expectedResponse, answer)
   336  
   337  		attempts++
   338  	}
   339  
   340  	return answer, false, NewUserInputError("expected response was not input. Retries exceeded", false)
   341  }
   342  
   343  // MoreRecent determines if file at path a was modified more recently than file
   344  // at path b. If one file does not exist, the other will be treated as most
   345  // recent. If both files do not exist or an error occurs, an error is returned.
   346  func MoreRecent(a, b string) (bool, error) {
   347  	fileA, errA := os.Stat(a)
   348  	if errA != nil && !os.IsNotExist(errA) {
   349  		return false, errA
   350  	}
   351  
   352  	fileB, errB := os.Stat(b)
   353  	if errB != nil && !os.IsNotExist(errB) {
   354  		return false, errB
   355  	}
   356  
   357  	switch {
   358  	case os.IsNotExist(errA) && os.IsNotExist(errB):
   359  		return false, errors.New("neither file exists")
   360  	case os.IsNotExist(errA):
   361  		return false, nil
   362  	case os.IsNotExist(errB):
   363  		return true, nil
   364  	}
   365  
   366  	return (fileA.ModTime().Unix() >= fileB.ModTime().Unix()), nil
   367  }
   368  
   369  func AddTagPrefix(tag string) string {
   370  	if strings.HasPrefix(tag, TagPrefix) {
   371  		return tag
   372  	}
   373  	return TagPrefix + tag
   374  }
   375  
   376  func TrimTagPrefix(tag string) string {
   377  	return strings.TrimPrefix(tag, TagPrefix)
   378  }
   379  
   380  func TagStringToSemver(tag string) (semver.Version, error) {
   381  	return semver.Make(TrimTagPrefix(tag))
   382  }
   383  
   384  func SemverToTagString(tag semver.Version) string {
   385  	return AddTagPrefix(tag.String())
   386  }
   387  
   388  // CopyFileLocal copies a local file from one local location to another.
   389  func CopyFileLocal(src, dst string, required bool) error {
   390  	logrus.Infof("Trying to copy file %s to %s (required: %v)", src, dst, required)
   391  	srcStat, err := os.Stat(src)
   392  	if err != nil && required {
   393  		return fmt.Errorf("source %s is required but does not exist: %w", src, err)
   394  	}
   395  	if os.IsNotExist(err) && !required {
   396  		logrus.Infof(
   397  			"File %s does not exist but is also not required",
   398  			filepath.Base(src),
   399  		)
   400  		return nil
   401  	}
   402  
   403  	if !srcStat.Mode().IsRegular() {
   404  		return errors.New("cannot copy non-regular file: IsRegular reports " +
   405  			"whether m describes a regular file. That is, it tests that no " +
   406  			"mode type bits are set")
   407  	}
   408  
   409  	source, err := os.Open(src)
   410  	if err != nil {
   411  		return fmt.Errorf("open source file %s: %w", src, err)
   412  	}
   413  	defer source.Close()
   414  
   415  	destination, err := os.Create(dst)
   416  	if err != nil {
   417  		return fmt.Errorf("create destination file %s: %w", dst, err)
   418  	}
   419  	defer destination.Close()
   420  	if _, err := io.Copy(destination, source); err != nil {
   421  		return fmt.Errorf("copy source %s to destination %s: %w", src, dst, err)
   422  	}
   423  	logrus.Infof("Copied %s", filepath.Base(dst))
   424  	return nil
   425  }
   426  
   427  // CopyDirContentsLocal copies local directory contents from one local location
   428  // to another.
   429  func CopyDirContentsLocal(src, dst string) error {
   430  	logrus.Infof("Trying to copy dir %s to %s", src, dst)
   431  	// If initial destination does not exist create it.
   432  	if _, err := os.Stat(dst); err != nil {
   433  		if err := os.MkdirAll(dst, os.FileMode(0o755)); err != nil {
   434  			return fmt.Errorf("create destination directory %s: %w", dst, err)
   435  		}
   436  	}
   437  	files, err := os.ReadDir(src)
   438  	if err != nil {
   439  		return fmt.Errorf("reading source dir %s: %w", src, err)
   440  	}
   441  	for _, file := range files {
   442  		srcPath := filepath.Join(src, file.Name())
   443  		dstPath := filepath.Join(dst, file.Name())
   444  
   445  		fileInfo, err := os.Stat(srcPath)
   446  		if err != nil {
   447  			return fmt.Errorf("stat source path %s: %w", srcPath, err)
   448  		}
   449  
   450  		switch fileInfo.Mode() & os.ModeType {
   451  		case os.ModeDir:
   452  			if !Exists(dstPath) {
   453  				if err := os.MkdirAll(dstPath, os.FileMode(0o755)); err != nil {
   454  					return fmt.Errorf("creating destination dir %s: %w", dstPath, err)
   455  				}
   456  			}
   457  			if err := CopyDirContentsLocal(srcPath, dstPath); err != nil {
   458  				return fmt.Errorf("copy %s to %s: %w", srcPath, dstPath, err)
   459  			}
   460  		default:
   461  			if err := CopyFileLocal(srcPath, dstPath, false); err != nil {
   462  				return fmt.Errorf("copy %s to %s: %w", srcPath, dstPath, err)
   463  			}
   464  		}
   465  	}
   466  	return nil
   467  }
   468  
   469  // RemoveAndReplaceDir removes a directory and its contents then recreates it.
   470  func RemoveAndReplaceDir(path string) error {
   471  	logrus.Infof("Removing %s", path)
   472  	if err := os.RemoveAll(path); err != nil {
   473  		return fmt.Errorf("remove %s: %w", path, err)
   474  	}
   475  	logrus.Infof("Creating %s", path)
   476  	if err := os.MkdirAll(path, os.FileMode(0o755)); err != nil {
   477  		return fmt.Errorf("create %s: %w", path, err)
   478  	}
   479  	return nil
   480  }
   481  
   482  // Exists indicates whether a file exists.
   483  func Exists(path string) bool {
   484  	if _, err := os.Stat(path); os.IsNotExist(err) {
   485  		return false
   486  	}
   487  
   488  	return true
   489  }
   490  
   491  // WrapText wraps a text
   492  func WrapText(originalText string, lineSize int) (wrappedText string) {
   493  	words := strings.Fields(strings.TrimSpace(originalText))
   494  	wrappedText = words[0]
   495  	spaceLeft := lineSize - len(wrappedText)
   496  	for _, word := range words[1:] {
   497  		if len(word)+1 > spaceLeft {
   498  			wrappedText += "\n" + word
   499  			spaceLeft = lineSize - len(word)
   500  		} else {
   501  			wrappedText += " " + word
   502  			spaceLeft -= 1 + len(word)
   503  		}
   504  	}
   505  
   506  	return wrappedText
   507  }
   508  
   509  // StripControlCharacters takes a slice of bytes and removes control
   510  // characters and bare line feeds (ported from the original bash anago)
   511  func StripControlCharacters(logData []byte) []byte {
   512  	return regexpCRLF.ReplaceAllLiteral(
   513  		regexpCtrlChar.ReplaceAllLiteral(logData, []byte{}), []byte{},
   514  	)
   515  }
   516  
   517  // StripSensitiveData removes data deemed sensitive or non public
   518  // from a byte slice (ported from the original bash anago)
   519  func StripSensitiveData(logData []byte) []byte {
   520  	// Remove OAuth tokens
   521  	logData = regexpOauthToken.ReplaceAllLiteral(logData, []byte("__SANITIZED__:x-oauth-basic"))
   522  	// Remove GitHub tokens
   523  	logData = regexpGitToken.ReplaceAllLiteral(logData, []byte("//git:__SANITIZED__:@github.com"))
   524  	return logData
   525  }
   526  
   527  // CleanLogFile cleans control characters and sensitive data from a file
   528  func CleanLogFile(logPath string) (err error) {
   529  	logrus.Debugf("Sanitizing logfile %s", logPath)
   530  
   531  	// Open a tempfile to write sanitized log
   532  	tempFile, err := os.CreateTemp("", "temp-release-log-")
   533  	if err != nil {
   534  		return fmt.Errorf("creating temp file for sanitizing log: %w", err)
   535  	}
   536  	defer func() {
   537  		err = tempFile.Close()
   538  		os.Remove(tempFile.Name())
   539  	}()
   540  
   541  	// Open the new logfile for reading
   542  	logFile, err := os.Open(logPath)
   543  	if err != nil {
   544  		return fmt.Errorf("while opening %s : %w", logPath, err)
   545  	}
   546  	// Scan the log and pass it through the cleaning funcs
   547  	scanner := bufio.NewScanner(logFile)
   548  	for scanner.Scan() {
   549  		chunk := scanner.Bytes()
   550  		chunk = StripControlCharacters(
   551  			StripSensitiveData(chunk),
   552  		)
   553  		chunk = append(chunk, []byte{10}...)
   554  		_, err := tempFile.Write(chunk)
   555  		if err != nil {
   556  			return fmt.Errorf("while writing buffer to file: %w", err)
   557  		}
   558  	}
   559  	if err := logFile.Close(); err != nil {
   560  		return fmt.Errorf("closing log file: %w", err)
   561  	}
   562  
   563  	if err := CopyFileLocal(tempFile.Name(), logPath, true); err != nil {
   564  		return fmt.Errorf("writing clean logfile: %w", err)
   565  	}
   566  
   567  	return err
   568  }
   569
View as plain text