env.go

Documentation: github.com/bazelbuild/rules_go/go/tools/builders

     1  // Copyright 2017 The Bazel Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"bytes"
    19  	"errors"
    20  	"flag"
    21  	"fmt"
    22  	"io"
    23  	"io/ioutil"
    24  	"log"
    25  	"os"
    26  	"os/exec"
    27  	"path/filepath"
    28  	"runtime"
    29  	"strconv"
    30  	"strings"
    31  )
    32  
    33  var (
    34  	// cgoEnvVars is the list of all cgo environment variable
    35  	cgoEnvVars = []string{"CGO_CFLAGS", "CGO_CXXFLAGS", "CGO_CPPFLAGS", "CGO_LDFLAGS"}
    36  	// cgoAbsEnvFlags are all the flags that need absolute path in cgoEnvVars
    37  	cgoAbsEnvFlags = []string{"-I", "-L", "-isysroot", "-isystem", "-iquote", "-include", "-gcc-toolchain", "--sysroot", "-resource-dir", "-fsanitize-blacklist", "-fsanitize-ignorelist"}
    38  )
    39  
    40  // env holds a small amount of Go environment and toolchain information
    41  // which is common to multiple builders. Most Bazel-agnostic build information
    42  // is collected in go/build.Default though.
    43  //
    44  // See ./README.rst for more information about handling arguments and
    45  // environment variables.
    46  type env struct {
    47  	// sdk is the path to the Go SDK, which contains tools for the host
    48  	// platform. This may be different than GOROOT.
    49  	sdk string
    50  
    51  	// installSuffix is the name of the directory below GOROOT/pkg that contains
    52  	// the .a files for the standard library we should build against.
    53  	// For example, linux_amd64_race.
    54  	installSuffix string
    55  
    56  	// verbose indicates whether subprocess command lines should be printed.
    57  	verbose bool
    58  
    59  	// workDirPath is a temporary work directory. It is created lazily.
    60  	workDirPath string
    61  
    62  	shouldPreserveWorkDir bool
    63  }
    64  
    65  // envFlags registers flags common to multiple builders and returns an env
    66  // configured with those flags.
    67  func envFlags(flags *flag.FlagSet) *env {
    68  	env := &env{}
    69  	flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.")
    70  	flags.Var(&tagFlag{}, "tags", "List of build tags considered true.")
    71  	flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg")
    72  	flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed")
    73  	flags.BoolVar(&env.shouldPreserveWorkDir, "work", false, "if true, the temporary work directory will be preserved")
    74  	return env
    75  }
    76  
    77  // checkFlags checks whether env flags were set to valid values. checkFlags
    78  // should be called after parsing flags.
    79  func (e *env) checkFlags() error {
    80  	if e.sdk == "" {
    81  		return errors.New("-sdk was not set")
    82  	}
    83  	return nil
    84  }
    85  
    86  // workDir returns a path to a temporary work directory. The same directory
    87  // is returned on multiple calls. The caller is responsible for cleaning
    88  // up the work directory by calling cleanup.
    89  func (e *env) workDir() (path string, cleanup func(), err error) {
    90  	if e.workDirPath != "" {
    91  		return e.workDirPath, func() {}, nil
    92  	}
    93  	// Keep the stem "rules_go_work" in sync with reproducible_binary_test.go.
    94  	e.workDirPath, err = ioutil.TempDir("", "rules_go_work-")
    95  	if err != nil {
    96  		return "", func() {}, err
    97  	}
    98  	if e.verbose {
    99  		log.Printf("WORK=%s\n", e.workDirPath)
   100  	}
   101  	if e.shouldPreserveWorkDir {
   102  		cleanup = func() {}
   103  	} else {
   104  		cleanup = func() { os.RemoveAll(e.workDirPath) }
   105  	}
   106  	return e.workDirPath, cleanup, nil
   107  }
   108  
   109  // goTool returns a slice containing the path to an executable at
   110  // $GOROOT/pkg/$GOOS_$GOARCH/$tool and additional arguments.
   111  func (e *env) goTool(tool string, args ...string) []string {
   112  	platform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
   113  	toolPath := filepath.Join(e.sdk, "pkg", "tool", platform, tool)
   114  	if runtime.GOOS == "windows" {
   115  		toolPath += ".exe"
   116  	}
   117  	return append([]string{toolPath}, args...)
   118  }
   119  
   120  // goCmd returns a slice containing the path to the go executable
   121  // and additional arguments.
   122  func (e *env) goCmd(cmd string, args ...string) []string {
   123  	exe := filepath.Join(e.sdk, "bin", "go")
   124  	if runtime.GOOS == "windows" {
   125  		exe += ".exe"
   126  	}
   127  	return append([]string{exe, cmd}, args...)
   128  }
   129  
   130  // runCommand executes a subprocess that inherits stdout, stderr, and the
   131  // environment from this process.
   132  func (e *env) runCommand(args []string) error {
   133  	cmd := exec.Command(args[0], args[1:]...)
   134  	// Redirecting stdout to stderr. This mirrors behavior in the go command:
   135  	// https://go.googlesource.com/go/+/refs/tags/go1.15.2/src/cmd/go/internal/work/exec.go#1958
   136  	buf := &bytes.Buffer{}
   137  	cmd.Stdout = buf
   138  	cmd.Stderr = buf
   139  	err := runAndLogCommand(cmd, e.verbose)
   140  	os.Stderr.Write(relativizePaths(buf.Bytes()))
   141  	return err
   142  }
   143  
   144  // runCommandToFile executes a subprocess and writes stdout/stderr to the given
   145  // writers.
   146  func (e *env) runCommandToFile(out, err io.Writer, args []string) error {
   147  	cmd := exec.Command(args[0], args[1:]...)
   148  	cmd.Stdout = out
   149  	cmd.Stderr = err
   150  	return runAndLogCommand(cmd, e.verbose)
   151  }
   152  
   153  func absEnv(envNameList []string, argList []string) error {
   154  	for _, envName := range envNameList {
   155  		splitedEnv := strings.Fields(os.Getenv(envName))
   156  		absArgs(splitedEnv, argList)
   157  		if err := os.Setenv(envName, strings.Join(splitedEnv, " ")); err != nil {
   158  			return err
   159  		}
   160  	}
   161  	return nil
   162  }
   163  
   164  func runAndLogCommand(cmd *exec.Cmd, verbose bool) error {
   165  	if verbose {
   166  		fmt.Fprintln(os.Stderr, formatCommand(cmd))
   167  	}
   168  	cleanup := passLongArgsInResponseFiles(cmd)
   169  	defer cleanup()
   170  	if err := cmd.Run(); err != nil {
   171  		return fmt.Errorf("error running subcommand %s: %w", formatCommand(cmd), err)
   172  	}
   173  	return nil
   174  }
   175  
   176  // expandParamsFiles looks for arguments in args of the form
   177  // "-param=filename". When it finds these arguments it reads the file "filename"
   178  // and replaces the argument with its content.
   179  // It returns the expanded arguments as well as a bool that is true if any param
   180  // files have been passed.
   181  func expandParamsFiles(args []string) ([]string, bool, error) {
   182  	var paramsIndices []int
   183  	for i, arg := range args {
   184  		if strings.HasPrefix(arg, "-param=") {
   185  			paramsIndices = append(paramsIndices, i)
   186  		}
   187  	}
   188  	if len(paramsIndices) == 0 {
   189  		return args, false, nil
   190  	}
   191  	var expandedArgs []string
   192  	last := 0
   193  	for _, pi := range paramsIndices {
   194  		expandedArgs = append(expandedArgs, args[last:pi]...)
   195  		last = pi + 1
   196  
   197  		fileName := args[pi][len("-param="):]
   198  		fileArgs, err := readParamsFile(fileName)
   199  		if err != nil {
   200  			return nil, true, err
   201  		}
   202  		expandedArgs = append(expandedArgs, fileArgs...)
   203  	}
   204  	expandedArgs = append(expandedArgs, args[last:]...)
   205  	return expandedArgs, true, nil
   206  }
   207  
   208  // readParamsFiles parses a Bazel params file in "shell" format. The file
   209  // should contain one argument per line. Arguments may be quoted with single
   210  // quotes. All characters within quoted strings are interpreted literally
   211  // including newlines and excepting single quotes. Characters outside quoted
   212  // strings may be escaped with a backslash.
   213  func readParamsFile(name string) ([]string, error) {
   214  	data, err := ioutil.ReadFile(name)
   215  	if err != nil {
   216  		return nil, err
   217  	}
   218  
   219  	var args []string
   220  	var arg []byte
   221  	quote := false
   222  	escape := false
   223  	for p := 0; p < len(data); p++ {
   224  		b := data[p]
   225  		switch {
   226  		case escape:
   227  			arg = append(arg, b)
   228  			escape = false
   229  
   230  		case b == '\'':
   231  			quote = !quote
   232  
   233  		case !quote && b == '\\':
   234  			escape = true
   235  
   236  		case !quote && b == '\n':
   237  			args = append(args, string(arg))
   238  			arg = arg[:0]
   239  
   240  		default:
   241  			arg = append(arg, b)
   242  		}
   243  	}
   244  	if quote {
   245  		return nil, fmt.Errorf("unterminated quote")
   246  	}
   247  	if escape {
   248  		return nil, fmt.Errorf("unterminated escape")
   249  	}
   250  	if len(arg) > 0 {
   251  		args = append(args, string(arg))
   252  	}
   253  	return args, nil
   254  }
   255  
   256  // writeParamsFile formats a list of arguments in Bazel's "shell" format and writes
   257  // it to a file.
   258  func writeParamsFile(path string, args []string) error {
   259  	buf := new(bytes.Buffer)
   260  	for _, arg := range args {
   261  		if !strings.ContainsAny(arg, "'\n\\") {
   262  			fmt.Fprintln(buf, arg)
   263  			continue
   264  		}
   265  		buf.WriteByte('\'')
   266  		for _, r := range arg {
   267  			if r == '\'' {
   268  				buf.WriteString(`'\''`)
   269  			} else {
   270  				buf.WriteRune(r)
   271  			}
   272  		}
   273  		buf.WriteString("'\n")
   274  	}
   275  	return ioutil.WriteFile(path, buf.Bytes(), 0666)
   276  }
   277  
   278  // splitArgs splits a list of command line arguments into two parts: arguments
   279  // that should be interpreted by the builder (before "--"), and arguments
   280  // that should be passed through to the underlying tool (after "--").
   281  func splitArgs(args []string) (builderArgs []string, toolArgs []string) {
   282  	for i, arg := range args {
   283  		if arg == "--" {
   284  			return args[:i], args[i+1:]
   285  		}
   286  	}
   287  	return args, nil
   288  }
   289  
   290  // abs returns the absolute representation of path. Some tools/APIs require
   291  // absolute paths to work correctly. Most notably, golang on Windows cannot
   292  // handle relative paths to files whose absolute path is > ~250 chars, while
   293  // it can handle absolute paths. See http://goo.gl/eqeWjm.
   294  //
   295  // Note that strings that begin with "__BAZEL_" are not absolutized. These are
   296  // used on macOS for paths that the compiler wrapper (wrapped_clang) is
   297  // supposed to know about.
   298  func abs(path string) string {
   299  	if strings.HasPrefix(path, "__BAZEL_") {
   300  		return path
   301  	}
   302  
   303  	if abs, err := filepath.Abs(path); err != nil {
   304  		return path
   305  	} else {
   306  		return abs
   307  	}
   308  }
   309  
   310  // absArgs applies abs to strings that appear in args. Only paths that are
   311  // part of options named by flags are modified.
   312  func absArgs(args []string, flags []string) {
   313  	absNext := false
   314  	for i := range args {
   315  		if absNext {
   316  			args[i] = abs(args[i])
   317  			absNext = false
   318  			continue
   319  		}
   320  		for _, f := range flags {
   321  			if !strings.HasPrefix(args[i], f) {
   322  				continue
   323  			}
   324  			possibleValue := args[i][len(f):]
   325  			if len(possibleValue) == 0 {
   326  				absNext = true
   327  				break
   328  			}
   329  			separator := ""
   330  			if possibleValue[0] == '=' {
   331  				possibleValue = possibleValue[1:]
   332  				separator = "="
   333  			}
   334  			args[i] = fmt.Sprintf("%s%s%s", f, separator, abs(possibleValue))
   335  			break
   336  		}
   337  	}
   338  }
   339  
   340  // relativizePaths converts absolute paths found in the given output string to
   341  // relative, if they are within the working directory.
   342  func relativizePaths(output []byte) []byte {
   343  	dir, err := os.Getwd()
   344  	if dir == "" || err != nil {
   345  		return output
   346  	}
   347  	dirBytes := make([]byte, len(dir), len(dir)+1)
   348  	copy(dirBytes, dir)
   349  	if bytes.HasSuffix(dirBytes, []byte{filepath.Separator}) {
   350  		return bytes.ReplaceAll(output, dirBytes, nil)
   351  	}
   352  
   353  	// This is the common case.
   354  	// Replace "$CWD/" with "" and "$CWD" with "."
   355  	dirBytes = append(dirBytes, filepath.Separator)
   356  	output = bytes.ReplaceAll(output, dirBytes, nil)
   357  	dirBytes = dirBytes[:len(dirBytes)-1]
   358  	return bytes.ReplaceAll(output, dirBytes, []byte{'.'})
   359  }
   360  
   361  // formatCommand formats cmd as a string that can be pasted into a shell.
   362  // Spaces in environment variables and arguments are escaped as needed.
   363  func formatCommand(cmd *exec.Cmd) string {
   364  	quoteIfNeeded := func(s string) string {
   365  		if strings.IndexByte(s, ' ') < 0 {
   366  			return s
   367  		}
   368  		return strconv.Quote(s)
   369  	}
   370  	quoteEnvIfNeeded := func(s string) string {
   371  		eq := strings.IndexByte(s, '=')
   372  		if eq < 0 {
   373  			return s
   374  		}
   375  		key, value := s[:eq], s[eq+1:]
   376  		if strings.IndexByte(value, ' ') < 0 {
   377  			return s
   378  		}
   379  		return fmt.Sprintf("%s=%s", key, strconv.Quote(value))
   380  	}
   381  	var w bytes.Buffer
   382  	environ := cmd.Env
   383  	if environ == nil {
   384  		environ = os.Environ()
   385  	}
   386  	for _, e := range environ {
   387  		fmt.Fprintf(&w, "%s \\\n", quoteEnvIfNeeded(e))
   388  	}
   389  
   390  	sep := ""
   391  	for _, arg := range cmd.Args {
   392  		fmt.Fprintf(&w, "%s%s", sep, quoteIfNeeded(arg))
   393  		sep = " "
   394  	}
   395  	return w.String()
   396  }
   397  
   398  // passLongArgsInResponseFiles modifies cmd such that, for
   399  // certain programs, long arguments are passed in "response files", a
   400  // file on disk with the arguments, with one arg per line. An actual
   401  // argument starting with '@' means that the rest of the argument is
   402  // a filename of arguments to expand.
   403  //
   404  // See https://github.com/golang/go/issues/18468 (Windows) and
   405  // https://github.com/golang/go/issues/37768 (Darwin).
   406  func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) {
   407  	cleanup = func() {} // no cleanup by default
   408  	var argLen int
   409  	for _, arg := range cmd.Args {
   410  		argLen += len(arg)
   411  	}
   412  	// If we're not approaching 32KB of args, just pass args normally.
   413  	// (use 30KB instead to be conservative; not sure how accounting is done)
   414  	if !useResponseFile(cmd.Path, argLen) {
   415  		return
   416  	}
   417  	tf, err := ioutil.TempFile("", "args")
   418  	if err != nil {
   419  		log.Fatalf("error writing long arguments to response file: %v", err)
   420  	}
   421  	cleanup = func() { os.Remove(tf.Name()) }
   422  	var buf bytes.Buffer
   423  	for _, arg := range cmd.Args[1:] {
   424  		fmt.Fprintf(&buf, "%s\n", arg)
   425  	}
   426  	if _, err := tf.Write(buf.Bytes()); err != nil {
   427  		tf.Close()
   428  		cleanup()
   429  		log.Fatalf("error writing long arguments to response file: %v", err)
   430  	}
   431  	if err := tf.Close(); err != nil {
   432  		cleanup()
   433  		log.Fatalf("error writing long arguments to response file: %v", err)
   434  	}
   435  	cmd.Args = []string{cmd.Args[0], "@" + tf.Name()}
   436  	return cleanup
   437  }
   438  
   439  // quotePathIfNeeded quotes path if it contains whitespace and isn't already quoted.
   440  // Use this for paths that will be passed through
   441  // https://github.com/golang/go/blob/06264b740e3bfe619f5e90359d8f0d521bd47806/src/cmd/internal/quoted/quoted.go#L25
   442  func quotePathIfNeeded(path string) string {
   443  	if strings.HasPrefix(path, "\"") || strings.HasPrefix(path, "'") {
   444  		// Assume already quoted
   445  		return path
   446  	}
   447  	// https://github.com/golang/go/blob/06264b740e3bfe619f5e90359d8f0d521bd47806/src/cmd/internal/quoted/quoted.go#L16
   448  	if strings.IndexAny(path, " \t\n\r") < 0 {
   449  		// Does not require quoting
   450  		return path
   451  	}
   452  	// Escaping quotes is not supported, so we can assume path doesn't contain any quotes.
   453  	return "'" + path + "'"
   454  }
   455  
   456  func useResponseFile(path string, argLen int) bool {
   457  	// Unless the program uses objabi.Flagparse, which understands
   458  	// response files, don't use response files.
   459  	// TODO: do we need more commands? asm? cgo? For now, no.
   460  	prog := strings.TrimSuffix(filepath.Base(path), ".exe")
   461  	switch prog {
   462  	case "compile", "link":
   463  	default:
   464  		return false
   465  	}
   466  	// Windows has a limit of 32 KB arguments. To be conservative and not
   467  	// worry about whether that includes spaces or not, just use 30 KB.
   468  	// Darwin's limit is less clear. The OS claims 256KB, but we've seen
   469  	// failures with arglen as small as 50KB.
   470  	if argLen > (30 << 10) {
   471  		return true
   472  	}
   473  	return false
   474  }
   475
View as plain text