cgo2.go

Documentation: github.com/bazelbuild/rules_go/go/tools/builders

     1  // Copyright 2019 The Bazel Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // cgo2.go provides new cgo functionality for use by the GoCompilePkg action.
    16  // We can't use the functionality in cgo.go, since it relies too heavily
    17  // on logic in cgo.bzl. Ideally, we'd be able to replace cgo.go with this
    18  // file eventually, but not until Bazel gives us enough toolchain information
    19  // to compile ObjC files.
    20  
    21  package main
    22  
    23  import (
    24  	"bytes"
    25  	"fmt"
    26  	"io"
    27  	"io/ioutil"
    28  	"os"
    29  	"path/filepath"
    30  	"runtime"
    31  	"strings"
    32  )
    33  
    34  // cgo2 processes a set of mixed source files with cgo.
    35  func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs []string, packagePath, packageName string, cc string, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags []string, cgoExportHPath string) (srcDir string, allGoSrcs, cObjs []string, err error) {
    36  	// Report an error if the C/C++ toolchain wasn't configured.
    37  	if cc == "" {
    38  		err := cgoError(cgoSrcs[:])
    39  		err = append(err, cSrcs...)
    40  		err = append(err, cxxSrcs...)
    41  		err = append(err, objcSrcs...)
    42  		err = append(err, objcxxSrcs...)
    43  		err = append(err, sSrcs...)
    44  		return "", nil, nil, err
    45  	}
    46  
    47  	// If we only have C/C++ sources without cgo, just compile and pack them
    48  	// without generating code. The Go command forbids this, but we've
    49  	// historically allowed it.
    50  	// TODO(jayconrod): this doesn't write CGO_LDFLAGS into the archive. We
    51  	// might miss dependencies like -lstdc++ if they aren't referenced in
    52  	// some other way.
    53  	if len(cgoSrcs) == 0 {
    54  		cObjs, err = compileCSources(goenv, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs, cc, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags)
    55  		return ".", nil, cObjs, err
    56  	}
    57  
    58  	workDir, cleanup, err := goenv.workDir()
    59  	if err != nil {
    60  		return "", nil, nil, err
    61  	}
    62  	defer cleanup()
    63  
    64  	// cgo2 will gather sources into a single temporary directory, since nogo
    65  	// scanners might want to include or exclude these sources we need to ensure
    66  	// that a fragment of the path is stable and human friendly enough to be
    67  	// referenced in nogo configuration.
    68  	workDir = filepath.Join(workDir, "cgo", packagePath)
    69  	if err := os.MkdirAll(workDir, 0700); err != nil {
    70  		return "", nil, nil, err
    71  	}
    72  
    73  	// Filter out -lstdc++ and -lc++ from ldflags if we don't have C++ sources,
    74  	// and set CGO_LDFLAGS. These flags get written as special comments into cgo
    75  	// generated sources. The compiler encodes those flags in the compiled .a
    76  	// file, and the linker passes them on to the external linker.
    77  	haveCxx := len(cxxSrcs)+len(objcxxSrcs) > 0
    78  	if !haveCxx {
    79  		for _, f := range ldFlags {
    80  			if strings.HasSuffix(f, ".a") {
    81  				// These flags come from cdeps options. Assume C++.
    82  				haveCxx = true
    83  				break
    84  			}
    85  		}
    86  	}
    87  	var combinedLdFlags []string
    88  	if haveCxx {
    89  		combinedLdFlags = append(combinedLdFlags, ldFlags...)
    90  	} else {
    91  		for _, f := range ldFlags {
    92  			if f != "-lc++" && f != "-lstdc++" {
    93  				combinedLdFlags = append(combinedLdFlags, f)
    94  			}
    95  		}
    96  	}
    97  	combinedLdFlags = append(combinedLdFlags, defaultLdFlags()...)
    98  	os.Setenv("CGO_LDFLAGS", strings.Join(combinedLdFlags, " "))
    99  
   100  	// If cgo sources are in different directories, gather them into a temporary
   101  	// directory so we can use -srcdir.
   102  	srcDir = filepath.Dir(cgoSrcs[0])
   103  	srcsInSingleDir := true
   104  	for _, src := range cgoSrcs[1:] {
   105  		if filepath.Dir(src) != srcDir {
   106  			srcsInSingleDir = false
   107  			break
   108  		}
   109  	}
   110  
   111  	if srcsInSingleDir {
   112  		for i := range cgoSrcs {
   113  			cgoSrcs[i] = filepath.Base(cgoSrcs[i])
   114  		}
   115  	} else {
   116  		srcDir = filepath.Join(workDir, "cgosrcs")
   117  		if err := os.Mkdir(srcDir, 0777); err != nil {
   118  			return "", nil, nil, err
   119  		}
   120  		copiedSrcs, err := gatherSrcs(srcDir, cgoSrcs)
   121  		if err != nil {
   122  			return "", nil, nil, err
   123  		}
   124  		cgoSrcs = copiedSrcs
   125  	}
   126  
   127  	// Generate Go and C code.
   128  	hdrDirs := map[string]bool{}
   129  	var hdrIncludes []string
   130  	for _, hdr := range hSrcs {
   131  		hdrDir := filepath.Dir(hdr)
   132  		if !hdrDirs[hdrDir] {
   133  			hdrDirs[hdrDir] = true
   134  			hdrIncludes = append(hdrIncludes, "-iquote", hdrDir)
   135  		}
   136  	}
   137  	hdrIncludes = append(hdrIncludes, "-iquote", workDir) // for _cgo_export.h
   138  
   139  	execRoot, err := bazelExecRoot()
   140  	if err != nil {
   141  		return "", nil, nil, err
   142  	}
   143  	// Trim the execroot from the //line comments emitted by cgo.
   144  	args := goenv.goTool("cgo", "-srcdir", srcDir, "-objdir", workDir, "-trimpath", execRoot)
   145  	if packagePath != "" {
   146  		args = append(args, "-importpath", packagePath)
   147  	}
   148  	args = append(args, "--")
   149  	args = append(args, cppFlags...)
   150  	args = append(args, hdrIncludes...)
   151  	args = append(args, cFlags...)
   152  	args = append(args, cgoSrcs...)
   153  	if err := goenv.runCommand(args); err != nil {
   154  		return "", nil, nil, err
   155  	}
   156  
   157  	if cgoExportHPath != "" {
   158  		if err := copyFile(filepath.Join(workDir, "_cgo_export.h"), cgoExportHPath); err != nil {
   159  			return "", nil, nil, err
   160  		}
   161  	}
   162  	genGoSrcs := make([]string, 1+len(cgoSrcs))
   163  	genGoSrcs[0] = filepath.Join(workDir, "_cgo_gotypes.go")
   164  	genCSrcs := make([]string, 1+len(cgoSrcs))
   165  	genCSrcs[0] = filepath.Join(workDir, "_cgo_export.c")
   166  	for i, src := range cgoSrcs {
   167  		stem := strings.TrimSuffix(filepath.Base(src), ".go")
   168  		genGoSrcs[i+1] = filepath.Join(workDir, stem+".cgo1.go")
   169  		genCSrcs[i+1] = filepath.Join(workDir, stem+".cgo2.c")
   170  	}
   171  	cgoMainC := filepath.Join(workDir, "_cgo_main.c")
   172  
   173  	// Compile C, C++, Objective-C/C++, and assembly code.
   174  	defaultCFlags := defaultCFlags(workDir)
   175  	combinedCFlags := combineFlags(cppFlags, hdrIncludes, cFlags, defaultCFlags)
   176  	for _, lang := range []struct{ srcs, flags []string }{
   177  		{genCSrcs, combinedCFlags},
   178  		{cSrcs, combinedCFlags},
   179  		{cxxSrcs, combineFlags(cppFlags, hdrIncludes, cxxFlags, defaultCFlags)},
   180  		{objcSrcs, combineFlags(cppFlags, hdrIncludes, objcFlags, defaultCFlags)},
   181  		{objcxxSrcs, combineFlags(cppFlags, hdrIncludes, objcxxFlags, defaultCFlags)},
   182  		{sSrcs, nil},
   183  	} {
   184  		for _, src := range lang.srcs {
   185  			obj := filepath.Join(workDir, fmt.Sprintf("_x%d.o", len(cObjs)))
   186  			cObjs = append(cObjs, obj)
   187  			if err := cCompile(goenv, src, cc, lang.flags, obj); err != nil {
   188  				return "", nil, nil, err
   189  			}
   190  		}
   191  	}
   192  
   193  	mainObj := filepath.Join(workDir, "_cgo_main.o")
   194  	if err := cCompile(goenv, cgoMainC, cc, combinedCFlags, mainObj); err != nil {
   195  		return "", nil, nil, err
   196  	}
   197  
   198  	// Link cgo binary and use the symbols to generate _cgo_import.go.
   199  	mainBin := filepath.Join(workDir, "_cgo_.o") // .o is a lie; it's an executable
   200  	args = append([]string{cc, "-o", mainBin, mainObj}, cObjs...)
   201  	args = append(args, combinedLdFlags...)
   202  	var originalErrBuf bytes.Buffer
   203  	if err := goenv.runCommandToFile(os.Stdout, &originalErrBuf, args); err != nil {
   204  		// If linking the binary for cgo fails, this is usually because the
   205  		// object files reference external symbols that can't be resolved yet.
   206  		// Since the binary is only produced to have its symbols read by the cgo
   207  		// command, there is no harm in trying to build it allowing unresolved
   208  		// symbols - the real link that happens at the end will fail if they
   209  		// rightfully can't be resolved.
   210  		var allowUnresolvedSymbolsLdFlag string
   211  		switch os.Getenv("GOOS") {
   212  		case "windows":
   213  			// MinGW's linker doesn't seem to support --unresolved-symbols
   214  			// and MSVC isn't supported at all.
   215  			return "", nil, nil, err
   216  		case "darwin", "ios":
   217  			allowUnresolvedSymbolsLdFlag = "-Wl,-undefined,dynamic_lookup"
   218  		default:
   219  			allowUnresolvedSymbolsLdFlag = "-Wl,--unresolved-symbols=ignore-all"
   220  		}
   221  		// Print and return the original error if we can't link the binary with
   222  		// the additional linker flags as they may simply be incorrect for the
   223  		// particular compiler/linker pair and would obscure the true reason for
   224  		// the failure of the original command.
   225  		if err2 := goenv.runCommandToFile(
   226  			os.Stdout,
   227  			ioutil.Discard,
   228  			append(args, allowUnresolvedSymbolsLdFlag),
   229  		); err2 != nil {
   230  			os.Stderr.Write(relativizePaths(originalErrBuf.Bytes()))
   231  			return "", nil, nil, err
   232  		}
   233  		// Do not print the original error - rerunning the command with the
   234  		// additional linker flag fixed it.
   235  	}
   236  
   237  	cgoImportsGo := filepath.Join(workDir, "_cgo_imports.go")
   238  	args = goenv.goTool("cgo", "-dynpackage", packageName, "-dynimport", mainBin, "-dynout", cgoImportsGo)
   239  	if err := goenv.runCommand(args); err != nil {
   240  		return "", nil, nil, err
   241  	}
   242  	genGoSrcs = append(genGoSrcs, cgoImportsGo)
   243  
   244  	// Copy regular Go source files into the work directory so that we can
   245  	// use -trimpath=workDir.
   246  	goBases, err := gatherSrcs(workDir, goSrcs)
   247  	if err != nil {
   248  		return "", nil, nil, err
   249  	}
   250  
   251  	allGoSrcs = make([]string, len(goSrcs)+len(genGoSrcs))
   252  	for i := range goSrcs {
   253  		allGoSrcs[i] = filepath.Join(workDir, goBases[i])
   254  	}
   255  	copy(allGoSrcs[len(goSrcs):], genGoSrcs)
   256  	return workDir, allGoSrcs, cObjs, nil
   257  }
   258  
   259  // compileCSources compiles a list of C, C++, Objective-C, Objective-C++,
   260  // and assembly sources into .o files to be packed into the archive.
   261  // It does not run cgo. This is used for packages with "cgo = True" but
   262  // without any .go files that import "C". The Go command forbids this,
   263  // but we have historically allowed it.
   264  func compileCSources(goenv *env, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs []string, cc string, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags []string) (cObjs []string, err error) {
   265  	workDir, cleanup, err := goenv.workDir()
   266  	if err != nil {
   267  		return nil, err
   268  	}
   269  	defer cleanup()
   270  
   271  	hdrDirs := map[string]bool{}
   272  	var hdrIncludes []string
   273  	for _, hdr := range hSrcs {
   274  		hdrDir := filepath.Dir(hdr)
   275  		if !hdrDirs[hdrDir] {
   276  			hdrDirs[hdrDir] = true
   277  			hdrIncludes = append(hdrIncludes, "-iquote", hdrDir)
   278  		}
   279  	}
   280  
   281  	defaultCFlags := defaultCFlags(workDir)
   282  	for _, lang := range []struct{ srcs, flags []string }{
   283  		{cSrcs, combineFlags(cppFlags, hdrIncludes, cFlags, defaultCFlags)},
   284  		{cxxSrcs, combineFlags(cppFlags, hdrIncludes, cxxFlags, defaultCFlags)},
   285  		{objcSrcs, combineFlags(cppFlags, hdrIncludes, objcFlags, defaultCFlags)},
   286  		{objcxxSrcs, combineFlags(cppFlags, hdrIncludes, objcxxFlags, defaultCFlags)},
   287  		{sSrcs, nil},
   288  	} {
   289  		for _, src := range lang.srcs {
   290  			obj := filepath.Join(workDir, fmt.Sprintf("_x%d.o", len(cObjs)))
   291  			cObjs = append(cObjs, obj)
   292  			if err := cCompile(goenv, src, cc, lang.flags, obj); err != nil {
   293  				return nil, err
   294  			}
   295  		}
   296  	}
   297  	return cObjs, nil
   298  }
   299  
   300  func combineFlags(lists ...[]string) []string {
   301  	n := 0
   302  	for _, list := range lists {
   303  		n += len(list)
   304  	}
   305  	flags := make([]string, 0, n)
   306  	for _, list := range lists {
   307  		flags = append(flags, list...)
   308  	}
   309  	return flags
   310  }
   311  
   312  func cCompile(goenv *env, src, cc string, flags []string, out string) error {
   313  	args := []string{cc}
   314  	args = append(args, flags...)
   315  	args = append(args, "-c", src, "-o", out)
   316  	return goenv.runCommand(args)
   317  }
   318  
   319  func defaultCFlags(workDir string) []string {
   320  	flags := []string{
   321  		"-fdebug-prefix-map=" + abs(".") + "=.",
   322  		"-fdebug-prefix-map=" + workDir + "=.",
   323  	}
   324  	goos, goarch := os.Getenv("GOOS"), os.Getenv("GOARCH")
   325  	switch {
   326  	case goos == "darwin" || goos == "ios":
   327  		return flags
   328  	case goos == "windows" && goarch == "amd64":
   329  		return append(flags, "-mthreads")
   330  	default:
   331  		return append(flags, "-pthread")
   332  	}
   333  }
   334  
   335  func defaultLdFlags() []string {
   336  	goos, goarch := os.Getenv("GOOS"), os.Getenv("GOARCH")
   337  	switch {
   338  	case goos == "android":
   339  		return []string{"-llog", "-ldl"}
   340  	case goos == "darwin" || goos == "ios":
   341  		return nil
   342  	case goos == "windows" && goarch == "amd64":
   343  		return []string{"-mthreads"}
   344  	default:
   345  		return []string{"-pthread"}
   346  	}
   347  }
   348  
   349  // gatherSrcs copies or links files listed in srcs into dir. This is needed
   350  // to effectively use -trimpath with generated sources. It's also needed by cgo.
   351  //
   352  // gatherSrcs returns the basenames of copied files in the directory.
   353  func gatherSrcs(dir string, srcs []string) ([]string, error) {
   354  	copiedBases := make([]string, len(srcs))
   355  	for i, src := range srcs {
   356  		base := filepath.Base(src)
   357  		ext := filepath.Ext(base)
   358  		stem := base[:len(base)-len(ext)]
   359  		var err error
   360  		for j := 1; j < 10000; j++ {
   361  			if err = copyOrLinkFile(src, filepath.Join(dir, base)); err == nil {
   362  				break
   363  			} else if !os.IsExist(err) {
   364  				return nil, err
   365  			} else {
   366  				base = fmt.Sprintf("%s_%d%s", stem, j, ext)
   367  			}
   368  		}
   369  		if err != nil {
   370  			return nil, fmt.Errorf("could not find unique name for file %s", src)
   371  		}
   372  		copiedBases[i] = base
   373  	}
   374  	return copiedBases, nil
   375  }
   376  
   377  func bazelExecRoot() (string, error) {
   378  	// Bazel executes the builder with a working directory of the form
   379  	// .../execroot/<workspace name>. By stripping the last segment, we obtain a
   380  	// prefix of all possible source files, even when contained in external
   381  	// repositories.
   382  	cwd, err := os.Getwd()
   383  	if err != nil {
   384  		return "", err
   385  	}
   386  	return filepath.Dir(cwd), nil
   387  }
   388  
   389  type cgoError []string
   390  
   391  func (e cgoError) Error() string {
   392  	b := &bytes.Buffer{}
   393  	fmt.Fprint(b, "CC is not set and files need to be processed with cgo:\n")
   394  	for _, f := range e {
   395  		fmt.Fprintf(b, "\t%s\n", f)
   396  	}
   397  	fmt.Fprintf(b, "Ensure that 'cgo = True' is set and the C/C++ toolchain is configured.")
   398  	return b.String()
   399  }
   400  
   401  func copyFile(inPath, outPath string) error {
   402  	inFile, err := os.Open(inPath)
   403  	if err != nil {
   404  		return err
   405  	}
   406  	defer inFile.Close()
   407  	outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
   408  	if err != nil {
   409  		return err
   410  	}
   411  	defer outFile.Close()
   412  	_, err = io.Copy(outFile, inFile)
   413  	return err
   414  }
   415  
   416  func linkFile(inPath, outPath string) error {
   417  	inPath, err := filepath.Abs(inPath)
   418  	if err != nil {
   419  		return err
   420  	}
   421  	return os.Symlink(inPath, outPath)
   422  }
   423  
   424  func copyOrLinkFile(inPath, outPath string) error {
   425  	if runtime.GOOS == "windows" {
   426  		return copyFile(inPath, outPath)
   427  	} else {
   428  		return linkFile(inPath, outPath)
   429  	}
   430  }
   431
View as plain text