stdliblist.go

Documentation: github.com/bazelbuild/rules_go/go/tools/builders

     1  // Copyright 2021 The Bazel Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/json"
    20  	"flag"
    21  	"fmt"
    22  	"go/build"
    23  	"os"
    24  	"path/filepath"
    25  	"strings"
    26  )
    27  
    28  // Copy and pasted from golang.org/x/tools/go/packages
    29  type flatPackagesError struct {
    30  	Pos  string // "file:line:col" or "file:line" or "" or "-"
    31  	Msg  string
    32  	Kind flatPackagesErrorKind
    33  }
    34  
    35  type flatPackagesErrorKind int
    36  
    37  const (
    38  	UnknownError flatPackagesErrorKind = iota
    39  	ListError
    40  	ParseError
    41  	TypeError
    42  )
    43  
    44  func (err flatPackagesError) Error() string {
    45  	pos := err.Pos
    46  	if pos == "" {
    47  		pos = "-" // like token.Position{}.String()
    48  	}
    49  	return pos + ": " + err.Msg
    50  }
    51  
    52  // flatPackage is the JSON form of Package
    53  // It drops all the type and syntax fields, and transforms the Imports
    54  type flatPackage struct {
    55  	ID              string
    56  	Name            string              `json:",omitempty"`
    57  	PkgPath         string              `json:",omitempty"`
    58  	Standard        bool                `json:",omitempty"`
    59  	Errors          []flatPackagesError `json:",omitempty"`
    60  	GoFiles         []string            `json:",omitempty"`
    61  	CompiledGoFiles []string            `json:",omitempty"`
    62  	OtherFiles      []string            `json:",omitempty"`
    63  	ExportFile      string              `json:",omitempty"`
    64  	Imports         map[string]string   `json:",omitempty"`
    65  }
    66  
    67  type goListPackage struct {
    68  	Dir        string // directory containing package sources
    69  	ImportPath string // import path of package in dir
    70  	Name       string // package name
    71  	Target     string // install path
    72  	Goroot     bool   // is this package in the Go root?
    73  	Standard   bool   // is this package part of the standard Go library?
    74  	Root       string // Go root or Go path dir containing this package
    75  	Export     string // file containing export data (when using -export)
    76  	// Source files
    77  	GoFiles           []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
    78  	CgoFiles          []string // .go source files that import "C"
    79  	CompiledGoFiles   []string // .go files presented to compiler (when using -compiled)
    80  	IgnoredGoFiles    []string // .go source files ignored due to build constraints
    81  	IgnoredOtherFiles []string // non-.go source files ignored due to build constraints
    82  	CFiles            []string // .c source files
    83  	CXXFiles          []string // .cc, .cxx and .cpp source files
    84  	MFiles            []string // .m source files
    85  	HFiles            []string // .h, .hh, .hpp and .hxx source files
    86  	FFiles            []string // .f, .F, .for and .f90 Fortran source files
    87  	SFiles            []string // .s source files
    88  	SwigFiles         []string // .swig files
    89  	SwigCXXFiles      []string // .swigcxx files
    90  	SysoFiles         []string // .syso object files to add to archive
    91  	TestGoFiles       []string // _test.go files in package
    92  	XTestGoFiles      []string // _test.go files outside package
    93  	// Embedded files
    94  	EmbedPatterns      []string // //go:embed patterns
    95  	EmbedFiles         []string // files matched by EmbedPatterns
    96  	TestEmbedPatterns  []string // //go:embed patterns in TestGoFiles
    97  	TestEmbedFiles     []string // files matched by TestEmbedPatterns
    98  	XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles
    99  	XTestEmbedFiles    []string // files matched by XTestEmbedPatterns
   100  	// Dependency information
   101  	Imports   []string          // import paths used by this package
   102  	ImportMap map[string]string // map from source import to ImportPath (identity entries omitted)
   103  	// Error information
   104  	Incomplete bool                 // this package or a dependency has an error
   105  	Error      *flatPackagesError   // error loading package
   106  	DepsErrors []*flatPackagesError // errors loading dependencies
   107  }
   108  
   109  var rulesGoStdlibPrefix string
   110  
   111  func init() {
   112  	if rulesGoStdlibPrefix == "" {
   113  		panic("rulesGoStdlibPrefix should have been set via -X")
   114  	}
   115  }
   116  
   117  func stdlibPackageID(importPath string) string {
   118  	return rulesGoStdlibPrefix + importPath
   119  }
   120  
   121  // outputBasePath replace the cloneBase with output base label
   122  func outputBasePath(cloneBase, p string) string {
   123  	dir, _ := filepath.Rel(cloneBase, p)
   124  	return filepath.Join("__BAZEL_OUTPUT_BASE__", dir)
   125  }
   126  
   127  // absoluteSourcesPaths replace cloneBase of the absolution
   128  // paths with the label for all source files in a package
   129  func absoluteSourcesPaths(cloneBase, pkgDir string, srcs []string) []string {
   130  	ret := make([]string, 0, len(srcs))
   131  	pkgDir = outputBasePath(cloneBase, pkgDir)
   132  	for _, src := range srcs {
   133  		absPath := src
   134  
   135  		// Generated files will already have an absolute path. These come from
   136  		// the compiler's cache.
   137  		if !filepath.IsAbs(src) {
   138  			absPath = filepath.Join(pkgDir, src)
   139  		}
   140  
   141  		ret = append(ret, absPath)
   142  	}
   143  	return ret
   144  }
   145  
   146  // filterGoFiles keeps only files either ending in .go or those without an
   147  // extension (which are from the cache). This is a work around for
   148  // https://golang.org/issue/28749: cmd/go puts assembly, C, and C++ files in
   149  // CompiledGoFiles.
   150  func filterGoFiles(srcs []string, pathReplaceFn func(p string) string) []string {
   151  	ret := make([]string, 0, len(srcs))
   152  	for _, f := range srcs {
   153  		if ext := filepath.Ext(f); ext == ".go" || ext == "" {
   154  			ret = append(ret, pathReplaceFn(f))
   155  		}
   156  	}
   157  
   158  	return ret
   159  }
   160  
   161  func flatPackageForStd(cloneBase string, pkg *goListPackage, pathReplaceFn func(p string) string) *flatPackage {
   162  	goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles)
   163  	compiledGoFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.CompiledGoFiles)
   164  
   165  	newPkg := &flatPackage{
   166  		ID:              stdlibPackageID(pkg.ImportPath),
   167  		Name:            pkg.Name,
   168  		PkgPath:         pkg.ImportPath,
   169  		ExportFile:      outputBasePath(cloneBase, pkg.Target),
   170  		Imports:         map[string]string{},
   171  		Standard:        pkg.Standard,
   172  		GoFiles:         goFiles,
   173  		CompiledGoFiles: filterGoFiles(compiledGoFiles, pathReplaceFn),
   174  	}
   175  
   176  	// imports
   177  	//
   178  	// Imports contains the IDs of all imported packages.
   179  	// ImportsMap records (path, ID) only where they differ.
   180  	ids := make(map[string]struct{})
   181  	for _, id := range pkg.Imports {
   182  		ids[id] = struct{}{}
   183  	}
   184  
   185  	for path, id := range pkg.ImportMap {
   186  		newPkg.Imports[path] = stdlibPackageID(id)
   187  		delete(ids, id)
   188  	}
   189  
   190  	for id := range ids {
   191  		if id != "C" {
   192  			newPkg.Imports[id] = stdlibPackageID(id)
   193  		}
   194  	}
   195  
   196  	return newPkg
   197  }
   198  
   199  // stdliblist runs `go list -json` on the standard library and saves it to a file.
   200  func stdliblist(args []string) error {
   201  	// process the args
   202  	flags := flag.NewFlagSet("stdliblist", flag.ExitOnError)
   203  	goenv := envFlags(flags)
   204  	out := flags.String("out", "", "Path to output go list json")
   205  	cachePath := flags.String("cache", "", "Path to use for GOCACHE")
   206  	if err := flags.Parse(args); err != nil {
   207  		return err
   208  	}
   209  	if err := goenv.checkFlags(); err != nil {
   210  		return err
   211  	}
   212  
   213  	if filepath.IsAbs(goenv.sdk) {
   214  		return fmt.Errorf("-sdk needs to be a relative path, but got %s", goenv.sdk)
   215  	}
   216  
   217  	// In Go 1.18, the standard library started using go:embed directives.
   218  	// When Bazel runs this action, it does so inside a sandbox where GOROOT points
   219  	// to an external/go_sdk directory that contains a symlink farm of all files in
   220  	// the Go SDK.
   221  	// If we run "go list" with that GOROOT, this action will fail because those
   222  	// go:embed directives will refuse to include the symlinks in the sandbox.
   223  	//
   224  	// To work around this, cloneGoRoot creates a copy of a subset of external/go_sdk
   225  	// that is sufficient to call "go list" into a new cloneBase directory, e.g.
   226  	// "go list" needs to call "compile", which needs "pkg/tool".
   227  	// We also need to retain the same relative path to the root directory, e.g.
   228  	// "$OUTPUT_BASE/external/go_sdk" becomes
   229  	// {cloneBase}/external/go_sdk", which will be set at GOROOT later. This ensures
   230  	// that file paths in the generated JSON are still valid.
   231  	//
   232  	// Here we replicate goRoot(absolute path of goenv.sdk) to newGoRoot.
   233  	cloneBase, cleanup, err := goenv.workDir()
   234  	if err != nil {
   235  		return err
   236  	}
   237  	defer func() { cleanup() }()
   238  
   239  	newGoRoot := filepath.Join(cloneBase, goenv.sdk)
   240  	if err := replicate(abs(goenv.sdk), abs(newGoRoot), replicatePaths("src", "pkg/tool", "pkg/include")); err != nil {
   241  		return err
   242  	}
   243  
   244  	// Ensure paths are absolute.
   245  	absPaths := []string{}
   246  	for _, path := range filepath.SplitList(os.Getenv("PATH")) {
   247  		absPaths = append(absPaths, abs(path))
   248  	}
   249  	os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator)))
   250  	os.Setenv("GOROOT", newGoRoot)
   251  
   252  	cgoEnabled := os.Getenv("CGO_ENABLED") == "1"
   253  	// Make sure we have an absolute path to the C compiler.
   254  	ccEnv, ok := os.LookupEnv("CC")
   255  	if cgoEnabled && !ok {
   256  		return fmt.Errorf("CC must be set")
   257  	}
   258  	os.Setenv("CC", quotePathIfNeeded(abs(ccEnv)))
   259  
   260  	// Modify CGO flags to use only absolute path
   261  	// because go is having its own sandbox, all CGO flags must use absolute path
   262  	if err := absEnv(cgoEnvVars, cgoAbsEnvFlags); err != nil {
   263  		return fmt.Errorf("error modifying cgo environment to absolute path: %v", err)
   264  	}
   265  
   266  	// We want to keep the cache around so that the processed files can be used by other tools.
   267  	absCachePath := abs(*cachePath)
   268  	os.Setenv("GOCACHE", absCachePath)
   269  	os.Setenv("GOMODCACHE", absCachePath)
   270  	os.Setenv("GOPATH", absCachePath)
   271  
   272  	listArgs := goenv.goCmd("list")
   273  	if len(build.Default.BuildTags) > 0 {
   274  		listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ","))
   275  	}
   276  
   277  	if cgoEnabled {
   278  		listArgs = append(listArgs, "-compiled=true")
   279  	}
   280  
   281  	listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo")
   282  
   283  	jsonFile, err := os.Create(*out)
   284  	if err != nil {
   285  		return err
   286  	}
   287  	defer jsonFile.Close()
   288  
   289  	jsonData := &bytes.Buffer{}
   290  	if err := goenv.runCommandToFile(jsonData, os.Stderr, listArgs); err != nil {
   291  		return err
   292  	}
   293  
   294  	encoder := json.NewEncoder(jsonFile)
   295  	decoder := json.NewDecoder(jsonData)
   296  	pathReplaceFn := func(s string) string {
   297  		if strings.HasPrefix(s, absCachePath) {
   298  			return strings.Replace(s, absCachePath, filepath.Join("__BAZEL_EXECROOT__", *cachePath), 1)
   299  		}
   300  
   301  		return s
   302  	}
   303  	for decoder.More() {
   304  		var pkg *goListPackage
   305  		if err := decoder.Decode(&pkg); err != nil {
   306  			return err
   307  		}
   308  		if err := encoder.Encode(flatPackageForStd(cloneBase, pkg, pathReplaceFn)); err != nil {
   309  			return err
   310  		}
   311  	}
   312  
   313  	return nil
   314  }
   315
View as plain text