Source file src/golang.org/x/tools/go/packages/golist.go

Documentation: golang.org/x/tools/go/packages

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package packages
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/json"
    11  	"fmt"
    12  	"log"
    13  	"os"
    14  	"os/exec"
    15  	"path"
    16  	"path/filepath"
    17  	"reflect"
    18  	"sort"
    19  	"strconv"
    20  	"strings"
    21  	"sync"
    22  	"unicode"
    23  
    24  	"golang.org/x/tools/go/internal/packagesdriver"
    25  	"golang.org/x/tools/internal/gocommand"
    26  	"golang.org/x/tools/internal/packagesinternal"
    27  )
    28  
    29  // debug controls verbose logging.
    30  var debug, _ = strconv.ParseBool(os.Getenv("GOPACKAGESDEBUG"))
    31  
    32  // A goTooOldError reports that the go command
    33  // found by exec.LookPath is too old to use the new go list behavior.
    34  type goTooOldError struct {
    35  	error
    36  }
    37  
    38  // responseDeduper wraps a DriverResponse, deduplicating its contents.
    39  type responseDeduper struct {
    40  	seenRoots    map[string]bool
    41  	seenPackages map[string]*Package
    42  	dr           *DriverResponse
    43  }
    44  
    45  func newDeduper() *responseDeduper {
    46  	return &responseDeduper{
    47  		dr:           &DriverResponse{},
    48  		seenRoots:    map[string]bool{},
    49  		seenPackages: map[string]*Package{},
    50  	}
    51  }
    52  
    53  // addAll fills in r with a DriverResponse.
    54  func (r *responseDeduper) addAll(dr *DriverResponse) {
    55  	for _, pkg := range dr.Packages {
    56  		r.addPackage(pkg)
    57  	}
    58  	for _, root := range dr.Roots {
    59  		r.addRoot(root)
    60  	}
    61  	r.dr.GoVersion = dr.GoVersion
    62  }
    63  
    64  func (r *responseDeduper) addPackage(p *Package) {
    65  	if r.seenPackages[p.ID] != nil {
    66  		return
    67  	}
    68  	r.seenPackages[p.ID] = p
    69  	r.dr.Packages = append(r.dr.Packages, p)
    70  }
    71  
    72  func (r *responseDeduper) addRoot(id string) {
    73  	if r.seenRoots[id] {
    74  		return
    75  	}
    76  	r.seenRoots[id] = true
    77  	r.dr.Roots = append(r.dr.Roots, id)
    78  }
    79  
    80  type golistState struct {
    81  	cfg *Config
    82  	ctx context.Context
    83  
    84  	envOnce    sync.Once
    85  	goEnvError error
    86  	goEnv      map[string]string
    87  
    88  	rootsOnce     sync.Once
    89  	rootDirsError error
    90  	rootDirs      map[string]string
    91  
    92  	goVersionOnce  sync.Once
    93  	goVersionError error
    94  	goVersion      int // The X in Go 1.X.
    95  
    96  	// vendorDirs caches the (non)existence of vendor directories.
    97  	vendorDirs map[string]bool
    98  }
    99  
   100  // getEnv returns Go environment variables. Only specific variables are
   101  // populated -- computing all of them is slow.
   102  func (state *golistState) getEnv() (map[string]string, error) {
   103  	state.envOnce.Do(func() {
   104  		var b *bytes.Buffer
   105  		b, state.goEnvError = state.invokeGo("env", "-json", "GOMOD", "GOPATH")
   106  		if state.goEnvError != nil {
   107  			return
   108  		}
   109  
   110  		state.goEnv = make(map[string]string)
   111  		decoder := json.NewDecoder(b)
   112  		if state.goEnvError = decoder.Decode(&state.goEnv); state.goEnvError != nil {
   113  			return
   114  		}
   115  	})
   116  	return state.goEnv, state.goEnvError
   117  }
   118  
   119  // mustGetEnv is a convenience function that can be used if getEnv has already succeeded.
   120  func (state *golistState) mustGetEnv() map[string]string {
   121  	env, err := state.getEnv()
   122  	if err != nil {
   123  		panic(fmt.Sprintf("mustGetEnv: %v", err))
   124  	}
   125  	return env
   126  }
   127  
   128  // goListDriver uses the go list command to interpret the patterns and produce
   129  // the build system package structure.
   130  // See driver for more details.
   131  func goListDriver(cfg *Config, patterns ...string) (_ *DriverResponse, err error) {
   132  	// Make sure that any asynchronous go commands are killed when we return.
   133  	parentCtx := cfg.Context
   134  	if parentCtx == nil {
   135  		parentCtx = context.Background()
   136  	}
   137  	ctx, cancel := context.WithCancel(parentCtx)
   138  	defer cancel()
   139  
   140  	response := newDeduper()
   141  
   142  	state := &golistState{
   143  		cfg:        cfg,
   144  		ctx:        ctx,
   145  		vendorDirs: map[string]bool{},
   146  	}
   147  
   148  	// Fill in response.Sizes asynchronously if necessary.
   149  	if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 {
   150  		errCh := make(chan error)
   151  		go func() {
   152  			compiler, arch, err := packagesdriver.GetSizesForArgsGolist(ctx, state.cfgInvocation(), cfg.gocmdRunner)
   153  			response.dr.Compiler = compiler
   154  			response.dr.Arch = arch
   155  			errCh <- err
   156  		}()
   157  		defer func() {
   158  			if sizesErr := <-errCh; sizesErr != nil {
   159  				err = sizesErr
   160  			}
   161  		}()
   162  	}
   163  
   164  	// Determine files requested in contains patterns
   165  	var containFiles []string
   166  	restPatterns := make([]string, 0, len(patterns))
   167  	// Extract file= and other [querytype]= patterns. Report an error if querytype
   168  	// doesn't exist.
   169  extractQueries:
   170  	for _, pattern := range patterns {
   171  		eqidx := strings.Index(pattern, "=")
   172  		if eqidx < 0 {
   173  			restPatterns = append(restPatterns, pattern)
   174  		} else {
   175  			query, value := pattern[:eqidx], pattern[eqidx+len("="):]
   176  			switch query {
   177  			case "file":
   178  				containFiles = append(containFiles, value)
   179  			case "pattern":
   180  				restPatterns = append(restPatterns, value)
   181  			case "": // not a reserved query
   182  				restPatterns = append(restPatterns, pattern)
   183  			default:
   184  				for _, rune := range query {
   185  					if rune < 'a' || rune > 'z' { // not a reserved query
   186  						restPatterns = append(restPatterns, pattern)
   187  						continue extractQueries
   188  					}
   189  				}
   190  				// Reject all other patterns containing "="
   191  				return nil, fmt.Errorf("invalid query type %q in query pattern %q", query, pattern)
   192  			}
   193  		}
   194  	}
   195  
   196  	// See if we have any patterns to pass through to go list. Zero initial
   197  	// patterns also requires a go list call, since it's the equivalent of
   198  	// ".".
   199  	if len(restPatterns) > 0 || len(patterns) == 0 {
   200  		dr, err := state.createDriverResponse(restPatterns...)
   201  		if err != nil {
   202  			return nil, err
   203  		}
   204  		response.addAll(dr)
   205  	}
   206  
   207  	if len(containFiles) != 0 {
   208  		if err := state.runContainsQueries(response, containFiles); err != nil {
   209  			return nil, err
   210  		}
   211  	}
   212  
   213  	// (We may yet return an error due to defer.)
   214  	return response.dr, nil
   215  }
   216  
   217  func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error {
   218  	for _, query := range queries {
   219  		// TODO(matloob): Do only one query per directory.
   220  		fdir := filepath.Dir(query)
   221  		// Pass absolute path of directory to go list so that it knows to treat it as a directory,
   222  		// not a package path.
   223  		pattern, err := filepath.Abs(fdir)
   224  		if err != nil {
   225  			return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
   226  		}
   227  		dirResponse, err := state.createDriverResponse(pattern)
   228  
   229  		// If there was an error loading the package, or no packages are returned,
   230  		// or the package is returned with errors, try to load the file as an
   231  		// ad-hoc package.
   232  		// Usually the error will appear in a returned package, but may not if we're
   233  		// in module mode and the ad-hoc is located outside a module.
   234  		if err != nil || len(dirResponse.Packages) == 0 || len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].GoFiles) == 0 &&
   235  			len(dirResponse.Packages[0].Errors) == 1 {
   236  			var queryErr error
   237  			if dirResponse, queryErr = state.adhocPackage(pattern, query); queryErr != nil {
   238  				return err // return the original error
   239  			}
   240  		}
   241  		isRoot := make(map[string]bool, len(dirResponse.Roots))
   242  		for _, root := range dirResponse.Roots {
   243  			isRoot[root] = true
   244  		}
   245  		for _, pkg := range dirResponse.Packages {
   246  			// Add any new packages to the main set
   247  			// We don't bother to filter packages that will be dropped by the changes of roots,
   248  			// that will happen anyway during graph construction outside this function.
   249  			// Over-reporting packages is not a problem.
   250  			response.addPackage(pkg)
   251  			// if the package was not a root one, it cannot have the file
   252  			if !isRoot[pkg.ID] {
   253  				continue
   254  			}
   255  			for _, pkgFile := range pkg.GoFiles {
   256  				if filepath.Base(query) == filepath.Base(pkgFile) {
   257  					response.addRoot(pkg.ID)
   258  					break
   259  				}
   260  			}
   261  		}
   262  	}
   263  	return nil
   264  }
   265  
   266  // adhocPackage attempts to load or construct an ad-hoc package for a given
   267  // query, if the original call to the driver produced inadequate results.
   268  func (state *golistState) adhocPackage(pattern, query string) (*DriverResponse, error) {
   269  	response, err := state.createDriverResponse(query)
   270  	if err != nil {
   271  		return nil, err
   272  	}
   273  	// If we get nothing back from `go list`,
   274  	// try to make this file into its own ad-hoc package.
   275  	// TODO(rstambler): Should this check against the original response?
   276  	if len(response.Packages) == 0 {
   277  		response.Packages = append(response.Packages, &Package{
   278  			ID:              "command-line-arguments",
   279  			PkgPath:         query,
   280  			GoFiles:         []string{query},
   281  			CompiledGoFiles: []string{query},
   282  			Imports:         make(map[string]*Package),
   283  		})
   284  		response.Roots = append(response.Roots, "command-line-arguments")
   285  	}
   286  	// Handle special cases.
   287  	if len(response.Packages) == 1 {
   288  		// golang/go#33482: If this is a file= query for ad-hoc packages where
   289  		// the file only exists on an overlay, and exists outside of a module,
   290  		// add the file to the package and remove the errors.
   291  		if response.Packages[0].ID == "command-line-arguments" ||
   292  			filepath.ToSlash(response.Packages[0].PkgPath) == filepath.ToSlash(query) {
   293  			if len(response.Packages[0].GoFiles) == 0 {
   294  				filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
   295  				// TODO(matloob): check if the file is outside of a root dir?
   296  				for path := range state.cfg.Overlay {
   297  					if path == filename {
   298  						response.Packages[0].Errors = nil
   299  						response.Packages[0].GoFiles = []string{path}
   300  						response.Packages[0].CompiledGoFiles = []string{path}
   301  					}
   302  				}
   303  			}
   304  		}
   305  	}
   306  	return response, nil
   307  }
   308  
   309  // Fields must match go list;
   310  // see $GOROOT/src/cmd/go/internal/load/pkg.go.
   311  type jsonPackage struct {
   312  	ImportPath        string
   313  	Dir               string
   314  	Name              string
   315  	Export            string
   316  	GoFiles           []string
   317  	CompiledGoFiles   []string
   318  	IgnoredGoFiles    []string
   319  	IgnoredOtherFiles []string
   320  	EmbedPatterns     []string
   321  	EmbedFiles        []string
   322  	CFiles            []string
   323  	CgoFiles          []string
   324  	CXXFiles          []string
   325  	MFiles            []string
   326  	HFiles            []string
   327  	FFiles            []string
   328  	SFiles            []string
   329  	SwigFiles         []string
   330  	SwigCXXFiles      []string
   331  	SysoFiles         []string
   332  	Imports           []string
   333  	ImportMap         map[string]string
   334  	Deps              []string
   335  	Module            *Module
   336  	TestGoFiles       []string
   337  	TestImports       []string
   338  	XTestGoFiles      []string
   339  	XTestImports      []string
   340  	ForTest           string // q in a "p [q.test]" package, else ""
   341  	DepOnly           bool
   342  
   343  	Error      *packagesinternal.PackageError
   344  	DepsErrors []*packagesinternal.PackageError
   345  }
   346  
   347  type jsonPackageError struct {
   348  	ImportStack []string
   349  	Pos         string
   350  	Err         string
   351  }
   352  
   353  func otherFiles(p *jsonPackage) [][]string {
   354  	return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles}
   355  }
   356  
   357  // createDriverResponse uses the "go list" command to expand the pattern
   358  // words and return a response for the specified packages.
   359  func (state *golistState) createDriverResponse(words ...string) (*DriverResponse, error) {
   360  	// go list uses the following identifiers in ImportPath and Imports:
   361  	//
   362  	// 	"p"			-- importable package or main (command)
   363  	// 	"q.test"		-- q's test executable
   364  	// 	"p [q.test]"		-- variant of p as built for q's test executable
   365  	// 	"q_test [q.test]"	-- q's external test package
   366  	//
   367  	// The packages p that are built differently for a test q.test
   368  	// are q itself, plus any helpers used by the external test q_test,
   369  	// typically including "testing" and all its dependencies.
   370  
   371  	// Run "go list" for complete
   372  	// information on the specified packages.
   373  	goVersion, err := state.getGoVersion()
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	buf, err := state.invokeGo("list", golistargs(state.cfg, words, goVersion)...)
   378  	if err != nil {
   379  		return nil, err
   380  	}
   381  
   382  	seen := make(map[string]*jsonPackage)
   383  	pkgs := make(map[string]*Package)
   384  	additionalErrors := make(map[string][]Error)
   385  	// Decode the JSON and convert it to Package form.
   386  	response := &DriverResponse{
   387  		GoVersion: goVersion,
   388  	}
   389  	for dec := json.NewDecoder(buf); dec.More(); {
   390  		p := new(jsonPackage)
   391  		if err := dec.Decode(p); err != nil {
   392  			return nil, fmt.Errorf("JSON decoding failed: %v", err)
   393  		}
   394  
   395  		if p.ImportPath == "" {
   396  			// The documentation for go list says that “[e]rroneous packages will have
   397  			// a non-empty ImportPath”. If for some reason it comes back empty, we
   398  			// prefer to error out rather than silently discarding data or handing
   399  			// back a package without any way to refer to it.
   400  			if p.Error != nil {
   401  				return nil, Error{
   402  					Pos: p.Error.Pos,
   403  					Msg: p.Error.Err,
   404  				}
   405  			}
   406  			return nil, fmt.Errorf("package missing import path: %+v", p)
   407  		}
   408  
   409  		// Work around https://golang.org/issue/33157:
   410  		// go list -e, when given an absolute path, will find the package contained at
   411  		// that directory. But when no package exists there, it will return a fake package
   412  		// with an error and the ImportPath set to the absolute path provided to go list.
   413  		// Try to convert that absolute path to what its package path would be if it's
   414  		// contained in a known module or GOPATH entry. This will allow the package to be
   415  		// properly "reclaimed" when overlays are processed.
   416  		if filepath.IsAbs(p.ImportPath) && p.Error != nil {
   417  			pkgPath, ok, err := state.getPkgPath(p.ImportPath)
   418  			if err != nil {
   419  				return nil, err
   420  			}
   421  			if ok {
   422  				p.ImportPath = pkgPath
   423  			}
   424  		}
   425  
   426  		if old, found := seen[p.ImportPath]; found {
   427  			// If one version of the package has an error, and the other doesn't, assume
   428  			// that this is a case where go list is reporting a fake dependency variant
   429  			// of the imported package: When a package tries to invalidly import another
   430  			// package, go list emits a variant of the imported package (with the same
   431  			// import path, but with an error on it, and the package will have a
   432  			// DepError set on it). An example of when this can happen is for imports of
   433  			// main packages: main packages can not be imported, but they may be
   434  			// separately matched and listed by another pattern.
   435  			// See golang.org/issue/36188 for more details.
   436  
   437  			// The plan is that eventually, hopefully in Go 1.15, the error will be
   438  			// reported on the importing package rather than the duplicate "fake"
   439  			// version of the imported package. Once all supported versions of Go
   440  			// have the new behavior this logic can be deleted.
   441  			// TODO(matloob): delete the workaround logic once all supported versions of
   442  			// Go return the errors on the proper package.
   443  
   444  			// There should be exactly one version of a package that doesn't have an
   445  			// error.
   446  			if old.Error == nil && p.Error == nil {
   447  				if !reflect.DeepEqual(p, old) {
   448  					return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath)
   449  				}
   450  				continue
   451  			}
   452  
   453  			// Determine if this package's error needs to be bubbled up.
   454  			// This is a hack, and we expect for go list to eventually set the error
   455  			// on the package.
   456  			if old.Error != nil {
   457  				var errkind string
   458  				if strings.Contains(old.Error.Err, "not an importable package") {
   459  					errkind = "not an importable package"
   460  				} else if strings.Contains(old.Error.Err, "use of internal package") && strings.Contains(old.Error.Err, "not allowed") {
   461  					errkind = "use of internal package not allowed"
   462  				}
   463  				if errkind != "" {
   464  					if len(old.Error.ImportStack) < 1 {
   465  						return nil, fmt.Errorf(`internal error: go list gave a %q error with empty import stack`, errkind)
   466  					}
   467  					importingPkg := old.Error.ImportStack[len(old.Error.ImportStack)-1]
   468  					if importingPkg == old.ImportPath {
   469  						// Using an older version of Go which put this package itself on top of import
   470  						// stack, instead of the importer. Look for importer in second from top
   471  						// position.
   472  						if len(old.Error.ImportStack) < 2 {
   473  							return nil, fmt.Errorf(`internal error: go list gave a %q error with an import stack without importing package`, errkind)
   474  						}
   475  						importingPkg = old.Error.ImportStack[len(old.Error.ImportStack)-2]
   476  					}
   477  					additionalErrors[importingPkg] = append(additionalErrors[importingPkg], Error{
   478  						Pos:  old.Error.Pos,
   479  						Msg:  old.Error.Err,
   480  						Kind: ListError,
   481  					})
   482  				}
   483  			}
   484  
   485  			// Make sure that if there's a version of the package without an error,
   486  			// that's the one reported to the user.
   487  			if old.Error == nil {
   488  				continue
   489  			}
   490  
   491  			// This package will replace the old one at the end of the loop.
   492  		}
   493  		seen[p.ImportPath] = p
   494  
   495  		pkg := &Package{
   496  			Name:            p.Name,
   497  			ID:              p.ImportPath,
   498  			GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles),
   499  			CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles),
   500  			OtherFiles:      absJoin(p.Dir, otherFiles(p)...),
   501  			EmbedFiles:      absJoin(p.Dir, p.EmbedFiles),
   502  			EmbedPatterns:   absJoin(p.Dir, p.EmbedPatterns),
   503  			IgnoredFiles:    absJoin(p.Dir, p.IgnoredGoFiles, p.IgnoredOtherFiles),
   504  			forTest:         p.ForTest,
   505  			depsErrors:      p.DepsErrors,
   506  			Module:          p.Module,
   507  		}
   508  
   509  		if (state.cfg.Mode&typecheckCgo) != 0 && len(p.CgoFiles) != 0 {
   510  			if len(p.CompiledGoFiles) > len(p.GoFiles) {
   511  				// We need the cgo definitions, which are in the first
   512  				// CompiledGoFile after the non-cgo ones. This is a hack but there
   513  				// isn't currently a better way to find it. We also need the pure
   514  				// Go files and unprocessed cgo files, all of which are already
   515  				// in pkg.GoFiles.
   516  				cgoTypes := p.CompiledGoFiles[len(p.GoFiles)]
   517  				pkg.CompiledGoFiles = append([]string{cgoTypes}, pkg.GoFiles...)
   518  			} else {
   519  				// golang/go#38990: go list silently fails to do cgo processing
   520  				pkg.CompiledGoFiles = nil
   521  				pkg.Errors = append(pkg.Errors, Error{
   522  					Msg:  "go list failed to return CompiledGoFiles. This may indicate failure to perform cgo processing; try building at the command line. See https://golang.org/issue/38990.",
   523  					Kind: ListError,
   524  				})
   525  			}
   526  		}
   527  
   528  		// Work around https://golang.org/issue/28749:
   529  		// cmd/go puts assembly, C, and C++ files in CompiledGoFiles.
   530  		// Remove files from CompiledGoFiles that are non-go files
   531  		// (or are not files that look like they are from the cache).
   532  		if len(pkg.CompiledGoFiles) > 0 {
   533  			out := pkg.CompiledGoFiles[:0]
   534  			for _, f := range pkg.CompiledGoFiles {
   535  				if ext := filepath.Ext(f); ext != ".go" && ext != "" { // ext == "" means the file is from the cache, so probably cgo-processed file
   536  					continue
   537  				}
   538  				out = append(out, f)
   539  			}
   540  			pkg.CompiledGoFiles = out
   541  		}
   542  
   543  		// Extract the PkgPath from the package's ID.
   544  		if i := strings.IndexByte(pkg.ID, ' '); i >= 0 {
   545  			pkg.PkgPath = pkg.ID[:i]
   546  		} else {
   547  			pkg.PkgPath = pkg.ID
   548  		}
   549  
   550  		if pkg.PkgPath == "unsafe" {
   551  			pkg.CompiledGoFiles = nil // ignore fake unsafe.go file (#59929)
   552  		} else if len(pkg.CompiledGoFiles) == 0 {
   553  			// Work around for pre-go.1.11 versions of go list.
   554  			// TODO(matloob): they should be handled by the fallback.
   555  			// Can we delete this?
   556  			pkg.CompiledGoFiles = pkg.GoFiles
   557  		}
   558  
   559  		// Assume go list emits only absolute paths for Dir.
   560  		if p.Dir != "" && !filepath.IsAbs(p.Dir) {
   561  			log.Fatalf("internal error: go list returned non-absolute Package.Dir: %s", p.Dir)
   562  		}
   563  
   564  		if p.Export != "" && !filepath.IsAbs(p.Export) {
   565  			pkg.ExportFile = filepath.Join(p.Dir, p.Export)
   566  		} else {
   567  			pkg.ExportFile = p.Export
   568  		}
   569  
   570  		// imports
   571  		//
   572  		// Imports contains the IDs of all imported packages.
   573  		// ImportsMap records (path, ID) only where they differ.
   574  		ids := make(map[string]bool)
   575  		for _, id := range p.Imports {
   576  			ids[id] = true
   577  		}
   578  		pkg.Imports = make(map[string]*Package)
   579  		for path, id := range p.ImportMap {
   580  			pkg.Imports[path] = &Package{ID: id} // non-identity import
   581  			delete(ids, id)
   582  		}
   583  		for id := range ids {
   584  			if id == "C" {
   585  				continue
   586  			}
   587  
   588  			pkg.Imports[id] = &Package{ID: id} // identity import
   589  		}
   590  		if !p.DepOnly {
   591  			response.Roots = append(response.Roots, pkg.ID)
   592  		}
   593  
   594  		// Temporary work-around for golang/go#39986. Parse filenames out of
   595  		// error messages. This happens if there are unrecoverable syntax
   596  		// errors in the source, so we can't match on a specific error message.
   597  		//
   598  		// TODO(rfindley): remove this heuristic, in favor of considering
   599  		// InvalidGoFiles from the list driver.
   600  		if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) {
   601  			addFilenameFromPos := func(pos string) bool {
   602  				split := strings.Split(pos, ":")
   603  				if len(split) < 1 {
   604  					return false
   605  				}
   606  				filename := strings.TrimSpace(split[0])
   607  				if filename == "" {
   608  					return false
   609  				}
   610  				if !filepath.IsAbs(filename) {
   611  					filename = filepath.Join(state.cfg.Dir, filename)
   612  				}
   613  				info, _ := os.Stat(filename)
   614  				if info == nil {
   615  					return false
   616  				}
   617  				pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename)
   618  				pkg.GoFiles = append(pkg.GoFiles, filename)
   619  				return true
   620  			}
   621  			found := addFilenameFromPos(err.Pos)
   622  			// In some cases, go list only reports the error position in the
   623  			// error text, not the error position. One such case is when the
   624  			// file's package name is a keyword (see golang.org/issue/39763).
   625  			if !found {
   626  				addFilenameFromPos(err.Err)
   627  			}
   628  		}
   629  
   630  		if p.Error != nil {
   631  			msg := strings.TrimSpace(p.Error.Err) // Trim to work around golang.org/issue/32363.
   632  			// Address golang.org/issue/35964 by appending import stack to error message.
   633  			if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 {
   634  				msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack)
   635  			}
   636  			pkg.Errors = append(pkg.Errors, Error{
   637  				Pos:  p.Error.Pos,
   638  				Msg:  msg,
   639  				Kind: ListError,
   640  			})
   641  		}
   642  
   643  		pkgs[pkg.ID] = pkg
   644  	}
   645  
   646  	for id, errs := range additionalErrors {
   647  		if p, ok := pkgs[id]; ok {
   648  			p.Errors = append(p.Errors, errs...)
   649  		}
   650  	}
   651  	for _, pkg := range pkgs {
   652  		response.Packages = append(response.Packages, pkg)
   653  	}
   654  	sort.Slice(response.Packages, func(i, j int) bool { return response.Packages[i].ID < response.Packages[j].ID })
   655  
   656  	return response, nil
   657  }
   658  
   659  func (state *golistState) shouldAddFilenameFromError(p *jsonPackage) bool {
   660  	if len(p.GoFiles) > 0 || len(p.CompiledGoFiles) > 0 {
   661  		return false
   662  	}
   663  
   664  	goV, err := state.getGoVersion()
   665  	if err != nil {
   666  		return false
   667  	}
   668  
   669  	// On Go 1.14 and earlier, only add filenames from errors if the import stack is empty.
   670  	// The import stack behaves differently for these versions than newer Go versions.
   671  	if goV < 15 {
   672  		return len(p.Error.ImportStack) == 0
   673  	}
   674  
   675  	// On Go 1.15 and later, only parse filenames out of error if there's no import stack,
   676  	// or the current package is at the top of the import stack. This is not guaranteed
   677  	// to work perfectly, but should avoid some cases where files in errors don't belong to this
   678  	// package.
   679  	return len(p.Error.ImportStack) == 0 || p.Error.ImportStack[len(p.Error.ImportStack)-1] == p.ImportPath
   680  }
   681  
   682  // getGoVersion returns the effective minor version of the go command.
   683  func (state *golistState) getGoVersion() (int, error) {
   684  	state.goVersionOnce.Do(func() {
   685  		state.goVersion, state.goVersionError = gocommand.GoVersion(state.ctx, state.cfgInvocation(), state.cfg.gocmdRunner)
   686  	})
   687  	return state.goVersion, state.goVersionError
   688  }
   689  
   690  // getPkgPath finds the package path of a directory if it's relative to a root
   691  // directory.
   692  func (state *golistState) getPkgPath(dir string) (string, bool, error) {
   693  	absDir, err := filepath.Abs(dir)
   694  	if err != nil {
   695  		return "", false, err
   696  	}
   697  	roots, err := state.determineRootDirs()
   698  	if err != nil {
   699  		return "", false, err
   700  	}
   701  
   702  	for rdir, rpath := range roots {
   703  		// Make sure that the directory is in the module,
   704  		// to avoid creating a path relative to another module.
   705  		if !strings.HasPrefix(absDir, rdir) {
   706  			continue
   707  		}
   708  		// TODO(matloob): This doesn't properly handle symlinks.
   709  		r, err := filepath.Rel(rdir, dir)
   710  		if err != nil {
   711  			continue
   712  		}
   713  		if rpath != "" {
   714  			// We choose only one root even though the directory even it can belong in multiple modules
   715  			// or GOPATH entries. This is okay because we only need to work with absolute dirs when a
   716  			// file is missing from disk, for instance when gopls calls go/packages in an overlay.
   717  			// Once the file is saved, gopls, or the next invocation of the tool will get the correct
   718  			// result straight from golist.
   719  			// TODO(matloob): Implement module tiebreaking?
   720  			return path.Join(rpath, filepath.ToSlash(r)), true, nil
   721  		}
   722  		return filepath.ToSlash(r), true, nil
   723  	}
   724  	return "", false, nil
   725  }
   726  
   727  // absJoin absolutizes and flattens the lists of files.
   728  func absJoin(dir string, fileses ...[]string) (res []string) {
   729  	for _, files := range fileses {
   730  		for _, file := range files {
   731  			if !filepath.IsAbs(file) {
   732  				file = filepath.Join(dir, file)
   733  			}
   734  			res = append(res, file)
   735  		}
   736  	}
   737  	return res
   738  }
   739  
   740  func jsonFlag(cfg *Config, goVersion int) string {
   741  	if goVersion < 19 {
   742  		return "-json"
   743  	}
   744  	var fields []string
   745  	added := make(map[string]bool)
   746  	addFields := func(fs ...string) {
   747  		for _, f := range fs {
   748  			if !added[f] {
   749  				added[f] = true
   750  				fields = append(fields, f)
   751  			}
   752  		}
   753  	}
   754  	addFields("Name", "ImportPath", "Error") // These fields are always needed
   755  	if cfg.Mode&NeedFiles != 0 || cfg.Mode&NeedTypes != 0 {
   756  		addFields("Dir", "GoFiles", "IgnoredGoFiles", "IgnoredOtherFiles", "CFiles",
   757  			"CgoFiles", "CXXFiles", "MFiles", "HFiles", "FFiles", "SFiles",
   758  			"SwigFiles", "SwigCXXFiles", "SysoFiles")
   759  		if cfg.Tests {
   760  			addFields("TestGoFiles", "XTestGoFiles")
   761  		}
   762  	}
   763  	if cfg.Mode&NeedTypes != 0 {
   764  		// CompiledGoFiles seems to be required for the test case TestCgoNoSyntax,
   765  		// even when -compiled isn't passed in.
   766  		// TODO(#52435): Should we make the test ask for -compiled, or automatically
   767  		// request CompiledGoFiles in certain circumstances?
   768  		addFields("Dir", "CompiledGoFiles")
   769  	}
   770  	if cfg.Mode&NeedCompiledGoFiles != 0 {
   771  		addFields("Dir", "CompiledGoFiles", "Export")
   772  	}
   773  	if cfg.Mode&NeedImports != 0 {
   774  		// When imports are requested, DepOnly is used to distinguish between packages
   775  		// explicitly requested and transitive imports of those packages.
   776  		addFields("DepOnly", "Imports", "ImportMap")
   777  		if cfg.Tests {
   778  			addFields("TestImports", "XTestImports")
   779  		}
   780  	}
   781  	if cfg.Mode&NeedDeps != 0 {
   782  		addFields("DepOnly")
   783  	}
   784  	if usesExportData(cfg) {
   785  		// Request Dir in the unlikely case Export is not absolute.
   786  		addFields("Dir", "Export")
   787  	}
   788  	if cfg.Mode&needInternalForTest != 0 {
   789  		addFields("ForTest")
   790  	}
   791  	if cfg.Mode&needInternalDepsErrors != 0 {
   792  		addFields("DepsErrors")
   793  	}
   794  	if cfg.Mode&NeedModule != 0 {
   795  		addFields("Module")
   796  	}
   797  	if cfg.Mode&NeedEmbedFiles != 0 {
   798  		addFields("EmbedFiles")
   799  	}
   800  	if cfg.Mode&NeedEmbedPatterns != 0 {
   801  		addFields("EmbedPatterns")
   802  	}
   803  	return "-json=" + strings.Join(fields, ",")
   804  }
   805  
   806  func golistargs(cfg *Config, words []string, goVersion int) []string {
   807  	const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo
   808  	fullargs := []string{
   809  		"-e", jsonFlag(cfg, goVersion),
   810  		fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypes|NeedTypesInfo|NeedTypesSizes) != 0),
   811  		fmt.Sprintf("-test=%t", cfg.Tests),
   812  		fmt.Sprintf("-export=%t", usesExportData(cfg)),
   813  		fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0),
   814  		// go list doesn't let you pass -test and -find together,
   815  		// probably because you'd just get the TestMain.
   816  		fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0 && !usesExportData(cfg)),
   817  	}
   818  
   819  	// golang/go#60456: with go1.21 and later, go list serves pgo variants, which
   820  	// can be costly to compute and may result in redundant processing for the
   821  	// caller. Disable these variants. If someone wants to add e.g. a NeedPGO
   822  	// mode flag, that should be a separate proposal.
   823  	if goVersion >= 21 {
   824  		fullargs = append(fullargs, "-pgo=off")
   825  	}
   826  
   827  	fullargs = append(fullargs, cfg.BuildFlags...)
   828  	fullargs = append(fullargs, "--")
   829  	fullargs = append(fullargs, words...)
   830  	return fullargs
   831  }
   832  
   833  // cfgInvocation returns an Invocation that reflects cfg's settings.
   834  func (state *golistState) cfgInvocation() gocommand.Invocation {
   835  	cfg := state.cfg
   836  	return gocommand.Invocation{
   837  		BuildFlags: cfg.BuildFlags,
   838  		ModFile:    cfg.modFile,
   839  		ModFlag:    cfg.modFlag,
   840  		CleanEnv:   cfg.Env != nil,
   841  		Env:        cfg.Env,
   842  		Logf:       cfg.Logf,
   843  		WorkingDir: cfg.Dir,
   844  	}
   845  }
   846  
   847  // invokeGo returns the stdout of a go command invocation.
   848  func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) {
   849  	cfg := state.cfg
   850  
   851  	inv := state.cfgInvocation()
   852  
   853  	// For Go versions 1.16 and above, `go list` accepts overlays directly via
   854  	// the -overlay flag. Set it, if it's available.
   855  	//
   856  	// The check for "list" is not necessarily required, but we should avoid
   857  	// getting the go version if possible.
   858  	if verb == "list" {
   859  		goVersion, err := state.getGoVersion()
   860  		if err != nil {
   861  			return nil, err
   862  		}
   863  		if goVersion >= 16 {
   864  			filename, cleanup, err := state.writeOverlays()
   865  			if err != nil {
   866  				return nil, err
   867  			}
   868  			defer cleanup()
   869  			inv.Overlay = filename
   870  		}
   871  	}
   872  	inv.Verb = verb
   873  	inv.Args = args
   874  	gocmdRunner := cfg.gocmdRunner
   875  	if gocmdRunner == nil {
   876  		gocmdRunner = &gocommand.Runner{}
   877  	}
   878  	stdout, stderr, friendlyErr, err := gocmdRunner.RunRaw(cfg.Context, inv)
   879  	if err != nil {
   880  		// Check for 'go' executable not being found.
   881  		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
   882  			return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound)
   883  		}
   884  
   885  		exitErr, ok := err.(*exec.ExitError)
   886  		if !ok {
   887  			// Catastrophic error:
   888  			// - context cancellation
   889  			return nil, fmt.Errorf("couldn't run 'go': %w", err)
   890  		}
   891  
   892  		// Old go version?
   893  		if strings.Contains(stderr.String(), "flag provided but not defined") {
   894  			return nil, goTooOldError{fmt.Errorf("unsupported version of go: %s: %s", exitErr, stderr)}
   895  		}
   896  
   897  		// Related to #24854
   898  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "unexpected directory layout") {
   899  			return nil, friendlyErr
   900  		}
   901  
   902  		// Is there an error running the C compiler in cgo? This will be reported in the "Error" field
   903  		// and should be suppressed by go list -e.
   904  		//
   905  		// This condition is not perfect yet because the error message can include other error messages than runtime/cgo.
   906  		isPkgPathRune := func(r rune) bool {
   907  			// From https://golang.org/ref/spec#Import_declarations:
   908  			//    Implementation restriction: A compiler may restrict ImportPaths to non-empty strings
   909  			//    using only characters belonging to Unicode's L, M, N, P, and S general categories
   910  			//    (the Graphic characters without spaces) and may also exclude the
   911  			//    characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD.
   912  			return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
   913  				!strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
   914  		}
   915  		// golang/go#36770: Handle case where cmd/go prints module download messages before the error.
   916  		msg := stderr.String()
   917  		for strings.HasPrefix(msg, "go: downloading") {
   918  			msg = msg[strings.IndexRune(msg, '\n')+1:]
   919  		}
   920  		if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
   921  			msg := msg[len("# "):]
   922  			if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") {
   923  				return stdout, nil
   924  			}
   925  			// Treat pkg-config errors as a special case (golang.org/issue/36770).
   926  			if strings.HasPrefix(msg, "pkg-config") {
   927  				return stdout, nil
   928  			}
   929  		}
   930  
   931  		// This error only appears in stderr. See golang.org/cl/166398 for a fix in go list to show
   932  		// the error in the Err section of stdout in case -e option is provided.
   933  		// This fix is provided for backwards compatibility.
   934  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") {
   935  			output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   936  				strings.Trim(stderr.String(), "\n"))
   937  			return bytes.NewBufferString(output), nil
   938  		}
   939  
   940  		// Similar to the previous error, but currently lacks a fix in Go.
   941  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must all be in one directory") {
   942  			output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   943  				strings.Trim(stderr.String(), "\n"))
   944  			return bytes.NewBufferString(output), nil
   945  		}
   946  
   947  		// Backwards compatibility for Go 1.11 because 1.12 and 1.13 put the directory in the ImportPath.
   948  		// If the package doesn't exist, put the absolute path of the directory into the error message,
   949  		// as Go 1.13 list does.
   950  		const noSuchDirectory = "no such directory"
   951  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), noSuchDirectory) {
   952  			errstr := stderr.String()
   953  			abspath := strings.TrimSpace(errstr[strings.Index(errstr, noSuchDirectory)+len(noSuchDirectory):])
   954  			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   955  				abspath, strings.Trim(stderr.String(), "\n"))
   956  			return bytes.NewBufferString(output), nil
   957  		}
   958  
   959  		// Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist.
   960  		// Note that the error message we look for in this case is different that the one looked for above.
   961  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") {
   962  			output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   963  				strings.Trim(stderr.String(), "\n"))
   964  			return bytes.NewBufferString(output), nil
   965  		}
   966  
   967  		// Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a
   968  		// directory outside any module.
   969  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") {
   970  			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   971  				// TODO(matloob): command-line-arguments isn't correct here.
   972  				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
   973  			return bytes.NewBufferString(output), nil
   974  		}
   975  
   976  		// Another variation of the previous error
   977  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
   978  			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
   979  				// TODO(matloob): command-line-arguments isn't correct here.
   980  				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
   981  			return bytes.NewBufferString(output), nil
   982  		}
   983  
   984  		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit
   985  		// status if there's a dependency on a package that doesn't exist. But it should return
   986  		// a zero exit status and set an error on that package.
   987  		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
   988  			// Don't clobber stdout if `go list` actually returned something.
   989  			if len(stdout.String()) > 0 {
   990  				return stdout, nil
   991  			}
   992  			// try to extract package name from string
   993  			stderrStr := stderr.String()
   994  			var importPath string
   995  			colon := strings.Index(stderrStr, ":")
   996  			if colon > 0 && strings.HasPrefix(stderrStr, "go build ") {
   997  				importPath = stderrStr[len("go build "):colon]
   998  			}
   999  			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
  1000  				importPath, strings.Trim(stderrStr, "\n"))
  1001  			return bytes.NewBufferString(output), nil
  1002  		}
  1003  
  1004  		// Export mode entails a build.
  1005  		// If that build fails, errors appear on stderr
  1006  		// (despite the -e flag) and the Export field is blank.
  1007  		// Do not fail in that case.
  1008  		// The same is true if an ad-hoc package given to go list doesn't exist.
  1009  		// TODO(matloob): Remove these once we can depend on go list to exit with a zero status with -e even when
  1010  		// packages don't exist or a build fails.
  1011  		if !usesExportData(cfg) && !containsGoFile(args) {
  1012  			return nil, friendlyErr
  1013  		}
  1014  	}
  1015  	return stdout, nil
  1016  }
  1017  
  1018  // OverlayJSON is the format overlay files are expected to be in.
  1019  // The Replace map maps from overlaid paths to replacement paths:
  1020  // the Go command will forward all reads trying to open
  1021  // each overlaid path to its replacement path, or consider the overlaid
  1022  // path not to exist if the replacement path is empty.
  1023  //
  1024  // From golang/go#39958.
  1025  type OverlayJSON struct {
  1026  	Replace map[string]string `json:"replace,omitempty"`
  1027  }
  1028  
  1029  // writeOverlays writes out files for go list's -overlay flag, as described
  1030  // above.
  1031  func (state *golistState) writeOverlays() (filename string, cleanup func(), err error) {
  1032  	// Do nothing if there are no overlays in the config.
  1033  	if len(state.cfg.Overlay) == 0 {
  1034  		return "", func() {}, nil
  1035  	}
  1036  	dir, err := os.MkdirTemp("", "gopackages-*")
  1037  	if err != nil {
  1038  		return "", nil, err
  1039  	}
  1040  	// The caller must clean up this directory, unless this function returns an
  1041  	// error.
  1042  	cleanup = func() {
  1043  		os.RemoveAll(dir)
  1044  	}
  1045  	defer func() {
  1046  		if err != nil {
  1047  			cleanup()
  1048  		}
  1049  	}()
  1050  	overlays := map[string]string{}
  1051  	for k, v := range state.cfg.Overlay {
  1052  		// Create a unique filename for the overlaid files, to avoid
  1053  		// creating nested directories.
  1054  		noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "")
  1055  		f, err := os.CreateTemp(dir, fmt.Sprintf("*-%s", noSeparator))
  1056  		if err != nil {
  1057  			return "", func() {}, err
  1058  		}
  1059  		if _, err := f.Write(v); err != nil {
  1060  			return "", func() {}, err
  1061  		}
  1062  		if err := f.Close(); err != nil {
  1063  			return "", func() {}, err
  1064  		}
  1065  		overlays[k] = f.Name()
  1066  	}
  1067  	b, err := json.Marshal(OverlayJSON{Replace: overlays})
  1068  	if err != nil {
  1069  		return "", func() {}, err
  1070  	}
  1071  	// Write out the overlay file that contains the filepath mappings.
  1072  	filename = filepath.Join(dir, "overlay.json")
  1073  	if err := os.WriteFile(filename, b, 0665); err != nil {
  1074  		return "", func() {}, err
  1075  	}
  1076  	return filename, cleanup, nil
  1077  }
  1078  
  1079  func containsGoFile(s []string) bool {
  1080  	for _, f := range s {
  1081  		if strings.HasSuffix(f, ".go") {
  1082  			return true
  1083  		}
  1084  	}
  1085  	return false
  1086  }
  1087  
  1088  func cmdDebugStr(cmd *exec.Cmd) string {
  1089  	env := make(map[string]string)
  1090  	for _, kv := range cmd.Env {
  1091  		split := strings.SplitN(kv, "=", 2)
  1092  		k, v := split[0], split[1]
  1093  		env[k] = v
  1094  	}
  1095  
  1096  	var args []string
  1097  	for _, arg := range cmd.Args {
  1098  		quoted := strconv.Quote(arg)
  1099  		if quoted[1:len(quoted)-1] != arg || strings.Contains(arg, " ") {
  1100  			args = append(args, quoted)
  1101  		} else {
  1102  			args = append(args, arg)
  1103  		}
  1104  	}
  1105  	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " "))
  1106  }
  1107  

View as plain text