Source file src/golang.org/x/tools/internal/imports/mod.go

Documentation: golang.org/x/tools/internal/imports

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package imports
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/json"
    11  	"fmt"
    12  	"os"
    13  	"path"
    14  	"path/filepath"
    15  	"regexp"
    16  	"sort"
    17  	"strconv"
    18  	"strings"
    19  
    20  	"golang.org/x/mod/module"
    21  	"golang.org/x/tools/internal/event"
    22  	"golang.org/x/tools/internal/gocommand"
    23  	"golang.org/x/tools/internal/gopathwalk"
    24  	"golang.org/x/tools/internal/stdlib"
    25  )
    26  
    27  // Notes(rfindley): ModuleResolver appears to be heavily optimized for scanning
    28  // as fast as possible, which is desirable for a call to goimports from the
    29  // command line, but it doesn't work as well for gopls, where it suffers from
    30  // slow startup (golang/go#44863) and intermittent hanging (golang/go#59216),
    31  // both caused by populating the cache, albeit in slightly different ways.
    32  //
    33  // A high level list of TODOs:
    34  //  - Optimize the scan itself, as there is some redundancy statting and
    35  //    reading go.mod files.
    36  //  - Invert the relationship between ProcessEnv and Resolver (see the
    37  //    docstring of ProcessEnv).
    38  //  - Make it easier to use an external resolver implementation.
    39  //
    40  // Smaller TODOs are annotated in the code below.
    41  
    42  // ModuleResolver implements the Resolver interface for a workspace using
    43  // modules.
    44  //
    45  // A goal of the ModuleResolver is to invoke the Go command as little as
    46  // possible. To this end, it runs the Go command only for listing module
    47  // information (i.e. `go list -m -e -json ...`). Package scanning, the process
    48  // of loading package information for the modules, is implemented internally
    49  // via the scan method.
    50  //
    51  // It has two types of state: the state derived from the go command, which
    52  // is populated by init, and the state derived from scans, which is populated
    53  // via scan. A root is considered scanned if it has been walked to discover
    54  // directories. However, if the scan did not require additional information
    55  // from the directory (such as package name or exports), the directory
    56  // information itself may be partially populated. It will be lazily filled in
    57  // as needed by scans, using the scanCallback.
    58  type ModuleResolver struct {
    59  	env *ProcessEnv
    60  
    61  	// Module state, populated during construction
    62  	dummyVendorMod *gocommand.ModuleJSON            // if vendoring is enabled, a pseudo-module to represent the /vendor directory
    63  	moduleCacheDir string                           // GOMODCACHE, inferred from GOPATH if unset
    64  	roots          []gopathwalk.Root                // roots to scan, in approximate order of importance
    65  	mains          []*gocommand.ModuleJSON          // main modules
    66  	mainByDir      map[string]*gocommand.ModuleJSON // module information by dir, to join with roots
    67  	modsByModPath  []*gocommand.ModuleJSON          // all modules, ordered by # of path components in their module path
    68  	modsByDir      []*gocommand.ModuleJSON          // ...or by the number of path components in their Dir.
    69  
    70  	// Scanning state, populated by scan
    71  
    72  	// scanSema prevents concurrent scans, and guards scannedRoots and the cache
    73  	// fields below (though the caches themselves are concurrency safe).
    74  	// Receive to acquire, send to release.
    75  	scanSema     chan struct{}
    76  	scannedRoots map[gopathwalk.Root]bool // if true, root has been walked
    77  
    78  	// Caches of directory info, populated by scans and scan callbacks
    79  	//
    80  	// moduleCacheCache stores cached information about roots in the module
    81  	// cache, which are immutable and therefore do not need to be invalidated.
    82  	//
    83  	// otherCache stores information about all other roots (even GOROOT), which
    84  	// may change.
    85  	moduleCacheCache *DirInfoCache
    86  	otherCache       *DirInfoCache
    87  }
    88  
    89  // newModuleResolver returns a new module-aware goimports resolver.
    90  //
    91  // Note: use caution when modifying this constructor: changes must also be
    92  // reflected in ModuleResolver.ClearForNewScan.
    93  func newModuleResolver(e *ProcessEnv, moduleCacheCache *DirInfoCache) (*ModuleResolver, error) {
    94  	r := &ModuleResolver{
    95  		env:      e,
    96  		scanSema: make(chan struct{}, 1),
    97  	}
    98  	r.scanSema <- struct{}{} // release
    99  
   100  	goenv, err := r.env.goEnv()
   101  	if err != nil {
   102  		return nil, err
   103  	}
   104  
   105  	// TODO(rfindley): can we refactor to share logic with r.env.invokeGo?
   106  	inv := gocommand.Invocation{
   107  		BuildFlags: r.env.BuildFlags,
   108  		ModFlag:    r.env.ModFlag,
   109  		Env:        r.env.env(),
   110  		Logf:       r.env.Logf,
   111  		WorkingDir: r.env.WorkingDir,
   112  	}
   113  
   114  	vendorEnabled := false
   115  	var mainModVendor *gocommand.ModuleJSON    // for module vendoring
   116  	var mainModsVendor []*gocommand.ModuleJSON // for workspace vendoring
   117  
   118  	goWork := r.env.Env["GOWORK"]
   119  	if len(goWork) == 0 {
   120  		// TODO(rfindley): VendorEnabled runs the go command to get GOFLAGS, but
   121  		// they should be available from the ProcessEnv. Can we avoid the redundant
   122  		// invocation?
   123  		vendorEnabled, mainModVendor, err = gocommand.VendorEnabled(context.TODO(), inv, r.env.GocmdRunner)
   124  		if err != nil {
   125  			return nil, err
   126  		}
   127  	} else {
   128  		vendorEnabled, mainModsVendor, err = gocommand.WorkspaceVendorEnabled(context.Background(), inv, r.env.GocmdRunner)
   129  		if err != nil {
   130  			return nil, err
   131  		}
   132  	}
   133  
   134  	if vendorEnabled {
   135  		if mainModVendor != nil {
   136  			// Module vendor mode is on, so all the non-Main modules are irrelevant,
   137  			// and we need to search /vendor for everything.
   138  			r.mains = []*gocommand.ModuleJSON{mainModVendor}
   139  			r.dummyVendorMod = &gocommand.ModuleJSON{
   140  				Path: "",
   141  				Dir:  filepath.Join(mainModVendor.Dir, "vendor"),
   142  			}
   143  			r.modsByModPath = []*gocommand.ModuleJSON{mainModVendor, r.dummyVendorMod}
   144  			r.modsByDir = []*gocommand.ModuleJSON{mainModVendor, r.dummyVendorMod}
   145  		} else {
   146  			// Workspace vendor mode is on, so all the non-Main modules are irrelevant,
   147  			// and we need to search /vendor for everything.
   148  			r.mains = mainModsVendor
   149  			r.dummyVendorMod = &gocommand.ModuleJSON{
   150  				Path: "",
   151  				Dir:  filepath.Join(filepath.Dir(goWork), "vendor"),
   152  			}
   153  			r.modsByModPath = append(append([]*gocommand.ModuleJSON{}, mainModsVendor...), r.dummyVendorMod)
   154  			r.modsByDir = append(append([]*gocommand.ModuleJSON{}, mainModsVendor...), r.dummyVendorMod)
   155  		}
   156  	} else {
   157  		// Vendor mode is off, so run go list -m ... to find everything.
   158  		err := r.initAllMods()
   159  		// We expect an error when running outside of a module with
   160  		// GO111MODULE=on. Other errors are fatal.
   161  		if err != nil {
   162  			if errMsg := err.Error(); !strings.Contains(errMsg, "working directory is not part of a module") && !strings.Contains(errMsg, "go.mod file not found") {
   163  				return nil, err
   164  			}
   165  		}
   166  	}
   167  
   168  	r.moduleCacheDir = gomodcacheForEnv(goenv)
   169  	if r.moduleCacheDir == "" {
   170  		return nil, fmt.Errorf("cannot resolve GOMODCACHE")
   171  	}
   172  
   173  	sort.Slice(r.modsByModPath, func(i, j int) bool {
   174  		count := func(x int) int {
   175  			return strings.Count(r.modsByModPath[x].Path, "/")
   176  		}
   177  		return count(j) < count(i) // descending order
   178  	})
   179  	sort.Slice(r.modsByDir, func(i, j int) bool {
   180  		count := func(x int) int {
   181  			return strings.Count(r.modsByDir[x].Dir, string(filepath.Separator))
   182  		}
   183  		return count(j) < count(i) // descending order
   184  	})
   185  
   186  	r.roots = []gopathwalk.Root{}
   187  	if goenv["GOROOT"] != "" { // "" happens in tests
   188  		r.roots = append(r.roots, gopathwalk.Root{Path: filepath.Join(goenv["GOROOT"], "/src"), Type: gopathwalk.RootGOROOT})
   189  	}
   190  	r.mainByDir = make(map[string]*gocommand.ModuleJSON)
   191  	for _, main := range r.mains {
   192  		r.roots = append(r.roots, gopathwalk.Root{Path: main.Dir, Type: gopathwalk.RootCurrentModule})
   193  		r.mainByDir[main.Dir] = main
   194  	}
   195  	if vendorEnabled {
   196  		r.roots = append(r.roots, gopathwalk.Root{Path: r.dummyVendorMod.Dir, Type: gopathwalk.RootOther})
   197  	} else {
   198  		addDep := func(mod *gocommand.ModuleJSON) {
   199  			if mod.Replace == nil {
   200  				// This is redundant with the cache, but we'll skip it cheaply enough
   201  				// when we encounter it in the module cache scan.
   202  				//
   203  				// Including it at a lower index in r.roots than the module cache dir
   204  				// helps prioritize matches from within existing dependencies.
   205  				r.roots = append(r.roots, gopathwalk.Root{Path: mod.Dir, Type: gopathwalk.RootModuleCache})
   206  			} else {
   207  				r.roots = append(r.roots, gopathwalk.Root{Path: mod.Dir, Type: gopathwalk.RootOther})
   208  			}
   209  		}
   210  		// Walk dependent modules before scanning the full mod cache, direct deps first.
   211  		for _, mod := range r.modsByModPath {
   212  			if !mod.Indirect && !mod.Main {
   213  				addDep(mod)
   214  			}
   215  		}
   216  		for _, mod := range r.modsByModPath {
   217  			if mod.Indirect && !mod.Main {
   218  				addDep(mod)
   219  			}
   220  		}
   221  		// If provided, share the moduleCacheCache.
   222  		//
   223  		// TODO(rfindley): The module cache is immutable. However, the loaded
   224  		// exports do depend on GOOS and GOARCH. Fortunately, the
   225  		// ProcessEnv.buildContext does not adjust these from build.DefaultContext
   226  		// (even though it should). So for now, this is OK to share, but we need to
   227  		// add logic for handling GOOS/GOARCH.
   228  		r.moduleCacheCache = moduleCacheCache
   229  		r.roots = append(r.roots, gopathwalk.Root{Path: r.moduleCacheDir, Type: gopathwalk.RootModuleCache})
   230  	}
   231  
   232  	r.scannedRoots = map[gopathwalk.Root]bool{}
   233  	if r.moduleCacheCache == nil {
   234  		r.moduleCacheCache = NewDirInfoCache()
   235  	}
   236  	r.otherCache = NewDirInfoCache()
   237  	return r, nil
   238  }
   239  
   240  // gomodcacheForEnv returns the GOMODCACHE value to use based on the given env
   241  // map, which must have GOMODCACHE and GOPATH populated.
   242  //
   243  // TODO(rfindley): this is defensive refactoring.
   244  //  1. Is this even relevant anymore? Can't we just read GOMODCACHE.
   245  //  2. Use this to separate module cache scanning from other scanning.
   246  func gomodcacheForEnv(goenv map[string]string) string {
   247  	if gmc := goenv["GOMODCACHE"]; gmc != "" {
   248  		return gmc
   249  	}
   250  	gopaths := filepath.SplitList(goenv["GOPATH"])
   251  	if len(gopaths) == 0 {
   252  		return ""
   253  	}
   254  	return filepath.Join(gopaths[0], "/pkg/mod")
   255  }
   256  
   257  func (r *ModuleResolver) initAllMods() error {
   258  	stdout, err := r.env.invokeGo(context.TODO(), "list", "-m", "-e", "-json", "...")
   259  	if err != nil {
   260  		return err
   261  	}
   262  	for dec := json.NewDecoder(stdout); dec.More(); {
   263  		mod := &gocommand.ModuleJSON{}
   264  		if err := dec.Decode(mod); err != nil {
   265  			return err
   266  		}
   267  		if mod.Dir == "" {
   268  			if r.env.Logf != nil {
   269  				r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path)
   270  			}
   271  			// Can't do anything with a module that's not downloaded.
   272  			continue
   273  		}
   274  		// golang/go#36193: the go command doesn't always clean paths.
   275  		mod.Dir = filepath.Clean(mod.Dir)
   276  		r.modsByModPath = append(r.modsByModPath, mod)
   277  		r.modsByDir = append(r.modsByDir, mod)
   278  		if mod.Main {
   279  			r.mains = append(r.mains, mod)
   280  		}
   281  	}
   282  	return nil
   283  }
   284  
   285  // ClearForNewScan invalidates the last scan.
   286  //
   287  // It preserves the set of roots, but forgets about the set of directories.
   288  // Though it forgets the set of module cache directories, it remembers their
   289  // contents, since they are assumed to be immutable.
   290  func (r *ModuleResolver) ClearForNewScan() Resolver {
   291  	<-r.scanSema // acquire r, to guard scannedRoots
   292  	r2 := &ModuleResolver{
   293  		env:            r.env,
   294  		dummyVendorMod: r.dummyVendorMod,
   295  		moduleCacheDir: r.moduleCacheDir,
   296  		roots:          r.roots,
   297  		mains:          r.mains,
   298  		mainByDir:      r.mainByDir,
   299  		modsByModPath:  r.modsByModPath,
   300  
   301  		scanSema:         make(chan struct{}, 1),
   302  		scannedRoots:     make(map[gopathwalk.Root]bool),
   303  		otherCache:       NewDirInfoCache(),
   304  		moduleCacheCache: r.moduleCacheCache,
   305  	}
   306  	r2.scanSema <- struct{}{} // r2 must start released
   307  	// Invalidate root scans. We don't need to invalidate module cache roots,
   308  	// because they are immutable.
   309  	// (We don't support a use case where GOMODCACHE is cleaned in the middle of
   310  	// e.g. a gopls session: the user must restart gopls to get accurate
   311  	// imports.)
   312  	//
   313  	// Scanning for new directories in GOMODCACHE should be handled elsewhere,
   314  	// via a call to ScanModuleCache.
   315  	for _, root := range r.roots {
   316  		if root.Type == gopathwalk.RootModuleCache && r.scannedRoots[root] {
   317  			r2.scannedRoots[root] = true
   318  		}
   319  	}
   320  	r.scanSema <- struct{}{} // release r
   321  	return r2
   322  }
   323  
   324  // ClearModuleInfo invalidates resolver state that depends on go.mod file
   325  // contents (essentially, the output of go list -m -json ...).
   326  //
   327  // Notably, it does not forget directory contents, which are reset
   328  // asynchronously via ClearForNewScan.
   329  //
   330  // If the ProcessEnv is a GOPATH environment, ClearModuleInfo is a no op.
   331  //
   332  // TODO(rfindley): move this to a new env.go, consolidating ProcessEnv methods.
   333  func (e *ProcessEnv) ClearModuleInfo() {
   334  	if r, ok := e.resolver.(*ModuleResolver); ok {
   335  		resolver, err := newModuleResolver(e, e.ModCache)
   336  		if err != nil {
   337  			e.resolver = nil
   338  			e.resolverErr = err
   339  			return
   340  		}
   341  
   342  		<-r.scanSema // acquire (guards caches)
   343  		resolver.moduleCacheCache = r.moduleCacheCache
   344  		resolver.otherCache = r.otherCache
   345  		r.scanSema <- struct{}{} // release
   346  
   347  		e.UpdateResolver(resolver)
   348  	}
   349  }
   350  
   351  // UpdateResolver sets the resolver for the ProcessEnv to use in imports
   352  // operations. Only for use with the result of [Resolver.ClearForNewScan].
   353  //
   354  // TODO(rfindley): this awkward API is a result of the (arguably) inverted
   355  // relationship between configuration and state described in the doc comment
   356  // for [ProcessEnv].
   357  func (e *ProcessEnv) UpdateResolver(r Resolver) {
   358  	e.resolver = r
   359  	e.resolverErr = nil
   360  }
   361  
   362  // findPackage returns the module and directory from within the main modules
   363  // and their dependencies that contains the package at the given import path,
   364  // or returns nil, "" if no module is in scope.
   365  func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, string) {
   366  	// This can't find packages in the stdlib, but that's harmless for all
   367  	// the existing code paths.
   368  	for _, m := range r.modsByModPath {
   369  		if !strings.HasPrefix(importPath, m.Path) {
   370  			continue
   371  		}
   372  		pathInModule := importPath[len(m.Path):]
   373  		pkgDir := filepath.Join(m.Dir, pathInModule)
   374  		if r.dirIsNestedModule(pkgDir, m) {
   375  			continue
   376  		}
   377  
   378  		if info, ok := r.cacheLoad(pkgDir); ok {
   379  			if loaded, err := info.reachedStatus(nameLoaded); loaded {
   380  				if err != nil {
   381  					continue // No package in this dir.
   382  				}
   383  				return m, pkgDir
   384  			}
   385  			if scanned, err := info.reachedStatus(directoryScanned); scanned && err != nil {
   386  				continue // Dir is unreadable, etc.
   387  			}
   388  			// This is slightly wrong: a directory doesn't have to have an
   389  			// importable package to count as a package for package-to-module
   390  			// resolution. package main or _test files should count but
   391  			// don't.
   392  			// TODO(heschi): fix this.
   393  			if _, err := r.cachePackageName(info); err == nil {
   394  				return m, pkgDir
   395  			}
   396  		}
   397  
   398  		// Not cached. Read the filesystem.
   399  		pkgFiles, err := os.ReadDir(pkgDir)
   400  		if err != nil {
   401  			continue
   402  		}
   403  		// A module only contains a package if it has buildable go
   404  		// files in that directory. If not, it could be provided by an
   405  		// outer module. See #29736.
   406  		for _, fi := range pkgFiles {
   407  			if ok, _ := r.env.matchFile(pkgDir, fi.Name()); ok {
   408  				return m, pkgDir
   409  			}
   410  		}
   411  	}
   412  	return nil, ""
   413  }
   414  
   415  func (r *ModuleResolver) cacheLoad(dir string) (directoryPackageInfo, bool) {
   416  	if info, ok := r.moduleCacheCache.Load(dir); ok {
   417  		return info, ok
   418  	}
   419  	return r.otherCache.Load(dir)
   420  }
   421  
   422  func (r *ModuleResolver) cacheStore(info directoryPackageInfo) {
   423  	if info.rootType == gopathwalk.RootModuleCache {
   424  		r.moduleCacheCache.Store(info.dir, info)
   425  	} else {
   426  		r.otherCache.Store(info.dir, info)
   427  	}
   428  }
   429  
   430  // cachePackageName caches the package name for a dir already in the cache.
   431  func (r *ModuleResolver) cachePackageName(info directoryPackageInfo) (string, error) {
   432  	if info.rootType == gopathwalk.RootModuleCache {
   433  		return r.moduleCacheCache.CachePackageName(info)
   434  	}
   435  	return r.otherCache.CachePackageName(info)
   436  }
   437  
   438  func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []stdlib.Symbol, error) {
   439  	if info.rootType == gopathwalk.RootModuleCache {
   440  		return r.moduleCacheCache.CacheExports(ctx, env, info)
   441  	}
   442  	return r.otherCache.CacheExports(ctx, env, info)
   443  }
   444  
   445  // findModuleByDir returns the module that contains dir, or nil if no such
   446  // module is in scope.
   447  func (r *ModuleResolver) findModuleByDir(dir string) *gocommand.ModuleJSON {
   448  	// This is quite tricky and may not be correct. dir could be:
   449  	// - a package in the main module.
   450  	// - a replace target underneath the main module's directory.
   451  	//    - a nested module in the above.
   452  	// - a replace target somewhere totally random.
   453  	//    - a nested module in the above.
   454  	// - in the mod cache.
   455  	// - in /vendor/ in -mod=vendor mode.
   456  	//    - nested module? Dunno.
   457  	// Rumor has it that replace targets cannot contain other replace targets.
   458  	//
   459  	// Note that it is critical here that modsByDir is sorted to have deeper dirs
   460  	// first. This ensures that findModuleByDir finds the innermost module.
   461  	// See also golang/go#56291.
   462  	for _, m := range r.modsByDir {
   463  		if !strings.HasPrefix(dir, m.Dir) {
   464  			continue
   465  		}
   466  
   467  		if r.dirIsNestedModule(dir, m) {
   468  			continue
   469  		}
   470  
   471  		return m
   472  	}
   473  	return nil
   474  }
   475  
   476  // dirIsNestedModule reports if dir is contained in a nested module underneath
   477  // mod, not actually in mod.
   478  func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON) bool {
   479  	if !strings.HasPrefix(dir, mod.Dir) {
   480  		return false
   481  	}
   482  	if r.dirInModuleCache(dir) {
   483  		// Nested modules in the module cache are pruned,
   484  		// so it cannot be a nested module.
   485  		return false
   486  	}
   487  	if mod != nil && mod == r.dummyVendorMod {
   488  		// The /vendor pseudomodule is flattened and doesn't actually count.
   489  		return false
   490  	}
   491  	modDir, _ := r.modInfo(dir)
   492  	if modDir == "" {
   493  		return false
   494  	}
   495  	return modDir != mod.Dir
   496  }
   497  
   498  func readModName(modFile string) string {
   499  	modBytes, err := os.ReadFile(modFile)
   500  	if err != nil {
   501  		return ""
   502  	}
   503  	return modulePath(modBytes)
   504  }
   505  
   506  func (r *ModuleResolver) modInfo(dir string) (modDir, modName string) {
   507  	if r.dirInModuleCache(dir) {
   508  		if matches := modCacheRegexp.FindStringSubmatch(dir); len(matches) == 3 {
   509  			index := strings.Index(dir, matches[1]+"@"+matches[2])
   510  			modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
   511  			return modDir, readModName(filepath.Join(modDir, "go.mod"))
   512  		}
   513  	}
   514  	for {
   515  		if info, ok := r.cacheLoad(dir); ok {
   516  			return info.moduleDir, info.moduleName
   517  		}
   518  		f := filepath.Join(dir, "go.mod")
   519  		info, err := os.Stat(f)
   520  		if err == nil && !info.IsDir() {
   521  			return dir, readModName(f)
   522  		}
   523  
   524  		d := filepath.Dir(dir)
   525  		if len(d) >= len(dir) {
   526  			return "", "" // reached top of file system, no go.mod
   527  		}
   528  		dir = d
   529  	}
   530  }
   531  
   532  func (r *ModuleResolver) dirInModuleCache(dir string) bool {
   533  	if r.moduleCacheDir == "" {
   534  		return false
   535  	}
   536  	return strings.HasPrefix(dir, r.moduleCacheDir)
   537  }
   538  
   539  func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) {
   540  	names := map[string]string{}
   541  	for _, path := range importPaths {
   542  		// TODO(rfindley): shouldn't this use the dirInfoCache?
   543  		_, packageDir := r.findPackage(path)
   544  		if packageDir == "" {
   545  			continue
   546  		}
   547  		name, err := packageDirToName(packageDir)
   548  		if err != nil {
   549  			continue
   550  		}
   551  		names[path] = name
   552  	}
   553  	return names, nil
   554  }
   555  
   556  func (r *ModuleResolver) scan(ctx context.Context, callback *scanCallback) error {
   557  	ctx, done := event.Start(ctx, "imports.ModuleResolver.scan")
   558  	defer done()
   559  
   560  	processDir := func(info directoryPackageInfo) {
   561  		// Skip this directory if we were not able to get the package information successfully.
   562  		if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
   563  			return
   564  		}
   565  		pkg, err := r.canonicalize(info)
   566  		if err != nil {
   567  			return
   568  		}
   569  		if !callback.dirFound(pkg) {
   570  			return
   571  		}
   572  
   573  		pkg.packageName, err = r.cachePackageName(info)
   574  		if err != nil {
   575  			return
   576  		}
   577  		if !callback.packageNameLoaded(pkg) {
   578  			return
   579  		}
   580  
   581  		_, exports, err := r.loadExports(ctx, pkg, false)
   582  		if err != nil {
   583  			return
   584  		}
   585  		callback.exportsLoaded(pkg, exports)
   586  	}
   587  
   588  	// Start processing everything in the cache, and listen for the new stuff
   589  	// we discover in the walk below.
   590  	stop1 := r.moduleCacheCache.ScanAndListen(ctx, processDir)
   591  	defer stop1()
   592  	stop2 := r.otherCache.ScanAndListen(ctx, processDir)
   593  	defer stop2()
   594  
   595  	// We assume cached directories are fully cached, including all their
   596  	// children, and have not changed. We can skip them.
   597  	skip := func(root gopathwalk.Root, dir string) bool {
   598  		if r.env.SkipPathInScan != nil && root.Type == gopathwalk.RootCurrentModule {
   599  			if root.Path == dir {
   600  				return false
   601  			}
   602  
   603  			if r.env.SkipPathInScan(filepath.Clean(dir)) {
   604  				return true
   605  			}
   606  		}
   607  
   608  		info, ok := r.cacheLoad(dir)
   609  		if !ok {
   610  			return false
   611  		}
   612  		// This directory can be skipped as long as we have already scanned it.
   613  		// Packages with errors will continue to have errors, so there is no need
   614  		// to rescan them.
   615  		packageScanned, _ := info.reachedStatus(directoryScanned)
   616  		return packageScanned
   617  	}
   618  
   619  	add := func(root gopathwalk.Root, dir string) {
   620  		r.cacheStore(r.scanDirForPackage(root, dir))
   621  	}
   622  
   623  	// r.roots and the callback are not necessarily safe to use in the
   624  	// goroutine below. Process them eagerly.
   625  	roots := filterRoots(r.roots, callback.rootFound)
   626  	// We can't cancel walks, because we need them to finish to have a usable
   627  	// cache. Instead, run them in a separate goroutine and detach.
   628  	scanDone := make(chan struct{})
   629  	go func() {
   630  		select {
   631  		case <-ctx.Done():
   632  			return
   633  		case <-r.scanSema: // acquire
   634  		}
   635  		defer func() { r.scanSema <- struct{}{} }() // release
   636  		// We have the lock on r.scannedRoots, and no other scans can run.
   637  		for _, root := range roots {
   638  			if ctx.Err() != nil {
   639  				return
   640  			}
   641  
   642  			if r.scannedRoots[root] {
   643  				continue
   644  			}
   645  			gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: r.env.Logf, ModulesEnabled: true})
   646  			r.scannedRoots[root] = true
   647  		}
   648  		close(scanDone)
   649  	}()
   650  	select {
   651  	case <-ctx.Done():
   652  	case <-scanDone:
   653  	}
   654  	return nil
   655  }
   656  
   657  func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) float64 {
   658  	if stdlib.HasPackage(path) {
   659  		return MaxRelevance
   660  	}
   661  	mod, _ := r.findPackage(path)
   662  	return modRelevance(mod)
   663  }
   664  
   665  func modRelevance(mod *gocommand.ModuleJSON) float64 {
   666  	var relevance float64
   667  	switch {
   668  	case mod == nil: // out of scope
   669  		return MaxRelevance - 4
   670  	case mod.Indirect:
   671  		relevance = MaxRelevance - 3
   672  	case !mod.Main:
   673  		relevance = MaxRelevance - 2
   674  	default:
   675  		relevance = MaxRelevance - 1 // main module ties with stdlib
   676  	}
   677  
   678  	_, versionString, ok := module.SplitPathVersion(mod.Path)
   679  	if ok {
   680  		index := strings.Index(versionString, "v")
   681  		if index == -1 {
   682  			return relevance
   683  		}
   684  		if versionNumber, err := strconv.ParseFloat(versionString[index+1:], 64); err == nil {
   685  			relevance += versionNumber / 1000
   686  		}
   687  	}
   688  
   689  	return relevance
   690  }
   691  
   692  // canonicalize gets the result of canonicalizing the packages using the results
   693  // of initializing the resolver from 'go list -m'.
   694  func (r *ModuleResolver) canonicalize(info directoryPackageInfo) (*pkg, error) {
   695  	// Packages in GOROOT are already canonical, regardless of the std/cmd modules.
   696  	if info.rootType == gopathwalk.RootGOROOT {
   697  		return &pkg{
   698  			importPathShort: info.nonCanonicalImportPath,
   699  			dir:             info.dir,
   700  			packageName:     path.Base(info.nonCanonicalImportPath),
   701  			relevance:       MaxRelevance,
   702  		}, nil
   703  	}
   704  
   705  	importPath := info.nonCanonicalImportPath
   706  	mod := r.findModuleByDir(info.dir)
   707  	// Check if the directory is underneath a module that's in scope.
   708  	if mod != nil {
   709  		// It is. If dir is the target of a replace directive,
   710  		// our guessed import path is wrong. Use the real one.
   711  		if mod.Dir == info.dir {
   712  			importPath = mod.Path
   713  		} else {
   714  			dirInMod := info.dir[len(mod.Dir)+len("/"):]
   715  			importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
   716  		}
   717  	} else if !strings.HasPrefix(importPath, info.moduleName) {
   718  		// The module's name doesn't match the package's import path. It
   719  		// probably needs a replace directive we don't have.
   720  		return nil, fmt.Errorf("package in %q is not valid without a replace statement", info.dir)
   721  	}
   722  
   723  	res := &pkg{
   724  		importPathShort: importPath,
   725  		dir:             info.dir,
   726  		relevance:       modRelevance(mod),
   727  	}
   728  	// We may have discovered a package that has a different version
   729  	// in scope already. Canonicalize to that one if possible.
   730  	if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" {
   731  		res.dir = canonicalDir
   732  	}
   733  	return res, nil
   734  }
   735  
   736  func (r *ModuleResolver) loadExports(ctx context.Context, pkg *pkg, includeTest bool) (string, []stdlib.Symbol, error) {
   737  	if info, ok := r.cacheLoad(pkg.dir); ok && !includeTest {
   738  		return r.cacheExports(ctx, r.env, info)
   739  	}
   740  	return loadExportsFromFiles(ctx, r.env, pkg.dir, includeTest)
   741  }
   742  
   743  func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) directoryPackageInfo {
   744  	subdir := ""
   745  	if dir != root.Path {
   746  		subdir = dir[len(root.Path)+len("/"):]
   747  	}
   748  	importPath := filepath.ToSlash(subdir)
   749  	if strings.HasPrefix(importPath, "vendor/") {
   750  		// Only enter vendor directories if they're explicitly requested as a root.
   751  		return directoryPackageInfo{
   752  			status: directoryScanned,
   753  			err:    fmt.Errorf("unwanted vendor directory"),
   754  		}
   755  	}
   756  	switch root.Type {
   757  	case gopathwalk.RootCurrentModule:
   758  		importPath = path.Join(r.mainByDir[root.Path].Path, filepath.ToSlash(subdir))
   759  	case gopathwalk.RootModuleCache:
   760  		matches := modCacheRegexp.FindStringSubmatch(subdir)
   761  		if len(matches) == 0 {
   762  			return directoryPackageInfo{
   763  				status: directoryScanned,
   764  				err:    fmt.Errorf("invalid module cache path: %v", subdir),
   765  			}
   766  		}
   767  		modPath, err := module.UnescapePath(filepath.ToSlash(matches[1]))
   768  		if err != nil {
   769  			if r.env.Logf != nil {
   770  				r.env.Logf("decoding module cache path %q: %v", subdir, err)
   771  			}
   772  			return directoryPackageInfo{
   773  				status: directoryScanned,
   774  				err:    fmt.Errorf("decoding module cache path %q: %v", subdir, err),
   775  			}
   776  		}
   777  		importPath = path.Join(modPath, filepath.ToSlash(matches[3]))
   778  	}
   779  
   780  	modDir, modName := r.modInfo(dir)
   781  	result := directoryPackageInfo{
   782  		status:                 directoryScanned,
   783  		dir:                    dir,
   784  		rootType:               root.Type,
   785  		nonCanonicalImportPath: importPath,
   786  		moduleDir:              modDir,
   787  		moduleName:             modName,
   788  	}
   789  	if root.Type == gopathwalk.RootGOROOT {
   790  		// stdlib packages are always in scope, despite the confusing go.mod
   791  		return result
   792  	}
   793  	return result
   794  }
   795  
   796  // modCacheRegexp splits a path in a module cache into module, module version, and package.
   797  var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
   798  
   799  var (
   800  	slashSlash = []byte("//")
   801  	moduleStr  = []byte("module")
   802  )
   803  
   804  // modulePath returns the module path from the gomod file text.
   805  // If it cannot find a module path, it returns an empty string.
   806  // It is tolerant of unrelated problems in the go.mod file.
   807  //
   808  // Copied from cmd/go/internal/modfile.
   809  func modulePath(mod []byte) string {
   810  	for len(mod) > 0 {
   811  		line := mod
   812  		mod = nil
   813  		if i := bytes.IndexByte(line, '\n'); i >= 0 {
   814  			line, mod = line[:i], line[i+1:]
   815  		}
   816  		if i := bytes.Index(line, slashSlash); i >= 0 {
   817  			line = line[:i]
   818  		}
   819  		line = bytes.TrimSpace(line)
   820  		if !bytes.HasPrefix(line, moduleStr) {
   821  			continue
   822  		}
   823  		line = line[len(moduleStr):]
   824  		n := len(line)
   825  		line = bytes.TrimSpace(line)
   826  		if len(line) == n || len(line) == 0 {
   827  			continue
   828  		}
   829  
   830  		if line[0] == '"' || line[0] == '`' {
   831  			p, err := strconv.Unquote(string(line))
   832  			if err != nil {
   833  				return "" // malformed quoted string or multiline module path
   834  			}
   835  			return p
   836  		}
   837  
   838  		return string(line)
   839  	}
   840  	return "" // missing module path
   841  }
   842  

View as plain text