pkgload.go

Documentation: cuelang.org/go/internal/mod/modpkgload

     1  package modpkgload
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"runtime"
     7  	"slices"
     8  	"sort"
     9  	"strings"
    10  	"sync/atomic"
    11  
    12  	"cuelang.org/go/internal/mod/modimports"
    13  	"cuelang.org/go/internal/mod/modrequirements"
    14  	"cuelang.org/go/internal/par"
    15  	"cuelang.org/go/mod/module"
    16  )
    17  
    18  // Registry represents a module registry, or at least this package's view of it.
    19  type Registry interface {
    20  	// Fetch returns the location of the contents for the given module
    21  	// version, downloading it if necessary.
    22  	Fetch(ctx context.Context, m module.Version) (module.SourceLoc, error)
    23  }
    24  
    25  // Flags is a set of flags tracking metadata about a package.
    26  type Flags int8
    27  
    28  const (
    29  	// PkgInAll indicates that the package is in the "all" package pattern,
    30  	// regardless of whether we are loading the "all" package pattern.
    31  	//
    32  	// When the PkgInAll flag and PkgImportsLoaded flags are both set, the caller
    33  	// who set the last of those flags must propagate the PkgInAll marking to all
    34  	// of the imports of the marked package.
    35  	PkgInAll Flags = 1 << iota
    36  
    37  	// PkgIsRoot indicates that the package matches one of the root package
    38  	// patterns requested by the caller.
    39  	PkgIsRoot
    40  
    41  	// PkgFromRoot indicates that the package is in the transitive closure of
    42  	// imports starting at the roots. (Note that every package marked as PkgIsRoot
    43  	// is also trivially marked PkgFromRoot.)
    44  	PkgFromRoot
    45  
    46  	// PkgImportsLoaded indicates that the Imports field of a
    47  	// Pkg have been populated.
    48  	PkgImportsLoaded
    49  )
    50  
    51  func (f Flags) String() string {
    52  	var buf strings.Builder
    53  	set := func(f1 Flags, s string) {
    54  		if (f & f1) == 0 {
    55  			return
    56  		}
    57  		if buf.Len() > 0 {
    58  			buf.WriteString(",")
    59  		}
    60  		buf.WriteString(s)
    61  		f &^= f1
    62  	}
    63  	set(PkgInAll, "inAll")
    64  	set(PkgIsRoot, "isRoot")
    65  	set(PkgFromRoot, "fromRoot")
    66  	set(PkgImportsLoaded, "importsLoaded")
    67  	if f != 0 {
    68  		set(f, fmt.Sprintf("extra%x", int(f)))
    69  	}
    70  	return buf.String()
    71  }
    72  
    73  // has reports whether all of the flags in cond are set in f.
    74  func (f Flags) has(cond Flags) bool {
    75  	return f&cond == cond
    76  }
    77  
    78  type Packages struct {
    79  	mainModuleVersion module.Version
    80  	mainModuleLoc     module.SourceLoc
    81  	pkgCache          par.Cache[string, *Package]
    82  	pkgs              []*Package
    83  	rootPkgs          []*Package
    84  	work              *par.Queue
    85  	requirements      *modrequirements.Requirements
    86  	registry          Registry
    87  }
    88  
    89  type Package struct {
    90  	// Populated at construction time:
    91  	path string // import path
    92  
    93  	// Populated at construction time and updated by [loader.applyPkgFlags]:
    94  	flags atomicLoadPkgFlags
    95  
    96  	// Populated by [loader.load].
    97  	mod          module.Version     // module providing package
    98  	locs         []module.SourceLoc // location of source code directories
    99  	err          error              // error loading package
   100  	imports      []*Package         // packages imported by this one
   101  	inStd        bool
   102  	fromExternal bool
   103  	altMods      []module.Version // modules that could have contained the package but did not
   104  
   105  	// Populated by postprocessing in [Packages.buildStacks]:
   106  	stack *Package // package importing this one in minimal import stack for this pkg
   107  }
   108  
   109  func (pkg *Package) ImportPath() string {
   110  	return pkg.path
   111  }
   112  
   113  func (pkg *Package) FromExternalModule() bool {
   114  	return pkg.fromExternal
   115  }
   116  
   117  func (pkg *Package) Locations() []module.SourceLoc {
   118  	return pkg.locs
   119  }
   120  
   121  func (pkg *Package) Error() error {
   122  	return pkg.err
   123  }
   124  
   125  func (pkg *Package) SetError(err error) {
   126  	pkg.err = err
   127  }
   128  
   129  func (pkg *Package) HasFlags(flags Flags) bool {
   130  	return pkg.flags.has(flags)
   131  }
   132  
   133  func (pkg *Package) Imports() []*Package {
   134  	return pkg.imports
   135  }
   136  
   137  func (pkg *Package) Flags() Flags {
   138  	return pkg.flags.get()
   139  }
   140  
   141  func (pkg *Package) Mod() module.Version {
   142  	return pkg.mod
   143  }
   144  
   145  // LoadPackages loads information about all the given packages and the
   146  // packages they import, recursively, using modules from the given
   147  // requirements to determine which modules they might be obtained from,
   148  // and reg to download module contents.
   149  func LoadPackages(
   150  	ctx context.Context,
   151  	mainModulePath string,
   152  	mainModuleLoc module.SourceLoc,
   153  	rs *modrequirements.Requirements,
   154  	reg Registry,
   155  	rootPkgPaths []string,
   156  ) *Packages {
   157  	pkgs := &Packages{
   158  		mainModuleVersion: module.MustNewVersion(mainModulePath, ""),
   159  		mainModuleLoc:     mainModuleLoc,
   160  		requirements:      rs,
   161  		registry:          reg,
   162  		work:              par.NewQueue(runtime.GOMAXPROCS(0)),
   163  	}
   164  	inRoots := map[*Package]bool{}
   165  	pkgs.rootPkgs = make([]*Package, 0, len(rootPkgPaths))
   166  	for _, p := range rootPkgPaths {
   167  		// TODO the original logic didn't add PkgInAll here. Not sure why,
   168  		// and that might be a lurking problem.
   169  		if root := pkgs.addPkg(ctx, p, PkgIsRoot|PkgInAll); !inRoots[root] {
   170  			pkgs.rootPkgs = append(pkgs.rootPkgs, root)
   171  			inRoots[root] = true
   172  		}
   173  	}
   174  	<-pkgs.work.Idle()
   175  	pkgs.buildStacks()
   176  	return pkgs
   177  }
   178  
   179  // buildStacks computes minimal import stacks for each package,
   180  // for use in error messages. When it completes, packages that
   181  // are part of the original root set have pkg.stack == nil,
   182  // and other packages have pkg.stack pointing at the next
   183  // package up the import stack in their minimal chain.
   184  // As a side effect, buildStacks also constructs ld.pkgs,
   185  // the list of all packages loaded.
   186  func (pkgs *Packages) buildStacks() {
   187  	for _, pkg := range pkgs.rootPkgs {
   188  		pkg.stack = pkg // sentinel to avoid processing in next loop
   189  		pkgs.pkgs = append(pkgs.pkgs, pkg)
   190  	}
   191  	for i := 0; i < len(pkgs.pkgs); i++ { // not range: appending to pkgs.pkgs in loop
   192  		pkg := pkgs.pkgs[i]
   193  		for _, next := range pkg.imports {
   194  			if next.stack == nil {
   195  				next.stack = pkg
   196  				pkgs.pkgs = append(pkgs.pkgs, next)
   197  			}
   198  		}
   199  	}
   200  	for _, pkg := range pkgs.rootPkgs {
   201  		pkg.stack = nil
   202  	}
   203  }
   204  
   205  func (pkgs *Packages) Roots() []*Package {
   206  	return slices.Clip(pkgs.rootPkgs)
   207  }
   208  
   209  func (pkgs *Packages) All() []*Package {
   210  	return slices.Clip(pkgs.pkgs)
   211  }
   212  
   213  func (pkgs *Packages) Pkg(pkgPath string) *Package {
   214  	pkg, _ := pkgs.pkgCache.Get(pkgPath)
   215  	return pkg
   216  }
   217  
   218  func (pkgs *Packages) addPkg(ctx context.Context, pkgPath string, flags Flags) *Package {
   219  	pkg := pkgs.pkgCache.Do(pkgPath, func() *Package {
   220  		pkg := &Package{
   221  			path: pkgPath,
   222  		}
   223  		pkgs.applyPkgFlags(ctx, pkg, flags)
   224  
   225  		pkgs.work.Add(func() { pkgs.load(ctx, pkg) })
   226  		return pkg
   227  	})
   228  
   229  	// Ensure the flags apply even if the package already existed.
   230  	pkgs.applyPkgFlags(ctx, pkg, flags)
   231  	return pkg
   232  }
   233  
   234  func (pkgs *Packages) load(ctx context.Context, pkg *Package) {
   235  	if IsStdlibPackage(pkg.path) {
   236  		pkg.inStd = true
   237  		return
   238  	}
   239  	pkg.fromExternal = pkg.mod != pkgs.mainModuleVersion
   240  	pkg.mod, pkg.locs, pkg.altMods, pkg.err = pkgs.importFromModules(ctx, pkg.path)
   241  	if pkg.err != nil {
   242  		return
   243  	}
   244  	if pkgs.mainModuleVersion.Path() == pkg.mod.Path() {
   245  		pkgs.applyPkgFlags(ctx, pkg, PkgInAll)
   246  	}
   247  	pkgQual := module.ParseImportPath(pkg.path).Qualifier
   248  	importsMap := make(map[string]bool)
   249  	for _, loc := range pkg.locs {
   250  		imports, err := modimports.AllImports(modimports.PackageFiles(loc.FS, loc.Dir, pkgQual))
   251  		if err != nil {
   252  			pkg.err = fmt.Errorf("cannot get imports: %v", err)
   253  			return
   254  		}
   255  		for _, imp := range imports {
   256  			importsMap[imp] = true
   257  		}
   258  	}
   259  	imports := make([]string, 0, len(importsMap))
   260  	for imp := range importsMap {
   261  		imports = append(imports, imp)
   262  	}
   263  	sort.Strings(imports) // Make the algorithm deterministic for tests.
   264  
   265  	pkg.imports = make([]*Package, 0, len(imports))
   266  	var importFlags Flags
   267  	if pkg.flags.has(PkgInAll) {
   268  		importFlags = PkgInAll
   269  	}
   270  	for _, path := range imports {
   271  		pkg.imports = append(pkg.imports, pkgs.addPkg(ctx, path, importFlags))
   272  	}
   273  	pkgs.applyPkgFlags(ctx, pkg, PkgImportsLoaded)
   274  }
   275  
   276  // applyPkgFlags updates pkg.flags to set the given flags and propagate the
   277  // (transitive) effects of those flags, possibly loading or enqueueing further
   278  // packages as a result.
   279  func (pkgs *Packages) applyPkgFlags(ctx context.Context, pkg *Package, flags Flags) {
   280  	if flags == 0 {
   281  		return
   282  	}
   283  
   284  	if flags.has(PkgInAll) {
   285  		// This package matches a root pattern by virtue of being in "all".
   286  		flags |= PkgIsRoot
   287  	}
   288  	if flags.has(PkgIsRoot) {
   289  		flags |= PkgFromRoot
   290  	}
   291  
   292  	old := pkg.flags.update(flags)
   293  	new := old | flags
   294  	if new == old || !new.has(PkgImportsLoaded) {
   295  		// We either didn't change the state of pkg, or we don't know anything about
   296  		// its dependencies yet. Either way, we can't usefully load its test or
   297  		// update its dependencies.
   298  		return
   299  	}
   300  
   301  	if new.has(PkgInAll) && !old.has(PkgInAll|PkgImportsLoaded) {
   302  		// We have just marked pkg with pkgInAll, or we have just loaded its
   303  		// imports, or both. Now is the time to propagate pkgInAll to the imports.
   304  		for _, dep := range pkg.imports {
   305  			pkgs.applyPkgFlags(ctx, dep, PkgInAll)
   306  		}
   307  	}
   308  
   309  	if new.has(PkgFromRoot) && !old.has(PkgFromRoot|PkgImportsLoaded) {
   310  		for _, dep := range pkg.imports {
   311  			pkgs.applyPkgFlags(ctx, dep, PkgFromRoot)
   312  		}
   313  	}
   314  }
   315  
   316  // An atomicLoadPkgFlags stores a loadPkgFlags for which individual flags can be
   317  // added atomically.
   318  type atomicLoadPkgFlags struct {
   319  	bits atomic.Int32
   320  }
   321  
   322  // update sets the given flags in af (in addition to any flags already set).
   323  //
   324  // update returns the previous flag state so that the caller may determine which
   325  // flags were newly-set.
   326  func (af *atomicLoadPkgFlags) update(flags Flags) (old Flags) {
   327  	for {
   328  		old := af.bits.Load()
   329  		new := old | int32(flags)
   330  		if new == old || af.bits.CompareAndSwap(old, new) {
   331  			return Flags(old)
   332  		}
   333  	}
   334  }
   335  
   336  func (af *atomicLoadPkgFlags) get() Flags {
   337  	return Flags(af.bits.Load())
   338  }
   339  
   340  // has reports whether all of the flags in cond are set in af.
   341  func (af *atomicLoadPkgFlags) has(cond Flags) bool {
   342  	return Flags(af.bits.Load())&cond == cond
   343  }
   344  
   345  func IsStdlibPackage(pkgPath string) bool {
   346  	firstElem, _, _ := strings.Cut(pkgPath, "/")
   347  	return !strings.Contains(firstElem, ".")
   348  }
   349
View as plain text