update-repos.go

Documentation: github.com/bazelbuild/bazel-gazelle/cmd/gazelle

     1  /* Copyright 2017 The Bazel Authors. All rights reserved.
     2  
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6  
     7     http://www.apache.org/licenses/LICENSE-2.0
     8  
     9  Unless required by applicable law or agreed to in writing, software
    10  distributed under the License is distributed on an "AS IS" BASIS,
    11  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  See the License for the specific language governing permissions and
    13  limitations under the License.
    14  */
    15  
    16  package main
    17  
    18  import (
    19  	"bytes"
    20  	"errors"
    21  	"flag"
    22  	"fmt"
    23  	"os"
    24  	"path/filepath"
    25  	"sort"
    26  	"strings"
    27  
    28  	"github.com/bazelbuild/bazel-gazelle/config"
    29  	"github.com/bazelbuild/bazel-gazelle/internal/wspace"
    30  	"github.com/bazelbuild/bazel-gazelle/label"
    31  	"github.com/bazelbuild/bazel-gazelle/language"
    32  	"github.com/bazelbuild/bazel-gazelle/merger"
    33  	"github.com/bazelbuild/bazel-gazelle/repo"
    34  	"github.com/bazelbuild/bazel-gazelle/rule"
    35  )
    36  
    37  type updateReposConfig struct {
    38  	repoFilePath  string
    39  	importPaths   []string
    40  	macroFileName string
    41  	macroDefName  string
    42  	pruneRules    bool
    43  	workspace     *rule.File
    44  	repoFileMap   map[string]*rule.File
    45  }
    46  
    47  const updateReposName = "_update-repos"
    48  
    49  func getUpdateReposConfig(c *config.Config) *updateReposConfig {
    50  	return c.Exts[updateReposName].(*updateReposConfig)
    51  }
    52  
    53  type updateReposConfigurer struct{}
    54  
    55  type macroFlag struct {
    56  	macroFileName *string
    57  	macroDefName  *string
    58  }
    59  
    60  func (f macroFlag) Set(value string) error {
    61  	args := strings.Split(value, "%")
    62  	if len(args) != 2 {
    63  		return fmt.Errorf("Failure parsing to_macro: %s, expected format is macroFile%%defName", value)
    64  	}
    65  	if strings.HasPrefix(args[0], "..") {
    66  		return fmt.Errorf("Failure parsing to_macro: %s, macro file path %s should not start with \"..\"", value, args[0])
    67  	}
    68  	*f.macroFileName = args[0]
    69  	*f.macroDefName = args[1]
    70  	return nil
    71  }
    72  
    73  func (f macroFlag) String() string {
    74  	return ""
    75  }
    76  
    77  func (*updateReposConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {
    78  	uc := &updateReposConfig{}
    79  	c.Exts[updateReposName] = uc
    80  	fs.StringVar(&uc.repoFilePath, "from_file", "", "Gazelle will translate repositories listed in this file into repository rules in WORKSPACE or a .bzl macro function. Gopkg.lock and go.mod files are supported")
    81  	fs.Var(macroFlag{macroFileName: &uc.macroFileName, macroDefName: &uc.macroDefName}, "to_macro", "Tells Gazelle to write repository rules into a .bzl macro function rather than the WORKSPACE file. . The expected format is: macroFile%defName")
    82  	fs.BoolVar(&uc.pruneRules, "prune", false, "When enabled, Gazelle will remove rules that no longer have equivalent repos in the go.mod file. Can only used with -from_file.")
    83  }
    84  
    85  func (*updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
    86  	uc := getUpdateReposConfig(c)
    87  	switch {
    88  	case uc.repoFilePath != "":
    89  		if len(fs.Args()) != 0 {
    90  			return fmt.Errorf("got %d positional arguments with -from_file; wanted 0.\nTry -help for more information.", len(fs.Args()))
    91  		}
    92  		if !filepath.IsAbs(uc.repoFilePath) {
    93  			uc.repoFilePath = filepath.Join(c.WorkDir, uc.repoFilePath)
    94  		}
    95  
    96  	default:
    97  		if len(fs.Args()) == 0 {
    98  			return fmt.Errorf("no repositories specified\nTry -help for more information.")
    99  		}
   100  		if uc.pruneRules {
   101  			return fmt.Errorf("the -prune option can only be used with -from_file")
   102  		}
   103  		uc.importPaths = fs.Args()
   104  	}
   105  
   106  	var err error
   107  	workspacePath := wspace.FindWORKSPACEFile(c.RepoRoot)
   108  	uc.workspace, err = rule.LoadWorkspaceFile(workspacePath, "")
   109  	if err != nil {
   110  		if c.Bzlmod {
   111  			return nil
   112  		} else {
   113  			return fmt.Errorf("loading WORKSPACE file: %v", err)
   114  		}
   115  	}
   116  	c.Repos, uc.repoFileMap, err = repo.ListRepositories(uc.workspace)
   117  	if err != nil {
   118  		return fmt.Errorf("loading WORKSPACE file: %v", err)
   119  	}
   120  
   121  	return nil
   122  }
   123  
   124  func (*updateReposConfigurer) KnownDirectives() []string { return nil }
   125  
   126  func (*updateReposConfigurer) Configure(c *config.Config, rel string, f *rule.File) {}
   127  
   128  func updateRepos(wd string, args []string) (err error) {
   129  	// Build configuration with all languages.
   130  	cexts := make([]config.Configurer, 0, len(languages)+2)
   131  	cexts = append(cexts, &config.CommonConfigurer{}, &updateReposConfigurer{})
   132  
   133  	for _, lang := range languages {
   134  		cexts = append(cexts, lang)
   135  	}
   136  
   137  	c, err := newUpdateReposConfiguration(wd, args, cexts)
   138  	if err != nil {
   139  		return err
   140  	}
   141  	uc := getUpdateReposConfig(c)
   142  
   143  	kinds := make(map[string]rule.KindInfo)
   144  	loads := []rule.LoadInfo{}
   145  	for _, lang := range languages {
   146  		if moduleAwareLang, ok := lang.(language.ModuleAwareLanguage); ok {
   147  			loads = append(loads, moduleAwareLang.ApparentLoads(c.ModuleToApparentName)...)
   148  		} else {
   149  			loads = append(loads, lang.Loads()...)
   150  		}
   151  		for kind, info := range lang.Kinds() {
   152  			kinds[kind] = info
   153  		}
   154  	}
   155  
   156  	// TODO(jayconrod): move Go-specific RemoteCache logic to language/go.
   157  	var knownRepos []repo.Repo
   158  
   159  	reposFromDirectives := make(map[string]bool)
   160  	for _, r := range c.Repos {
   161  		if repo.IsFromDirective(r) {
   162  			reposFromDirectives[r.Name()] = true
   163  		}
   164  
   165  		if r.Kind() == "go_repository" {
   166  			knownRepos = append(knownRepos, repo.Repo{
   167  				Name:     r.Name(),
   168  				GoPrefix: r.AttrString("importpath"),
   169  				Remote:   r.AttrString("remote"),
   170  				VCS:      r.AttrString("vcs"),
   171  			})
   172  		}
   173  	}
   174  	rc, cleanup := repo.NewRemoteCache(knownRepos)
   175  	defer func() {
   176  		if cerr := cleanup(); err == nil && cerr != nil {
   177  			err = cerr
   178  		}
   179  	}()
   180  
   181  	// Fix the workspace file with each language.
   182  	for _, lang := range filterLanguages(c, languages) {
   183  		lang.Fix(c, uc.workspace)
   184  	}
   185  
   186  	// Generate rules from command language arguments or by importing a file.
   187  	var gen, empty []*rule.Rule
   188  	if uc.repoFilePath == "" {
   189  		gen, err = updateRepoImports(c, rc)
   190  	} else {
   191  		gen, empty, err = importRepos(c, rc)
   192  	}
   193  	if err != nil {
   194  		return err
   195  	}
   196  
   197  	// Organize generated and empty rules by file. A rule should go into the file
   198  	// it came from (by name). New rules should go into WORKSPACE or the file
   199  	// specified with -to_macro.
   200  	var newGen []*rule.Rule
   201  	genForFiles := make(map[*rule.File][]*rule.Rule)
   202  	emptyForFiles := make(map[*rule.File][]*rule.Rule)
   203  	genNames := make(map[string]*rule.Rule)
   204  	for _, r := range gen {
   205  
   206  		// Skip generation of rules that are defined as directives.
   207  		if reposFromDirectives[r.Name()] {
   208  			continue
   209  		}
   210  
   211  		if existingRule := genNames[r.Name()]; existingRule != nil {
   212  			import1 := existingRule.AttrString("importpath")
   213  			import2 := r.AttrString("importpath")
   214  			return fmt.Errorf("imports %s and %s resolve to the same repository rule name %s",
   215  				import1, import2, r.Name())
   216  		} else {
   217  			genNames[r.Name()] = r
   218  		}
   219  		f := uc.repoFileMap[r.Name()]
   220  		if f != nil {
   221  			genForFiles[f] = append(genForFiles[f], r)
   222  		} else {
   223  			newGen = append(newGen, r)
   224  		}
   225  	}
   226  	for _, r := range empty {
   227  		f := uc.repoFileMap[r.Name()]
   228  		if f == nil {
   229  			panic(fmt.Sprintf("empty rule %q for deletion that was not found", r.Name()))
   230  		}
   231  		emptyForFiles[f] = append(emptyForFiles[f], r)
   232  	}
   233  
   234  	var macroPath string
   235  	if uc.macroFileName != "" {
   236  		macroPath = filepath.Join(c.RepoRoot, filepath.Clean(uc.macroFileName))
   237  	}
   238  	// If we are in bzlmod mode, then do not update the workspace. However, if a macro file was
   239  	// specified, proceed with generating the macro file. This is useful for rule repositories that
   240  	// build with bzlmod enabled, but support clients that use legacy WORKSPACE dependency loading.
   241  	if !c.Bzlmod || macroPath != "" {
   242  		var newGenFile *rule.File
   243  		for f := range genForFiles {
   244  			if macroPath == "" && wspace.IsWORKSPACE(f.Path) ||
   245  				macroPath != "" && f.Path == macroPath && f.DefName == uc.macroDefName {
   246  				newGenFile = f
   247  				break
   248  			}
   249  		}
   250  		if newGenFile == nil {
   251  			if uc.macroFileName == "" {
   252  				newGenFile = uc.workspace
   253  			} else {
   254  				var err error
   255  				newGenFile, err = rule.LoadMacroFile(macroPath, "", uc.macroDefName)
   256  				if os.IsNotExist(err) {
   257  					newGenFile, err = rule.EmptyMacroFile(macroPath, "", uc.macroDefName)
   258  					if err != nil {
   259  						return fmt.Errorf("error creating %q: %v", macroPath, err)
   260  					}
   261  				} else if err != nil {
   262  					return fmt.Errorf("error loading %q: %v", macroPath, err)
   263  				}
   264  			}
   265  		}
   266  		genForFiles[newGenFile] = append(genForFiles[newGenFile], newGen...)
   267  	}
   268  
   269  	workspaceInsertIndex := findWorkspaceInsertIndex(uc.workspace, kinds, loads)
   270  	for _, r := range genForFiles[uc.workspace] {
   271  		r.SetPrivateAttr(merger.UnstableInsertIndexKey, workspaceInsertIndex)
   272  	}
   273  
   274  	// Merge rules and fix loads in each file.
   275  	seenFile := make(map[*rule.File]bool)
   276  	sortedFiles := make([]*rule.File, 0, len(genForFiles))
   277  	for f := range genForFiles {
   278  		if !seenFile[f] {
   279  			seenFile[f] = true
   280  			sortedFiles = append(sortedFiles, f)
   281  		}
   282  	}
   283  	for f := range emptyForFiles {
   284  		if !seenFile[f] {
   285  			seenFile[f] = true
   286  			sortedFiles = append(sortedFiles, f)
   287  		}
   288  	}
   289  	// If we are in bzlmod mode, then do not update the workspace.
   290  	if !c.Bzlmod && ensureMacroInWorkspace(uc, workspaceInsertIndex) {
   291  		if !seenFile[uc.workspace] {
   292  			seenFile[uc.workspace] = true
   293  			sortedFiles = append(sortedFiles, uc.workspace)
   294  		}
   295  	}
   296  	sort.Slice(sortedFiles, func(i, j int) bool {
   297  		if cmp := strings.Compare(sortedFiles[i].Path, sortedFiles[j].Path); cmp != 0 {
   298  			return cmp < 0
   299  		}
   300  		return sortedFiles[i].DefName < sortedFiles[j].DefName
   301  	})
   302  
   303  	updatedFiles := make(map[string]*rule.File)
   304  	for _, f := range sortedFiles {
   305  		merger.MergeFile(f, emptyForFiles[f], genForFiles[f], merger.PreResolve, kinds)
   306  		merger.FixLoads(f, loads)
   307  		if f == uc.workspace && !c.Bzlmod {
   308  			if err := merger.CheckGazelleLoaded(f); err != nil {
   309  				return err
   310  			}
   311  		}
   312  		f.Sync()
   313  		if uf, ok := updatedFiles[f.Path]; ok {
   314  			uf.SyncMacroFile(f)
   315  		} else {
   316  			updatedFiles[f.Path] = f
   317  		}
   318  	}
   319  
   320  	// Write updated files to disk.
   321  	for _, f := range sortedFiles {
   322  		if uf := updatedFiles[f.Path]; uf != nil {
   323  			if f.DefName != "" {
   324  				uf.SortMacro()
   325  			}
   326  			newContent := f.Format()
   327  			if !bytes.Equal(f.Content, newContent) {
   328  				if err := uf.Save(uf.Path); err != nil {
   329  					return err
   330  				}
   331  			}
   332  			delete(updatedFiles, f.Path)
   333  		}
   334  	}
   335  
   336  	return nil
   337  }
   338  
   339  func newUpdateReposConfiguration(wd string, args []string, cexts []config.Configurer) (*config.Config, error) {
   340  	c := config.New()
   341  	c.WorkDir = wd
   342  	fs := flag.NewFlagSet("gazelle", flag.ContinueOnError)
   343  	// Flag will call this on any parse error. Don't print usage unless
   344  	// -h or -help were passed explicitly.
   345  	fs.Usage = func() {}
   346  	for _, cext := range cexts {
   347  		cext.RegisterFlags(fs, "update-repos", c)
   348  	}
   349  	if err := fs.Parse(args); err != nil {
   350  		if err == flag.ErrHelp {
   351  			updateReposUsage(fs)
   352  			return nil, err
   353  		}
   354  		// flag already prints the error; don't print it again.
   355  		return nil, errors.New("Try -help for more information")
   356  	}
   357  	for _, cext := range cexts {
   358  		if err := cext.CheckFlags(fs, c); err != nil {
   359  			return nil, err
   360  		}
   361  	}
   362  	return c, nil
   363  }
   364  
   365  func updateReposUsage(fs *flag.FlagSet) {
   366  	fmt.Fprint(os.Stderr, `usage:
   367  
   368  # Add/update repositories by import path
   369  gazelle update-repos example.com/repo1 example.com/repo2
   370  
   371  # Import repositories from lock file
   372  gazelle update-repos -from_file=file
   373  
   374  The update-repos command updates repository rules in the WORKSPACE file.
   375  update-repos can add or update repositories explicitly by import path.
   376  update-repos can also import repository rules from a vendoring tool's lock
   377  file (currently only deps' Gopkg.lock is supported).
   378  
   379  FLAGS:
   380  
   381  `)
   382  	fs.PrintDefaults()
   383  }
   384  
   385  func updateRepoImports(c *config.Config, rc *repo.RemoteCache) (gen []*rule.Rule, err error) {
   386  	// TODO(jayconrod): let the user pick the language with a command line flag.
   387  	// For now, only use the first language that implements the interface.
   388  	uc := getUpdateReposConfig(c)
   389  	var updater language.RepoUpdater
   390  	for _, lang := range filterLanguages(c, languages) {
   391  		if u, ok := lang.(language.RepoUpdater); ok {
   392  			updater = u
   393  			break
   394  		}
   395  	}
   396  	if updater == nil {
   397  		return nil, fmt.Errorf("no languages can update repositories")
   398  	}
   399  	res := updater.UpdateRepos(language.UpdateReposArgs{
   400  		Config:  c,
   401  		Imports: uc.importPaths,
   402  		Cache:   rc,
   403  	})
   404  	return res.Gen, res.Error
   405  }
   406  
   407  func importRepos(c *config.Config, rc *repo.RemoteCache) (gen, empty []*rule.Rule, err error) {
   408  	uc := getUpdateReposConfig(c)
   409  	importSupported := false
   410  	var importer language.RepoImporter
   411  	for _, lang := range filterLanguages(c, languages) {
   412  		if i, ok := lang.(language.RepoImporter); ok {
   413  			importSupported = true
   414  			if i.CanImport(uc.repoFilePath) {
   415  				importer = i
   416  				break
   417  			}
   418  		}
   419  	}
   420  	if importer == nil {
   421  		if importSupported {
   422  			return nil, nil, fmt.Errorf("unknown file format: %s", uc.repoFilePath)
   423  		} else {
   424  			return nil, nil, fmt.Errorf("no supported languages can import configuration files")
   425  		}
   426  	}
   427  	res := importer.ImportRepos(language.ImportReposArgs{
   428  		Config: c,
   429  		Path:   uc.repoFilePath,
   430  		Prune:  uc.pruneRules,
   431  		Cache:  rc,
   432  	})
   433  	return res.Gen, res.Empty, res.Error
   434  }
   435  
   436  // findWorkspaceInsertIndex reads a WORKSPACE file and finds an index within
   437  // f.File.Stmt where new direct dependencies should be inserted. In general, new
   438  // dependencies should be inserted after repository rules are loaded (described
   439  // by kinds) but before macros declaring indirect dependencies.
   440  func findWorkspaceInsertIndex(f *rule.File, kinds map[string]rule.KindInfo, loads []rule.LoadInfo) int {
   441  	loadFiles := make(map[string]struct{})
   442  	loadRepos := make(map[string]struct{})
   443  	for _, li := range loads {
   444  		name, err := label.Parse(li.Name)
   445  		if err != nil {
   446  			continue
   447  		}
   448  		loadFiles[li.Name] = struct{}{}
   449  		loadRepos[name.Repo] = struct{}{}
   450  	}
   451  
   452  	// Find the first index after load statements from files that contain
   453  	// repository rules (for example, "@bazel_gazelle//:deps.bzl") and after
   454  	// repository rules declaring those files (http_archive for bazel_gazelle).
   455  	// It doesn't matter whether the repository rules are actually loaded.
   456  	insertAfter := 0
   457  
   458  	for _, ld := range f.Loads {
   459  		if _, ok := loadFiles[ld.Name()]; !ok {
   460  			continue
   461  		}
   462  		if idx := ld.Index(); idx > insertAfter {
   463  			insertAfter = idx
   464  		}
   465  	}
   466  
   467  	for _, r := range f.Rules {
   468  		if _, ok := loadRepos[r.Name()]; !ok {
   469  			continue
   470  		}
   471  		if idx := r.Index(); idx > insertAfter {
   472  			insertAfter = idx
   473  		}
   474  	}
   475  
   476  	// There may be many direct dependencies after that index (perhaps
   477  	// 'update-repos' inserted them previously). We want to insert after those.
   478  	// So find the highest index after insertAfter before a call to something
   479  	// that doesn't look like a direct dependency.
   480  	insertBefore := len(f.File.Stmt)
   481  	for _, r := range f.Rules {
   482  		kind := r.Kind()
   483  		if kind == "local_repository" || kind == "http_archive" || kind == "git_repository" {
   484  			// Built-in or well-known repository rules.
   485  			continue
   486  		}
   487  		if _, ok := kinds[kind]; ok {
   488  			// Repository rule Gazelle might generate.
   489  			continue
   490  		}
   491  		if r.Name() != "" {
   492  			// Has a name attribute, probably still a repository rule.
   493  			continue
   494  		}
   495  		if idx := r.Index(); insertAfter < idx && idx < insertBefore {
   496  			insertBefore = idx
   497  		}
   498  	}
   499  
   500  	return insertBefore
   501  }
   502  
   503  // ensureMacroInWorkspace adds a call to the repository macro if the -to_macro
   504  // flag was used, and the macro was not called or declared with a
   505  // '# gazelle:repository_macro' directive.
   506  //
   507  // ensureMacroInWorkspace returns true if the WORKSPACE file was updated
   508  // and should be saved.
   509  func ensureMacroInWorkspace(uc *updateReposConfig, insertIndex int) (updated bool) {
   510  	if uc.macroFileName == "" {
   511  		return false
   512  	}
   513  
   514  	// Check whether the macro is already declared.
   515  	// We won't add a call if the macro is declared but not called. It might
   516  	// be called somewhere else.
   517  	macroValue := uc.macroFileName + "%" + uc.macroDefName
   518  	for _, d := range uc.workspace.Directives {
   519  		if d.Key == "repository_macro" {
   520  			if parsed, _ := repo.ParseRepositoryMacroDirective(d.Value); parsed != nil && parsed.Path == uc.macroFileName && parsed.DefName == uc.macroDefName {
   521  				return false
   522  			}
   523  		}
   524  	}
   525  
   526  	// Try to find a load and a call.
   527  	var load *rule.Load
   528  	var call *rule.Rule
   529  	var loadedDefName string
   530  	for _, l := range uc.workspace.Loads {
   531  		switch l.Name() {
   532  		case ":" + uc.macroFileName, "//:" + uc.macroFileName, "@//:" + uc.macroFileName:
   533  			load = l
   534  			pairs := l.SymbolPairs()
   535  			for _, pair := range pairs {
   536  				if pair.From == uc.macroDefName {
   537  					loadedDefName = pair.To
   538  				}
   539  			}
   540  		}
   541  	}
   542  
   543  	for _, r := range uc.workspace.Rules {
   544  		if r.Kind() == loadedDefName {
   545  			call = r
   546  		}
   547  	}
   548  
   549  	// Add the load and call if they're missing.
   550  	if call == nil {
   551  		if load == nil {
   552  			load = rule.NewLoad("//:" + uc.macroFileName)
   553  			load.Insert(uc.workspace, insertIndex)
   554  		}
   555  		if loadedDefName == "" {
   556  			load.Add(uc.macroDefName)
   557  		}
   558  
   559  		call = rule.NewRule(uc.macroDefName, "")
   560  		call.InsertAt(uc.workspace, insertIndex)
   561  	}
   562  
   563  	// Add the directive to the call.
   564  	call.AddComment("# gazelle:repository_macro " + macroValue)
   565  
   566  	return true
   567  }
   568
View as plain text