/* Copyright 2017 The Bazel Authors. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "bytes" "context" "errors" "flag" "fmt" "log" "os" "path/filepath" "sort" "strings" "syscall" "github.com/bazelbuild/bazel-gazelle/config" gzflag "github.com/bazelbuild/bazel-gazelle/flag" "github.com/bazelbuild/bazel-gazelle/internal/wspace" "github.com/bazelbuild/bazel-gazelle/label" "github.com/bazelbuild/bazel-gazelle/language" "github.com/bazelbuild/bazel-gazelle/merger" "github.com/bazelbuild/bazel-gazelle/repo" "github.com/bazelbuild/bazel-gazelle/resolve" "github.com/bazelbuild/bazel-gazelle/rule" "github.com/bazelbuild/bazel-gazelle/walk" ) // updateConfig holds configuration information needed to run the fix and // update commands. This includes everything in config.Config, but it also // includes some additional fields that aren't relevant to other packages. type updateConfig struct { dirs []string emit emitFunc repos []repo.Repo workspaceFiles []*rule.File walkMode walk.Mode patchPath string patchBuffer bytes.Buffer print0 bool profile profiler } type emitFunc func(c *config.Config, f *rule.File) error var modeFromName = map[string]emitFunc{ "print": printFile, "fix": fixFile, "diff": diffFile, } const updateName = "_update" func getUpdateConfig(c *config.Config) *updateConfig { return c.Exts[updateName].(*updateConfig) } type updateConfigurer struct { mode string recursive bool knownImports []string repoConfigPath string cpuProfile string memProfile string } func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { uc := &updateConfig{} c.Exts[updateName] = uc c.ShouldFix = cmd == "fix" fs.StringVar(&ucr.mode, "mode", "fix", "print: prints all of the updated BUILD files\n\tfix: rewrites all of the BUILD files in place\n\tdiff: computes the rewrite but then just does a diff") fs.BoolVar(&ucr.recursive, "r", true, "when true, gazelle will update subdirectories recursively") fs.StringVar(&uc.patchPath, "patch", "", "when set with -mode=diff, gazelle will write to a file instead of stdout") fs.BoolVar(&uc.print0, "print0", false, "when set with -mode=fix, gazelle will print the names of rewritten files separated with \\0 (NULL)") fs.StringVar(&ucr.cpuProfile, "cpuprofile", "", "write cpu profile to `file`") fs.StringVar(&ucr.memProfile, "memprofile", "", "write memory profile to `file`") fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)") fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.") } func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { uc := getUpdateConfig(c) var ok bool uc.emit, ok = modeFromName[ucr.mode] if !ok { return fmt.Errorf("unrecognized emit mode: %q", ucr.mode) } if uc.patchPath != "" && ucr.mode != "diff" { return fmt.Errorf("-patch set but -mode is %s, not diff", ucr.mode) } if uc.patchPath != "" && !filepath.IsAbs(uc.patchPath) { uc.patchPath = filepath.Join(c.WorkDir, uc.patchPath) } p, err := newProfiler(ucr.cpuProfile, ucr.memProfile) if err != nil { return err } uc.profile = p dirs := fs.Args() if len(dirs) == 0 { dirs = []string{"."} } uc.dirs = make([]string, len(dirs)) for i, arg := range dirs { dir := arg if !filepath.IsAbs(dir) { dir = filepath.Join(c.WorkDir, dir) } dir, err := filepath.EvalSymlinks(dir) if err != nil { return fmt.Errorf("%s: failed to resolve symlinks: %v", arg, err) } if !isDescendingDir(dir, c.RepoRoot) { return fmt.Errorf("%s: not a subdirectory of repo root %s", arg, c.RepoRoot) } uc.dirs[i] = dir } if ucr.recursive && c.IndexLibraries { uc.walkMode = walk.VisitAllUpdateSubdirsMode } else if c.IndexLibraries { uc.walkMode = walk.VisitAllUpdateDirsMode } else if ucr.recursive { uc.walkMode = walk.UpdateSubdirsMode } else { uc.walkMode = walk.UpdateDirsMode } // Load the repo configuration file (WORKSPACE by default) to find out // names and prefixes of other go_repositories. This affects external // dependency resolution for Go. // TODO(jayconrod): Go-specific code should be moved to language/go. if ucr.repoConfigPath == "" { ucr.repoConfigPath = wspace.FindWORKSPACEFile(c.RepoRoot) } repoConfigFile, err := rule.LoadWorkspaceFile(ucr.repoConfigPath, "") if err != nil && !os.IsNotExist(err) && !isDirErr(err) { return err } else if err == nil { c.Repos, _, err = repo.ListRepositories(repoConfigFile) if err != nil { return err } } for _, imp := range ucr.knownImports { uc.repos = append(uc.repos, repo.Repo{ Name: label.ImportPathToBazelRepoName(imp), GoPrefix: imp, }) } for _, r := range c.Repos { if r.Kind() == "go_repository" { var name string if apparentName := c.ModuleToApparentName(r.AttrString("module_name")); apparentName != "" { name = apparentName } else { name = r.Name() } uc.repos = append(uc.repos, repo.Repo{ Name: name, GoPrefix: r.AttrString("importpath"), }) } } // If the repo configuration file is not WORKSPACE, also load WORKSPACE // and any declared macro files so we can apply fixes. workspacePath := wspace.FindWORKSPACEFile(c.RepoRoot) var workspace *rule.File if ucr.repoConfigPath == workspacePath { workspace = repoConfigFile } else { workspace, err = rule.LoadWorkspaceFile(workspacePath, "") if err != nil && !os.IsNotExist(err) && !isDirErr(err) { return err } } if workspace != nil { c.RepoName = findWorkspaceName(workspace) _, repoFileMap, err := repo.ListRepositories(workspace) if err != nil { return err } seen := make(map[*rule.File]bool) for _, f := range repoFileMap { if !seen[f] { uc.workspaceFiles = append(uc.workspaceFiles, f) seen[f] = true } } sort.Slice(uc.workspaceFiles, func(i, j int) bool { return uc.workspaceFiles[i].Path < uc.workspaceFiles[j].Path }) } return nil } func (ucr *updateConfigurer) KnownDirectives() []string { return nil } func (ucr *updateConfigurer) Configure(c *config.Config, rel string, f *rule.File) {} // visitRecord stores information about about a directory visited with // packages.Walk. type visitRecord struct { // pkgRel is the slash-separated path to the visited directory, relative to // the repository root. "" for the repository root itself. pkgRel string // c is the configuration for the directory with directives applied. c *config.Config // rules is a list of generated Go rules. rules []*rule.Rule // imports contains opaque import information for each rule in rules. imports []interface{} // empty is a list of empty Go rules that may be deleted. empty []*rule.Rule // file is the build file being processed. file *rule.File // mappedKinds are mapped kinds used during this visit. mappedKinds []config.MappedKind mappedKindInfo map[string]rule.KindInfo } var genericLoads = []rule.LoadInfo{ { Name: "@bazel_gazelle//:def.bzl", Symbols: []string{"gazelle"}, }, } func runFixUpdate(wd string, cmd command, args []string) (err error) { cexts := make([]config.Configurer, 0, len(languages)+4) cexts = append(cexts, &config.CommonConfigurer{}, &updateConfigurer{}, &walk.Configurer{}, &resolve.Configurer{}) for _, lang := range languages { cexts = append(cexts, lang) } c, err := newFixUpdateConfiguration(wd, cmd, args, cexts) if err != nil { return err } mrslv := newMetaResolver() kinds := make(map[string]rule.KindInfo) loads := genericLoads exts := make([]interface{}, 0, len(languages)) for _, lang := range languages { for kind, info := range lang.Kinds() { mrslv.AddBuiltin(kind, lang) kinds[kind] = info } if moduleAwareLang, ok := lang.(language.ModuleAwareLanguage); ok { loads = append(loads, moduleAwareLang.ApparentLoads(c.ModuleToApparentName)...) } else { loads = append(loads, lang.Loads()...) } exts = append(exts, lang) } ruleIndex := resolve.NewRuleIndex(mrslv.Resolver, exts...) if err := fixRepoFiles(c, loads); err != nil { return err } ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, lang := range languages { if life, ok := lang.(language.LifecycleManager); ok { life.Before(ctx) } } // Visit all directories in the repository. var visits []visitRecord uc := getUpdateConfig(c) defer func() { if err := uc.profile.stop(); err != nil { log.Printf("stopping profiler: %v", err) } }() var errorsFromWalk []error walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { // If this file is ignored or if Gazelle was not asked to update this // directory, just index the build file and move on. if !update { if c.IndexLibraries && f != nil { for _, repl := range c.KindMap { mrslv.MappedKind(rel, repl) } for _, r := range f.Rules { ruleIndex.AddRule(c, r, f) } } return } // Fix any problems in the file. if f != nil { for _, l := range filterLanguages(c, languages) { l.Fix(c, f) } } // Generate rules. var empty, gen []*rule.Rule var imports []interface{} for _, l := range filterLanguages(c, languages) { res := l.GenerateRules(language.GenerateArgs{ Config: c, Dir: dir, Rel: rel, File: f, Subdirs: subdirs, RegularFiles: regularFiles, GenFiles: genFiles, OtherEmpty: empty, OtherGen: gen, }) if len(res.Gen) != len(res.Imports) { log.Panicf("%s: language %s generated %d rules but returned %d imports", rel, l.Name(), len(res.Gen), len(res.Imports)) } empty = append(empty, res.Empty...) gen = append(gen, res.Gen...) imports = append(imports, res.Imports...) } if f == nil && len(gen) == 0 { return } // Apply and record relevant kind mappings. var ( mappedKinds []config.MappedKind mappedKindInfo = make(map[string]rule.KindInfo) ) // We apply map_kind to all rules, including pre-existing ones. var allRules []*rule.Rule allRules = append(allRules, gen...) if f != nil { allRules = append(allRules, f.Rules...) } for _, r := range allRules { repl, err := lookupMapKindReplacement(c.KindMap, r.Kind()) if err != nil { errorsFromWalk = append(errorsFromWalk, fmt.Errorf("looking up mapped kind: %w", err)) continue } if repl != nil { mappedKindInfo[repl.KindName] = kinds[r.Kind()] mappedKinds = append(mappedKinds, *repl) mrslv.MappedKind(rel, *repl) r.SetKind(repl.KindName) } } for _, r := range empty { if repl, ok := c.KindMap[r.Kind()]; ok { mappedKindInfo[repl.KindName] = kinds[r.Kind()] mappedKinds = append(mappedKinds, repl) mrslv.MappedKind(rel, repl) r.SetKind(repl.KindName) } } // Insert or merge rules into the build file. if f == nil { f = rule.EmptyFile(filepath.Join(dir, c.DefaultBuildFileName()), rel) for _, r := range gen { r.Insert(f) } } else { merger.MergeFile(f, empty, gen, merger.PreResolve, unionKindInfoMaps(kinds, mappedKindInfo)) } visits = append(visits, visitRecord{ pkgRel: rel, c: c, rules: gen, imports: imports, empty: empty, file: f, mappedKinds: mappedKinds, mappedKindInfo: mappedKindInfo, }) // Add library rules to the dependency resolution table. if c.IndexLibraries { for _, r := range f.Rules { ruleIndex.AddRule(c, r, f) } } }) for _, lang := range languages { if finishable, ok := lang.(language.FinishableLanguage); ok { finishable.DoneGeneratingRules() } } if len(errorsFromWalk) == 1 { return errorsFromWalk[0] } if len(errorsFromWalk) > 1 { var additionalErrors []string for _, error := range errorsFromWalk[1:] { additionalErrors = append(additionalErrors, error.Error()) } return fmt.Errorf("encountered multiple errors: %w, %v", errorsFromWalk[0], strings.Join(additionalErrors, ", ")) } // Finish building the index for dependency resolution. ruleIndex.Finish() // Resolve dependencies. rc, cleanupRc := repo.NewRemoteCache(uc.repos) defer func() { if cerr := cleanupRc(); err == nil && cerr != nil { err = cerr } }() if err := maybePopulateRemoteCacheFromGoMod(c, rc); err != nil { log.Print(err) } for _, v := range visits { for i, r := range v.rules { from := label.New(c.RepoName, v.pkgRel, r.Name()) if rslv := mrslv.Resolver(r, v.pkgRel); rslv != nil { rslv.Resolve(v.c, ruleIndex, rc, r, v.imports[i], from) } } merger.MergeFile(v.file, v.empty, v.rules, merger.PostResolve, unionKindInfoMaps(kinds, v.mappedKindInfo)) } for _, lang := range languages { if life, ok := lang.(language.LifecycleManager); ok { life.AfterResolvingDeps(ctx) } } // Emit merged files. var exit error for _, v := range visits { merger.FixLoads(v.file, applyKindMappings(v.mappedKinds, loads)) if err := uc.emit(v.c, v.file); err != nil { if err == errExit { exit = err } else { log.Print(err) } } } if uc.patchPath != "" { if err := os.WriteFile(uc.patchPath, uc.patchBuffer.Bytes(), 0o666); err != nil { return err } } return exit } // lookupMapKindReplacement finds a mapped replacement for rule kind `kind`, resolving transitively. // i.e. if go_library is mapped to custom_go_library, and custom_go_library is mapped to other_go_library, // looking up go_library will return other_go_library. // It returns an error on a loop, and may return nil if no remapping should be performed. func lookupMapKindReplacement(kindMap map[string]config.MappedKind, kind string) (*config.MappedKind, error) { var mapped *config.MappedKind seenKinds := make(map[string]struct{}) seenKindPath := []string{kind} for { replacement, ok := kindMap[kind] if !ok { break } seenKindPath = append(seenKindPath, replacement.KindName) if _, alreadySeen := seenKinds[replacement.KindName]; alreadySeen { return nil, fmt.Errorf("found loop of map_kind replacements: %s", strings.Join(seenKindPath, " -> ")) } seenKinds[replacement.KindName] = struct{}{} mapped = &replacement if kind == replacement.KindName { break } kind = replacement.KindName } return mapped, nil } func newFixUpdateConfiguration(wd string, cmd command, args []string, cexts []config.Configurer) (*config.Config, error) { c := config.New() c.WorkDir = wd fs := flag.NewFlagSet("gazelle", flag.ContinueOnError) // Flag will call this on any parse error. Don't print usage unless // -h or -help were passed explicitly. fs.Usage = func() {} for _, cext := range cexts { cext.RegisterFlags(fs, cmd.String(), c) } if err := fs.Parse(args); err != nil { if err == flag.ErrHelp { fixUpdateUsage(fs) return nil, err } // flag already prints the error; don't print it again. log.Fatal("Try -help for more information.") } for _, cext := range cexts { if err := cext.CheckFlags(fs, c); err != nil { return nil, err } } return c, nil } func fixUpdateUsage(fs *flag.FlagSet) { fmt.Fprint(os.Stderr, `usage: gazelle [fix|update] [flags...] [package-dirs...] The update command creates new build files and update existing BUILD files when needed. The fix command also creates and updates build files, and in addition, it may make potentially breaking updates to usage of rules. For example, it may delete obsolete rules or rename existing rules. There are several output modes which can be selected with the -mode flag. The output mode determines what Gazelle does with updated BUILD files. fix (default) - write updated BUILD files back to disk. print - print updated BUILD files to stdout. diff - diff updated BUILD files against existing files in unified format. Gazelle accepts a list of paths to Go package directories to process (defaults to the working directory if none are given). It recursively traverses subdirectories. All directories must be under the directory specified by -repo_root; if -repo_root is not given, this is the directory containing the WORKSPACE file. FLAGS: `) fs.PrintDefaults() } func fixRepoFiles(c *config.Config, loads []rule.LoadInfo) error { uc := getUpdateConfig(c) if !c.ShouldFix { return nil } shouldFix := false for _, d := range uc.dirs { if d == c.RepoRoot { shouldFix = true } } if !shouldFix { return nil } for _, f := range uc.workspaceFiles { merger.FixLoads(f, loads) workspaceFile := wspace.FindWORKSPACEFile(c.RepoRoot) if f.Path == workspaceFile { removeLegacyGoRepository(f) if err := merger.CheckGazelleLoaded(f); err != nil { return err } } if err := uc.emit(c, f); err != nil { return err } } return nil } // removeLegacyGoRepository removes loads of go_repository from // @io_bazel_rules_go. FixLoads should be called after this; it will load from // @bazel_gazelle. func removeLegacyGoRepository(f *rule.File) { for _, l := range f.Loads { if l.Name() == "@io_bazel_rules_go//go:def.bzl" { l.Remove("go_repository") if l.IsEmpty() { l.Delete() } } } } func findWorkspaceName(f *rule.File) string { var name string for _, r := range f.Rules { if r.Kind() == "workspace" { name = r.Name() break } } // HACK(bazelbuild/rules_go#2355, bazelbuild/rules_go#2387): // We can't patch the WORKSPACE file with the correct name because Bazel // writes it first; our patches won't apply. if name == "com_google_googleapis" { return "go_googleapis" } return name } func isDescendingDir(dir, root string) bool { rel, err := filepath.Rel(root, dir) if err != nil { return false } if rel == "." { return true } return !strings.HasPrefix(rel, "..") } func findOutputPath(c *config.Config, f *rule.File) string { if c.ReadBuildFilesDir == "" && c.WriteBuildFilesDir == "" { return f.Path } baseDir := c.WriteBuildFilesDir if c.WriteBuildFilesDir == "" { baseDir = c.RepoRoot } outputDir := filepath.Join(baseDir, filepath.FromSlash(f.Pkg)) defaultOutputPath := filepath.Join(outputDir, c.DefaultBuildFileName()) ents, err := os.ReadDir(outputDir) if err != nil { // Ignore error. Directory probably doesn't exist. return defaultOutputPath } outputPath := rule.MatchBuildFile(outputDir, c.ValidBuildFileNames, ents) if outputPath == "" { return defaultOutputPath } return outputPath } // maybePopulateRemoteCacheFromGoMod reads go.mod and adds a root to rc for each // module requirement. This lets the Go extension avoid a network lookup for // unknown imports with -external=external, and it lets dependency resolution // succeed with -external=static when it might not otherwise. // // This function does not override roots added from WORKSPACE (or some other // configuration file), but it's useful when there is no such file. In most // cases, a user of Gazelle with indirect Go dependencies does not need to add // '# gazelle:repository' or '# gazelle:repository_macro' directives to their // WORKSPACE file. This need was frustrating for developers in non-Go // repositories with go_repository dependencies declared in macros. It wasn't // obvious that Gazelle was missing these. // // This function is regrettably Go specific and does not belong here, but it // can't be moved to //language/go until //repo is broken up and moved there. func maybePopulateRemoteCacheFromGoMod(c *config.Config, rc *repo.RemoteCache) error { haveGo := false for name := range c.Exts { if name == "go" { haveGo = true break } } if !haveGo { return nil } goModPath := filepath.Join(c.RepoRoot, "go.mod") if _, err := os.Stat(goModPath); err != nil { return nil } return rc.PopulateFromGoMod(goModPath) } func unionKindInfoMaps(a, b map[string]rule.KindInfo) map[string]rule.KindInfo { if len(a) == 0 { return b } if len(b) == 0 { return a } result := make(map[string]rule.KindInfo, len(a)+len(b)) for _, m := range []map[string]rule.KindInfo{a, b} { for k, v := range m { result[k] = v } } return result } // applyKindMappings returns a copy of LoadInfo that includes c.KindMap. func applyKindMappings(mappedKinds []config.MappedKind, loads []rule.LoadInfo) []rule.LoadInfo { if len(mappedKinds) == 0 { return loads } // Add new RuleInfos or replace existing ones with merged ones. mappedLoads := make([]rule.LoadInfo, len(loads)) copy(mappedLoads, loads) for _, mappedKind := range mappedKinds { mappedLoads = appendOrMergeKindMapping(mappedLoads, mappedKind) } return mappedLoads } // appendOrMergeKindMapping adds LoadInfo for the given replacement. func appendOrMergeKindMapping(mappedLoads []rule.LoadInfo, mappedKind config.MappedKind) []rule.LoadInfo { // If mappedKind.KindLoad already exists in the list, create a merged copy. for i, load := range mappedLoads { if load.Name == mappedKind.KindLoad { mappedLoads[i].Symbols = append(load.Symbols, mappedKind.KindName) return mappedLoads } } // Add a new LoadInfo. return append(mappedLoads, rule.LoadInfo{ Name: mappedKind.KindLoad, Symbols: []string{mappedKind.KindName}, }) } func isDirErr(err error) bool { if err == nil { return false } var pe *os.PathError if errors.As(err, &pe) { return pe.Err == syscall.EISDIR } return false }