fix.go

Documentation: github.com/bazelbuild/buildtools/edit

     1  /*
     2  Copyright 2016 Google LLC
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      https://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Functions to clean and fix BUILD files
    18  
    19  package edit
    20  
    21  import (
    22  	"regexp"
    23  	"sort"
    24  	"strings"
    25  
    26  	"github.com/bazelbuild/buildtools/build"
    27  	"github.com/bazelbuild/buildtools/labels"
    28  )
    29  
    30  // splitOptionsWithSpaces is a cleanup function.
    31  // It splits options strings that contain a space. This change
    32  // should be safe as Blaze is splitting those strings, but we will
    33  // eventually get rid of this misfeature.
    34  //
    35  //	eg. it converts from:
    36  //	  copts = ["-Dfoo -Dbar"]
    37  //	to:
    38  //	  copts = ["-Dfoo", "-Dbar"]
    39  func splitOptionsWithSpaces(_ *build.File, r *build.Rule, _ string) bool {
    40  	var attrToRewrite = []string{
    41  		"copts",
    42  		"linkopts",
    43  	}
    44  	fixed := false
    45  	for _, attrName := range attrToRewrite {
    46  		attr := r.Attr(attrName)
    47  		if attr != nil {
    48  			for _, li := range AllLists(attr) {
    49  				fixed = splitStrings(li) || fixed
    50  			}
    51  		}
    52  	}
    53  	return fixed
    54  }
    55  
    56  func splitStrings(list *build.ListExpr) bool {
    57  	var all []build.Expr
    58  	fixed := false
    59  	for _, e := range list.List {
    60  		str, ok := e.(*build.StringExpr)
    61  		if !ok {
    62  			all = append(all, e)
    63  			continue
    64  		}
    65  		if strings.Contains(str.Value, " ") && !strings.Contains(str.Value, "'\"") && !strings.Contains(str.Value, "$(location") {
    66  			fixed = true
    67  			for i, substr := range strings.Fields(str.Value) {
    68  				item := &build.StringExpr{Value: substr}
    69  				if i == 0 {
    70  					item.Comments = str.Comments
    71  				}
    72  				all = append(all, item)
    73  			}
    74  		} else {
    75  			all = append(all, str)
    76  		}
    77  	}
    78  	list.List = all
    79  	return fixed
    80  }
    81  
    82  // shortenLabels rewrites the labels in the rule using the short notation.
    83  func shortenLabels(_ *build.File, r *build.Rule, pkg string) bool {
    84  	fixed := false
    85  	for _, attr := range r.AttrKeys() {
    86  		e := r.Attr(attr)
    87  		if !ContainsLabels(r.Kind(), attr) {
    88  			continue
    89  		}
    90  		for _, li := range AllLists(e) {
    91  			for _, elem := range li.List {
    92  				str, ok := elem.(*build.StringExpr)
    93  				if ok && str.Value != labels.Shorten(str.Value, pkg) {
    94  					str.Value = labels.Shorten(str.Value, pkg)
    95  					fixed = true
    96  				}
    97  			}
    98  		}
    99  	}
   100  	return fixed
   101  }
   102  
   103  // removeVisibility removes useless visibility attributes.
   104  func removeVisibility(f *build.File, r *build.Rule, pkg string) bool {
   105  	// If no default_visibility is given, it is implicitly private.
   106  	defaultVisibility := []string{"//visibility:private"}
   107  	if pkgDecl := ExistingPackageDeclaration(f); pkgDecl != nil {
   108  		if pkgDecl.Attr("default_visibility") != nil {
   109  			defaultVisibility = pkgDecl.AttrStrings("default_visibility")
   110  		}
   111  	}
   112  
   113  	visibility := r.AttrStrings("visibility")
   114  	if len(visibility) == 0 || len(visibility) != len(defaultVisibility) {
   115  		return false
   116  	}
   117  	sort.Strings(defaultVisibility)
   118  	sort.Strings(visibility)
   119  	for i, vis := range visibility {
   120  		if vis != defaultVisibility[i] {
   121  			return false
   122  		}
   123  	}
   124  	r.DelAttr("visibility")
   125  	return true
   126  }
   127  
   128  // removeTestOnly removes the useless testonly attributes.
   129  func removeTestOnly(f *build.File, r *build.Rule, pkg string) bool {
   130  	pkgDecl := ExistingPackageDeclaration(f)
   131  
   132  	def := strings.HasSuffix(r.Kind(), "_test") || r.Kind() == "test_suite"
   133  	if !def {
   134  		if pkgDecl == nil || pkgDecl.Attr("default_testonly") == nil {
   135  			def = strings.HasPrefix(pkg, "javatests/")
   136  		} else if pkgDecl.AttrLiteral("default_testonly") == "1" {
   137  			def = true
   138  		} else if pkgDecl.AttrLiteral("default_testonly") != "0" {
   139  			// Non-literal value: it's not safe to do a change.
   140  			return false
   141  		}
   142  	}
   143  
   144  	testonly := r.AttrLiteral("testonly")
   145  	if def && testonly == "1" {
   146  		r.DelAttr("testonly")
   147  		return true
   148  	}
   149  	if !def && testonly == "0" {
   150  		r.DelAttr("testonly")
   151  		return true
   152  	}
   153  	return false
   154  }
   155  
   156  func genruleRenameDepsTools(_ *build.File, r *build.Rule, _ string) bool {
   157  	return r.Kind() == "genrule" && RenameAttribute(r, "deps", "tools") == nil
   158  }
   159  
   160  // explicitHeuristicLabels adds $(location ...) for each label in the string s.
   161  func explicitHeuristicLabels(s string, labels map[string]bool) string {
   162  	// Regexp comes from LABEL_CHAR_MATCHER in
   163  	//   java/com/google/devtools/build/lib/analysis/LabelExpander.java
   164  	re := regexp.MustCompile("[a-zA-Z0-9:/_.+-]+|[^a-zA-Z0-9:/_.+-]+")
   165  	parts := re.FindAllString(s, -1)
   166  	changed := false
   167  	canChange := true
   168  	for i, part := range parts {
   169  		// We don't want to add $(location when it's already present.
   170  		// So we skip the next label when we see location(s).
   171  		if part == "location" || part == "locations" {
   172  			canChange = false
   173  		}
   174  		if !labels[part] {
   175  			if labels[":"+part] { // leading colon is often missing
   176  				part = ":" + part
   177  			} else {
   178  				continue
   179  			}
   180  		}
   181  
   182  		if !canChange {
   183  			canChange = true
   184  			continue
   185  		}
   186  		parts[i] = "$(location " + part + ")"
   187  		changed = true
   188  	}
   189  	if changed {
   190  		return strings.Join(parts, "")
   191  	}
   192  	return s
   193  }
   194  
   195  func addLabels(r *build.Rule, attr string, labels map[string]bool) {
   196  	a := r.Attr(attr)
   197  	if a == nil {
   198  		return
   199  	}
   200  	for _, li := range AllLists(a) {
   201  		for _, item := range li.List {
   202  			if str, ok := item.(*build.StringExpr); ok {
   203  				labels[str.Value] = true
   204  			}
   205  		}
   206  	}
   207  }
   208  
   209  // genruleFixHeuristicLabels modifies the cmd attribute of genrules, so
   210  // that they don't rely on heuristic label expansion anymore.
   211  // Label expansion is made explicit with the $(location ...) command.
   212  func genruleFixHeuristicLabels(_ *build.File, r *build.Rule, _ string) bool {
   213  	if r.Kind() != "genrule" {
   214  		return false
   215  	}
   216  
   217  	cmd := r.Attr("cmd")
   218  	if cmd == nil {
   219  		return false
   220  	}
   221  	labels := make(map[string]bool)
   222  	addLabels(r, "tools", labels)
   223  	addLabels(r, "srcs", labels)
   224  
   225  	fixed := false
   226  	for _, str := range AllStrings(cmd) {
   227  		newVal := explicitHeuristicLabels(str.Value, labels)
   228  		if newVal != str.Value {
   229  			fixed = true
   230  			str.Value = newVal
   231  		}
   232  	}
   233  	return fixed
   234  }
   235  
   236  // sortExportsFiles sorts the first argument of exports_files if it is a list.
   237  func sortExportsFiles(_ *build.File, r *build.Rule, _ string) bool {
   238  	if r.Kind() != "exports_files" || len(r.Call.List) == 0 {
   239  		return false
   240  	}
   241  	build.SortStringList(r.Call.List[0])
   242  	return true
   243  }
   244  
   245  // removeVarref replaces all varref('x') with '$(x)'.
   246  // The goal is to eventually remove varref from the build language.
   247  func removeVarref(_ *build.File, r *build.Rule, _ string) bool {
   248  	fixed := false
   249  	EditFunction(r.Call, "varref", func(call *build.CallExpr, stk []build.Expr) build.Expr {
   250  		if len(call.List) != 1 {
   251  			return nil
   252  		}
   253  		str, ok := (call.List[0]).(*build.StringExpr)
   254  		if !ok {
   255  			return nil
   256  		}
   257  		fixed = true
   258  		str.Value = "$(" + str.Value + ")"
   259  		// Preserve suffix comments from the function call
   260  		str.Comment().Suffix = append(str.Comment().Suffix, call.Comment().Suffix...)
   261  		return str
   262  	})
   263  	return fixed
   264  }
   265  
   266  // sortGlob sorts the list argument to glob.
   267  func sortGlob(_ *build.File, r *build.Rule, _ string) bool {
   268  	fixed := false
   269  	EditFunction(r.Call, "glob", func(call *build.CallExpr, stk []build.Expr) build.Expr {
   270  		if len(call.List) == 0 {
   271  			return nil
   272  		}
   273  		build.SortStringList(call.List[0])
   274  		fixed = true
   275  		return call
   276  	})
   277  	return fixed
   278  }
   279  
   280  func evaluateListConcatenation(expr build.Expr) build.Expr {
   281  	if _, ok := expr.(*build.ListExpr); ok {
   282  		return expr
   283  	}
   284  	bin, ok := expr.(*build.BinaryExpr)
   285  	if !ok || bin.Op != "+" {
   286  		return expr
   287  	}
   288  	li1, ok1 := evaluateListConcatenation(bin.X).(*build.ListExpr)
   289  	li2, ok2 := evaluateListConcatenation(bin.Y).(*build.ListExpr)
   290  	if !ok1 || !ok2 {
   291  		return expr
   292  	}
   293  	res := *li1
   294  	res.List = append(li1.List, li2.List...)
   295  	return &res
   296  }
   297  
   298  // mergeLiteralLists evaluates the concatenation of two literal lists.
   299  // e.g. [1, 2] + [3, 4]  ->  [1, 2, 3, 4]
   300  func mergeLiteralLists(_ *build.File, r *build.Rule, _ string) bool {
   301  	fixed := false
   302  	build.Edit(r.Call, func(expr build.Expr, stk []build.Expr) build.Expr {
   303  		newexpr := evaluateListConcatenation(expr)
   304  		fixed = fixed || (newexpr != expr)
   305  		return newexpr
   306  	})
   307  	return fixed
   308  }
   309  
   310  // usePlusEqual replaces uses of extend and append with the += operator.
   311  // e.g. foo.extend(bar)  =>  foo += bar
   312  //
   313  //	foo.append(bar)  =>  foo += [bar]
   314  func usePlusEqual(f *build.File) bool {
   315  	fixed := false
   316  	for i, stmt := range f.Stmt {
   317  		call, ok := stmt.(*build.CallExpr)
   318  		if !ok {
   319  			continue
   320  		}
   321  		dot, ok := call.X.(*build.DotExpr)
   322  		if !ok || len(call.List) != 1 {
   323  			continue
   324  		}
   325  		obj, ok := dot.X.(*build.Ident)
   326  		if !ok {
   327  			continue
   328  		}
   329  
   330  		var fix *build.AssignExpr
   331  		if dot.Name == "extend" {
   332  			fix = &build.AssignExpr{LHS: obj, Op: "+=", RHS: call.List[0]}
   333  		} else if dot.Name == "append" {
   334  			list := &build.ListExpr{List: []build.Expr{call.List[0]}}
   335  			fix = &build.AssignExpr{LHS: obj, Op: "+=", RHS: list}
   336  		} else {
   337  			continue
   338  		}
   339  		fix.Comments = call.Comments // Keep original comments
   340  		f.Stmt[i] = fix
   341  		fixed = true
   342  	}
   343  	return fixed
   344  }
   345  
   346  // cleanUnusedLoads removes symbols from load statements that are not used in the file.
   347  // It also cleans symbols loaded multiple times, sorts symbol list, and removes load
   348  // statements when the list is empty.
   349  func cleanUnusedLoads(f *build.File) bool {
   350  	symbols := UsedSymbols(f)
   351  	fixed := false
   352  
   353  	// Map of symbol in this file -> modules it's loaded from
   354  	symbolsToModules := make(map[string][]string)
   355  
   356  	var all []build.Expr
   357  	for _, stmt := range f.Stmt {
   358  		load, ok := stmt.(*build.LoadStmt)
   359  		if !ok || ContainsComments(load, "@unused") {
   360  			all = append(all, stmt)
   361  			continue
   362  		}
   363  		var fromSymbols, toSymbols []*build.Ident
   364  		for i := range load.From {
   365  			fromSymbol := load.From[i]
   366  			toSymbol := load.To[i]
   367  			if symbols[toSymbol.Name] {
   368  				// The symbol is actually used
   369  
   370  				// If the most recent load for this symbol was from the same file, remove it.
   371  				previousModules := symbolsToModules[toSymbol.Name]
   372  				if len(previousModules) > 0 {
   373  					if previousModules[len(previousModules)-1] == load.Module.Value {
   374  						fixed = true
   375  						continue
   376  					}
   377  				}
   378  				symbolsToModules[toSymbol.Name] = append(symbolsToModules[toSymbol.Name], load.Module.Value)
   379  
   380  				fromSymbols = append(fromSymbols, fromSymbol)
   381  				toSymbols = append(toSymbols, toSymbol)
   382  			} else {
   383  				fixed = true
   384  			}
   385  		}
   386  		if len(toSymbols) > 0 { // Keep the load statement if it loads at least one symbol.
   387  			sort.Sort(loadArgs{fromSymbols, toSymbols})
   388  			load.From = fromSymbols
   389  			load.To = toSymbols
   390  			all = append(all, load)
   391  		} else {
   392  			fixed = true
   393  			// If the load statement contains before- or after-comments,
   394  			// keep them by re-attaching to a new CommentBlock node.
   395  			if len(load.Comment().Before) == 0 && len(load.Comment().After) == 0 {
   396  				continue
   397  			}
   398  			cb := &build.CommentBlock{}
   399  			cb.Comment().After = load.Comment().Before
   400  			cb.Comment().After = append(cb.Comment().After, load.Comment().After...)
   401  			all = append(all, cb)
   402  		}
   403  	}
   404  	f.Stmt = all
   405  	return fixed
   406  }
   407  
   408  // movePackageDeclarationToTheTop ensures that the call to package() is done
   409  // before everything else (except comments).
   410  func movePackageDeclarationToTheTop(f *build.File) bool {
   411  	pkg := ExistingPackageDeclaration(f)
   412  	if pkg == nil {
   413  		return false
   414  	}
   415  	all := []build.Expr{}
   416  	inserted := false // true when the package declaration has been inserted
   417  	for _, stmt := range f.Stmt {
   418  		_, isComment := stmt.(*build.CommentBlock)
   419  		_, isString := stmt.(*build.StringExpr)     // typically a docstring
   420  		_, isAssignExpr := stmt.(*build.AssignExpr) // e.g. variable declaration
   421  		_, isLoad := stmt.(*build.LoadStmt)
   422  		if isComment || isString || isAssignExpr || isLoad {
   423  			all = append(all, stmt)
   424  			continue
   425  		}
   426  		if stmt == pkg.Call {
   427  			if inserted {
   428  				// remove the old package
   429  				continue
   430  			}
   431  			return false // the file was ok
   432  		}
   433  		if !inserted {
   434  			all = append(all, pkg.Call)
   435  			inserted = true
   436  		}
   437  		all = append(all, stmt)
   438  	}
   439  	f.Stmt = all
   440  	return true
   441  }
   442  
   443  // moveToPackage is an auxiliary function used by moveLicenses.
   444  // The function shouldn't appear more than once in the file (depot cleanup has
   445  // been done).
   446  func moveToPackage(f *build.File, attrname string) bool {
   447  	var all []build.Expr
   448  	fixed := false
   449  	for _, stmt := range f.Stmt {
   450  		rule, ok := ExprToRule(stmt, attrname)
   451  		if !ok || len(rule.Call.List) != 1 {
   452  			all = append(all, stmt)
   453  			continue
   454  		}
   455  		pkgDecl := PackageDeclaration(f)
   456  		pkgDecl.SetAttr(attrname, rule.Call.List[0])
   457  		pkgDecl.AttrDefn(attrname).Comments = *stmt.Comment()
   458  		fixed = true
   459  	}
   460  	f.Stmt = all
   461  	return fixed
   462  }
   463  
   464  // moveLicenses replaces the 'licenses' function with an attribute
   465  // in package.
   466  // Before:  licenses(["notice"])
   467  // After:   package(licenses = ["notice"])
   468  func moveLicenses(f *build.File) bool {
   469  	return moveToPackage(f, "licenses")
   470  }
   471  
   472  // AllRuleFixes is a list of all Buildozer fixes that can be applied on a rule.
   473  var AllRuleFixes = []struct {
   474  	Name    string
   475  	Fn      func(file *build.File, rule *build.Rule, pkg string) bool
   476  	Message string
   477  }{
   478  	{"sortGlob", sortGlob,
   479  		"Sort the list in a call to glob"},
   480  	{"splitOptions", splitOptionsWithSpaces,
   481  		"Each option should be given separately in the list"},
   482  	{"shortenLabels", shortenLabels,
   483  		"Style: Use the canonical label notation"},
   484  	{"removeVisibility", removeVisibility,
   485  		"This visibility attribute is useless (it corresponds to the default value)"},
   486  	{"removeTestOnly", removeTestOnly,
   487  		"This testonly attribute is useless (it corresponds to the default value)"},
   488  	{"genruleRenameDepsTools", genruleRenameDepsTools,
   489  		"'deps' attribute in genrule has been renamed 'tools'"},
   490  	{"genruleFixHeuristicLabels", genruleFixHeuristicLabels,
   491  		"$(location) should be called explicitly"},
   492  	{"sortExportsFiles", sortExportsFiles,
   493  		"Files in exports_files should be sorted"},
   494  	{"varref", removeVarref,
   495  		"All varref('foo') should be replaced with '$foo'"},
   496  	{"mergeLiteralLists", mergeLiteralLists,
   497  		"Remove useless list concatenation"},
   498  }
   499  
   500  // FileLevelFixes is a list of all Buildozer fixes that apply on the whole file.
   501  var FileLevelFixes = []struct {
   502  	Name    string
   503  	Fn      func(file *build.File) bool
   504  	Message string
   505  }{
   506  	{"movePackageToTop", movePackageDeclarationToTheTop,
   507  		"The package declaration should be the first rule in a file"},
   508  	{"usePlusEqual", usePlusEqual,
   509  		"Prefer '+=' over 'extend' or 'append'"},
   510  	{"unusedLoads", cleanUnusedLoads,
   511  		"Remove unused symbols from load statements"},
   512  	{"moveLicenses", moveLicenses,
   513  		"Move licenses to the package function"},
   514  }
   515  
   516  // FixRule aims to fix errors in BUILD files, remove deprecated features, and
   517  // simplify the code.
   518  func FixRule(f *build.File, pkg string, rule *build.Rule, fixes []string) *build.File {
   519  	fixesAsMap := make(map[string]bool)
   520  	for _, fix := range fixes {
   521  		fixesAsMap[fix] = true
   522  	}
   523  	fixed := false
   524  	for _, fix := range AllRuleFixes {
   525  		if len(fixes) == 0 || fixesAsMap[fix.Name] {
   526  			fixed = fix.Fn(f, rule, pkg) || fixed
   527  		}
   528  	}
   529  	if !fixed {
   530  		return nil
   531  	}
   532  	return f
   533  }
   534  
   535  // FixFile fixes everything it can in the BUILD file.
   536  func FixFile(f *build.File, pkg string, fixes []string) *build.File {
   537  	fixesAsMap := make(map[string]bool)
   538  	for _, fix := range fixes {
   539  		fixesAsMap[fix] = true
   540  	}
   541  	fixed := false
   542  	for _, rule := range f.Rules("") {
   543  		res := FixRule(f, pkg, rule, fixes)
   544  		if res != nil {
   545  			fixed = true
   546  			f = res
   547  		}
   548  	}
   549  	for _, fix := range FileLevelFixes {
   550  		if len(fixes) == 0 || fixesAsMap[fix.Name] {
   551  			fixed = fix.Fn(f) || fixed
   552  		}
   553  	}
   554  	if !fixed {
   555  		return nil
   556  	}
   557  	return f
   558  }
   559  
   560  // A wrapper for a LoadStmt's From and To slices for consistent sorting of their contents.
   561  // It's assumed that the following slices have the same length, the contents are sorted by
   562  // the `To` attribute, the items of `From` are swapped exactly the same way as the items of `To`.
   563  type loadArgs struct {
   564  	From []*build.Ident
   565  	To   []*build.Ident
   566  }
   567  
   568  func (args loadArgs) Len() int {
   569  	return len(args.From)
   570  }
   571  
   572  func (args loadArgs) Swap(i, j int) {
   573  	args.From[i], args.From[j] = args.From[j], args.From[i]
   574  	args.To[i], args.To[j] = args.To[j], args.To[i]
   575  }
   576  
   577  func (args loadArgs) Less(i, j int) bool {
   578  	return args.To[i].Name < args.To[j].Name
   579  }
   580
View as plain text