Source file src/github.com/bazelbuild/buildtools/unused_deps/unused_deps.go

Documentation: github.com/bazelbuild/buildtools/unused_deps

     1  /*
     2  Copyright 2017 Google LLC
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      https://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // The unused_deps binary prints out buildozer commands for removing
    18  // unused Java dependencies from java_library Bazel rules.
    19  package main
    20  
    21  import (
    22  	"bufio"
    23  	"bytes"
    24  	"errors"
    25  	"flag"
    26  	"fmt"
    27  	"io/ioutil"
    28  	"log"
    29  	"os"
    30  	"os/exec"
    31  	"path"
    32  	"strings"
    33  
    34  	"github.com/bazelbuild/buildtools/build"
    35  	"github.com/bazelbuild/buildtools/config"
    36  	depspb "github.com/bazelbuild/buildtools/deps_proto"
    37  	"github.com/bazelbuild/buildtools/edit"
    38  	eapb "github.com/bazelbuild/buildtools/extra_actions_base_proto"
    39  	"github.com/bazelbuild/buildtools/labels"
    40  	"github.com/golang/protobuf/proto"
    41  )
    42  
    43  var (
    44  	buildVersion     = "redacted"
    45  	buildScmRevision = "redacted"
    46  
    47  	version             = flag.Bool("version", false, "Print the version of unused_deps")
    48  	cQuery              = flag.Bool("cquery", false, "Use 'cquery' command instead of 'query'")
    49  	buildTool           = flag.String("build_tool", config.DefaultBuildTool, config.BuildToolHelp)
    50  	extraActionFileName = flag.String("extra_action_file", "", config.ExtraActionFileNameHelp)
    51  	outputFileName      = flag.String("output_file", "", "used only with extra_action_file")
    52  	buildOptions        = stringList("extra_build_flags", "Extra build flags to use when building the targets.")
    53  
    54  	blazeFlags = []string{"--tool_tag=unused_deps", "--keep_going", "--color=yes", "--curses=yes"}
    55  
    56  	aspect = `
    57  # Explicitly creates a params file for a Javac action.
    58  def _javac_params(target, ctx):
    59      params = []
    60      for action in target.actions:
    61          if not action.mnemonic == "Javac" and not action.mnemonic == "KotlinCompile":
    62              continue
    63          output = ctx.actions.declare_file("%s.javac_params" % target.label.name)
    64          args = ctx.actions.args()
    65          args.add_all(action.argv)
    66          ctx.actions.write(
    67              output = output,
    68              content = args,
    69          )
    70          params.append(output)
    71          break
    72      return [OutputGroupInfo(unused_deps_outputs = depset(params))]
    73  
    74  javac_params = aspect(
    75      implementation = _javac_params,
    76  )
    77  `
    78  )
    79  
    80  func stringList(name, help string) func() []string {
    81  	f := flag.String(name, "", help)
    82  	return func() []string {
    83  		if *f == "" {
    84  			return nil
    85  		}
    86  		res := strings.Split(*f, ",")
    87  		for i := range res {
    88  			res[i] = strings.TrimSpace(res[i])
    89  		}
    90  		return res
    91  	}
    92  }
    93  
    94  // getJarPath prints the path to the output jar file specified in the extra_action file at path.
    95  func getJarPath(path string) (string, error) {
    96  	data, err := ioutil.ReadFile(path)
    97  	if err != nil {
    98  		return "", err
    99  	}
   100  	i := &eapb.ExtraActionInfo{}
   101  	if err := proto.Unmarshal(data, i); err != nil {
   102  		return "", err
   103  	}
   104  	ext, err := proto.GetExtension(i, eapb.E_JavaCompileInfo_JavaCompileInfo)
   105  	if err != nil {
   106  		return "", err
   107  	}
   108  	jci, ok := ext.(*eapb.JavaCompileInfo)
   109  	if !ok {
   110  		return "", errors.New("no JavaCompileInfo in " + path)
   111  	}
   112  	return jci.GetOutputjar(), nil
   113  }
   114  
   115  // writeUnusedDeps writes the labels of unused direct deps, one per line, to outputFileName.
   116  func writeUnusedDeps(jarPath, outputFileName string) {
   117  	depsPath := strings.Replace(jarPath, ".jar", ".jdeps", 1)
   118  	paramsPath := jarPath + "-2.params"
   119  	file, _ := os.Create(outputFileName)
   120  	for dep := range unusedDeps(depsPath, directDepParams(paramsPath)) {
   121  		file.WriteString(dep + "\n")
   122  	}
   123  }
   124  
   125  func cmdWithStderr(name string, arg ...string) *exec.Cmd {
   126  	cmd := exec.Command(name, arg...)
   127  	cmd.Stderr = os.Stderr
   128  	return cmd
   129  }
   130  
   131  // blazeInfo retrieves the blaze info value for a given key.
   132  func blazeInfo(key string) (value string) {
   133  	out, err := cmdWithStderr(*buildTool, "info", key).Output()
   134  	if err != nil {
   135  		log.Printf("'%s info %s' failed: %s", *buildTool, key, err)
   136  	}
   137  	return strings.TrimSpace(bytes.NewBuffer(out).String())
   138  }
   139  
   140  // inputFileName returns a blaze output file name from which to read input.
   141  func inputFileName(blazeBin, pkg, ruleName, extension string) string {
   142  	name := fmt.Sprintf("%s/%s/lib%s.%s", blazeBin, pkg, ruleName, extension) // *_library
   143  	if _, err := os.Stat(name); err == nil {
   144  		return name
   145  	}
   146  	// lazily let the caller handle it if this doesn't exist
   147  	return fmt.Sprintf("%s/%s/%s.%s", blazeBin, pkg, ruleName, extension) // *_{binary,test}
   148  }
   149  
   150  // directDepParams reads the jar-2.params files, looking for a
   151  // "--direct_dependencies" argument.  When found, the direct dependencies are
   152  // returned as a map from jar file names to labels.
   153  func directDepParams(blazeOutputPath string, paramsFileNames ...string) (depsByJar map[string]string) {
   154  	depsByJar = make(map[string]string)
   155  	errs := make([]error, 0)
   156  	for _, paramsFileName := range paramsFileNames {
   157  		data, err := ioutil.ReadFile(paramsFileName)
   158  		if err != nil {
   159  			errs = append(errs, err)
   160  			continue
   161  		}
   162  		// The classpath param exceeds MaxScanTokenSize, so we scan just the
   163  		// dependencies section.
   164  		directDepsFlag := []byte("--direct_dependencies")
   165  		arg := bytes.Index(data, directDepsFlag)
   166  		if arg < 0 {
   167  			continue
   168  		}
   169  		first := arg + len(directDepsFlag) + 1
   170  
   171  		scanner := bufio.NewScanner(bytes.NewReader(data[first:]))
   172  		for scanner.Scan() {
   173  			jar := scanner.Text()
   174  			if strings.HasPrefix(jar, "--") {
   175  				break
   176  			}
   177  			label, err := jarManifestValue(blazeOutputPath+strings.TrimPrefix(jar, "bazel-out"), "Target-Label")
   178  			if err != nil {
   179  				continue
   180  			}
   181  			if strings.HasPrefix(label, "@@") || strings.HasPrefix(label, "@/") {
   182  				label = label[1:]
   183  			}
   184  			depsByJar[jar] = label
   185  		}
   186  		if err := scanner.Err(); err != nil {
   187  			log.Printf("reading %s: %s", paramsFileName, err)
   188  		}
   189  	}
   190  	if len(errs) == len(paramsFileNames) {
   191  		for _, err := range errs {
   192  			log.Println(err)
   193  		}
   194  	}
   195  	return depsByJar
   196  }
   197  
   198  // unusedDeps returns a set of labels that are unused deps.
   199  // It reads Dependencies proto messages from depsFileName (a jdeps file), which indicate deps used
   200  // at compile time, and returns those values in the depsByJar map that aren't used at compile time.
   201  func unusedDeps(depsFileName string, depsByJar map[string]string) (unusedDeps map[string]bool) {
   202  	unusedDeps = make(map[string]bool)
   203  	data, err := ioutil.ReadFile(depsFileName)
   204  	if err != nil {
   205  		log.Println(err)
   206  		return unusedDeps
   207  	}
   208  	dependencies := &depspb.Dependencies{}
   209  	if err := proto.Unmarshal(data, dependencies); err != nil {
   210  		log.Println(err)
   211  		return unusedDeps
   212  	}
   213  	for _, label := range depsByJar {
   214  		unusedDeps[label] = true
   215  	}
   216  	for _, dependency := range dependencies.Dependency {
   217  		if *dependency.Kind == depspb.Dependency_EXPLICIT {
   218  			delete(unusedDeps, depsByJar[*dependency.Path])
   219  		}
   220  	}
   221  	return unusedDeps
   222  }
   223  
   224  // parseBuildFile tries to read and parse the contents of buildFileName.
   225  func parseBuildFile(buildFileName string) (buildFile *build.File, err error) {
   226  	data, err := ioutil.ReadFile(buildFileName)
   227  	if err != nil {
   228  		return nil, err
   229  	}
   230  	return build.Parse(buildFileName, data)
   231  }
   232  
   233  // getDepsExpr tries to parse the content of buildFileName and return the deps Expr for ruleName.
   234  func getDepsExpr(buildFileName string, ruleName string) build.Expr {
   235  	buildFile, err := parseBuildFile(buildFileName)
   236  	if buildFile == nil {
   237  		log.Printf("%s when parsing %s", err, buildFileName)
   238  		return nil
   239  	}
   240  	rule := edit.FindRuleByName(buildFile, ruleName)
   241  	if rule == nil {
   242  		log.Printf("%s not found in %s", ruleName, buildFileName)
   243  		return nil
   244  	}
   245  	depsExpr := rule.Attr("deps")
   246  	if depsExpr == nil {
   247  		log.Printf("no deps attribute for %s in %s", ruleName, buildFileName)
   248  	}
   249  	return depsExpr
   250  }
   251  
   252  // hasRuntimeComment returns true if expr has an EOL comment containing the word "runtime".
   253  // TODO(bazel-team): delete when this comment convention is extinct
   254  func hasRuntimeComment(expr build.Expr) bool {
   255  	for _, comment := range expr.Comment().Suffix {
   256  		if strings.Contains(strings.ToLower(comment.Token), "runtime") {
   257  			return true
   258  		}
   259  	}
   260  	return false
   261  }
   262  
   263  // printCommands prints, for each key in the deps map, a buildozer command
   264  // to remove that entry from the deps attribute of the rule identified by label.
   265  // Returns true if at least one command was printed, or false otherwise.
   266  func printCommands(label string, deps map[string]bool) (anyCommandPrinted bool) {
   267  	buildFileName, repo, pkg, ruleName := edit.InterpretLabelWithRepo(label)
   268  	if repo != "" {
   269  		outputBase := blazeInfo(config.DefaultOutputBase)
   270  		buildFileName = fmt.Sprintf("%s/external/%s/%s", outputBase, repo, buildFileName)
   271  	}
   272  
   273  	depsExpr := getDepsExpr(buildFileName, ruleName)
   274  	for _, li := range edit.AllLists(depsExpr) {
   275  		for _, elem := range li.List {
   276  			for dep := range deps {
   277  				str, ok := elem.(*build.StringExpr)
   278  				if !ok {
   279  					continue
   280  				}
   281  				buildLabel := str.Value
   282  				if repo != "" && buildLabel[:2] == "//" {
   283  					buildLabel = fmt.Sprintf("@%s%s", repo, str.Value)
   284  				}
   285  				if !labels.Equal(buildLabel, dep, pkg) {
   286  					continue
   287  				}
   288  				if hasRuntimeComment(str) {
   289  					fmt.Printf("buildozer 'move deps runtime_deps %s' %s\n", str.Value, label)
   290  				} else {
   291  					// add dep's exported dependencies to label before removing dep
   292  					fmt.Printf("buildozer \"add deps $(%s query 'labels(exports, %s)' | tr '\\n' ' ')\" %s\n", *buildTool, str.Value, label)
   293  					fmt.Printf("buildozer 'remove deps %s' %s\n", str.Value, label)
   294  				}
   295  				anyCommandPrinted = true
   296  			}
   297  		}
   298  	}
   299  	return anyCommandPrinted
   300  }
   301  
   302  // setupAspect creates a workspace in a tmpdir and populates it with an aspect,
   303  // which is used with --override_repository below.
   304  func setupAspect() (string, error) {
   305  	tmp, err := ioutil.TempDir(os.TempDir(), "unused_deps")
   306  	if err != nil {
   307  		return "", err
   308  	}
   309  	for _, f := range []string{"WORKSPACE", "BUILD"} {
   310  		if err := ioutil.WriteFile(path.Join(tmp, f), []byte{}, 0666); err != nil {
   311  			return "", err
   312  		}
   313  	}
   314  	if err := ioutil.WriteFile(path.Join(tmp, "unused_deps.bzl"), []byte(aspect), 0666); err != nil {
   315  		return "", err
   316  	}
   317  	return tmp, nil
   318  }
   319  
   320  func usage() {
   321  	fmt.Fprintf(os.Stderr, `usage: unused_deps TARGET...
   322  
   323  For Java rules in TARGETs, prints commands to delete deps unused at compile time.
   324  Note these may be used at run time; see documentation for more information.
   325  `)
   326  	os.Exit(2)
   327  }
   328  
   329  func main() {
   330  	flag.Usage = usage
   331  	flag.Parse()
   332  	if *version {
   333  		fmt.Printf("unused_deps version: %s \n", buildVersion)
   334  		fmt.Printf("unused_deps scm revision: %s \n", buildScmRevision)
   335  		os.Exit(0)
   336  	}
   337  
   338  	if *extraActionFileName != "" {
   339  		jarPath, err := getJarPath(*extraActionFileName)
   340  		if err != nil {
   341  			log.Fatal(err)
   342  		}
   343  		writeUnusedDeps(jarPath, *outputFileName)
   344  		return
   345  	}
   346  	targetPatterns := flag.Args()
   347  	if len(targetPatterns) == 0 {
   348  		targetPatterns = []string{"//..."}
   349  	}
   350  	queryCmd := []string{}
   351  	if *cQuery {
   352  		queryCmd = append(queryCmd, "cquery")
   353  	} else {
   354  		queryCmd = append(queryCmd, "query")
   355  	}
   356  	queryCmd = append(queryCmd, blazeFlags...)
   357  	queryCmd = append(
   358  		queryCmd, fmt.Sprintf("kind('(kt|java|android)_*', %s)", strings.Join(targetPatterns, " + ")))
   359  
   360  	log.Printf("running: %s %s", *buildTool, strings.Join(queryCmd, " "))
   361  	queryOut, err := cmdWithStderr(*buildTool, queryCmd...).Output()
   362  	if err != nil {
   363  		log.Print(err)
   364  	}
   365  	if len(queryOut) == 0 {
   366  		fmt.Fprintln(os.Stderr, "found no targets of kind (kt|java|android)_*")
   367  		usage()
   368  	}
   369  
   370  	aspectDir, err := setupAspect()
   371  	if err != nil {
   372  		log.Print(err)
   373  		os.Exit(1)
   374  	}
   375  	defer func() {
   376  		os.RemoveAll(aspectDir)
   377  	}()
   378  
   379  	buildCmd := []string{"build"}
   380  	buildCmd = append(buildCmd, blazeFlags...)
   381  	buildCmd = append(buildCmd, config.DefaultExtraBuildFlags...)
   382  	buildCmd = append(buildCmd, "--output_groups=+unused_deps_outputs")
   383  	buildCmd = append(buildCmd, "--override_repository=unused_deps="+aspectDir)
   384  	buildCmd = append(buildCmd, "--aspects=@unused_deps//:unused_deps.bzl%javac_params")
   385  	buildCmd = append(buildCmd, buildOptions()...)
   386  
   387  	blazeArgs := append(buildCmd, targetPatterns...)
   388  
   389  	log.Printf("running: %s %s", *buildTool, strings.Join(blazeArgs, " "))
   390  	cmdWithStderr(*buildTool, blazeArgs...).Run()
   391  	binDir := blazeInfo(config.DefaultBinDir)
   392  	blazeOutputPath := blazeInfo(config.DefaultOutputPath)
   393  	fmt.Fprintf(os.Stderr, "\n") // vertical space between build output and unused_deps output
   394  
   395  	anyCommandPrinted := false
   396  	for _, label := range strings.Fields(string(queryOut)) {
   397  		if *cQuery && strings.HasPrefix(label, "(") {
   398  			// cquery output includes the target's configuration ID.  Skip it.
   399  			// https://docs.bazel.build/versions/main/cquery.html#configurations
   400  			continue
   401  		}
   402  		_, repo, pkg, ruleName := edit.InterpretLabelWithRepo(label)
   403  		blazeBin := binDir
   404  		if repo != "" {
   405  			blazeBin = fmt.Sprintf("%s/external/%s", binDir, repo)
   406  		}
   407  		depsByJar := directDepParams(blazeOutputPath, inputFileName(blazeBin, pkg, ruleName, "javac_params"))
   408  		depsToRemove := unusedDeps(inputFileName(blazeBin, pkg, ruleName, "jdeps"), depsByJar)
   409  		// TODO(bazel-team): instead of printing, have buildifier-like modes?
   410  		anyCommandPrinted = printCommands(label, depsToRemove) || anyCommandPrinted
   411  	}
   412  	if !anyCommandPrinted {
   413  		fmt.Fprintln(os.Stderr, "No unused deps found.")
   414  	}
   415  }
   416  

View as plain text