separate_test.go

Documentation: golang.org/x/tools/go/analysis/unitchecker

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package unitchecker_test
     6  
     7  // This file illustrates separate analysis with an example.
     8  
     9  import (
    10  	"bytes"
    11  	"encoding/json"
    12  	"fmt"
    13  	"go/token"
    14  	"go/types"
    15  	"io"
    16  	"os"
    17  	"path/filepath"
    18  	"strings"
    19  	"sync/atomic"
    20  	"testing"
    21  
    22  	"golang.org/x/tools/go/analysis/passes/printf"
    23  	"golang.org/x/tools/go/analysis/unitchecker"
    24  	"golang.org/x/tools/go/gcexportdata"
    25  	"golang.org/x/tools/go/packages"
    26  	"golang.org/x/tools/internal/testenv"
    27  	"golang.org/x/tools/internal/testfiles"
    28  	"golang.org/x/tools/txtar"
    29  )
    30  
    31  // TestExampleSeparateAnalysis demonstrates the principle of separate
    32  // analysis, the distribution of units of type-checking and analysis
    33  // work across several processes, using serialized summaries to
    34  // communicate between them.
    35  //
    36  // It uses two different kinds of task, "manager" and "worker":
    37  //
    38  //   - The manager computes the graph of package dependencies, and makes
    39  //     a request to the worker for each package. It does not parse,
    40  //     type-check, or analyze Go code. It is analogous "go vet".
    41  //
    42  //   - The worker, which contains the Analyzers, reads each request,
    43  //     loads, parses, and type-checks the files of one package,
    44  //     applies all necessary analyzers to the package, then writes
    45  //     its results to a file. It is a unitchecker-based driver,
    46  //     analogous to the program specified by go vet -vettool= flag.
    47  //
    48  // In practice these would be separate executables, but for simplicity
    49  // of this example they are provided by one executable in two
    50  // different modes: the Example function is the manager, and the same
    51  // executable invoked with ENTRYPOINT=worker is the worker.
    52  // (See TestIntegration for how this happens.)
    53  //
    54  // Unfortunately this can't be a true Example because of the skip,
    55  // which requires a testing.T.
    56  func TestExampleSeparateAnalysis(t *testing.T) {
    57  	testenv.NeedsGoPackages(t)
    58  
    59  	// src is an archive containing a module with a printf mistake.
    60  	const src = `
    61  -- go.mod --
    62  module separate
    63  go 1.18
    64  
    65  -- main/main.go --
    66  package main
    67  
    68  import "separate/lib"
    69  
    70  func main() {
    71  	lib.MyPrintf("%s", 123)
    72  }
    73  
    74  -- lib/lib.go --
    75  package lib
    76  
    77  import "fmt"
    78  
    79  func MyPrintf(format string, args ...any) {
    80  	fmt.Printf(format, args...)
    81  }
    82  `
    83  
    84  	// Expand archive into tmp tree.
    85  	tmpdir := t.TempDir()
    86  	if err := testfiles.ExtractTxtar(tmpdir, txtar.Parse([]byte(src))); err != nil {
    87  		t.Fatal(err)
    88  	}
    89  
    90  	// Load metadata for the main package and all its dependencies.
    91  	cfg := &packages.Config{
    92  		Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedModule,
    93  		Dir:  tmpdir,
    94  		Env: append(os.Environ(),
    95  			"GOPROXY=off", // disable network
    96  			"GOWORK=off",  // an ambient GOWORK value would break package loading
    97  		),
    98  		Logf: t.Logf,
    99  	}
   100  	pkgs, err := packages.Load(cfg, "separate/main")
   101  	if err != nil {
   102  		t.Fatal(err)
   103  	}
   104  	// Stop if any package had a metadata error.
   105  	if packages.PrintErrors(pkgs) > 0 {
   106  		t.Fatal("there were errors among loaded packages")
   107  	}
   108  
   109  	// Now we have loaded the import graph,
   110  	// let's begin the proper work of the manager.
   111  
   112  	// Gather root packages. They will get all analyzers,
   113  	// whereas dependencies get only the subset that
   114  	// produce facts or are required by them.
   115  	roots := make(map[*packages.Package]bool)
   116  	for _, pkg := range pkgs {
   117  		roots[pkg] = true
   118  	}
   119  
   120  	// nextID generates sequence numbers for each unit of work.
   121  	// We use it to create names of temporary files.
   122  	var nextID atomic.Int32
   123  
   124  	var allDiagnostics []string
   125  
   126  	// Visit all packages in postorder: dependencies first.
   127  	// TODO(adonovan): opt: use parallel postorder.
   128  	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
   129  		if pkg.PkgPath == "unsafe" {
   130  			return
   131  		}
   132  
   133  		// Choose a unique prefix for temporary files
   134  		// (.cfg .types .facts) produced by this package.
   135  		// We stow it in an otherwise unused field of
   136  		// Package so it can be accessed by our importers.
   137  		prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1))
   138  		pkg.ExportFile = prefix
   139  
   140  		// Construct the request to the worker.
   141  		var (
   142  			importMap   = make(map[string]string)
   143  			packageFile = make(map[string]string)
   144  			packageVetx = make(map[string]string)
   145  		)
   146  		for importPath, dep := range pkg.Imports {
   147  			importMap[importPath] = dep.PkgPath
   148  			if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe"
   149  				packageFile[dep.PkgPath] = depPrefix + ".types"
   150  				packageVetx[dep.PkgPath] = depPrefix + ".facts"
   151  			}
   152  		}
   153  		cfg := unitchecker.Config{
   154  			ID:           pkg.ID,
   155  			ImportPath:   pkg.PkgPath,
   156  			GoFiles:      pkg.CompiledGoFiles,
   157  			NonGoFiles:   pkg.OtherFiles,
   158  			IgnoredFiles: pkg.IgnoredFiles,
   159  			ImportMap:    importMap,
   160  			PackageFile:  packageFile,
   161  			PackageVetx:  packageVetx,
   162  			VetxOnly:     !roots[pkg],
   163  			VetxOutput:   prefix + ".facts",
   164  		}
   165  		if pkg.Module != nil {
   166  			if v := pkg.Module.GoVersion; v != "" {
   167  				cfg.GoVersion = "go" + v
   168  			}
   169  		}
   170  
   171  		// Write the JSON configuration message to a file.
   172  		cfgData, err := json.Marshal(cfg)
   173  		if err != nil {
   174  			t.Fatalf("internal error in json.Marshal: %v", err)
   175  		}
   176  		cfgFile := prefix + ".cfg"
   177  		if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil {
   178  			t.Fatal(err)
   179  		}
   180  
   181  		// Send the request to the worker.
   182  		cmd := testenv.Command(t, os.Args[0], "-json", cfgFile)
   183  		cmd.Stderr = os.Stderr
   184  		cmd.Stdout = new(bytes.Buffer)
   185  		cmd.Env = append(os.Environ(), "ENTRYPOINT=worker")
   186  		if err := cmd.Run(); err != nil {
   187  			t.Fatal(err)
   188  		}
   189  
   190  		// Parse JSON output and gather in allDiagnostics.
   191  		dec := json.NewDecoder(cmd.Stdout.(io.Reader))
   192  		for {
   193  			type jsonDiagnostic struct {
   194  				Posn    string `json:"posn"`
   195  				Message string `json:"message"`
   196  			}
   197  			// 'results' maps Package.Path -> Analyzer.Name -> diagnostics
   198  			var results map[string]map[string][]jsonDiagnostic
   199  			if err := dec.Decode(&results); err != nil {
   200  				if err == io.EOF {
   201  					break
   202  				}
   203  				t.Fatalf("internal error decoding JSON: %v", err)
   204  			}
   205  			for _, result := range results {
   206  				for analyzer, diags := range result {
   207  					for _, diag := range diags {
   208  						rel := strings.ReplaceAll(diag.Posn, tmpdir, "")
   209  						rel = filepath.ToSlash(rel)
   210  						msg := fmt.Sprintf("%s: [%s] %s", rel, analyzer, diag.Message)
   211  						allDiagnostics = append(allDiagnostics, msg)
   212  					}
   213  				}
   214  			}
   215  		}
   216  	})
   217  
   218  	// Observe that the example produces a fact-based diagnostic
   219  	// from separate analysis of "main", "lib", and "fmt":
   220  
   221  	const want = `/main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int`
   222  	if got := strings.Join(allDiagnostics, "\n"); got != want {
   223  		t.Errorf("Got: %s\nWant: %s", got, want)
   224  	}
   225  }
   226  
   227  // -- worker process --
   228  
   229  // worker is the main entry point for a unitchecker-based driver
   230  // with only a single analyzer, for illustration.
   231  func worker() {
   232  	// Currently the unitchecker API doesn't allow clients to
   233  	// control exactly how and where fact and type information
   234  	// is produced and consumed.
   235  	//
   236  	// So, for example, it assumes that type information has
   237  	// already been produced by the compiler, which is true when
   238  	// running under "go vet", but isn't necessary. It may be more
   239  	// convenient and efficient for a distributed analysis system
   240  	// if the worker generates both of them, which is the approach
   241  	// taken in this example; they could even be saved as two
   242  	// sections of a single file.
   243  	//
   244  	// Consequently, this test currently needs special access to
   245  	// private hooks in unitchecker to control how and where facts
   246  	// and types are produced and consumed. In due course this
   247  	// will become a respectable public API. In the meantime, it
   248  	// should at least serve as a demonstration of how one could
   249  	// fork unitchecker to achieve separate analysis without go vet.
   250  	unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes)
   251  
   252  	unitchecker.Main(printf.Analyzer)
   253  }
   254  
   255  func makeTypesImporter(cfg *unitchecker.Config, fset *token.FileSet) types.Importer {
   256  	imports := make(map[string]*types.Package)
   257  	return importerFunc(func(importPath string) (*types.Package, error) {
   258  		// Resolve import path to package path (vendoring, etc)
   259  		path, ok := cfg.ImportMap[importPath]
   260  		if !ok {
   261  			return nil, fmt.Errorf("can't resolve import %q", path)
   262  		}
   263  		if path == "unsafe" {
   264  			return types.Unsafe, nil
   265  		}
   266  
   267  		// Find, read, and decode file containing type information.
   268  		file, ok := cfg.PackageFile[path]
   269  		if !ok {
   270  			return nil, fmt.Errorf("no package file for %q", path)
   271  		}
   272  		f, err := os.Open(file)
   273  		if err != nil {
   274  			return nil, err
   275  		}
   276  		defer f.Close() // ignore error
   277  		return gcexportdata.Read(f, fset, imports, path)
   278  	})
   279  }
   280  
   281  func exportTypes(cfg *unitchecker.Config, fset *token.FileSet, pkg *types.Package) error {
   282  	var out bytes.Buffer
   283  	if err := gcexportdata.Write(&out, fset, pkg); err != nil {
   284  		return err
   285  	}
   286  	typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types"
   287  	return os.WriteFile(typesFile, out.Bytes(), 0666)
   288  }
   289  
   290  // -- helpers --
   291  
   292  type importerFunc func(path string) (*types.Package, error)
   293  
   294  func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
   295
View as plain text