files.go

Documentation: github.com/bazelbuild/bazel-gazelle/testtools

     1  /* Copyright 2018 The Bazel Authors. All rights reserved.
     2  
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6  
     7     http://www.apache.org/licenses/LICENSE-2.0
     8  
     9  Unless required by applicable law or agreed to in writing, software
    10  distributed under the License is distributed on an "AS IS" BASIS,
    11  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  See the License for the specific language governing permissions and
    13  limitations under the License.
    14  */
    15  
    16  package testtools
    17  
    18  import (
    19  	"bytes"
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"io"
    24  	"io/fs"
    25  	"os"
    26  	"os/exec"
    27  	"path"
    28  	"path/filepath"
    29  	"strconv"
    30  	"strings"
    31  	"testing"
    32  	"time"
    33  
    34  	"github.com/google/go-cmp/cmp"
    35  )
    36  
    37  const cmdTimeoutOrInterruptExitCode = -1
    38  
    39  // FileSpec specifies the content of a test file.
    40  type FileSpec struct {
    41  	// Path is a slash-separated path relative to the test directory. If Path
    42  	// ends with a slash, it indicates a directory should be created
    43  	// instead of a file.
    44  	Path string
    45  
    46  	// Symlink is a slash-separated path relative to the test directory. If set,
    47  	// it indicates a symbolic link should be created with this path instead of a
    48  	// file.
    49  	Symlink string
    50  
    51  	// Content is the content of the test file.
    52  	Content string
    53  
    54  	// NotExist asserts that no file at this path exists.
    55  	// It is only valid in CheckFiles.
    56  	NotExist bool
    57  }
    58  
    59  // CreateFiles creates a directory of test files. This is a more compact
    60  // alternative to testdata directories. CreateFiles returns a canonical path
    61  // to the directory and a function to call to clean up the directory
    62  // after the test.
    63  func CreateFiles(t *testing.T, files []FileSpec) (dir string, cleanup func()) {
    64  	t.Helper()
    65  	dir, err := os.MkdirTemp(os.Getenv("TEST_TEMPDIR"), "gazelle_test")
    66  	if err != nil {
    67  		t.Fatal(err)
    68  	}
    69  	dir, err = filepath.EvalSymlinks(dir)
    70  	if err != nil {
    71  		t.Fatal(err)
    72  	}
    73  
    74  	for _, f := range files {
    75  		if f.NotExist {
    76  			t.Fatalf("CreateFiles: NotExist may not be set: %s", f.Path)
    77  		}
    78  		path := filepath.Join(dir, filepath.FromSlash(f.Path))
    79  		if strings.HasSuffix(f.Path, "/") {
    80  			if err := os.MkdirAll(path, 0o700); err != nil {
    81  				os.RemoveAll(dir)
    82  				t.Fatal(err)
    83  			}
    84  			continue
    85  		}
    86  		if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
    87  			os.RemoveAll(dir)
    88  			t.Fatal(err)
    89  		}
    90  		if f.Symlink != "" {
    91  			if err := os.Symlink(f.Symlink, path); err != nil {
    92  				t.Fatal(err)
    93  			}
    94  			continue
    95  		}
    96  		if err := os.WriteFile(path, []byte(f.Content), 0o600); err != nil {
    97  			os.RemoveAll(dir)
    98  			t.Fatal(err)
    99  		}
   100  	}
   101  
   102  	return dir, func() { os.RemoveAll(dir) }
   103  }
   104  
   105  // CheckFiles checks that files in "dir" exist and have the content specified
   106  // in "files". Files not listed in "files" are not tested, so extra files
   107  // are allowed.
   108  func CheckFiles(t *testing.T, dir string, files []FileSpec) {
   109  	t.Helper()
   110  	for _, f := range files {
   111  		path := filepath.Join(dir, f.Path)
   112  
   113  		st, err := os.Stat(path)
   114  		if f.NotExist {
   115  			if err == nil {
   116  				t.Errorf("asserted to not exist, but does: %s", f.Path)
   117  			} else if !os.IsNotExist(err) {
   118  				t.Errorf("could not stat %s: %v", f.Path, err)
   119  			}
   120  			continue
   121  		}
   122  
   123  		if strings.HasSuffix(f.Path, "/") {
   124  			if err != nil {
   125  				t.Errorf("could not stat %s: %v", f.Path, err)
   126  			} else if !st.IsDir() {
   127  				t.Errorf("not a directory: %s", f.Path)
   128  			}
   129  		} else {
   130  			want := strings.TrimSpace(f.Content)
   131  			gotBytes, err := os.ReadFile(filepath.Join(dir, f.Path))
   132  			if err != nil {
   133  				t.Errorf("could not read %s: %v", f.Path, err)
   134  				continue
   135  			}
   136  			got := strings.TrimSpace(string(gotBytes))
   137  			if diff := cmp.Diff(want, got); diff != "" {
   138  				t.Errorf("%s diff (-want,+got):\n%s", f.Path, diff)
   139  			}
   140  		}
   141  	}
   142  }
   143  
   144  type TestGazelleGenerationArgs struct {
   145  	// Name is the name of the test.
   146  	Name string
   147  	// TestDataPathAbsolute is the absolute path to the test data directory.
   148  	// For example, /home/user/workspace/path/to/test_data/my_testcase.
   149  	TestDataPathAbsolute string
   150  	// TestDataPathRealtive is the workspace relative path to the test data directory.
   151  	// For example, path/to/test_data/my_testcase.
   152  	TestDataPathRelative string
   153  	// GazelleBinaryPath is the workspace relative path to the location of the gazelle binary
   154  	// we want to test.
   155  	GazelleBinaryPath string
   156  
   157  	// BuildInSuffix is the suffix for all test input build files. Includes the ".".
   158  	// Default: ".in", so input BUILD files should be named BUILD.in.
   159  	BuildInSuffix string
   160  
   161  	// BuildOutSuffix is the suffix for all test output build files. Includes the ".".
   162  	// Default: ".out", so out BUILD files should be named BUILD.out.
   163  	BuildOutSuffix string
   164  
   165  	// Timeout is the duration after which the generation process will be killed.
   166  	Timeout time.Duration
   167  }
   168  
   169  var (
   170  	argumentsFilename        = "arguments.txt"
   171  	expectedStdoutFilename   = "expectedStdout.txt"
   172  	expectedStderrFilename   = "expectedStderr.txt"
   173  	expectedExitCodeFilename = "expectedExitCode.txt"
   174  )
   175  
   176  // TestGazelleGenerationOnPath runs a full gazelle binary on a testdata directory.
   177  // With a test data directory of the form:
   178  // └── <testDataPath>
   179  //
   180  //	└── some_test
   181  //	    ├── WORKSPACE
   182  //	    ├── README.md --> README describing what the test does.
   183  //	    ├── arguments.txt --> newline delimited list of arguments to pass in (ignored if empty).
   184  //	    ├── expectedStdout.txt --> Expected stdout for this test.
   185  //	    ├── expectedStderr.txt --> Expected stderr for this test.
   186  //	    ├── expectedExitCode.txt --> Expected exit code for this test.
   187  //	    └── app
   188  //	        └── sourceFile.foo
   189  //	        └── BUILD.in --> BUILD file prior to running gazelle.
   190  //	        └── BUILD.out --> BUILD file expected after running gazelle.
   191  func TestGazelleGenerationOnPath(t *testing.T, args *TestGazelleGenerationArgs) {
   192  	t.Run(args.Name, func(t *testing.T) {
   193  		t.Helper() // Make the stack trace a little bit more clear.
   194  		if args.BuildInSuffix == "" {
   195  			args.BuildInSuffix = ".in"
   196  		}
   197  		if args.BuildOutSuffix == "" {
   198  			args.BuildOutSuffix = ".out"
   199  		}
   200  		var inputs []FileSpec
   201  		var goldens []FileSpec
   202  
   203  		config := &testConfig{}
   204  		f := func(path string, d fs.DirEntry, err error) error {
   205  			if err != nil {
   206  				t.Fatalf("File walk error on path %q. Error: %v", path, err)
   207  			}
   208  
   209  			shortPath := strings.TrimPrefix(path, args.TestDataPathAbsolute)
   210  
   211  			info, err := d.Info()
   212  			if err != nil {
   213  				t.Fatalf("File info error on path %q. Error: %v", path, err)
   214  			}
   215  
   216  			if info.IsDir() {
   217  				return nil
   218  			}
   219  
   220  			content, err := os.ReadFile(path)
   221  			if err != nil {
   222  				t.Errorf("os.ReadFile(%q) error: %v", path, err)
   223  			}
   224  
   225  			// Read in expected stdout, stderr, and exit code files.
   226  			if d.Name() == argumentsFilename {
   227  				config.Args = strings.Split(string(content), "\n")
   228  				return nil
   229  			}
   230  			if d.Name() == expectedStdoutFilename {
   231  				config.Stdout = string(content)
   232  				return nil
   233  			}
   234  			if d.Name() == expectedStderrFilename {
   235  				config.Stderr = string(content)
   236  				return nil
   237  			}
   238  			if d.Name() == expectedExitCodeFilename {
   239  				config.ExitCode, err = strconv.Atoi(string(content))
   240  				if err != nil {
   241  					// Set the ExitCode to a sentinel value (-1) to ensure that if the caller is updating the files on disk the value is updated.
   242  					config.ExitCode = -1
   243  					t.Errorf("Failed to parse expected exit code (%q) error: %v", path, err)
   244  				}
   245  				return nil
   246  			}
   247  
   248  			if strings.HasSuffix(shortPath, args.BuildInSuffix) {
   249  				inputs = append(inputs, FileSpec{
   250  					Path:    filepath.Join(args.Name, strings.TrimSuffix(shortPath, args.BuildInSuffix)+".bazel"),
   251  					Content: string(content),
   252  				})
   253  			} else if strings.HasSuffix(shortPath, args.BuildOutSuffix) {
   254  				goldens = append(goldens, FileSpec{
   255  					Path:    filepath.Join(args.Name, strings.TrimSuffix(shortPath, args.BuildOutSuffix)+".bazel"),
   256  					Content: string(content),
   257  				})
   258  			} else {
   259  				inputs = append(inputs, FileSpec{
   260  					Path:    filepath.Join(args.Name, shortPath),
   261  					Content: string(content),
   262  				})
   263  				goldens = append(goldens, FileSpec{
   264  					Path:    filepath.Join(args.Name, shortPath),
   265  					Content: string(content),
   266  				})
   267  			}
   268  			return nil
   269  		}
   270  		if err := filepath.WalkDir(args.TestDataPathAbsolute, f); err != nil {
   271  			t.Fatal(err)
   272  		}
   273  
   274  		testdataDir, cleanup := CreateFiles(t, inputs)
   275  		workspaceRoot := filepath.Join(testdataDir, args.Name)
   276  
   277  		var stdout, stderr bytes.Buffer
   278  		var actualExitCode int
   279  		defer cleanup()
   280  		defer func() {
   281  			if t.Failed() {
   282  				shouldUpdate := os.Getenv("UPDATE_SNAPSHOTS") != ""
   283  				buildWorkspaceDirectory := os.Getenv("BUILD_WORKSPACE_DIRECTORY")
   284  				updateCommand := fmt.Sprintf("UPDATE_SNAPSHOTS=true bazel run %s", os.Getenv("TEST_TARGET"))
   285  				// srcTestDirectory is the directory of the source code of the test case.
   286  				srcTestDirectory := path.Join(buildWorkspaceDirectory, path.Dir(args.TestDataPathRelative), args.Name)
   287  				if shouldUpdate {
   288  					// Update stdout, stderr, exit code.
   289  					updateExpectedConfig(t, config.Stdout, redactWorkspacePath(stdout.String(), workspaceRoot), srcTestDirectory, expectedStdoutFilename)
   290  					updateExpectedConfig(t, config.Stderr, redactWorkspacePath(stderr.String(), workspaceRoot), srcTestDirectory, expectedStderrFilename)
   291  					updateExpectedConfig(t, fmt.Sprintf("%d", config.ExitCode), fmt.Sprintf("%d", actualExitCode), srcTestDirectory, expectedExitCodeFilename)
   292  
   293  					err := filepath.Walk(testdataDir, func(walkedPath string, info os.FileInfo, err error) error {
   294  						if err != nil {
   295  							return err
   296  						}
   297  						relativePath := strings.TrimPrefix(walkedPath, testdataDir)
   298  						if shouldUpdate {
   299  							if buildWorkspaceDirectory == "" {
   300  								t.Fatalf("Tried to update snapshots but no BUILD_WORKSPACE_DIRECTORY specified.\n Try %s.", updateCommand)
   301  							}
   302  
   303  							if info.Name() == "BUILD.bazel" {
   304  								destFile := strings.TrimSuffix(path.Join(buildWorkspaceDirectory, path.Dir(args.TestDataPathRelative)+relativePath), ".bazel") + args.BuildOutSuffix
   305  
   306  								err := copyFile(walkedPath, destFile)
   307  								if err != nil {
   308  									t.Fatalf("Failed to copy file %v to %v. Error: %v\n", walkedPath, destFile, err)
   309  								}
   310  							}
   311  
   312  						}
   313  						t.Logf("%q exists in %v", relativePath, testdataDir)
   314  						return nil
   315  					})
   316  					if err != nil {
   317  						t.Fatalf("Failed to walk file: %v", err)
   318  					}
   319  
   320  				} else {
   321  					t.Logf(`
   322  =====================================================================================
   323  Run %s to update BUILD.out and expected{Stdout,Stderr,ExitCode}.txt files.
   324  =====================================================================================
   325  `, updateCommand)
   326  				}
   327  			}
   328  		}()
   329  
   330  		ctx, cancel := context.WithTimeout(context.Background(), args.Timeout)
   331  		defer cancel()
   332  		cmd := exec.CommandContext(ctx, args.GazelleBinaryPath, config.Args...)
   333  		cmd.Stdout = &stdout
   334  		cmd.Stderr = &stderr
   335  		cmd.Dir = workspaceRoot
   336  		cmd.Env = append(os.Environ(), fmt.Sprintf("BUILD_WORKSPACE_DIRECTORY=%v", workspaceRoot))
   337  		if err := cmd.Run(); err != nil {
   338  			var e *exec.ExitError
   339  			if !errors.As(err, &e) {
   340  				t.Fatal(err)
   341  			}
   342  		}
   343  		errs := make([]error, 0)
   344  		actualExitCode = cmd.ProcessState.ExitCode()
   345  		if config.ExitCode != actualExitCode {
   346  			if actualExitCode == cmdTimeoutOrInterruptExitCode {
   347  				errs = append(errs, fmt.Errorf("gazelle exceeded the timeout or was interrupted"))
   348  			} else {
   349  
   350  				errs = append(errs, fmt.Errorf("expected gazelle exit code: %d\ngot: %d",
   351  					config.ExitCode, actualExitCode,
   352  				))
   353  			}
   354  		}
   355  		actualStdout := redactWorkspacePath(stdout.String(), workspaceRoot)
   356  		if strings.TrimSpace(config.Stdout) != strings.TrimSpace(actualStdout) {
   357  			errs = append(errs, fmt.Errorf("expected gazelle stdout: %s\ngot: %s",
   358  				config.Stdout, actualStdout,
   359  			))
   360  		}
   361  		actualStderr := redactWorkspacePath(stderr.String(), workspaceRoot)
   362  		if strings.TrimSpace(config.Stderr) != strings.TrimSpace(actualStderr) {
   363  			errs = append(errs, fmt.Errorf("expected gazelle stderr: %s\ngot: %s",
   364  				config.Stderr, actualStderr,
   365  			))
   366  		}
   367  		if len(errs) > 0 {
   368  			for _, err := range errs {
   369  				t.Log(err)
   370  			}
   371  			t.FailNow()
   372  		}
   373  
   374  		CheckFiles(t, testdataDir, goldens)
   375  	})
   376  }
   377  
   378  func copyFile(src string, dest string) error {
   379  	srcFile, err := os.Open(src)
   380  	if err != nil {
   381  		return err
   382  	}
   383  	defer srcFile.Close()
   384  
   385  	destFile, err := os.Create(dest)
   386  	if err != nil {
   387  		return err
   388  	}
   389  	defer destFile.Close()
   390  
   391  	_, err = io.Copy(destFile, srcFile)
   392  	if err != nil {
   393  		return err
   394  	}
   395  	err = destFile.Sync()
   396  	if err != nil {
   397  		return err
   398  	}
   399  	return nil
   400  }
   401  
   402  type testConfig struct {
   403  	Args     []string
   404  	ExitCode int
   405  	Stdout   string
   406  	Stderr   string
   407  }
   408  
   409  // updateExpectedConfig writes to an expected stdout, stderr, or exit code file
   410  // with the latest results of a test.
   411  func updateExpectedConfig(t *testing.T, expected string, actual string, srcTestDirectory string, expectedFilename string) {
   412  	if expected != actual {
   413  		destFile := path.Join(srcTestDirectory, expectedFilename)
   414  
   415  		err := os.WriteFile(destFile, []byte(actual), 0o644)
   416  		if err != nil {
   417  			t.Fatalf("Failed to write file %v. Error: %v\n", destFile, err)
   418  		}
   419  	}
   420  }
   421  
   422  // redactWorkspacePath replaces workspace path with a constant to make the test
   423  // output reproducible.
   424  func redactWorkspacePath(s, wsPath string) string {
   425  	return strings.ReplaceAll(s, wsPath, "%WORKSPACEPATH%")
   426  }
   427
View as plain text