frameworktestutil.go

Documentation: sigs.k8s.io/kustomize/kyaml/fn/framework/frameworktestutil

     1  // Copyright 2020 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  // Package frameworktestutil contains utilities for testing functions written using the framework.
     5  package frameworktestutil
     6  
     7  import (
     8  	"bytes"
     9  	"os"
    10  	"path/filepath"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/spf13/cobra"
    16  	"github.com/stretchr/testify/assert"
    17  	"github.com/stretchr/testify/require"
    18  	"sigs.k8s.io/kustomize/kyaml/fn/framework"
    19  	"sigs.k8s.io/kustomize/kyaml/kio"
    20  )
    21  
    22  const (
    23  	DefaultTestDataDirectory   = "testdata"
    24  	DefaultConfigInputFilename = "config.yaml"
    25  	DefaultInputFilename       = "input.yaml"
    26  	DefaultInputFilenameGlob   = "input*.yaml"
    27  	DefaultOutputFilename      = "expected.yaml"
    28  	DefaultErrorFilename       = "errors.txt"
    29  )
    30  
    31  // CommandResultsChecker tests a command-wrapped function by running it with predefined inputs
    32  // and comparing the outputs to expected results.
    33  type CommandResultsChecker struct {
    34  	// TestDataDirectory is the directory containing the testdata subdirectories.
    35  	// CommandResultsChecker will recurse into each test directory and run the Command
    36  	// if the directory contains at least one of ExpectedOutputFilename or ExpectedErrorFilename.
    37  	// Defaults to "testdata"
    38  	TestDataDirectory string
    39  
    40  	// ExpectedOutputFilename is the file with the expected output of the function
    41  	// Defaults to "expected.yaml".  Directories containing neither this file
    42  	// nor ExpectedErrorFilename will be skipped.
    43  	ExpectedOutputFilename string
    44  
    45  	// ExpectedErrorFilename is the file containing elements of an expected error message.
    46  	// Each line of the file will be treated as a regex that must match the actual error.
    47  	// Defaults to "errors.txt".  Directories containing neither this file
    48  	// nor ExpectedOutputFilename will be skipped.
    49  	ExpectedErrorFilename string
    50  
    51  	// UpdateExpectedFromActual if set to true will write the actual results to the
    52  	// expected testdata files.  This is useful for updating test data.
    53  	UpdateExpectedFromActual bool
    54  
    55  	// OutputAssertionFunc allows you to swap out the logic used to compare the expected output
    56  	// from the fixture file to the actual output.
    57  	// By default, it performs a string comparison after normalizing whitespace.
    58  	OutputAssertionFunc AssertionFunc
    59  
    60  	// ErrorAssertionFunc allows you to swap out the logic used to compare the expected error
    61  	// message from the fixture file to the actual error message.
    62  	// By default, it interprets each line of the fixture as a regex that the actual error must match.
    63  	ErrorAssertionFunc AssertionFunc
    64  
    65  	// ConfigInputFilename is the name of the config file provided as the first
    66  	// argument to the function.  Directories without this file will be skipped.
    67  	// Defaults to "config.yaml"
    68  	ConfigInputFilename string
    69  
    70  	// InputFilenameGlob matches function inputs
    71  	// Defaults to "input*.yaml"
    72  	InputFilenameGlob string
    73  
    74  	// Command provides the function to run.
    75  	Command func() *cobra.Command
    76  
    77  	*checkerCore
    78  }
    79  
    80  // Assert runs the command with the input provided in each valid test directory
    81  // and verifies that the actual output and error match the fixtures in the directory.
    82  func (rc *CommandResultsChecker) Assert(t *testing.T) bool {
    83  	t.Helper()
    84  	if rc.ConfigInputFilename == "" {
    85  		rc.ConfigInputFilename = DefaultConfigInputFilename
    86  	}
    87  	if rc.InputFilenameGlob == "" {
    88  		rc.InputFilenameGlob = DefaultInputFilenameGlob
    89  	}
    90  	rc.checkerCore = &checkerCore{
    91  		testDataDirectory:        rc.TestDataDirectory,
    92  		expectedOutputFilename:   rc.ExpectedOutputFilename,
    93  		expectedErrorFilename:    rc.ExpectedErrorFilename,
    94  		updateExpectedFromActual: rc.UpdateExpectedFromActual,
    95  		outputAssertionFunc:      rc.OutputAssertionFunc,
    96  		errorAssertionFunc:       rc.ErrorAssertionFunc,
    97  	}
    98  	rc.checkerCore.setDefaults()
    99  	runAllTestCases(t, rc)
   100  	return true
   101  }
   102  
   103  func (rc *CommandResultsChecker) isTestDir(path string) bool {
   104  	return atLeastOneFileExists(
   105  		filepath.Join(path, rc.ConfigInputFilename),
   106  		filepath.Join(path, rc.checkerCore.expectedOutputFilename),
   107  		filepath.Join(path, rc.checkerCore.expectedErrorFilename),
   108  	)
   109  }
   110  
   111  func (rc *CommandResultsChecker) runInCurrentDir(t *testing.T) (string, string) {
   112  	t.Helper()
   113  	_, err := os.Stat(rc.ConfigInputFilename)
   114  	if os.IsNotExist(err) {
   115  		t.Errorf("Test case is missing FunctionConfig input file (default: %s)", DefaultConfigInputFilename)
   116  	}
   117  	require.NoError(t, err)
   118  	args := []string{rc.ConfigInputFilename}
   119  
   120  	if rc.InputFilenameGlob != "" {
   121  		inputs, err := filepath.Glob(rc.InputFilenameGlob)
   122  		require.NoError(t, err)
   123  		args = append(args, inputs...)
   124  	}
   125  
   126  	var stdOut, stdErr bytes.Buffer
   127  	cmd := rc.Command()
   128  	cmd.SetArgs(args)
   129  	cmd.SetOut(&stdOut)
   130  	cmd.SetErr(&stdErr)
   131  
   132  	_ = cmd.Execute()
   133  	return stdOut.String(), stdErr.String()
   134  }
   135  
   136  // ProcessorResultsChecker tests a processor function by running it with predefined inputs
   137  // and comparing the outputs to expected results.
   138  type ProcessorResultsChecker struct {
   139  	// TestDataDirectory is the directory containing the testdata subdirectories.
   140  	// ProcessorResultsChecker will recurse into each test directory and run the Command
   141  	// if the directory contains at least one of ExpectedOutputFilename or ExpectedErrorFilename.
   142  	// Defaults to "testdata"
   143  	TestDataDirectory string
   144  
   145  	// ExpectedOutputFilename is the file with the expected output of the function
   146  	// Defaults to "expected.yaml".  Directories containing neither this file
   147  	// nor ExpectedErrorFilename will be skipped.
   148  	ExpectedOutputFilename string
   149  
   150  	// ExpectedErrorFilename is the file containing elements of an expected error message.
   151  	// Each line of the file will be treated as a regex that must match the actual error.
   152  	// Defaults to "errors.txt".  Directories containing neither this file
   153  	// nor ExpectedOutputFilename will be skipped.
   154  	ExpectedErrorFilename string
   155  
   156  	// UpdateExpectedFromActual if set to true will write the actual results to the
   157  	// expected testdata files.  This is useful for updating test data.
   158  	UpdateExpectedFromActual bool
   159  
   160  	// InputFilename is the name of the file containing the ResourceList input.
   161  	// Directories without this file will be skipped.
   162  	// Defaults to "input.yaml"
   163  	InputFilename string
   164  
   165  	// OutputAssertionFunc allows you to swap out the logic used to compare the expected output
   166  	// from the fixture file to the actual output.
   167  	// By default, it performs a string comparison after normalizing whitespace.
   168  	OutputAssertionFunc AssertionFunc
   169  
   170  	// ErrorAssertionFunc allows you to swap out the logic used to compare the expected error
   171  	// message from the fixture file to the actual error message.
   172  	// By default, it interprets each line of the fixture as a regex that the actual error must match.
   173  	ErrorAssertionFunc AssertionFunc
   174  
   175  	// Processor returns a ResourceListProcessor to run.
   176  	Processor func() framework.ResourceListProcessor
   177  
   178  	*checkerCore
   179  }
   180  
   181  // Assert runs the processor with the input provided in each valid test directory
   182  // and verifies that the actual output and error match the fixtures in the directory.
   183  func (rc *ProcessorResultsChecker) Assert(t *testing.T) bool {
   184  	t.Helper()
   185  	if rc.InputFilename == "" {
   186  		rc.InputFilename = DefaultInputFilename
   187  	}
   188  	rc.checkerCore = &checkerCore{
   189  		testDataDirectory:        rc.TestDataDirectory,
   190  		expectedOutputFilename:   rc.ExpectedOutputFilename,
   191  		expectedErrorFilename:    rc.ExpectedErrorFilename,
   192  		updateExpectedFromActual: rc.UpdateExpectedFromActual,
   193  		outputAssertionFunc:      rc.OutputAssertionFunc,
   194  		errorAssertionFunc:       rc.ErrorAssertionFunc,
   195  	}
   196  	rc.checkerCore.setDefaults()
   197  	runAllTestCases(t, rc)
   198  	return true
   199  }
   200  
   201  func (rc *ProcessorResultsChecker) isTestDir(path string) bool {
   202  	return atLeastOneFileExists(
   203  		filepath.Join(path, rc.InputFilename),
   204  		filepath.Join(path, rc.checkerCore.expectedOutputFilename),
   205  		filepath.Join(path, rc.checkerCore.expectedErrorFilename),
   206  	)
   207  }
   208  
   209  func (rc *ProcessorResultsChecker) runInCurrentDir(t *testing.T) (string, string) {
   210  	t.Helper()
   211  	_, err := os.Stat(rc.InputFilename)
   212  	if os.IsNotExist(err) {
   213  		t.Errorf("Test case is missing input file (default: %s)", DefaultInputFilename)
   214  	}
   215  	require.NoError(t, err)
   216  
   217  	actualOutput := bytes.NewBuffer([]byte{})
   218  	rlBytes, err := os.ReadFile(rc.InputFilename)
   219  	require.NoError(t, err)
   220  
   221  	rw := kio.ByteReadWriter{
   222  		Reader: bytes.NewBuffer(rlBytes),
   223  		Writer: actualOutput,
   224  	}
   225  	err = framework.Execute(rc.Processor(), &rw)
   226  	if err != nil {
   227  		require.NotEmptyf(t, err.Error(), "processor returned error with empty message")
   228  		return actualOutput.String(), err.Error()
   229  	}
   230  	return actualOutput.String(), ""
   231  }
   232  
   233  type AssertionFunc func(t *testing.T, expected string, actual string)
   234  
   235  // RequireEachLineMatches is an AssertionFunc that treats each line of expected string
   236  // as a regex that must match the actual string.
   237  func RequireEachLineMatches(t *testing.T, expected, actual string) {
   238  	t.Helper()
   239  	// Check that each expected line matches the output
   240  	actual = standardizeSpacing(actual)
   241  	for _, msg := range strings.Split(standardizeSpacing(expected), "\n") {
   242  		require.Regexp(t, regexp.MustCompile(msg), actual)
   243  	}
   244  }
   245  
   246  // RequireStrippedStringsEqual is an AssertionFunc that does a simple string comparison
   247  // of expected and actual after normalizing whitespace.
   248  func RequireStrippedStringsEqual(t *testing.T, expected, actual string) {
   249  	t.Helper()
   250  	require.Equal(t,
   251  		standardizeSpacing(expected),
   252  		standardizeSpacing(actual))
   253  }
   254  
   255  func standardizeSpacing(s string) string {
   256  	// remove extra whitespace and convert Windows line endings
   257  	return strings.ReplaceAll(strings.TrimSpace(s), "\r\n", "\n")
   258  }
   259  
   260  // resultsChecker is implemented by ProcessorResultsChecker and CommandResultsChecker, partially via checkerCore
   261  type resultsChecker interface {
   262  	// TestCasesRun returns a list of the test case directories that have been seen.
   263  	TestCasesRun() (paths []string)
   264  
   265  	// rootDir is the root of the tree of test directories to be searched for fixtures.
   266  	rootDir() string
   267  	// isTestDir takes the name of directory and returns whether or not it contains the files required to be a test case.
   268  	isTestDir(dir string) bool
   269  	// runInCurrentDir executes the code the checker is checking in the context of the current working directory.
   270  	runInCurrentDir(t *testing.T) (stdOut, stdErr string)
   271  	// resetTestCasesRun resets the list of test case directories seen.
   272  	resetTestCasesRun()
   273  	// recordTestCase adds to the list of test case directories seen.
   274  	recordTestCase(path string)
   275  	// readAssertionFiles returns the contents of the output and error fixtures
   276  	readAssertionFiles(t *testing.T) (expectedOutput string, expectedError string)
   277  	// shouldUpdateFixtures indicates whether or not this checker is currently being used to update fixtures instead of run them.
   278  	shouldUpdateFixtures() bool
   279  	// updateFixtures modifies the test fixture files to match the given content
   280  	updateFixtures(t *testing.T, actualOutput string, actualError string)
   281  	// assertOutputMatches compares the expected output to the output received.
   282  	assertOutputMatches(t *testing.T, expected string, actual string)
   283  	// assertErrorMatches compares the expected error to the error received.
   284  	assertErrorMatches(t *testing.T, expected string, actual string)
   285  }
   286  
   287  // checkerCore implements the resultsChecker methods that are shared between the two checker types
   288  type checkerCore struct {
   289  	testDataDirectory        string
   290  	expectedOutputFilename   string
   291  	expectedErrorFilename    string
   292  	updateExpectedFromActual bool
   293  	outputAssertionFunc      AssertionFunc
   294  	errorAssertionFunc       AssertionFunc
   295  
   296  	testsCasesRun []string
   297  }
   298  
   299  func (rc *checkerCore) setDefaults() {
   300  	if rc.testDataDirectory == "" {
   301  		rc.testDataDirectory = DefaultTestDataDirectory
   302  	}
   303  	if rc.expectedOutputFilename == "" {
   304  		rc.expectedOutputFilename = DefaultOutputFilename
   305  	}
   306  	if rc.expectedErrorFilename == "" {
   307  		rc.expectedErrorFilename = DefaultErrorFilename
   308  	}
   309  	if rc.outputAssertionFunc == nil {
   310  		rc.outputAssertionFunc = RequireStrippedStringsEqual
   311  	}
   312  	if rc.errorAssertionFunc == nil {
   313  		rc.errorAssertionFunc = RequireEachLineMatches
   314  	}
   315  }
   316  
   317  func (rc *checkerCore) rootDir() string {
   318  	return rc.testDataDirectory
   319  }
   320  
   321  func (rc *checkerCore) TestCasesRun() []string {
   322  	return rc.testsCasesRun
   323  }
   324  
   325  func (rc *checkerCore) resetTestCasesRun() {
   326  	rc.testsCasesRun = []string{}
   327  }
   328  
   329  func (rc *checkerCore) recordTestCase(s string) {
   330  	rc.testsCasesRun = append(rc.testsCasesRun, s)
   331  }
   332  
   333  func (rc *checkerCore) shouldUpdateFixtures() bool {
   334  	return rc.updateExpectedFromActual
   335  }
   336  
   337  func (rc *checkerCore) updateFixtures(t *testing.T, actualOutput string, actualError string) {
   338  	t.Helper()
   339  	if actualError != "" {
   340  		require.NoError(t, os.WriteFile(rc.expectedErrorFilename, []byte(actualError), 0600))
   341  	}
   342  	if len(actualOutput) > 0 {
   343  		require.NoError(t, os.WriteFile(rc.expectedOutputFilename, []byte(actualOutput), 0600))
   344  	}
   345  	t.Skip("Updated fixtures for test case")
   346  }
   347  
   348  func (rc *checkerCore) assertOutputMatches(t *testing.T, expected string, actual string) {
   349  	t.Helper()
   350  	rc.outputAssertionFunc(t, expected, actual)
   351  }
   352  
   353  func (rc *checkerCore) assertErrorMatches(t *testing.T, expected string, actual string) {
   354  	t.Helper()
   355  	rc.errorAssertionFunc(t, expected, actual)
   356  }
   357  
   358  func (rc *checkerCore) readAssertionFiles(t *testing.T) (string, string) {
   359  	t.Helper()
   360  	// read the expected results
   361  	var expectedOutput, expectedError string
   362  	if rc.expectedOutputFilename != "" {
   363  		_, err := os.Stat(rc.expectedOutputFilename)
   364  		if !os.IsNotExist(err) && err != nil {
   365  			t.FailNow()
   366  		}
   367  		if err == nil {
   368  			// only read the file if it exists
   369  			b, err := os.ReadFile(rc.expectedOutputFilename)
   370  			if !assert.NoError(t, err) {
   371  				t.FailNow()
   372  			}
   373  			expectedOutput = string(b)
   374  		}
   375  	}
   376  	if rc.expectedErrorFilename != "" {
   377  		_, err := os.Stat(rc.expectedErrorFilename)
   378  		if !os.IsNotExist(err) && err != nil {
   379  			t.FailNow()
   380  		}
   381  		if err == nil {
   382  			// only read the file if it exists
   383  			b, err := os.ReadFile(rc.expectedErrorFilename)
   384  			if !assert.NoError(t, err) {
   385  				t.FailNow()
   386  			}
   387  			expectedError = string(b)
   388  		}
   389  	}
   390  	return expectedOutput, expectedError
   391  }
   392  
   393  // runAllTestCases traverses rootDir to find test cases, calls getResult to invoke the function
   394  // under test in each directory, and then runs assertions on the returned output and error results.
   395  // It triggers a test failure if no valid test directories were found.
   396  func runAllTestCases(t *testing.T, checker resultsChecker) {
   397  	t.Helper()
   398  	checker.resetTestCasesRun()
   399  	err := filepath.Walk(checker.rootDir(),
   400  		func(path string, info os.FileInfo, err error) error {
   401  			require.NoError(t, err)
   402  			if info.IsDir() && checker.isTestDir(path) {
   403  				runDirectoryTestCase(t, path, checker)
   404  			}
   405  			return nil
   406  		})
   407  	require.NoError(t, err)
   408  	require.NotZero(t, len(checker.TestCasesRun()), "No complete test cases found in %s", checker.rootDir())
   409  }
   410  
   411  func runDirectoryTestCase(t *testing.T, path string, checker resultsChecker) {
   412  	t.Helper()
   413  	// cd into the directory so we can test functions that refer to local files by relative paths
   414  	d, err := os.Getwd()
   415  	require.NoError(t, err)
   416  
   417  	defer func() { require.NoError(t, os.Chdir(d)) }()
   418  	require.NoError(t, os.Chdir(path))
   419  
   420  	expectedOutput, expectedError := checker.readAssertionFiles(t)
   421  	if expectedError == "" && expectedOutput == "" && !checker.shouldUpdateFixtures() {
   422  		t.Fatalf("test directory %s must include either expected output or expected error fixture", path)
   423  	}
   424  
   425  	// run the test
   426  	t.Run(path, func(t *testing.T) {
   427  		checker.recordTestCase(path)
   428  		actualOutput, actualError := checker.runInCurrentDir(t)
   429  
   430  		// Configured to update the assertion files instead of comparing them
   431  		if checker.shouldUpdateFixtures() {
   432  			checker.updateFixtures(t, actualOutput, actualError)
   433  		}
   434  
   435  		// Compare the results to the assertion files
   436  		if expectedError != "" {
   437  			// We expected an error, so make sure there was one
   438  			require.NotEmptyf(t, actualError, "test expected an error but message was empty, and output was:\n%s", actualOutput)
   439  			checker.assertErrorMatches(t, expectedError, actualError)
   440  		} else {
   441  			// We didn't expect an error, and the output should match
   442  			require.Emptyf(t, actualError, "test expected no error but got an error message, and output was:\n%s", actualOutput)
   443  			checker.assertOutputMatches(t, expectedOutput, actualOutput)
   444  		}
   445  	})
   446  }
   447  
   448  func atLeastOneFileExists(files ...string) bool {
   449  	for _, file := range files {
   450  		if _, err := os.Stat(file); err == nil {
   451  			return true
   452  		}
   453  	}
   454  	return false
   455  }
   456
View as plain text