Source file src/github.com/go-openapi/validate/messages_test.go

Documentation: github.com/go-openapi/validate

     1  // Copyright 2015 go-swagger maintainers
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package validate
    16  
    17  import (
    18  	"fmt"
    19  	"os"
    20  	"path/filepath"
    21  	"regexp"
    22  	"sort"
    23  	"strings"
    24  	"testing"
    25  
    26  	"github.com/go-openapi/loads"
    27  	"github.com/go-openapi/strfmt"
    28  	"github.com/stretchr/testify/assert"
    29  	"github.com/stretchr/testify/require"
    30  	"gopkg.in/yaml.v3"
    31  )
    32  
    33  var (
    34  	// This debug environment variable allows to report and capture actual validation messages
    35  	// during testing. It should be disabled (undefined) during CI tests.
    36  	DebugTest = os.Getenv("SWAGGER_DEBUG_TEST") != ""
    37  )
    38  
    39  type ExpectedMessage struct {
    40  	Message              string `yaml:"message"`
    41  	WithContinueOnErrors bool   `yaml:"withContinueOnErrors"` // should be expected only when SetContinueOnErrors(true)
    42  	IsRegexp             bool   `yaml:"isRegexp"`             // expected message is interpreted as regexp (with regexp.MatchString())
    43  }
    44  
    45  type ExpectedFixture struct {
    46  	Comment           string            `yaml:"comment,omitempty"`
    47  	Todo              string            `yaml:"todo,omitempty"`
    48  	ExpectedLoadError bool              `yaml:"expectedLoadError"` // expect error on load: skip validate step
    49  	ExpectedValid     bool              `yaml:"expectedValid"`     // expect valid spec
    50  	ExpectedMessages  []ExpectedMessage `yaml:"expectedMessages"`
    51  	ExpectedWarnings  []ExpectedMessage `yaml:"expectedWarnings"`
    52  	Tested            bool              `yaml:"-"`
    53  	Failed            bool              `yaml:"-"`
    54  }
    55  
    56  type ExpectedMap map[string]*ExpectedFixture
    57  
    58  func (m ExpectedMap) Get(key string) (*ExpectedFixture, bool) {
    59  	v, ok := m[key] // no need to lock this map for now
    60  	return v, ok
    61  }
    62  
    63  // Test message improvements, issue #44 and some more
    64  // ContinueOnErrors mode on
    65  // WARNING: this test is very demanding and constructed with varied scenarios,
    66  // which are not necessarily "unitary". Expect multiple changes in messages whenever
    67  // altering the validator.
    68  func Test_MessageQualityContinueOnErrors_Issue44(t *testing.T) {
    69  	if !enableLongTests {
    70  		skipNotify(t)
    71  		t.SkipNow()
    72  	}
    73  	errs := testMessageQuality(t, true, true) /* set haltOnErrors=true to iterate spec by spec */
    74  	assert.Zero(t, errs, "Message testing didn't match expectations")
    75  }
    76  
    77  // ContinueOnErrors mode off
    78  func Test_MessageQualityStopOnErrors_Issue44(t *testing.T) {
    79  	if !enableLongTests {
    80  		skipNotify(t)
    81  		t.SkipNow()
    82  	}
    83  	errs := testMessageQuality(t, true, false) /* set haltOnErrors=true to iterate spec by spec */
    84  	assert.Zero(t, errs, "Message testing didn't match expectations")
    85  }
    86  
    87  func loadTestConfig(t *testing.T, fp string) ExpectedMap {
    88  	expectedConfig, err := os.ReadFile(fp)
    89  	require.NoErrorf(t, err, "cannot read expected messages config file: %v", err)
    90  
    91  	tested := make(ExpectedMap, 200)
    92  
    93  	err = yaml.Unmarshal(expectedConfig, &tested)
    94  	require.NoErrorf(t, err, "cannot unmarshall expected messages from config file : %v", err)
    95  
    96  	// Check config
    97  	for fixture, expected := range tested {
    98  		require.Nil(t, UniqueItems("", "", expected.ExpectedMessages), "duplicate error messages configured for %s", fixture)
    99  		require.Nil(t, UniqueItems("", "", expected.ExpectedWarnings), "duplicate warning messages configured for %s", fixture)
   100  	}
   101  	return tested
   102  }
   103  
   104  func testMessageQuality(t *testing.T, haltOnErrors bool, continueOnErrors bool) int {
   105  	// Verifies the production of validation error messages in multiple
   106  	// spec scenarios.
   107  	//
   108  	// The objective is to demonstrate that:
   109  	//   - messages are stable
   110  	//   - validation continues as much as possible, even in presence of many errors
   111  	//
   112  	// haltOnErrors is used in dev mode to study and fix testcases step by step (output is pretty verbose)
   113  	//
   114  	// set SWAGGER_DEBUG_TEST=1 env to get a report of messages at the end of each test.
   115  	// expectedMessage{"", false, false},
   116  	//
   117  	// expected messages and warnings are configured in ./fixtures/validation/expected_messages.yaml
   118  	//
   119  	var errs int // error count
   120  
   121  	tested := loadTestConfig(t, filepath.Join("fixtures", "validation", "expected_messages.yaml"))
   122  
   123  	if err := filepath.Walk(filepath.Join("fixtures", "validation"), testWalkSpecs(t, tested, haltOnErrors, continueOnErrors)); err != nil {
   124  		t.Logf("%v", err)
   125  		errs++
   126  	}
   127  	recapTest(t, tested)
   128  	return errs
   129  }
   130  
   131  func testDebugLog(t *testing.T, thisTest *ExpectedFixture) {
   132  	if DebugTest {
   133  		if thisTest.Comment != "" {
   134  			t.Logf("\tDEVMODE: Comment: %s", thisTest.Comment)
   135  		}
   136  		if thisTest.Todo != "" {
   137  			t.Logf("\tDEVMODE: Todo: %s", thisTest.Todo)
   138  		}
   139  	}
   140  }
   141  
   142  func expectInvalid(t *testing.T, path string, thisTest *ExpectedFixture, continueOnErrors bool) {
   143  	// Checking invalid specs
   144  	t.Logf("Testing messages for invalid spec: %s", path)
   145  	testDebugLog(t, thisTest)
   146  
   147  	doc, err := loads.Spec(path)
   148  
   149  	// Check specs with load errors (error is located in pkg loads or spec)
   150  	if thisTest.ExpectedLoadError {
   151  		// Expect a load error: no further validation may possibly be conducted.
   152  		require.Error(t, err, "expected this spec to return a load error")
   153  		assert.Equal(t, 0, verifyLoadErrors(t, err, thisTest.ExpectedMessages))
   154  		return
   155  	}
   156  
   157  	require.NoError(t, err, "expected this spec to load properly")
   158  
   159  	// Validate the spec document
   160  	validator := NewSpecValidator(doc.Schema(), strfmt.Default)
   161  	validator.SetContinueOnErrors(continueOnErrors)
   162  	res, warn := validator.Validate(doc)
   163  
   164  	// Check specs with load errors (error is located in pkg loads or spec)
   165  	require.False(t, res.IsValid(), "expected this spec to be invalid")
   166  
   167  	errs := verifyErrorsVsWarnings(t, res, warn)
   168  	errs += verifyErrors(t, res, thisTest.ExpectedMessages, "error", continueOnErrors)
   169  	errs += verifyErrors(t, warn, thisTest.ExpectedWarnings, "warning", continueOnErrors)
   170  	assert.Equal(t, 0, errs)
   171  
   172  	if errs > 0 {
   173  		t.Logf("Message qualification on spec validation failed for %s", path)
   174  		// DEVMODE allows developers to experiment and tune expected results
   175  		if DebugTest {
   176  			reportTest(t, path, res, thisTest.ExpectedMessages, "error", continueOnErrors)
   177  			reportTest(t, path, warn, thisTest.ExpectedWarnings, "warning", continueOnErrors)
   178  		}
   179  	}
   180  }
   181  
   182  func expectValid(t *testing.T, path string, thisTest *ExpectedFixture, continueOnErrors bool) {
   183  	// Expecting no message (e.g.valid spec): 0 message expected
   184  	t.Logf("Testing valid spec: %s", path)
   185  	testDebugLog(t, thisTest)
   186  
   187  	doc, err := loads.Spec(path)
   188  	require.NoError(t, err, "expected this spec to load without error")
   189  
   190  	validator := NewSpecValidator(doc.Schema(), strfmt.Default)
   191  	validator.SetContinueOnErrors(continueOnErrors)
   192  	res, warn := validator.Validate(doc)
   193  	assert.True(t, res.IsValid(), "expected this spec to be valid")
   194  	assert.Emptyf(t, res.Errors, "expected no returned errors")
   195  
   196  	// check warnings
   197  	errs := verifyErrors(t, warn, thisTest.ExpectedWarnings, "warning", continueOnErrors)
   198  	assert.Equal(t, 0, errs)
   199  
   200  	if DebugTest && errs > 0 {
   201  		reportTest(t, path, res, thisTest.ExpectedMessages, "error", continueOnErrors)
   202  		reportTest(t, path, warn, thisTest.ExpectedWarnings, "warning", continueOnErrors)
   203  	}
   204  }
   205  
   206  func checkMustHalt(t *testing.T, haltOnErrors bool) {
   207  	if t.Failed() && haltOnErrors {
   208  		assert.FailNow(t, "test halted: stop testing on message checking error mode")
   209  		return
   210  	}
   211  }
   212  
   213  func testWalkSpecs(t *testing.T, tested ExpectedMap, haltOnErrors, continueOnErrors bool) filepath.WalkFunc {
   214  	return func(path string, info os.FileInfo, _ error) error {
   215  		thisTest, found := tested.Get(info.Name())
   216  
   217  		if info.IsDir() || !found { // skip
   218  			return nil
   219  		}
   220  
   221  		t.Run(path, func(t *testing.T) {
   222  			if !DebugTest { // when running in dev mode, run serially
   223  				t.Parallel()
   224  			}
   225  			defer func() {
   226  				thisTest.Tested = true
   227  				thisTest.Failed = t.Failed()
   228  			}()
   229  
   230  			if !thisTest.ExpectedValid {
   231  				expectInvalid(t, path, thisTest, continueOnErrors)
   232  				checkMustHalt(t, haltOnErrors)
   233  			} else {
   234  				expectValid(t, path, thisTest, continueOnErrors)
   235  				checkMustHalt(t, haltOnErrors)
   236  			}
   237  		})
   238  		return nil
   239  	}
   240  }
   241  
   242  func recapTest(t *testing.T, config ExpectedMap) {
   243  	recapFailed := false
   244  	for k, v := range config {
   245  		if !v.Tested {
   246  			t.Logf("WARNING: %s configured but not tested (fixture not found)", k)
   247  			recapFailed = true
   248  		} else if v.Failed {
   249  			t.Logf("ERROR: %s failed passing messages verification", k)
   250  			recapFailed = true
   251  		}
   252  	}
   253  	if !recapFailed {
   254  		t.Log("INFO:We are good")
   255  	}
   256  }
   257  func reportTest(t *testing.T, path string, res *Result, expectedMessages []ExpectedMessage, msgtype string, continueOnErrors bool) {
   258  	const expected = "Expected "
   259  	// Prints out a recap of error messages. To be enabled during development / test iterations
   260  	verifiedErrors := make([]string, 0, 50)
   261  	lines := make([]string, 0, 50)
   262  	for _, e := range res.Errors {
   263  		verifiedErrors = append(verifiedErrors, e.Error())
   264  	}
   265  	t.Logf("DEVMODE:Recap of returned %s messages while validating %s ", msgtype, path)
   266  	for _, v := range verifiedErrors {
   267  		status := "Unexpected " + msgtype
   268  		for _, s := range expectedMessages {
   269  			if (s.WithContinueOnErrors && continueOnErrors) || !s.WithContinueOnErrors {
   270  				if s.IsRegexp {
   271  					if matched, _ := regexp.MatchString(s.Message, v); matched {
   272  						status = expected + msgtype
   273  						break
   274  					}
   275  				} else {
   276  					if strings.Contains(v, s.Message) {
   277  						status = expected + msgtype
   278  						break
   279  					}
   280  				}
   281  			}
   282  		}
   283  		lines = append(lines, fmt.Sprintf("[%s]%s", status, v))
   284  	}
   285  
   286  	for _, s := range expectedMessages {
   287  		if (s.WithContinueOnErrors && continueOnErrors) || !s.WithContinueOnErrors {
   288  			status := "Missing " + msgtype
   289  			for _, v := range verifiedErrors {
   290  				if s.IsRegexp {
   291  					if matched, _ := regexp.MatchString(s.Message, v); matched {
   292  						status = expected + msgtype
   293  						break
   294  					}
   295  				} else {
   296  					if strings.Contains(v, s.Message) {
   297  						status = expected + msgtype
   298  						break
   299  					}
   300  				}
   301  			}
   302  			if status != expected+msgtype {
   303  				lines = append(lines, fmt.Sprintf("[%s]%s", status, s.Message))
   304  			}
   305  		}
   306  	}
   307  	if len(lines) > 0 {
   308  		sort.Strings(lines)
   309  		for _, line := range lines {
   310  			t.Logf(line)
   311  		}
   312  	}
   313  }
   314  
   315  func verifyErrorsVsWarnings(t *testing.T, res, warn *Result) int {
   316  	// First verification of result conventions: results are redundant, just a matter of presentation
   317  	w := len(warn.Errors)
   318  	if !assert.Len(t, res.Warnings, w) ||
   319  		!assert.Empty(t, warn.Warnings) ||
   320  		!assert.Subset(t, res.Warnings, warn.Errors) ||
   321  		!assert.Subset(t, warn.Errors, res.Warnings) {
   322  		t.Log("Result equivalence errors vs warnings not verified")
   323  		return 1
   324  	}
   325  	return 0
   326  }
   327  
   328  func verifyErrors(t *testing.T, res *Result, expectedMessages []ExpectedMessage, msgtype string, continueOnErrors bool) int {
   329  	var numExpected, errs int
   330  	verifiedErrors := make([]string, 0, 50)
   331  
   332  	for _, e := range res.Errors {
   333  		verifiedErrors = append(verifiedErrors, e.Error())
   334  	}
   335  	for _, s := range expectedMessages {
   336  		if (s.WithContinueOnErrors == true && continueOnErrors == true) || s.WithContinueOnErrors == false {
   337  			numExpected++
   338  		}
   339  	}
   340  
   341  	// We got the expected number of messages (e.g. no duplicates, no uncontrolled side-effect, ...)
   342  	if !assert.Len(t, verifiedErrors, numExpected, "unexpected number of %s messages returned. Wanted %d, got %d", msgtype, numExpected, len(verifiedErrors)) {
   343  		errs++
   344  	}
   345  
   346  	// Check that all expected messages are here
   347  	for _, s := range expectedMessages {
   348  		found := false
   349  		if (s.WithContinueOnErrors == true && continueOnErrors == true) || s.WithContinueOnErrors == false {
   350  			for _, v := range verifiedErrors {
   351  				if s.IsRegexp {
   352  					if matched, _ := regexp.MatchString(s.Message, v); matched {
   353  						found = true
   354  						break
   355  					}
   356  				} else {
   357  					if strings.Contains(v, s.Message) {
   358  						found = true
   359  						break
   360  					}
   361  				}
   362  			}
   363  			if !assert.True(t, found, "Missing expected %s message: %s", msgtype, s.Message) {
   364  				errs++
   365  			}
   366  		}
   367  	}
   368  
   369  	// Check for no unexpected message
   370  	for _, v := range verifiedErrors {
   371  		found := false
   372  		for _, s := range expectedMessages {
   373  			if (s.WithContinueOnErrors == true && continueOnErrors == true) || s.WithContinueOnErrors == false {
   374  				if s.IsRegexp {
   375  					if matched, _ := regexp.MatchString(s.Message, v); matched {
   376  						found = true
   377  						break
   378  					}
   379  				} else {
   380  					if strings.Contains(v, s.Message) {
   381  						found = true
   382  						break
   383  					}
   384  				}
   385  			}
   386  		}
   387  		if !assert.True(t, found, "unexpected %s message: %s", msgtype, v) {
   388  			errs++
   389  		}
   390  	}
   391  	return errs
   392  }
   393  
   394  func verifyLoadErrors(t *testing.T, err error, expectedMessages []ExpectedMessage) int {
   395  	var errs int
   396  
   397  	// Perform several matches on single error message
   398  	// Process here error messages from loads (normally unit tested in the load package:
   399  	// we just want to figure out how all this is captured at the validate package level.
   400  	v := err.Error()
   401  	for _, s := range expectedMessages {
   402  		var found bool
   403  		if s.IsRegexp {
   404  			if found, _ = regexp.MatchString(s.Message, v); found {
   405  				break
   406  			}
   407  		} else {
   408  			if found = strings.Contains(v, s.Message); found {
   409  				break
   410  			}
   411  		}
   412  		if !assert.True(t, found, "unexpected load error: %s", v) {
   413  			t.Logf("Expecting one of the following:")
   414  			for _, s := range expectedMessages {
   415  				smode := "Contains"
   416  				if s.IsRegexp {
   417  					smode = "MatchString"
   418  				}
   419  				t.Logf("[%s]:%s", smode, s.Message)
   420  			}
   421  			errs++
   422  		}
   423  	}
   424  	return errs
   425  }
   426  
   427  func testIssue(t *testing.T, path string, expectedNumErrors, expectedNumWarnings int) {
   428  	res, _ := loadAndValidate(t, path)
   429  	if expectedNumErrors > -1 && !assert.Len(t, res.Errors, expectedNumErrors) {
   430  		t.Log("Returned errors:")
   431  		for _, e := range res.Errors {
   432  			t.Logf("%v", e)
   433  		}
   434  	}
   435  	if expectedNumWarnings > -1 && !assert.Len(t, res.Warnings, expectedNumWarnings) {
   436  		t.Log("Returned warnings:")
   437  		for _, e := range res.Warnings {
   438  			t.Logf("%v", e)
   439  		}
   440  	}
   441  }
   442  
   443  // Test unitary fixture for dev and bug fixing
   444  func Test_SingleFixture(t *testing.T) {
   445  	t.SkipNow()
   446  	path := filepath.Join("fixtures", "validation", "fixture-1231.yaml")
   447  	testIssue(t, path, -1, -1)
   448  }
   449  

View as plain text