1
16
17 package roundtrip
18
19 import (
20 "bytes"
21 gojson "encoding/json"
22 "io/ioutil"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "reflect"
27 "sort"
28 "strings"
29 "testing"
30
31 "github.com/google/go-cmp/cmp"
32
33 apiequality "k8s.io/apimachinery/pkg/api/equality"
34 "k8s.io/apimachinery/pkg/runtime"
35 "k8s.io/apimachinery/pkg/runtime/schema"
36 "k8s.io/apimachinery/pkg/runtime/serializer/json"
37 "k8s.io/apimachinery/pkg/runtime/serializer/protobuf"
38 "k8s.io/apimachinery/pkg/util/sets"
39 )
40
41
42
43
44
45 type CompatibilityTestOptions struct {
46
47
48 Scheme *runtime.Scheme
49
50
51
52 TestDataDir string
53
54
55
56
57
58
59
60
61 TestDataDirCurrentVersion string
62
63
64
65
66
67
68
69 TestDataDirsPreviousVersions []string
70
71
72
73 Kinds []schema.GroupVersionKind
74
75
76
77
78 FilledObjects map[schema.GroupVersionKind]runtime.Object
79
80
81 FillFuncs map[reflect.Type]FillFunc
82
83 JSON runtime.Serializer
84 YAML runtime.Serializer
85 Proto runtime.Serializer
86 }
87
88
89
90
91
92 type FillFunc func(s string, i int, obj interface{})
93
94 func NewCompatibilityTestOptions(scheme *runtime.Scheme) *CompatibilityTestOptions {
95 return &CompatibilityTestOptions{Scheme: scheme}
96 }
97
98
99 var coreKinds = sets.NewString(
100 "CreateOptions", "UpdateOptions", "PatchOptions", "DeleteOptions",
101 "GetOptions", "ListOptions", "ExportOptions",
102 "WatchEvent",
103 )
104
105 func (c *CompatibilityTestOptions) Complete(t *testing.T) *CompatibilityTestOptions {
106 t.Helper()
107
108
109 if c.Scheme == nil {
110 t.Fatal("scheme is required")
111 }
112
113
114 if c.TestDataDir == "" {
115 c.TestDataDir = "testdata"
116 }
117 if c.TestDataDirCurrentVersion == "" {
118 c.TestDataDirCurrentVersion = filepath.Join(c.TestDataDir, "HEAD")
119 }
120 if c.TestDataDirsPreviousVersions == nil {
121 dirs, err := filepath.Glob(filepath.Join(c.TestDataDir, "v*"))
122 if err != nil {
123 t.Fatal(err)
124 }
125 sort.Strings(dirs)
126 c.TestDataDirsPreviousVersions = dirs
127 }
128
129
130 if len(c.Kinds) == 0 {
131 gvks := []schema.GroupVersionKind{}
132 for gvk := range c.Scheme.AllKnownTypes() {
133 if gvk.Version == "" || gvk.Version == runtime.APIVersionInternal {
134
135 continue
136 }
137 if strings.HasSuffix(gvk.Kind, "List") {
138
139 continue
140 }
141 if gvk.Group != "" && coreKinds.Has(gvk.Kind) {
142
143 continue
144 }
145 gvks = append(gvks, gvk)
146 }
147 c.Kinds = gvks
148 }
149
150
151 sort.Slice(c.Kinds, func(i, j int) bool {
152 if c.Kinds[i].Group != c.Kinds[j].Group {
153 return c.Kinds[i].Group < c.Kinds[j].Group
154 }
155 if c.Kinds[i].Version != c.Kinds[j].Version {
156 return c.Kinds[i].Version < c.Kinds[j].Version
157 }
158 if c.Kinds[i].Kind != c.Kinds[j].Kind {
159 return c.Kinds[i].Kind < c.Kinds[j].Kind
160 }
161 return false
162 })
163
164
165 if c.FilledObjects == nil {
166 c.FilledObjects = map[schema.GroupVersionKind]runtime.Object{}
167 }
168 fillFuncs := defaultFillFuncs()
169 for k, v := range c.FillFuncs {
170 fillFuncs[k] = v
171 }
172 for _, gvk := range c.Kinds {
173 if _, ok := c.FilledObjects[gvk]; ok {
174 continue
175 }
176 obj, err := CompatibilityTestObject(c.Scheme, gvk, fillFuncs)
177 if err != nil {
178 t.Fatal(err)
179 }
180 c.FilledObjects[gvk] = obj
181 }
182
183 if c.JSON == nil {
184 c.JSON = json.NewSerializer(json.DefaultMetaFactory, c.Scheme, c.Scheme, true)
185 }
186 if c.YAML == nil {
187 c.YAML = json.NewYAMLSerializer(json.DefaultMetaFactory, c.Scheme, c.Scheme)
188 }
189 if c.Proto == nil {
190 c.Proto = protobuf.NewSerializer(c.Scheme, c.Scheme)
191 }
192
193 return c
194 }
195
196 func (c *CompatibilityTestOptions) Run(t *testing.T) {
197 usedHEADFixtures := sets.NewString()
198
199 for _, gvk := range c.Kinds {
200 t.Run(makeName(gvk), func(t *testing.T) {
201
202 t.Run("HEAD", func(t *testing.T) {
203 c.runCurrentVersionTest(t, gvk, usedHEADFixtures)
204 })
205
206 for _, previousVersionDir := range c.TestDataDirsPreviousVersions {
207 t.Run(filepath.Base(previousVersionDir), func(t *testing.T) {
208 c.runPreviousVersionTest(t, gvk, previousVersionDir, nil)
209 })
210 }
211
212 })
213 }
214
215
216 t.Run("unused_fixtures", func(t *testing.T) {
217 files, err := os.ReadDir(c.TestDataDirCurrentVersion)
218 if err != nil {
219 t.Fatal(err)
220 }
221 allFixtures := sets.NewString()
222 for _, file := range files {
223 allFixtures.Insert(file.Name())
224 }
225
226 if unused := allFixtures.Difference(usedHEADFixtures); len(unused) > 0 {
227 t.Fatalf("remove unused fixtures from %s:\n%s", c.TestDataDirCurrentVersion, strings.Join(unused.List(), "\n"))
228 }
229 })
230 }
231
232 func (c *CompatibilityTestOptions) runCurrentVersionTest(t *testing.T, gvk schema.GroupVersionKind, usedFiles sets.String) {
233 expectedObject := c.FilledObjects[gvk]
234 expectedJSON, expectedYAML, expectedProto := c.encode(t, expectedObject)
235
236 actualJSON, actualYAML, actualProto, err := read(c.TestDataDirCurrentVersion, gvk, "", usedFiles)
237 if err != nil && !os.IsNotExist(err) {
238 t.Fatal(err)
239 }
240
241 needsUpdate := false
242 if os.IsNotExist(err) {
243 t.Errorf("current version compatibility files did not exist: %v", err)
244 needsUpdate = true
245 } else {
246 if !bytes.Equal(expectedJSON, actualJSON) {
247 t.Errorf("json differs")
248 t.Log(cmp.Diff(string(actualJSON), string(expectedJSON)))
249 needsUpdate = true
250 }
251
252 if !bytes.Equal(expectedYAML, actualYAML) {
253 t.Errorf("yaml differs")
254 t.Log(cmp.Diff(string(actualYAML), string(expectedYAML)))
255 needsUpdate = true
256 }
257
258 if !bytes.Equal(expectedProto, actualProto) {
259 t.Errorf("proto differs")
260 needsUpdate = true
261 t.Log(cmp.Diff(dumpProto(t, actualProto[4:]), dumpProto(t, expectedProto[4:])))
262
263 }
264 }
265
266 if needsUpdate {
267 const updateEnvVar = "UPDATE_COMPATIBILITY_FIXTURE_DATA"
268 if os.Getenv(updateEnvVar) == "true" {
269 writeFile(t, c.TestDataDirCurrentVersion, gvk, "", "json", expectedJSON)
270 writeFile(t, c.TestDataDirCurrentVersion, gvk, "", "yaml", expectedYAML)
271 writeFile(t, c.TestDataDirCurrentVersion, gvk, "", "pb", expectedProto)
272 t.Logf("wrote expected compatibility data... verify, commit, and rerun tests")
273 } else {
274 t.Logf("if the diff is expected because of a new type or a new field, re-run with %s=true to update the compatibility data", updateEnvVar)
275 }
276 return
277 }
278
279 emptyObj, err := c.Scheme.New(gvk)
280 if err != nil {
281 t.Fatal(err)
282 }
283 {
284
285 compacted := &bytes.Buffer{}
286 if err := gojson.Compact(compacted, actualJSON); err != nil {
287 t.Error(err)
288 }
289
290 jsonDecoded := emptyObj.DeepCopyObject()
291 jsonDecoded, _, err = c.JSON.Decode(compacted.Bytes(), &gvk, jsonDecoded)
292 if err != nil {
293 t.Error(err)
294 } else if !apiequality.Semantic.DeepEqual(expectedObject, jsonDecoded) {
295 t.Errorf("expected and decoded json objects differed:\n%s", cmp.Diff(expectedObject, jsonDecoded))
296 }
297 }
298 {
299 yamlDecoded := emptyObj.DeepCopyObject()
300 yamlDecoded, _, err = c.YAML.Decode(actualYAML, &gvk, yamlDecoded)
301 if err != nil {
302 t.Error(err)
303 } else if !apiequality.Semantic.DeepEqual(expectedObject, yamlDecoded) {
304 t.Errorf("expected and decoded yaml objects differed:\n%s", cmp.Diff(expectedObject, yamlDecoded))
305 }
306 }
307 {
308 protoDecoded := emptyObj.DeepCopyObject()
309 protoDecoded, _, err = c.Proto.Decode(actualProto, &gvk, protoDecoded)
310 if err != nil {
311 t.Error(err)
312 } else if !apiequality.Semantic.DeepEqual(expectedObject, protoDecoded) {
313 t.Errorf("expected and decoded proto objects differed:\n%s", cmp.Diff(expectedObject, protoDecoded))
314 }
315 }
316 }
317
318 func (c *CompatibilityTestOptions) encode(t *testing.T, obj runtime.Object) (json, yaml, proto []byte) {
319 jsonBytes := bytes.NewBuffer(nil)
320 if err := c.JSON.Encode(obj, jsonBytes); err != nil {
321 t.Fatalf("error encoding json: %v", err)
322 }
323 yamlBytes := bytes.NewBuffer(nil)
324 if err := c.YAML.Encode(obj, yamlBytes); err != nil {
325 t.Fatalf("error encoding yaml: %v", err)
326 }
327 protoBytes := bytes.NewBuffer(nil)
328 if err := c.Proto.Encode(obj, protoBytes); err != nil {
329 t.Fatalf("error encoding proto: %v", err)
330 }
331 return jsonBytes.Bytes(), yamlBytes.Bytes(), protoBytes.Bytes()
332 }
333
334 func read(dir string, gvk schema.GroupVersionKind, suffix string, usedFiles sets.String) (json, yaml, proto []byte, err error) {
335 jsonFilename := makeName(gvk) + suffix + ".json"
336 actualJSON, jsonErr := ioutil.ReadFile(filepath.Join(dir, jsonFilename))
337 yamlFilename := makeName(gvk) + suffix + ".yaml"
338 actualYAML, yamlErr := ioutil.ReadFile(filepath.Join(dir, yamlFilename))
339 protoFilename := makeName(gvk) + suffix + ".pb"
340 actualProto, protoErr := ioutil.ReadFile(filepath.Join(dir, protoFilename))
341 if usedFiles != nil {
342 usedFiles.Insert(jsonFilename)
343 usedFiles.Insert(yamlFilename)
344 usedFiles.Insert(protoFilename)
345 }
346 if jsonErr != nil {
347 return actualJSON, actualYAML, actualProto, jsonErr
348 }
349 if yamlErr != nil {
350 return actualJSON, actualYAML, actualProto, yamlErr
351 }
352 if protoErr != nil {
353 return actualJSON, actualYAML, actualProto, protoErr
354 }
355 return actualJSON, actualYAML, actualProto, nil
356 }
357
358 func writeFile(t *testing.T, dir string, gvk schema.GroupVersionKind, suffix, extension string, data []byte) {
359 if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
360 t.Fatal("error making directory", err)
361 }
362 if err := ioutil.WriteFile(filepath.Join(dir, makeName(gvk)+suffix+"."+extension), data, os.FileMode(0644)); err != nil {
363 t.Fatalf("error writing %s: %v", extension, err)
364 }
365 }
366
367 func deleteFile(t *testing.T, dir string, gvk schema.GroupVersionKind, suffix, extension string) {
368 if err := os.Remove(filepath.Join(dir, makeName(gvk)+suffix+"."+extension)); err != nil {
369 t.Fatalf("error removing %s: %v", extension, err)
370 }
371 }
372
373 func (c *CompatibilityTestOptions) runPreviousVersionTest(t *testing.T, gvk schema.GroupVersionKind, previousVersionDir string, usedFiles sets.String) {
374 jsonBeforeRoundTrip, yamlBeforeRoundTrip, protoBeforeRoundTrip, err := read(previousVersionDir, gvk, "", usedFiles)
375 if os.IsNotExist(err) || (len(jsonBeforeRoundTrip) == 0 && len(yamlBeforeRoundTrip) == 0 && len(protoBeforeRoundTrip) == 0) {
376 t.SkipNow()
377 return
378 }
379 if err != nil {
380 t.Fatal(err)
381 }
382
383 emptyObj, err := c.Scheme.New(gvk)
384 if err != nil {
385 t.Fatal(err)
386 }
387
388
389 compacted := &bytes.Buffer{}
390 if err := gojson.Compact(compacted, jsonBeforeRoundTrip); err != nil {
391 t.Fatal(err)
392 }
393
394 jsonDecoded := emptyObj.DeepCopyObject()
395 jsonDecoded, _, err = c.JSON.Decode(compacted.Bytes(), &gvk, jsonDecoded)
396 if err != nil {
397 t.Fatal(err)
398 }
399 jsonBytes := bytes.NewBuffer(nil)
400 if err := c.JSON.Encode(jsonDecoded, jsonBytes); err != nil {
401 t.Fatalf("error encoding json: %v", err)
402 }
403 jsonAfterRoundTrip := jsonBytes.Bytes()
404
405 yamlDecoded := emptyObj.DeepCopyObject()
406 yamlDecoded, _, err = c.YAML.Decode(yamlBeforeRoundTrip, &gvk, yamlDecoded)
407 if err != nil {
408 t.Fatal(err)
409 } else if !apiequality.Semantic.DeepEqual(jsonDecoded, yamlDecoded) {
410 t.Errorf("decoded json and yaml objects differ:\n%s", cmp.Diff(jsonDecoded, yamlDecoded))
411 }
412 yamlBytes := bytes.NewBuffer(nil)
413 if err := c.YAML.Encode(yamlDecoded, yamlBytes); err != nil {
414 t.Fatalf("error encoding yaml: %v", err)
415 }
416 yamlAfterRoundTrip := yamlBytes.Bytes()
417
418 protoDecoded := emptyObj.DeepCopyObject()
419 protoDecoded, _, err = c.Proto.Decode(protoBeforeRoundTrip, &gvk, protoDecoded)
420 if err != nil {
421 t.Fatal(err)
422 } else if !apiequality.Semantic.DeepEqual(jsonDecoded, protoDecoded) {
423 t.Errorf("decoded json and proto objects differ:\n%s", cmp.Diff(jsonDecoded, protoDecoded))
424 }
425 protoBytes := bytes.NewBuffer(nil)
426 if err := c.Proto.Encode(protoDecoded, protoBytes); err != nil {
427 t.Fatalf("error encoding proto: %v", err)
428 }
429 protoAfterRoundTrip := protoBytes.Bytes()
430
431 jsonNeedsRemove := false
432 yamlNeedsRemove := false
433 protoNeedsRemove := false
434
435 expectedJSONAfterRoundTrip, expectedYAMLAfterRoundTrip, expectedProtoAfterRoundTrip, _ := read(previousVersionDir, gvk, ".after_roundtrip", usedFiles)
436 if len(expectedJSONAfterRoundTrip) == 0 {
437 expectedJSONAfterRoundTrip = jsonBeforeRoundTrip
438 } else if bytes.Equal(jsonBeforeRoundTrip, expectedJSONAfterRoundTrip) {
439 t.Errorf("JSON after_roundtrip file is identical and should be removed")
440 jsonNeedsRemove = true
441 }
442 if len(expectedYAMLAfterRoundTrip) == 0 {
443 expectedYAMLAfterRoundTrip = yamlBeforeRoundTrip
444 } else if bytes.Equal(yamlBeforeRoundTrip, expectedYAMLAfterRoundTrip) {
445 t.Errorf("YAML after_roundtrip file is identical and should be removed")
446 yamlNeedsRemove = true
447 }
448 if len(expectedProtoAfterRoundTrip) == 0 {
449 expectedProtoAfterRoundTrip = protoBeforeRoundTrip
450 } else if bytes.Equal(protoBeforeRoundTrip, expectedProtoAfterRoundTrip) {
451 t.Errorf("Proto after_roundtrip file is identical and should be removed")
452 protoNeedsRemove = true
453 }
454
455 jsonNeedsUpdate := false
456 yamlNeedsUpdate := false
457 protoNeedsUpdate := false
458
459 if !bytes.Equal(expectedJSONAfterRoundTrip, jsonAfterRoundTrip) {
460 t.Errorf("json differs")
461 t.Log(cmp.Diff(string(expectedJSONAfterRoundTrip), string(jsonAfterRoundTrip)))
462 jsonNeedsUpdate = true
463 }
464
465 if !bytes.Equal(expectedYAMLAfterRoundTrip, yamlAfterRoundTrip) {
466 t.Errorf("yaml differs")
467 t.Log(cmp.Diff(string(expectedYAMLAfterRoundTrip), string(yamlAfterRoundTrip)))
468 yamlNeedsUpdate = true
469 }
470
471 if !bytes.Equal(expectedProtoAfterRoundTrip, protoAfterRoundTrip) {
472 t.Errorf("proto differs")
473 protoNeedsUpdate = true
474 t.Log(cmp.Diff(dumpProto(t, expectedProtoAfterRoundTrip[4:]), dumpProto(t, protoAfterRoundTrip[4:])))
475
476 }
477
478 if jsonNeedsUpdate || yamlNeedsUpdate || protoNeedsUpdate || jsonNeedsRemove || yamlNeedsRemove || protoNeedsRemove {
479 const updateEnvVar = "UPDATE_COMPATIBILITY_FIXTURE_DATA"
480 if os.Getenv(updateEnvVar) == "true" {
481 if jsonNeedsUpdate {
482 writeFile(t, previousVersionDir, gvk, ".after_roundtrip", "json", jsonAfterRoundTrip)
483 } else if jsonNeedsRemove {
484 deleteFile(t, previousVersionDir, gvk, ".after_roundtrip", "json")
485 }
486
487 if yamlNeedsUpdate {
488 writeFile(t, previousVersionDir, gvk, ".after_roundtrip", "yaml", yamlAfterRoundTrip)
489 } else if yamlNeedsRemove {
490 deleteFile(t, previousVersionDir, gvk, ".after_roundtrip", "yaml")
491 }
492
493 if protoNeedsUpdate {
494 writeFile(t, previousVersionDir, gvk, ".after_roundtrip", "pb", protoAfterRoundTrip)
495 } else if protoNeedsRemove {
496 deleteFile(t, previousVersionDir, gvk, ".after_roundtrip", "pb")
497 }
498 t.Logf("wrote expected compatibility data... verify, commit, and rerun tests")
499 } else {
500 t.Logf("if the diff is expected because of a new type or a new field, re-run with %s=true to update the compatibility data", updateEnvVar)
501 }
502 return
503 }
504 }
505
506 func makeName(gvk schema.GroupVersionKind) string {
507 g := gvk.Group
508 if g == "" {
509 g = "core"
510 }
511 return g + "." + gvk.Version + "." + gvk.Kind
512 }
513
514 func dumpProto(t *testing.T, data []byte) string {
515 t.Helper()
516 protoc, err := exec.LookPath("protoc")
517 if err != nil {
518 t.Log(err)
519 return ""
520 }
521 cmd := exec.Command(protoc, "--decode_raw")
522 cmd.Stdin = bytes.NewBuffer(data)
523 d, err := cmd.CombinedOutput()
524 if err != nil {
525 t.Log(err)
526 return ""
527 }
528 return string(d)
529 }
530
View as plain text