1
2
3
4
5
6 package gcsfs
7
8 import (
9 "context"
10 "errors"
11 "fmt"
12 "io"
13 "os"
14 "path/filepath"
15 "reflect"
16 "strings"
17 "syscall"
18 "testing"
19
20 "golang.org/x/oauth2/google"
21
22 "cloud.google.com/go/storage"
23 "github.com/googleapis/google-cloud-go-testing/storage/stiface"
24 "github.com/spf13/afero"
25 )
26
27 const (
28 testBytes = 8
29 dirSize = 42
30 )
31
32 var bucketName = "a-test-bucket"
33
34 var files = []struct {
35 name string
36 exists bool
37 isdir bool
38 size int64
39 content string
40 offset int64
41 contentAtOffset string
42 }{
43 {"sub", true, true, dirSize, "", 0, ""},
44 {"sub/testDir2", true, true, dirSize, "", 0, ""},
45 {"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"},
46 {"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"},
47 {"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"},
48
49 {"", false, true, dirSize, "", 0, ""},
50
51 {"nonExisting", false, false, dirSize, "", 0, ""},
52 }
53
54 var dirs = []struct {
55 name string
56 children []string
57 }{
58 {"", []string{"sub", "testDir1", "testFile"}},
59 {"sub", []string{"testDir2"}},
60 {"sub/testDir2", []string{"testFile"}},
61 {"testDir1", []string{"testFile"}},
62 }
63
64 var gcsAfs *afero.Afero
65
66 func TestMain(m *testing.M) {
67 ctx := context.Background()
68 var err error
69
70
71 var exitCode int
72 defer os.Exit(exitCode)
73
74 defer func() {
75 err := recover()
76 if err != nil {
77 fmt.Print(err)
78 exitCode = 2
79 }
80 }()
81
82
83
84 cred, err := google.FindDefaultCredentials(ctx)
85 if err != nil && !strings.HasPrefix(err.Error(), "google: could not find default credentials") {
86 panic(err)
87 }
88
89 if cred == nil {
90 var fakeCredentialsAbsPath string
91 fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json")
92 if err != nil {
93 panic(err)
94 }
95
96 err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath)
97 if err != nil {
98 panic(err)
99 }
100
101
102 defer func() {
103 err = os.Remove("GOOGLE_APPLICATION_CREDENTIALS")
104 if err != nil {
105
106 fmt.Print("failed to clear fake GOOGLE_APPLICATION_CREDENTIALS", err)
107 }
108 }()
109 }
110
111 var c *storage.Client
112 c, err = storage.NewClient(ctx)
113 if err != nil {
114 panic(err)
115 }
116 client := stiface.AdaptClient(c)
117
118
119 mockClient := newClientMock()
120 mockClient.Client = client
121
122 gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}}
123
124
125
126
127 exitCode = m.Run()
128 }
129
130 func createFiles(t *testing.T) {
131 t.Helper()
132 var err error
133
134
135 for _, f := range files {
136 if !f.isdir && f.exists {
137 name := filepath.Join(bucketName, f.name)
138
139 var freshFile afero.File
140 freshFile, err = gcsAfs.Create(name)
141 if err != nil {
142 t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
143 }
144
145 var written int
146 var totalWritten int64
147 for totalWritten < f.size {
148 if totalWritten < f.offset {
149 writeBuf := []byte(strings.Repeat(f.content, int(f.offset)))
150 written, err = freshFile.WriteAt(writeBuf, totalWritten)
151 } else {
152 writeBuf := []byte(strings.Repeat(f.contentAtOffset, int(f.size-f.offset)))
153 written, err = freshFile.WriteAt(writeBuf, totalWritten)
154 }
155 if err != nil {
156 t.Fatalf("failed to write a file \"%s\": %s", f.name, err)
157 }
158
159 totalWritten += int64(written)
160 }
161
162 err = freshFile.Close()
163 if err != nil {
164 t.Fatalf("failed to close a file \"%s\": %s", f.name, err)
165 }
166 }
167 }
168 }
169
170 func removeFiles(t *testing.T) {
171 t.Helper()
172 var err error
173
174
175 for _, f := range files {
176 if !f.isdir && f.exists {
177 name := filepath.Join(bucketName, f.name)
178
179 err = gcsAfs.Remove(name)
180 if err != nil && err == syscall.ENOENT {
181 t.Errorf("failed to remove file \"%s\": %s", f.name, err)
182 }
183 }
184 }
185 }
186
187 func TestGcsFsOpen(t *testing.T) {
188 createFiles(t)
189 defer removeFiles(t)
190
191 for _, f := range files {
192 nameBase := filepath.Join(bucketName, f.name)
193
194 names := []string{
195 nameBase,
196 string(os.PathSeparator) + nameBase,
197 }
198 if f.name == "" {
199 names = []string{f.name}
200 }
201
202 for _, name := range names {
203 file, err := gcsAfs.Open(name)
204 if (err == nil) != f.exists {
205 t.Errorf("%v exists = %v, but got err = %v", name, f.exists, err)
206 }
207
208 if !f.exists {
209 continue
210 }
211 if err != nil {
212 t.Fatalf("%v: %v", name, err)
213 }
214
215 if file.Name() != filepath.FromSlash(nameBase) {
216 t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(nameBase))
217 }
218
219 s, err := file.Stat()
220 if err != nil {
221 t.Fatalf("stat %v: got error '%v'", file.Name(), err)
222 }
223
224 if isdir := s.IsDir(); isdir != f.isdir {
225 t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir)
226 }
227
228 if size := s.Size(); size != f.size {
229 t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size)
230 }
231 }
232 }
233 }
234
235 func TestGcsRead(t *testing.T) {
236 createFiles(t)
237 defer removeFiles(t)
238
239 for _, f := range files {
240 if !f.exists {
241 continue
242 }
243
244 nameBase := filepath.Join(bucketName, f.name)
245
246 names := []string{
247 nameBase,
248 string(os.PathSeparator) + nameBase,
249 }
250 if f.name == "" {
251 names = []string{f.name}
252 }
253
254 for _, name := range names {
255 file, err := gcsAfs.Open(name)
256 if err != nil {
257 t.Fatalf("opening %v: %v", name, err)
258 }
259
260 buf := make([]byte, 8)
261 n, err := file.Read(buf)
262 if err != nil {
263 if f.isdir && (err != syscall.EISDIR) {
264 t.Errorf("%v got error %v, expected EISDIR", name, err)
265 } else if !f.isdir {
266 t.Errorf("%v: %v", name, err)
267 }
268 } else if n != 8 {
269 t.Errorf("%v: got %d read bytes, expected 8", name, n)
270 } else if string(buf) != strings.Repeat(f.content, testBytes) {
271 t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
272 }
273 }
274 }
275 }
276
277 func TestGcsReadAt(t *testing.T) {
278 createFiles(t)
279 defer removeFiles(t)
280
281 for _, f := range files {
282 if !f.exists {
283 continue
284 }
285
286 nameBase := filepath.Join(bucketName, f.name)
287
288 names := []string{
289 nameBase,
290 string(os.PathSeparator) + nameBase,
291 }
292 if f.name == "" {
293 names = []string{f.name}
294 }
295
296 for _, name := range names {
297 file, err := gcsAfs.Open(name)
298 if err != nil {
299 t.Fatalf("opening %v: %v", name, err)
300 }
301
302 buf := make([]byte, testBytes)
303 n, err := file.ReadAt(buf, f.offset-testBytes/2)
304 if err != nil {
305 if f.isdir && (err != syscall.EISDIR) {
306 t.Errorf("%v got error %v, expected EISDIR", name, err)
307 } else if !f.isdir {
308 t.Errorf("%v: %v", name, err)
309 }
310 } else if n != 8 {
311 t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
312 } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) {
313 t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf))
314 }
315 }
316 }
317 }
318
319 func TestGcsSeek(t *testing.T) {
320 createFiles(t)
321 defer removeFiles(t)
322
323 for _, f := range files {
324 if !f.exists {
325 continue
326 }
327
328 nameBase := filepath.Join(bucketName, f.name)
329
330 names := []string{
331 nameBase,
332 string(os.PathSeparator) + nameBase,
333 }
334 if f.name == "" {
335 names = []string{f.name}
336 }
337
338 for _, name := range names {
339 file, err := gcsAfs.Open(name)
340 if err != nil {
341 t.Fatalf("opening %v: %v", name, err)
342 }
343
344 tests := []struct {
345 offIn int64
346 whence int
347 offOut int64
348 }{
349 {0, io.SeekStart, 0},
350 {10, io.SeekStart, 10},
351 {1, io.SeekCurrent, 11},
352 {10, io.SeekCurrent, 21},
353 {0, io.SeekEnd, f.size},
354 {-1, io.SeekEnd, f.size - 1},
355 }
356
357 for _, s := range tests {
358 n, err := file.Seek(s.offIn, s.whence)
359 if err != nil {
360 if f.isdir && err == syscall.EISDIR {
361 continue
362 }
363
364 t.Errorf("%v: %v", name, err)
365 }
366
367 if n != s.offOut {
368 t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut)
369 }
370 }
371 }
372
373 }
374 }
375
376 func TestGcsName(t *testing.T) {
377 createFiles(t)
378 defer removeFiles(t)
379
380 for _, f := range files {
381 if !f.exists {
382 continue
383 }
384
385 nameBase := filepath.Join(bucketName, f.name)
386
387 names := []string{
388 nameBase,
389 string(os.PathSeparator) + nameBase,
390 }
391 if f.name == "" {
392 names = []string{f.name}
393 }
394
395 for _, name := range names {
396 file, err := gcsAfs.Open(name)
397 if err != nil {
398 t.Fatalf("opening %v: %v", name, err)
399 }
400
401 n := file.Name()
402 if n != filepath.FromSlash(nameBase) {
403 t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(nameBase))
404 }
405 }
406
407 }
408 }
409
410 func TestGcsClose(t *testing.T) {
411 createFiles(t)
412 defer removeFiles(t)
413
414 for _, f := range files {
415 if !f.exists {
416 continue
417 }
418
419 nameBase := filepath.Join(bucketName, f.name)
420
421 names := []string{
422 nameBase,
423 string(os.PathSeparator) + nameBase,
424 }
425 if f.name == "" {
426 names = []string{f.name}
427 }
428
429 for _, name := range names {
430 file, err := gcsAfs.Open(name)
431 if err != nil {
432 t.Fatalf("opening %v: %v", name, err)
433 }
434
435 err = file.Close()
436 if err != nil {
437 t.Errorf("%v: %v", name, err)
438 }
439
440 err = file.Close()
441 if err == nil {
442 t.Errorf("%v: closing twice should return an error", name)
443 }
444
445 buf := make([]byte, 8)
446 n, err := file.Read(buf)
447 if n != 0 || err == nil {
448 t.Errorf("%v: could read from a closed file", name)
449 }
450
451 n, err = file.ReadAt(buf, 256)
452 if n != 0 || err == nil {
453 t.Errorf("%v: could readAt from a closed file", name)
454 }
455
456 off, err := file.Seek(0, io.SeekStart)
457 if off != 0 || err == nil {
458 t.Errorf("%v: could seek from a closed file", name)
459 }
460 }
461 }
462 }
463
464 func TestGcsOpenFile(t *testing.T) {
465 createFiles(t)
466 defer removeFiles(t)
467
468 for _, f := range files {
469 nameBase := filepath.Join(bucketName, f.name)
470
471 names := []string{
472 nameBase,
473 string(os.PathSeparator) + nameBase,
474 }
475 if f.name == "" {
476 names = []string{f.name}
477 }
478
479 for _, name := range names {
480 file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0o400)
481 if !f.exists {
482 if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
483 (f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
484 t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
485 }
486
487 continue
488 }
489
490 if err != nil {
491 t.Fatalf("%v: %v", name, err)
492 }
493
494 err = file.Close()
495 if err != nil {
496 t.Fatalf("failed to close a file \"%s\": %s", name, err)
497 }
498
499 _, err = gcsAfs.OpenFile(name, os.O_CREATE, 0o600)
500 if !errors.Is(err, syscall.EPERM) {
501 t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM)
502 }
503 }
504 }
505 }
506
507 func TestGcsFsStat(t *testing.T) {
508 createFiles(t)
509 defer removeFiles(t)
510
511 for _, f := range files {
512 nameBase := filepath.Join(bucketName, f.name)
513
514 names := []string{
515 nameBase,
516 string(os.PathSeparator) + nameBase,
517 }
518 if f.name == "" {
519 names = []string{f.name}
520 }
521
522 for _, name := range names {
523 fi, err := gcsAfs.Stat(name)
524 if !f.exists {
525 if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
526 (f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
527 t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
528 }
529
530 continue
531 }
532
533 if err != nil {
534 t.Fatalf("stat %v: got error '%v'", name, err)
535 }
536
537 if isdir := fi.IsDir(); isdir != f.isdir {
538 t.Errorf("%v directory, got: %v, expected: %v", name, isdir, f.isdir)
539 }
540
541 if size := fi.Size(); size != f.size {
542 t.Errorf("%v size, got: %v, expected: %v", name, size, f.size)
543 }
544 }
545 }
546 }
547
548 func TestGcsReaddir(t *testing.T) {
549 createFiles(t)
550 defer removeFiles(t)
551
552 for _, d := range dirs {
553 nameBase := filepath.Join(bucketName, d.name)
554
555 names := []string{
556 nameBase,
557 string(os.PathSeparator) + nameBase,
558 }
559
560 for _, name := range names {
561 dir, err := gcsAfs.Open(name)
562 if err != nil {
563 t.Fatal(err)
564 }
565
566 fi, err := dir.Readdir(0)
567 if err != nil {
568 t.Fatal(err)
569 }
570 var fileNames []string
571 for _, f := range fi {
572 fileNames = append(fileNames, f.Name())
573 }
574
575 if !reflect.DeepEqual(fileNames, d.children) {
576 t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
577 }
578
579 fi, err = dir.Readdir(1)
580 if err != nil {
581 t.Fatal(err)
582 }
583
584 fileNames = []string{}
585 for _, f := range fi {
586 fileNames = append(fileNames, f.Name())
587 }
588
589 if !reflect.DeepEqual(fileNames, d.children[0:1]) {
590 t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
591 }
592 }
593 }
594
595 nameBase := filepath.Join(bucketName, "testFile")
596
597 names := []string{
598 nameBase,
599 string(os.PathSeparator) + nameBase,
600 }
601
602 for _, name := range names {
603 dir, err := gcsAfs.Open(name)
604 if err != nil {
605 t.Fatal(err)
606 }
607
608 _, err = dir.Readdir(-1)
609 if err != syscall.ENOTDIR {
610 t.Fatal("Expected error")
611 }
612 }
613 }
614
615 func TestGcsReaddirnames(t *testing.T) {
616 createFiles(t)
617 defer removeFiles(t)
618
619 for _, d := range dirs {
620 nameBase := filepath.Join(bucketName, d.name)
621
622 names := []string{
623 nameBase,
624 string(os.PathSeparator) + nameBase,
625 }
626
627 for _, name := range names {
628 dir, err := gcsAfs.Open(name)
629 if err != nil {
630 t.Fatal(err)
631 }
632
633 fileNames, err := dir.Readdirnames(0)
634 if err != nil {
635 t.Fatal(err)
636 }
637
638 if !reflect.DeepEqual(fileNames, d.children) {
639 t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
640 }
641
642 fileNames, err = dir.Readdirnames(1)
643 if err != nil {
644 t.Fatal(err)
645 }
646
647 if !reflect.DeepEqual(fileNames, d.children[0:1]) {
648 t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
649 }
650 }
651 }
652
653 nameBase := filepath.Join(bucketName, "testFile")
654
655 names := []string{
656 nameBase,
657 string(os.PathSeparator) + nameBase,
658 }
659
660 for _, name := range names {
661 dir, err := gcsAfs.Open(name)
662 if err != nil {
663 t.Fatal(err)
664 }
665
666 _, err = dir.Readdirnames(-1)
667 if err != syscall.ENOTDIR {
668 t.Fatal("Expected error")
669 }
670 }
671 }
672
673 func TestGcsGlob(t *testing.T) {
674 createFiles(t)
675 defer removeFiles(t)
676
677 for _, s := range []struct {
678 glob string
679 entries []string
680 }{
681 {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
682 {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
683 {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
684 {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
685 } {
686 nameBase := filepath.Join(bucketName, s.glob)
687
688 prefixedGlobs := []string{
689 nameBase,
690 string(os.PathSeparator) + nameBase,
691 }
692
693 prefixedEntries := [][]string{{}, {}}
694 for _, entry := range s.entries {
695 prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry))
696 prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry))
697 }
698
699 for i, prefixedGlob := range prefixedGlobs {
700 entries, err := afero.Glob(gcsAfs.Fs, prefixedGlob)
701 if err != nil {
702 t.Error(err)
703 }
704 if reflect.DeepEqual(entries, prefixedEntries[i]) {
705 t.Logf("glob: %s: glob ok", prefixedGlob)
706 } else {
707 t.Errorf("glob: %s: got %#v, expected %#v", prefixedGlob, entries, prefixedEntries)
708 }
709 }
710 }
711 }
712
713 func TestGcsMkdir(t *testing.T) {
714 t.Run("empty", func(t *testing.T) {
715 emptyDirName := bucketName
716
717 err := gcsAfs.Mkdir(emptyDirName, 0o755)
718 if err == nil {
719 t.Fatal("did not fail upon creation of an empty folder")
720 }
721 })
722 t.Run("success", func(t *testing.T) {
723 dirName := filepath.Join(bucketName, "a-test-dir")
724 var err error
725
726 err = gcsAfs.Mkdir(dirName, 0o755)
727 if err != nil {
728 t.Fatal("failed to create a folder with error", err)
729 }
730
731 info, err := gcsAfs.Stat(dirName)
732 if err != nil {
733 t.Fatal("failed to get info", err)
734 }
735 if !info.IsDir() {
736 t.Fatalf("%s: not a dir", dirName)
737 }
738 if !info.Mode().IsDir() {
739 t.Errorf("%s: mode is not directory", dirName)
740 }
741
742 if info.Mode() != os.ModeDir|0o755 {
743 t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
744 }
745
746 err = gcsAfs.Remove(dirName)
747 if err != nil {
748 t.Fatalf("could not delete the folder %s after the test with error: %s", dirName, err)
749 }
750 })
751 }
752
753 func TestGcsMkdirAll(t *testing.T) {
754 t.Run("empty", func(t *testing.T) {
755 emptyDirName := bucketName
756
757 err := gcsAfs.MkdirAll(emptyDirName, 0o755)
758 if err == nil {
759 t.Fatal("did not fail upon creation of an empty folder")
760 }
761 })
762 t.Run("success", func(t *testing.T) {
763 dirName := filepath.Join(bucketName, "a/b/c")
764
765 err := gcsAfs.MkdirAll(dirName, 0o755)
766 if err != nil {
767 t.Fatal(err)
768 }
769
770 info, err := gcsAfs.Stat(filepath.Join(bucketName, "a"))
771 if err != nil {
772 t.Fatal(err)
773 }
774 if !info.Mode().IsDir() {
775 t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a"))
776 }
777 if info.Mode() != os.ModeDir|0o755 {
778 t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode())
779 }
780 info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b"))
781 if err != nil {
782 t.Fatal(err)
783 }
784 if !info.Mode().IsDir() {
785 t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b"))
786 }
787 if info.Mode() != os.ModeDir|0o755 {
788 t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode())
789 }
790 info, err = gcsAfs.Stat(dirName)
791 if err != nil {
792 t.Fatal(err)
793 }
794 if !info.Mode().IsDir() {
795 t.Errorf("%s: mode is not directory", dirName)
796 }
797 if info.Mode() != os.ModeDir|0o755 {
798 t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
799 }
800
801 err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a"))
802 if err != nil {
803 t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err)
804 }
805 })
806 }
807
808 func TestGcsRemoveAll(t *testing.T) {
809 t.Run("non-existent", func(t *testing.T) {
810 err := gcsAfs.RemoveAll(filepath.Join(bucketName, "a"))
811 if err != nil {
812 t.Fatal("error should be nil when removing non-existent file")
813 }
814 })
815 t.Run("success", func(t *testing.T) {
816 aDir := filepath.Join(bucketName, "a")
817 bDir := filepath.Join(aDir, "b")
818
819 err := gcsAfs.MkdirAll(bDir, 0o755)
820 if err != nil {
821 t.Fatal(err)
822 }
823 _, err = gcsAfs.Stat(bDir)
824 if err != nil {
825 t.Fatal(err)
826 }
827
828 err = gcsAfs.RemoveAll(aDir)
829 if err != nil {
830 t.Fatalf("failed to remove the folder %s with error: %s", aDir, err)
831 }
832
833 _, err = gcsAfs.Stat(aDir)
834 if err == nil {
835 t.Fatalf("folder %s wasn't removed", aDir)
836 }
837 })
838 }
839
View as plain text