fs.go

Documentation: github.com/spf13/afero/gcsfs

     1  // Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
     2  //
     3  // The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
     4  // licensed under Apache License 2.0.
     5  //
     6  // Licensed under the Apache License, Version 2.0 (the "License");
     7  // you may not use this file except in compliance with the License.
     8  // You may obtain a copy of the License at
     9  // http://www.apache.org/licenses/LICENSE-2.0
    10  //
    11  // Unless required by applicable law or agreed to in writing, software
    12  // distributed under the License is distributed on an "AS IS" BASIS,
    13  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  // See the License for the specific language governing permissions and
    15  // limitations under the License.
    16  
    17  package gcsfs
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"os"
    23  	"path/filepath"
    24  	"strings"
    25  	"syscall"
    26  	"time"
    27  
    28  	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
    29  )
    30  
    31  const (
    32  	defaultFileMode = 0o755
    33  	gsPrefix        = "gs://"
    34  )
    35  
    36  // Fs is a Fs implementation that uses functions provided by google cloud storage
    37  type Fs struct {
    38  	ctx       context.Context
    39  	client    stiface.Client
    40  	separator string
    41  
    42  	buckets       map[string]stiface.BucketHandle
    43  	rawGcsObjects map[string]*GcsFile
    44  
    45  	autoRemoveEmptyFolders bool // trigger for creating "virtual folders" (not required by GCSs)
    46  }
    47  
    48  func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
    49  	return NewGcsFsWithSeparator(ctx, client, "/")
    50  }
    51  
    52  func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
    53  	return &Fs{
    54  		ctx:           ctx,
    55  		client:        client,
    56  		separator:     folderSep,
    57  		rawGcsObjects: make(map[string]*GcsFile),
    58  
    59  		autoRemoveEmptyFolders: true,
    60  	}
    61  }
    62  
    63  // normSeparators will normalize all "\\" and "/" to the provided separator
    64  func (fs *Fs) normSeparators(s string) string {
    65  	return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
    66  }
    67  
    68  func (fs *Fs) ensureTrailingSeparator(s string) string {
    69  	if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
    70  		return s + fs.separator
    71  	}
    72  	return s
    73  }
    74  
    75  func (fs *Fs) ensureNoLeadingSeparator(s string) string {
    76  	if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
    77  		s = s[len(fs.separator):]
    78  	}
    79  
    80  	return s
    81  }
    82  
    83  func ensureNoPrefix(s string) string {
    84  	if len(s) > 0 && strings.HasPrefix(s, gsPrefix) {
    85  		return s[len(gsPrefix):]
    86  	}
    87  	return s
    88  }
    89  
    90  func validateName(s string) error {
    91  	if len(s) == 0 {
    92  		return ErrNoBucketInName
    93  	}
    94  	return nil
    95  }
    96  
    97  // Splits provided name into bucket name and path
    98  func (fs *Fs) splitName(name string) (bucketName string, path string) {
    99  	splitName := strings.Split(name, fs.separator)
   100  
   101  	return splitName[0], strings.Join(splitName[1:], fs.separator)
   102  }
   103  
   104  func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
   105  	bucket := fs.buckets[name]
   106  	if bucket == nil {
   107  		bucket = fs.client.Bucket(name)
   108  		_, err := bucket.Attrs(fs.ctx)
   109  		if err != nil {
   110  			return nil, err
   111  		}
   112  	}
   113  	return bucket, nil
   114  }
   115  
   116  func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
   117  	bucketName, path := fs.splitName(name)
   118  
   119  	bucket, err := fs.getBucket(bucketName)
   120  	if err != nil {
   121  		return nil, err
   122  	}
   123  
   124  	return bucket.Object(path), nil
   125  }
   126  
   127  func (fs *Fs) Name() string { return "GcsFs" }
   128  
   129  func (fs *Fs) Create(name string) (*GcsFile, error) {
   130  	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
   131  	if err := validateName(name); err != nil {
   132  		return nil, err
   133  	}
   134  
   135  	if !fs.autoRemoveEmptyFolders {
   136  		baseDir := filepath.Base(name)
   137  		if stat, err := fs.Stat(baseDir); err != nil || !stat.IsDir() {
   138  			err = fs.MkdirAll(baseDir, 0)
   139  			if err != nil {
   140  				return nil, err
   141  			}
   142  		}
   143  	}
   144  
   145  	obj, err := fs.getObj(name)
   146  	if err != nil {
   147  		return nil, err
   148  	}
   149  	w := obj.NewWriter(fs.ctx)
   150  	err = w.Close()
   151  	if err != nil {
   152  		return nil, err
   153  	}
   154  	file := NewGcsFile(fs.ctx, fs, obj, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0, name)
   155  
   156  	fs.rawGcsObjects[name] = file
   157  	return file, nil
   158  }
   159  
   160  func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
   161  	name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
   162  	if err := validateName(name); err != nil {
   163  		return err
   164  	}
   165  	// folder creation logic has to additionally check for folder name presence
   166  	bucketName, path := fs.splitName(name)
   167  	if bucketName == "" {
   168  		return ErrNoBucketInName
   169  	}
   170  	if path == "" {
   171  		// the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent
   172  		return ErrEmptyObjectName
   173  	}
   174  
   175  	obj, err := fs.getObj(name)
   176  	if err != nil {
   177  		return err
   178  	}
   179  	w := obj.NewWriter(fs.ctx)
   180  	return w.Close()
   181  }
   182  
   183  func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
   184  	path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
   185  	if err := validateName(path); err != nil {
   186  		return err
   187  	}
   188  	// folder creation logic has to additionally check for folder name presence
   189  	bucketName, splitPath := fs.splitName(path)
   190  	if bucketName == "" {
   191  		return ErrNoBucketInName
   192  	}
   193  	if splitPath == "" {
   194  		// the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent
   195  		return ErrEmptyObjectName
   196  	}
   197  
   198  	root := ""
   199  	folders := strings.Split(path, fs.separator)
   200  	for i, f := range folders {
   201  		if f == "" && i != 0 {
   202  			continue // it's the last item - it should be empty
   203  		}
   204  		// Don't force a delimiter prefix
   205  		if root != "" {
   206  			root = root + fs.separator + f
   207  		} else {
   208  			// we have to have at least bucket name + folder name to create successfully
   209  			root = f
   210  			continue
   211  		}
   212  
   213  		if err := fs.Mkdir(root, perm); err != nil {
   214  			return err
   215  		}
   216  	}
   217  	return nil
   218  }
   219  
   220  func (fs *Fs) Open(name string) (*GcsFile, error) {
   221  	return fs.OpenFile(name, os.O_RDONLY, 0)
   222  }
   223  
   224  func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
   225  	var file *GcsFile
   226  	var err error
   227  
   228  	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
   229  	if err = validateName(name); err != nil {
   230  		return nil, err
   231  	}
   232  
   233  	f, found := fs.rawGcsObjects[name]
   234  	if found {
   235  		file = NewGcsFileFromOldFH(flag, fileMode, f.resource)
   236  	} else {
   237  		var obj stiface.ObjectHandle
   238  		obj, err = fs.getObj(name)
   239  		if err != nil {
   240  			return nil, err
   241  		}
   242  		file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name)
   243  	}
   244  
   245  	if flag == os.O_RDONLY {
   246  		_, err = file.Stat()
   247  		if err != nil {
   248  			return nil, err
   249  		}
   250  	}
   251  
   252  	if flag&os.O_TRUNC != 0 {
   253  		err = file.resource.obj.Delete(fs.ctx)
   254  		if err != nil {
   255  			return nil, err
   256  		}
   257  		return fs.Create(name)
   258  	}
   259  
   260  	if flag&os.O_APPEND != 0 {
   261  		_, err = file.Seek(0, 2)
   262  		if err != nil {
   263  			return nil, err
   264  		}
   265  	}
   266  
   267  	if flag&os.O_CREATE != 0 {
   268  		_, err = file.Stat()
   269  		if err == nil { // the file actually exists
   270  			return nil, syscall.EPERM
   271  		}
   272  
   273  		_, err = file.WriteString("")
   274  		if err != nil {
   275  			return nil, err
   276  		}
   277  	}
   278  	return file, nil
   279  }
   280  
   281  func (fs *Fs) Remove(name string) error {
   282  	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
   283  	if err := validateName(name); err != nil {
   284  		return err
   285  	}
   286  
   287  	obj, err := fs.getObj(name)
   288  	if err != nil {
   289  		return err
   290  	}
   291  	info, err := fs.Stat(name)
   292  	if err != nil {
   293  		return err
   294  	}
   295  	delete(fs.rawGcsObjects, name)
   296  
   297  	if info.IsDir() {
   298  		// it's a folder, we ha to check its contents - it cannot be removed, if not empty
   299  		var dir *GcsFile
   300  		dir, err = fs.Open(name)
   301  		if err != nil {
   302  			return err
   303  		}
   304  		var infos []os.FileInfo
   305  		infos, err = dir.Readdir(0)
   306  		if err != nil {
   307  			return err
   308  		}
   309  		if len(infos) > 0 {
   310  			return syscall.ENOTEMPTY
   311  		}
   312  
   313  		// it's an empty folder, we can continue
   314  		name = fs.ensureTrailingSeparator(name)
   315  		obj, err = fs.getObj(name)
   316  		if err != nil {
   317  			return err
   318  		}
   319  
   320  		return obj.Delete(fs.ctx)
   321  	}
   322  	return obj.Delete(fs.ctx)
   323  }
   324  
   325  func (fs *Fs) RemoveAll(path string) error {
   326  	path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
   327  	if err := validateName(path); err != nil {
   328  		return err
   329  	}
   330  
   331  	pathInfo, err := fs.Stat(path)
   332  	if errors.Is(err, ErrFileNotFound) {
   333  		// return early if file doesn't exist
   334  		return nil
   335  	}
   336  	if err != nil {
   337  		return err
   338  	}
   339  
   340  	if !pathInfo.IsDir() {
   341  		return fs.Remove(path)
   342  	}
   343  
   344  	var dir *GcsFile
   345  	dir, err = fs.Open(path)
   346  	if err != nil {
   347  		return err
   348  	}
   349  
   350  	var infos []os.FileInfo
   351  	infos, err = dir.Readdir(0)
   352  	if err != nil {
   353  		return err
   354  	}
   355  	for _, info := range infos {
   356  		nameToRemove := fs.normSeparators(info.Name())
   357  		err = fs.RemoveAll(path + fs.separator + nameToRemove)
   358  		if err != nil {
   359  			return err
   360  		}
   361  	}
   362  
   363  	return fs.Remove(path)
   364  }
   365  
   366  func (fs *Fs) Rename(oldName, newName string) error {
   367  	oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
   368  	if err := validateName(oldName); err != nil {
   369  		return err
   370  	}
   371  
   372  	newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName)))
   373  	if err := validateName(newName); err != nil {
   374  		return err
   375  	}
   376  
   377  	src, err := fs.getObj(oldName)
   378  	if err != nil {
   379  		return err
   380  	}
   381  	dst, err := fs.getObj(newName)
   382  	if err != nil {
   383  		return err
   384  	}
   385  
   386  	if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil {
   387  		return err
   388  	}
   389  	delete(fs.rawGcsObjects, oldName)
   390  	return src.Delete(fs.ctx)
   391  }
   392  
   393  func (fs *Fs) Stat(name string) (os.FileInfo, error) {
   394  	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
   395  	if err := validateName(name); err != nil {
   396  		return nil, err
   397  	}
   398  
   399  	return newFileInfo(name, fs, defaultFileMode)
   400  }
   401  
   402  func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
   403  	return errors.New("method Chmod is not implemented in GCS")
   404  }
   405  
   406  func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
   407  	return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
   408  }
   409  
   410  func (fs *Fs) Chown(_ string, _, _ int) error {
   411  	return errors.New("method Chown is not implemented for GCS")
   412  }
   413
View as plain text