DefaultSignedURLExpiry is the default duration for SignedURLOptions.Expiry.
const (
DefaultSignedURLExpiry = 1 * time.Hour
DefaultSignedURLMethod = "GET"
)
const (
StatOpenTotal = "open.total"
StatOpenErrors = "open.errors"
StatAttrsTotal = "attrs.total"
StatAttrsErrors = "attrs.errors"
StatCreateTotal = "create.total"
StatCreateErrors = "create.errors"
StatDeleteTotal = "delete.total"
StatDeleteErrors = "delete.errors"
StatURLTotal = "url.total"
StatURLErrors = "url.errors"
)
DefaultLocalCreatePathMode is the default os.FileMode used when creating directories during a localFS.Create call.
const DefaultLocalCreatePathMode = os.FileMode(0o755)
var ErrNotImplemented = errors.New("not implemented")
LocalCreatePathMode is the os.FileMode used when creating directories via localFS.Create
var LocalCreatePathMode = DefaultLocalCreatePathMode
func Exists(ctx context.Context, fs FS, path string) bool
func IsNotExist(err error) bool
IsNotExist returns a boolean indicating whether the error is known to report that a path does not exist.
func List(ctx context.Context, w Walker, path string) ([]string, error)
List runs the Walker on the given path and returns the list of visited paths.
func Read(ctx context.Context, fs FS, path string, options *ReaderOptions) ([]byte, error)
func WalkN(ctx context.Context, w Walker, path string, n int, fn WalkFn) error
WalkN creates n workers which accept paths from the Walker. If a WalkFn returns non-nil error we wait for other running WalkFns to finish before returning.
func Write(ctx context.Context, fs FS, path string, data []byte, options *WriterOptions) error
Attributes represents the metadata of a File Inspired from gocloud.dev/blob.Attributes
type Attributes struct {
// ContentType is the MIME type of the blob object. It will not be empty.
ContentType string
// ContentEncoding specifies the encoding used for the blob's content, if any.
ContentEncoding string
// Metadata holds key/value pairs associated with the blob.
// Keys are guaranteed to be in lowercase, even if the backend provider
// has case-sensitive keys (although note that Metadata written via
// this package will always be lowercased). If there are duplicate
// case-insensitive keys (e.g., "foo" and "FOO"), only one value
// will be kept, and it is undefined which one.
Metadata map[string]string
// ModTime is the time the blob object was last modified.
ModTime time.Time
// CreationTime is the time the blob object was created.
CreationTime time.Time
// Size is the size of the object in bytes.
Size int64
}
type CacheOptions struct {
// MaxAge is the maximum time allowed since the underlying File's ModTime
// This means that if the cache is older than MaxAge, the Cache will fetch from the src again.
// If the expired File is still present on the src (i.e. not updated), it will be ignored.
MaxAge time.Duration
// DefaultExpired makes the cache treat a File as expired if its CreationTime cannot be checked.
// By default, it is false, which means the cache will treat zero-CreationTime files as valid.
// Only useful if MaxAge is set.
DefaultExpired bool
// NoData disables caching of the contents of the entries, it only stores the metadata.
NoData bool
}
FS is an interface which defines a virtual filesystem.
type FS interface {
Walker
// Open opens an existing file at path in the filesystem. Callers must close the
// File when done to release all underlying resources.
Open(ctx context.Context, path string, options *ReaderOptions) (*File, error)
// Attributes returns attributes about a path
Attributes(ctx context.Context, path string, options *ReaderOptions) (*Attributes, error)
// Create makes a new file at path in the filesystem. Callers must close the
// returned WriteCloser and check the error to be sure that the file
// was successfully written.
Create(ctx context.Context, path string, options *WriterOptions) (io.WriteCloser, error)
// Delete removes a path from the filesystem.
Delete(ctx context.Context, path string) error
// URL resolves a path to an addressable URL
URL(ctx context.Context, path string, options *SignedURLOptions) (string, error)
}
func NewCacheWrapper(src, cache FS, options *CacheOptions) FS
NewCacheWrapper creates an FS implementation which caches files opened from src into cache.
func NewCloudStorageFS(bucket string, credentials *google.Credentials) FS
NewCloudStorageFS creates a Google Cloud Storage FS credentials can be nil to use the default GOOGLE_APPLICATION_CREDENTIALS
func NewHashWrapper(h hash.Hash, fs FS, gs GetSetter) FS
NewHashWrapper creates a content addressable filesystem using hash.Hash to sum the content and store it using that name.
func NewLocalFS(path string) FS
func NewLoggerWrapper(fs FS, name string, l Logger) FS
NewLoggerWrapper creates a new FS which logs all calls to FS.
func NewMemoryFS() FS
NewMemoryFS creates a a basic in-memory implementation of FS.
func NewPrefixWrapper(fs FS, prefix string) FS
NewPrefixWrapper creates a FS which wraps fs and prefixes all paths with prefix.
func NewSlowWrapper(fs FS, readDelay time.Duration, writeDelay time.Duration) FS
NewSlowWrapper creates an artificially slow FS. Probably only useful for testing.
func NewStatsWrapper(fs FS, name string) FS
NewStatsWrapper creates an FS which records accesses for an FS. To retrieve the stats: stats := expvar.Get(name).(*expvar.Map) total := stats.Get("open.total").(*expvar.Int).Value() // int64
func NewTimeoutWrapper(fs FS, read time.Duration, write time.Duration) FS
NewTimeoutWrapper creates a FS which wraps fs and adds a timeout to most operations: read: Open, Attributes, URL write: Create, Delete
Note that the Open and Create methods are only for resolving the object, NOT actually reading or writing the contents. These operations should be fairly quick, on the same order as Attribute and Delete, respectively.
This depends on the underlying implementation to honour context's errors. It is at least supported on the CloudStorageFS.
Walk is not covered, since its duration is highly unpredictable.
File contains the metadata required to define a file (for reading).
type File struct {
io.ReadCloser // Underlying data.
Attributes
}
GetSetter implements a key-value store which is concurrency safe (can be used in multiple go-routines concurrently).
type GetSetter interface {
Get(key string) (string, error)
Set(key string, value string) error
Delete(key string) error
}
Logger can be a *log.Logger
type Logger interface {
Print(v ...interface{})
}
type MockFS struct {
mock.Mock
}
func NewMockFS() *MockFS
NewMockFS creates an FS where each method can be mocked. To be used in tests.
func (m *MockFS) Attributes(ctx context.Context, path string, options *ReaderOptions) (*Attributes, error)
func (m *MockFS) Create(ctx context.Context, path string, options *WriterOptions) (io.WriteCloser, error)
func (m *MockFS) Delete(ctx context.Context, path string) error
func (m *MockFS) Open(ctx context.Context, path string, options *ReaderOptions) (*File, error)
func (m *MockFS) URL(ctx context.Context, path string, options *SignedURLOptions) (string, error)
func (m *MockFS) Walk(ctx context.Context, path string, fn WalkFn) error
ReaderOptions are used to modify the behaviour of read operations. Inspired from gocloud.dev/blob.ReaderOptions It is provided for future extensibility.
type ReaderOptions struct {
// ReadCompressed controls whether the file must be uncompressed based on Content-Encoding.
// Only respected by Google Cloud Storage: https://cloud.google.com/storage/docs/transcoding
// Common pitfall: https://github.com/googleapis/google-cloud-go/issues/1743
ReadCompressed bool
}
type Scope int
const (
ScopeRead Scope = 1 << iota
ScopeWrite
ScopeDelete
ScopeSignURL
ScopeRW = ScopeRead | ScopeWrite
ScopeRWD = ScopeRW | ScopeDelete
)
func ResolveCloudStorageScope(scope Scope) Scope
func (s Scope) Has(s2 Scope) bool
func (s Scope) String() string
SignedURLOptions are used to modify the behaviour of write operations. Inspired from gocloud.dev/blob.SignedURLOptions Not all options are supported by all FS
type SignedURLOptions struct {
// Expiry sets how long the returned URL is valid for.
// Defaults to DefaultSignedURLExpiry.
Expiry time.Duration
// Method is the HTTP method that can be used on the URL; one of "GET", "PUT",
// or "DELETE". Defaults to "GET".
Method string
}
WalkFn is a function type which is passed to Walk.
type WalkFn func(path string) error
Walker is an interface which defines the Walk method.
type Walker interface {
// Walk traverses a path listing by prefix, calling fn with each path.
Walk(ctx context.Context, path string, fn WalkFn) error
}
WriterOptions are used to modify the behaviour of write operations. Inspired from gocloud.dev/blob.WriterOptions Not all options are supported by all FS
type WriterOptions struct {
Attributes Attributes
// BufferSize changes the default size in bytes of the chunks that
// Writer will upload in a single request; larger blobs will be split into
// multiple requests.
//
// This option may be ignored by some drivers.
//
// If 0, the driver will choose a reasonable default.
//
// If the Writer is used to do many small writes concurrently, using a
// smaller BufferSize may reduce memory usage.
BufferSize int
}
| Name | Synopsis |
|---|---|
| .. | |