file.go

Documentation: oras.land/oras-go/pkg/content

     1  /*
     2  Copyright The ORAS Authors.
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6  
     7  http://www.apache.org/licenses/LICENSE-2.0
     8  
     9  Unless required by applicable law or agreed to in writing, software
    10  distributed under the License is distributed on an "AS IS" BASIS,
    11  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  See the License for the specific language governing permissions and
    13  limitations under the License.
    14  */
    15  
    16  package content
    17  
    18  import (
    19  	"bytes"
    20  	"compress/gzip"
    21  	"context"
    22  	_ "crypto/sha256"
    23  	"fmt"
    24  	"io"
    25  	"io/ioutil"
    26  	"os"
    27  	"path/filepath"
    28  	"strings"
    29  	"sync"
    30  	"time"
    31  
    32  	"github.com/containerd/containerd/content"
    33  	"github.com/containerd/containerd/errdefs"
    34  	"github.com/containerd/containerd/remotes"
    35  	digest "github.com/opencontainers/go-digest"
    36  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    37  	"github.com/pkg/errors"
    38  )
    39  
    40  // File provides content via files from the file system
    41  type File struct {
    42  	DisableOverwrite          bool
    43  	AllowPathTraversalOnWrite bool
    44  
    45  	// Reproducible enables stripping times from added files
    46  	Reproducible bool
    47  
    48  	root         string
    49  	descriptor   *sync.Map // map[digest.Digest]ocispec.Descriptor
    50  	pathMap      *sync.Map // map[name string](file string)
    51  	memoryMap    *sync.Map // map[digest.Digest]([]byte)
    52  	refMap       *sync.Map // map[string]ocispec.Descriptor
    53  	tmpFiles     *sync.Map
    54  	ignoreNoName bool
    55  }
    56  
    57  // NewFile creats a new file target. It represents a single root reference and all of its components.
    58  func NewFile(rootPath string, opts ...WriterOpt) *File {
    59  	// we have to process the opts to find if they told us to change defaults
    60  	wOpts := DefaultWriterOpts()
    61  	for _, opt := range opts {
    62  		if err := opt(&wOpts); err != nil {
    63  			continue
    64  		}
    65  	}
    66  	return &File{
    67  		root:         rootPath,
    68  		descriptor:   &sync.Map{},
    69  		pathMap:      &sync.Map{},
    70  		memoryMap:    &sync.Map{},
    71  		refMap:       &sync.Map{},
    72  		tmpFiles:     &sync.Map{},
    73  		ignoreNoName: wOpts.IgnoreNoName,
    74  	}
    75  }
    76  
    77  func (s *File) Resolver() remotes.Resolver {
    78  	return s
    79  }
    80  
    81  func (s *File) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
    82  	desc, ok := s.getRef(ref)
    83  	if !ok {
    84  		return "", ocispec.Descriptor{}, fmt.Errorf("unknown reference: %s", ref)
    85  	}
    86  	return ref, desc, nil
    87  }
    88  
    89  func (s *File) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
    90  	if _, ok := s.refMap.Load(ref); !ok {
    91  		return nil, fmt.Errorf("unknown reference: %s", ref)
    92  	}
    93  	return s, nil
    94  }
    95  
    96  // Fetch get an io.ReadCloser for the specific content
    97  func (s *File) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
    98  	// first see if it is in the in-memory manifest map
    99  	manifest, ok := s.getMemory(desc)
   100  	if ok {
   101  		return ioutil.NopCloser(bytes.NewReader(manifest)), nil
   102  	}
   103  	desc, ok = s.get(desc)
   104  	if !ok {
   105  		return nil, ErrNotFound
   106  	}
   107  	name, ok := ResolveName(desc)
   108  	if !ok {
   109  		return nil, ErrNoName
   110  	}
   111  	path := s.ResolvePath(name)
   112  	return os.Open(path)
   113  }
   114  
   115  func (s *File) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
   116  	var tag, hash string
   117  	parts := strings.SplitN(ref, "@", 2)
   118  	if len(parts) > 0 {
   119  		tag = parts[0]
   120  	}
   121  	if len(parts) > 1 {
   122  		hash = parts[1]
   123  	}
   124  	return &filePusher{
   125  		store: s,
   126  		ref:   tag,
   127  		hash:  hash,
   128  	}, nil
   129  }
   130  
   131  type filePusher struct {
   132  	store *File
   133  	ref   string
   134  	hash  string
   135  }
   136  
   137  func (s *filePusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) {
   138  	name, ok := ResolveName(desc)
   139  	now := time.Now()
   140  	if !ok {
   141  		// if we were not told to ignore NoName, then return an error
   142  		if !s.store.ignoreNoName {
   143  			return nil, ErrNoName
   144  		}
   145  
   146  		// just return a nil writer - we do not want to calculate the hash, so just use
   147  		// whatever was passed in the descriptor
   148  		return NewIoContentWriter(ioutil.Discard, WithOutputHash(desc.Digest)), nil
   149  	}
   150  	path, err := s.store.resolveWritePath(name)
   151  	if err != nil {
   152  		return nil, err
   153  	}
   154  	file, afterCommit, err := s.store.createWritePath(path, desc, name)
   155  	if err != nil {
   156  		return nil, err
   157  	}
   158  
   159  	return &fileWriter{
   160  		store:    s.store,
   161  		file:     file,
   162  		desc:     desc,
   163  		digester: digest.Canonical.Digester(),
   164  		status: content.Status{
   165  			Ref:       name,
   166  			Total:     desc.Size,
   167  			StartedAt: now,
   168  			UpdatedAt: now,
   169  		},
   170  		afterCommit: afterCommit,
   171  	}, nil
   172  }
   173  
   174  // Add adds a file reference from a path, either directory or single file,
   175  // and returns the reference descriptor.
   176  func (s *File) Add(name, mediaType, path string) (ocispec.Descriptor, error) {
   177  	if path == "" {
   178  		path = name
   179  	}
   180  	path = s.MapPath(name, path)
   181  
   182  	fileInfo, err := os.Stat(path)
   183  	if err != nil {
   184  		return ocispec.Descriptor{}, err
   185  	}
   186  
   187  	var desc ocispec.Descriptor
   188  	if fileInfo.IsDir() {
   189  		desc, err = s.descFromDir(name, mediaType, path)
   190  	} else {
   191  		desc, err = s.descFromFile(fileInfo, mediaType, path)
   192  	}
   193  	if err != nil {
   194  		return ocispec.Descriptor{}, err
   195  	}
   196  	if desc.Annotations == nil {
   197  		desc.Annotations = make(map[string]string)
   198  	}
   199  	desc.Annotations[ocispec.AnnotationTitle] = name
   200  
   201  	s.set(desc)
   202  	return desc, nil
   203  }
   204  
   205  // Load is a lower-level memory-only version of Add. Rather than taking a path,
   206  // generating a descriptor and creating a reference, it takes raw data and a descriptor
   207  // that describes that data and stores it in memory. It will disappear at process
   208  // termination.
   209  //
   210  // It is especially useful for adding ephemeral data, such as config, that must
   211  // exist in order to walk a manifest.
   212  func (s *File) Load(desc ocispec.Descriptor, data []byte) error {
   213  	s.memoryMap.Store(desc.Digest, data)
   214  	return nil
   215  }
   216  
   217  // Ref gets a reference's descriptor and content
   218  func (s *File) Ref(ref string) (ocispec.Descriptor, []byte, error) {
   219  	desc, ok := s.getRef(ref)
   220  	if !ok {
   221  		return ocispec.Descriptor{}, nil, ErrNotFound
   222  	}
   223  	// first see if it is in the in-memory manifest map
   224  	manifest, ok := s.getMemory(desc)
   225  	if !ok {
   226  		return ocispec.Descriptor{}, nil, ErrNotFound
   227  	}
   228  	return desc, manifest, nil
   229  }
   230  
   231  func (s *File) descFromFile(info os.FileInfo, mediaType, path string) (ocispec.Descriptor, error) {
   232  	file, err := os.Open(path)
   233  	if err != nil {
   234  		return ocispec.Descriptor{}, err
   235  	}
   236  	defer file.Close()
   237  	digest, err := digest.FromReader(file)
   238  	if err != nil {
   239  		return ocispec.Descriptor{}, err
   240  	}
   241  
   242  	if mediaType == "" {
   243  		mediaType = DefaultBlobMediaType
   244  	}
   245  	return ocispec.Descriptor{
   246  		MediaType: mediaType,
   247  		Digest:    digest,
   248  		Size:      info.Size(),
   249  	}, nil
   250  }
   251  
   252  func (s *File) descFromDir(name, mediaType, root string) (ocispec.Descriptor, error) {
   253  	// generate temp file
   254  	file, err := s.tempFile()
   255  	if err != nil {
   256  		return ocispec.Descriptor{}, err
   257  	}
   258  	defer file.Close()
   259  	s.MapPath(name, file.Name())
   260  
   261  	// compress directory
   262  	digester := digest.Canonical.Digester()
   263  	zw := gzip.NewWriter(io.MultiWriter(file, digester.Hash()))
   264  	defer zw.Close()
   265  	tarDigester := digest.Canonical.Digester()
   266  	if err := tarDirectory(root, name, io.MultiWriter(zw, tarDigester.Hash()), s.Reproducible); err != nil {
   267  		return ocispec.Descriptor{}, err
   268  	}
   269  
   270  	// flush all
   271  	if err := zw.Close(); err != nil {
   272  		return ocispec.Descriptor{}, err
   273  	}
   274  	if err := file.Sync(); err != nil {
   275  		return ocispec.Descriptor{}, err
   276  	}
   277  
   278  	// generate descriptor
   279  	if mediaType == "" {
   280  		mediaType = DefaultBlobDirMediaType
   281  	}
   282  	info, err := file.Stat()
   283  	if err != nil {
   284  		return ocispec.Descriptor{}, err
   285  	}
   286  	return ocispec.Descriptor{
   287  		MediaType: mediaType,
   288  		Digest:    digester.Digest(),
   289  		Size:      info.Size(),
   290  		Annotations: map[string]string{
   291  			AnnotationDigest: tarDigester.Digest().String(),
   292  			AnnotationUnpack: "true",
   293  		},
   294  	}, nil
   295  }
   296  
   297  func (s *File) tempFile() (*os.File, error) {
   298  	file, err := ioutil.TempFile("", TempFilePattern)
   299  	if err != nil {
   300  		return nil, err
   301  	}
   302  	s.tmpFiles.Store(file.Name(), file)
   303  	return file, nil
   304  }
   305  
   306  // Close frees up resources used by the file store
   307  func (s *File) Close() error {
   308  	var errs []string
   309  	s.tmpFiles.Range(func(name, _ interface{}) bool {
   310  		if err := os.Remove(name.(string)); err != nil {
   311  			errs = append(errs, err.Error())
   312  		}
   313  		return true
   314  	})
   315  	if len(errs) > 0 {
   316  		return errors.New(strings.Join(errs, "; "))
   317  	}
   318  	return nil
   319  }
   320  
   321  func (s *File) resolveWritePath(name string) (string, error) {
   322  	path := s.ResolvePath(name)
   323  	if !s.AllowPathTraversalOnWrite {
   324  		base, err := filepath.Abs(s.root)
   325  		if err != nil {
   326  			return "", err
   327  		}
   328  		target, err := filepath.Abs(path)
   329  		if err != nil {
   330  			return "", err
   331  		}
   332  		rel, err := filepath.Rel(base, target)
   333  		if err != nil {
   334  			return "", ErrPathTraversalDisallowed
   335  		}
   336  		rel = filepath.ToSlash(rel)
   337  		if strings.HasPrefix(rel, "../") || rel == ".." {
   338  			return "", ErrPathTraversalDisallowed
   339  		}
   340  	}
   341  	if s.DisableOverwrite {
   342  		if _, err := os.Stat(path); err == nil {
   343  			return "", ErrOverwriteDisallowed
   344  		} else if !os.IsNotExist(err) {
   345  			return "", err
   346  		}
   347  	}
   348  	return path, nil
   349  }
   350  
   351  func (s *File) createWritePath(path string, desc ocispec.Descriptor, prefix string) (*os.File, func() error, error) {
   352  	if value, ok := desc.Annotations[AnnotationUnpack]; !ok || value != "true" {
   353  		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
   354  			return nil, nil, err
   355  		}
   356  		file, err := os.Create(path)
   357  		return file, nil, err
   358  	}
   359  
   360  	if err := os.MkdirAll(path, 0755); err != nil {
   361  		return nil, nil, err
   362  	}
   363  	file, err := s.tempFile()
   364  	checksum := desc.Annotations[AnnotationDigest]
   365  	afterCommit := func() error {
   366  		return extractTarGzip(path, prefix, file.Name(), checksum)
   367  	}
   368  	return file, afterCommit, err
   369  }
   370  
   371  // MapPath maps name to path
   372  func (s *File) MapPath(name, path string) string {
   373  	path = s.resolvePath(path)
   374  	s.pathMap.Store(name, path)
   375  	return path
   376  }
   377  
   378  // ResolvePath returns the path by name
   379  func (s *File) ResolvePath(name string) string {
   380  	if value, ok := s.pathMap.Load(name); ok {
   381  		if path, ok := value.(string); ok {
   382  			return path
   383  		}
   384  	}
   385  
   386  	// using the name as a fallback solution
   387  	return s.resolvePath(name)
   388  }
   389  
   390  func (s *File) resolvePath(path string) string {
   391  	if filepath.IsAbs(path) {
   392  		return path
   393  	}
   394  	return filepath.Join(s.root, path)
   395  }
   396  
   397  func (s *File) set(desc ocispec.Descriptor) {
   398  	s.descriptor.Store(desc.Digest, desc)
   399  }
   400  
   401  func (s *File) get(desc ocispec.Descriptor) (ocispec.Descriptor, bool) {
   402  	value, ok := s.descriptor.Load(desc.Digest)
   403  	if !ok {
   404  		return ocispec.Descriptor{}, false
   405  	}
   406  	desc, ok = value.(ocispec.Descriptor)
   407  	return desc, ok
   408  }
   409  
   410  func (s *File) getMemory(desc ocispec.Descriptor) ([]byte, bool) {
   411  	value, ok := s.memoryMap.Load(desc.Digest)
   412  	if !ok {
   413  		return nil, false
   414  	}
   415  	content, ok := value.([]byte)
   416  	return content, ok
   417  }
   418  
   419  func (s *File) getRef(ref string) (ocispec.Descriptor, bool) {
   420  	value, ok := s.refMap.Load(ref)
   421  	if !ok {
   422  		return ocispec.Descriptor{}, false
   423  	}
   424  	desc, ok := value.(ocispec.Descriptor)
   425  	return desc, ok
   426  }
   427  
   428  // StoreManifest stores a manifest linked to by the provided ref. The children of the
   429  // manifest, such as layers and config, should already exist in the file store, either
   430  // as files linked via Add(), or via Load(). If they do not exist, then a typical
   431  // Fetcher that walks the manifest will hit an unresolved hash.
   432  //
   433  // StoreManifest does *not* validate their presence.
   434  func (s *File) StoreManifest(ref string, desc ocispec.Descriptor, manifest []byte) error {
   435  	s.refMap.Store(ref, desc)
   436  	s.memoryMap.Store(desc.Digest, manifest)
   437  	return nil
   438  }
   439  
   440  type fileWriter struct {
   441  	store       *File
   442  	file        *os.File
   443  	desc        ocispec.Descriptor
   444  	digester    digest.Digester
   445  	status      content.Status
   446  	afterCommit func() error
   447  }
   448  
   449  func (w *fileWriter) Status() (content.Status, error) {
   450  	return w.status, nil
   451  }
   452  
   453  // Digest returns the current digest of the content, up to the current write.
   454  //
   455  // Cannot be called concurrently with `Write`.
   456  func (w *fileWriter) Digest() digest.Digest {
   457  	return w.digester.Digest()
   458  }
   459  
   460  // Write p to the transaction.
   461  func (w *fileWriter) Write(p []byte) (n int, err error) {
   462  	n, err = w.file.Write(p)
   463  	w.digester.Hash().Write(p[:n])
   464  	w.status.Offset += int64(len(p))
   465  	w.status.UpdatedAt = time.Now()
   466  	return n, err
   467  }
   468  
   469  func (w *fileWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
   470  	var base content.Info
   471  	for _, opt := range opts {
   472  		if err := opt(&base); err != nil {
   473  			return err
   474  		}
   475  	}
   476  
   477  	if w.file == nil {
   478  		return errors.Wrap(errdefs.ErrFailedPrecondition, "cannot commit on closed writer")
   479  	}
   480  	file := w.file
   481  	w.file = nil
   482  
   483  	if err := file.Sync(); err != nil {
   484  		file.Close()
   485  		return errors.Wrap(err, "sync failed")
   486  	}
   487  
   488  	fileInfo, err := file.Stat()
   489  	if err != nil {
   490  		file.Close()
   491  		return errors.Wrap(err, "stat failed")
   492  	}
   493  	if err := file.Close(); err != nil {
   494  		return errors.Wrap(err, "failed to close file")
   495  	}
   496  
   497  	if size > 0 && size != fileInfo.Size() {
   498  		return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit size %d, expected %d", fileInfo.Size(), size)
   499  	}
   500  	if dgst := w.digester.Digest(); expected != "" && expected != dgst {
   501  		return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit digest %s, expected %s", dgst, expected)
   502  	}
   503  
   504  	w.store.set(w.desc)
   505  	if w.afterCommit != nil {
   506  		return w.afterCommit()
   507  	}
   508  	return nil
   509  }
   510  
   511  // Close the writer, flushing any unwritten data and leaving the progress in
   512  // tact.
   513  func (w *fileWriter) Close() error {
   514  	if w.file == nil {
   515  		return nil
   516  	}
   517  
   518  	w.file.Sync()
   519  	err := w.file.Close()
   520  	w.file = nil
   521  	return err
   522  }
   523  
   524  func (w *fileWriter) Truncate(size int64) error {
   525  	if size != 0 {
   526  		return ErrUnsupportedSize
   527  	}
   528  	w.status.Offset = 0
   529  	w.digester.Hash().Reset()
   530  	if _, err := w.file.Seek(0, io.SeekStart); err != nil {
   531  		return err
   532  	}
   533  	return w.file.Truncate(0)
   534  }
   535
View as plain text