Source file src/oras.land/oras-go/pkg/oras/oras_test.go

Documentation: oras.land/oras-go/pkg/oras

     1  /*
     2  Copyright The ORAS Authors.
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6  
     7  http://www.apache.org/licenses/LICENSE-2.0
     8  
     9  Unless required by applicable law or agreed to in writing, software
    10  distributed under the License is distributed on an "AS IS" BASIS,
    11  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  See the License for the specific language governing permissions and
    13  limitations under the License.
    14  */
    15  
    16  package oras
    17  
    18  import (
    19  	"archive/tar"
    20  	"bytes"
    21  	"compress/gzip"
    22  	"context"
    23  	_ "crypto/sha256"
    24  	"fmt"
    25  	"io"
    26  	"io/ioutil"
    27  	"os"
    28  	"path/filepath"
    29  	"testing"
    30  	"time"
    31  
    32  	orascontent "oras.land/oras-go/pkg/content"
    33  	"oras.land/oras-go/pkg/target"
    34  
    35  	"github.com/containerd/containerd/images"
    36  	"github.com/distribution/distribution/v3/configuration"
    37  	"github.com/distribution/distribution/v3/registry"
    38  	_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
    39  	digest "github.com/opencontainers/go-digest"
    40  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    41  	"github.com/phayes/freeport"
    42  
    43  	"github.com/stretchr/testify/suite"
    44  )
    45  
    46  var (
    47  	testTarball  = "../../testdata/charts/chartmuseum-1.8.2.tgz"
    48  	testDir      = "../../testdata/charts/chartmuseum"
    49  	testDirFiles = []string{
    50  		"Chart.yaml",
    51  		"values.yaml",
    52  		"README.md",
    53  		"templates/_helpers.tpl",
    54  		"templates/NOTES.txt",
    55  		"templates/service.yaml",
    56  		".helmignore",
    57  	}
    58  )
    59  
    60  type ORASTestSuite struct {
    61  	suite.Suite
    62  	DockerRegistryHost string
    63  }
    64  
    65  func newContext() context.Context {
    66  	return context.Background()
    67  }
    68  
    69  func newResolver() target.Target {
    70  	reg, _ := orascontent.NewRegistry(orascontent.RegistryOptions{})
    71  	return reg
    72  }
    73  
    74  // Start Docker registry
    75  func (suite *ORASTestSuite) SetupSuite() {
    76  	config := &configuration.Configuration{}
    77  	port, err := freeport.GetFreePort()
    78  	if err != nil {
    79  		suite.Nil(err, "no error finding free port for test registry")
    80  	}
    81  	suite.DockerRegistryHost = fmt.Sprintf("localhost:%d", port)
    82  	config.HTTP.Addr = fmt.Sprintf(":%d", port)
    83  	config.HTTP.DrainTimeout = time.Duration(10) * time.Second
    84  	config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
    85  	dockerRegistry, err := registry.NewRegistry(context.Background(), config)
    86  	suite.Nil(err, "no error finding free port for test registry")
    87  
    88  	go dockerRegistry.ListenAndServe()
    89  }
    90  
    91  // Push files to docker registry
    92  func (suite *ORASTestSuite) Test_0_Copy() {
    93  	var (
    94  		err         error
    95  		ref         string
    96  		desc        ocispec.Descriptor
    97  		descriptors []ocispec.Descriptor
    98  		store       *orascontent.File
    99  		memStore    *orascontent.Memory
   100  	)
   101  
   102  	_, err = Copy(newContext(), nil, ref, nil, ref)
   103  	suite.NotNil(err, "error pushing with empty resolver")
   104  
   105  	_, err = Copy(newContext(), orascontent.NewMemory(), ref, newResolver(), "")
   106  	suite.NotNil(err, "error pushing when ref missing hostname")
   107  
   108  	ref = fmt.Sprintf("%s/empty:test", suite.DockerRegistryHost)
   109  
   110  	memStore = orascontent.NewMemory()
   111  	config, configDesc, err := orascontent.GenerateConfig(nil)
   112  	suite.Nil(err, "no error generating config")
   113  	memStore.Set(configDesc, config)
   114  	emptyManifest, emptyManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil)
   115  	suite.Nil(err, "no error creating manifest with empty descriptors")
   116  	err = memStore.StoreManifest(ref, emptyManifestDesc, emptyManifest)
   117  	suite.Nil(err, "no error pushing manifest with empty descriptors")
   118  	_, err = Copy(newContext(), memStore, ref, newResolver(), "")
   119  	suite.Nil(err, "no error pushing with empty descriptors")
   120  
   121  	// Load descriptors with test chart tgz (as single layer)
   122  	ref = fmt.Sprintf("%s/chart-tgz:test", suite.DockerRegistryHost)
   123  
   124  	store = orascontent.NewFile("")
   125  	err = store.Load(configDesc, config)
   126  	suite.Nil(err, "no error loading config for test chart")
   127  	basename := filepath.Base(testTarball)
   128  	desc, err = store.Add(basename, "", testTarball)
   129  	suite.Nil(err, "no error loading test chart")
   130  	testChartManifest, testChartManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil, desc)
   131  	suite.Nil(err, "no error creating manifest with test chart descriptor")
   132  	err = store.StoreManifest(ref, testChartManifestDesc, testChartManifest)
   133  	suite.Nil(err, "no error pushing manifest with test chart descriptor")
   134  	fmt.Printf("%s\n", testChartManifest)
   135  
   136  	_, err = Copy(newContext(), store, ref, newResolver(), "")
   137  	suite.Nil(err, "no error pushing test chart tgz (as single layer)")
   138  
   139  	// Load descriptors with test chart dir (each file as layer)
   140  	testDirAbs, err := filepath.Abs(testDir)
   141  	suite.Nil(err, "no error parsing test directory")
   142  	store = orascontent.NewFile(testDirAbs)
   143  	err = store.Load(configDesc, config)
   144  	suite.Nil(err, "no error saving config for test dir")
   145  	descriptors = []ocispec.Descriptor{}
   146  	var ff = func(pathX string, infoX os.FileInfo, errX error) error {
   147  		if !infoX.IsDir() {
   148  			filename := filepath.Join(filepath.Dir(pathX), infoX.Name())
   149  			name := filepath.ToSlash(filename)
   150  			desc, err = store.Add(name, "", filename)
   151  			if err != nil {
   152  				return err
   153  			}
   154  			descriptors = append(descriptors, desc)
   155  		}
   156  		return nil
   157  	}
   158  
   159  	cwd, _ := os.Getwd()
   160  	os.Chdir(testDir)
   161  	filepath.Walk(".", ff)
   162  	os.Chdir(cwd)
   163  
   164  	ref = fmt.Sprintf("%s/chart-dir:test", suite.DockerRegistryHost)
   165  	testChartDirManifest, testChartDirManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil, descriptors...)
   166  	suite.Nil(err, "no error creating manifest with test chart dir (each file as layer)")
   167  	err = store.StoreManifest(ref, testChartDirManifestDesc, testChartDirManifest)
   168  	suite.Nil(err, "no error pushing manifest with test chart dir (each file as layer)")
   169  	_, err = Copy(newContext(), store, ref, newResolver(), "")
   170  	suite.Nil(err, "no error pushing test chart dir (each file as layer)")
   171  }
   172  
   173  // Pull files and verify descriptors
   174  func (suite *ORASTestSuite) Test_1_Pull() {
   175  	var (
   176  		err       error
   177  		ref       string
   178  		desc      ocispec.Descriptor
   179  		store     *orascontent.Memory
   180  		emptyDesc ocispec.Descriptor
   181  	)
   182  
   183  	desc, err = Copy(newContext(), nil, ref, nil, ref)
   184  	suite.NotNil(err, "error pulling with empty resolver")
   185  	suite.Equal(desc, emptyDesc, "descriptor empty pulling with empty resolver")
   186  
   187  	// Pull non-existent
   188  	store = orascontent.NewMemory()
   189  	ref = fmt.Sprintf("%s/nonexistent:test", suite.DockerRegistryHost)
   190  	desc, err = Copy(newContext(), newResolver(), ref, store, ref)
   191  	suite.NotNil(err, "error pulling non-existent ref")
   192  	suite.Equal(desc, emptyDesc, "descriptor empty with error")
   193  
   194  	// Pull chart-tgz
   195  	store = orascontent.NewMemory()
   196  	ref = fmt.Sprintf("%s/chart-tgz:test", suite.DockerRegistryHost)
   197  	_, err = Copy(newContext(), newResolver(), ref, store, ref)
   198  	suite.Nil(err, "no error pulling chart-tgz ref")
   199  
   200  	// Verify the descriptors, single layer/file
   201  	content, err := ioutil.ReadFile(testTarball)
   202  	suite.Nil(err, "no error loading test chart")
   203  	name := filepath.Base(testTarball)
   204  	_, actualContent, ok := store.GetByName(name)
   205  	suite.True(ok, "find in memory")
   206  	suite.Equal(content, actualContent, ".tgz content matches on pull")
   207  
   208  	// Pull chart-dir
   209  	store = orascontent.NewMemory()
   210  	ref = fmt.Sprintf("%s/chart-dir:test", suite.DockerRegistryHost)
   211  	desc, err = Copy(newContext(), newResolver(), ref, store, ref)
   212  	suite.Nil(err, "no error pulling chart-dir ref")
   213  
   214  	// Verify the descriptors, multiple layers/files
   215  	cwd, _ := os.Getwd()
   216  	os.Chdir(testDir)
   217  	for _, filename := range testDirFiles {
   218  		content, err = ioutil.ReadFile(filename)
   219  		suite.Nil(err, fmt.Sprintf("no error loading %s", filename))
   220  		_, actualContent, ok := store.GetByName(filename)
   221  		suite.True(ok, "find in memory")
   222  		suite.Equal(content, actualContent, fmt.Sprintf("%s content matches on pull", filename))
   223  	}
   224  	os.Chdir(cwd)
   225  }
   226  
   227  // Push and pull with customized media types
   228  func (suite *ORASTestSuite) Test_2_MediaType() {
   229  	var (
   230  		testData = [][]string{
   231  			{"hi.txt", "application/vnd.me.hi", "hi"},
   232  			{"bye.txt", "application/vnd.me.bye", "bye"},
   233  		}
   234  		err         error
   235  		ref         string
   236  		descriptors []ocispec.Descriptor
   237  		store       *orascontent.Memory
   238  	)
   239  
   240  	// Push content with customized media types
   241  	store = orascontent.NewMemory()
   242  	descriptors = nil
   243  	for _, data := range testData {
   244  		desc, _ := store.Add(data[0], data[1], []byte(data[2]))
   245  		descriptors = append(descriptors, desc)
   246  	}
   247  	ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost)
   248  	config, configDesc, err := orascontent.GenerateConfig(nil)
   249  	suite.Nil(err, "no error generating config")
   250  	store.Set(configDesc, config)
   251  	emptyManifest, emptyManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil, descriptors...)
   252  	suite.Nil(err, "no error creating manifest with empty descriptors")
   253  	err = store.StoreManifest(ref, emptyManifestDesc, emptyManifest)
   254  	suite.Nil(err, "no error pushing manifest with empty descriptors")
   255  
   256  	_, err = Copy(newContext(), store, ref, newResolver(), ref)
   257  	suite.Nil(err, "no error pushing test data with customized media type")
   258  
   259  	// Pull with all media types
   260  	store = orascontent.NewMemory()
   261  	store.Set(configDesc, config)
   262  	ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost)
   263  	_, err = Copy(newContext(), newResolver(), ref, store, ref)
   264  	suite.Nil(err, "no error pulling media-type ref")
   265  	for _, data := range testData {
   266  		_, actualContent, ok := store.GetByName(data[0])
   267  		suite.True(ok, "find in memory")
   268  		content := []byte(data[2])
   269  		suite.Equal(content, actualContent, "test content matches on pull")
   270  	}
   271  
   272  	// Pull with specified media type
   273  	store = orascontent.NewMemory()
   274  	store.Set(configDesc, config)
   275  	ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost)
   276  	_, err = Copy(newContext(), newResolver(), ref, store, ref, WithAllowedMediaType(testData[0][1]))
   277  	suite.Nil(err, "no error pulling media-type ref")
   278  	for _, data := range testData[:1] {
   279  		_, actualContent, ok := store.GetByName(data[0])
   280  		suite.True(ok, "find in memory")
   281  		content := []byte(data[2])
   282  		suite.Equal(content, actualContent, "test content matches on pull")
   283  	}
   284  
   285  	// Pull with non-existing media type, so only should do root manifest
   286  	store = orascontent.NewMemory()
   287  	store.Set(configDesc, config)
   288  	ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost)
   289  	_, err = Copy(newContext(), newResolver(), ref, store, ref, WithAllowedMediaType("non.existing.media.type"))
   290  	suite.Nil(err, "no error pulling media-type ref")
   291  }
   292  
   293  // Pull with condition
   294  func (suite *ORASTestSuite) Test_3_Conditional_Pull() {
   295  	var (
   296  		testData = [][]string{
   297  			{"version.txt", "edge"},
   298  			{"content.txt", "hello world"},
   299  		}
   300  		err         error
   301  		ref         string
   302  		descriptors []ocispec.Descriptor
   303  		store       *orascontent.Memory
   304  		stop        bool
   305  	)
   306  
   307  	// Push test content
   308  	store = orascontent.NewMemory()
   309  	descriptors = nil
   310  	for _, data := range testData {
   311  		desc, _ := store.Add(data[0], "", []byte(data[1]))
   312  		descriptors = append(descriptors, desc)
   313  	}
   314  	ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost)
   315  	config, configDesc, err := orascontent.GenerateConfig(nil)
   316  	suite.Nil(err, "no error generating config")
   317  	store.Set(configDesc, config)
   318  	testManifest, testManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil, descriptors...)
   319  	suite.Nil(err, "no error creating manifest with test descriptors")
   320  	err = store.StoreManifest(ref, testManifestDesc, testManifest)
   321  	suite.Nil(err, "no error pushing manifest with test descriptors")
   322  	_, err = Copy(newContext(), store, ref, newResolver(), ref)
   323  	suite.Nil(err, "no error pushing test data")
   324  
   325  	// Pull all contents in sequence
   326  	store = orascontent.NewMemory()
   327  	store.Set(configDesc, config)
   328  	ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost)
   329  	_, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS)
   330  	suite.Nil(err, "no error pulling ref")
   331  	for i, data := range testData {
   332  		_, actualContent, ok := store.GetByName(data[0])
   333  		suite.True(ok, "find in memory")
   334  		content := []byte(data[1])
   335  		suite.Equal(content, actualContent, "test content matches on pull")
   336  		name, _ := orascontent.ResolveName(descriptors[i])
   337  		suite.Equal(data[0], name, "content sequence matches on pull")
   338  	}
   339  
   340  	// Selective pull contents: stop at the very beginning
   341  	store = orascontent.NewMemory()
   342  	store.Set(configDesc, config)
   343  	ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost)
   344  	_, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS,
   345  		WithPullBaseHandler(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   346  			if name, ok := orascontent.ResolveName(desc); ok && name == testData[0][0] {
   347  				return nil, ErrStopProcessing
   348  			}
   349  			return nil, nil
   350  		})))
   351  	suite.Nil(err, "no error pulling ref")
   352  
   353  	// Selective pull contents: stop in the middle
   354  	store = orascontent.NewMemory()
   355  	store.Set(configDesc, config)
   356  	ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost)
   357  	stop = false
   358  	_, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS,
   359  		WithPullBaseHandler(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   360  			if stop {
   361  				return nil, ErrStopProcessing
   362  			}
   363  			if name, ok := orascontent.ResolveName(desc); ok && name == testData[0][0] {
   364  				stop = true
   365  			}
   366  			return nil, nil
   367  		})))
   368  	suite.Nil(err, "no error pulling ref")
   369  	for _, data := range testData[:1] {
   370  		_, actualContent, ok := store.GetByName(data[0])
   371  		suite.True(ok, "find in memory")
   372  		content := []byte(data[1])
   373  		suite.Equal(content, actualContent, "test content matches on pull")
   374  	}
   375  }
   376  
   377  // Test for vulnerability GHSA-g5v4-5x39-vwhx
   378  func (suite *ORASTestSuite) Test_4_GHSA_g5v4_5x39_vwhx() {
   379  	var testVulnerability = func(headers []tar.Header, tag string, expectedError string) {
   380  		// Step 1: build malicious tar+gzip
   381  		buf := bytes.NewBuffer(nil)
   382  		digester := digest.Canonical.Digester()
   383  		zw := gzip.NewWriter(io.MultiWriter(buf, digester.Hash()))
   384  		tarDigester := digest.Canonical.Digester()
   385  		tw := tar.NewWriter(io.MultiWriter(zw, tarDigester.Hash()))
   386  		for _, header := range headers {
   387  			err := tw.WriteHeader(&header)
   388  			suite.Nil(err, "error writing header")
   389  		}
   390  		err := tw.Close()
   391  		suite.Nil(err, "error closing tar")
   392  		err = zw.Close()
   393  		suite.Nil(err, "error closing gzip")
   394  
   395  		// Step 2: construct malicious descriptor
   396  		evilDesc := ocispec.Descriptor{
   397  			MediaType: ocispec.MediaTypeImageLayerGzip,
   398  			Digest:    digester.Digest(),
   399  			Size:      int64(buf.Len()),
   400  			Annotations: map[string]string{
   401  				orascontent.AnnotationDigest: tarDigester.Digest().String(),
   402  				orascontent.AnnotationUnpack: "true",
   403  				ocispec.AnnotationTitle:      "foo",
   404  			},
   405  		}
   406  
   407  		// Step 3: upload malicious artifact to registry
   408  		memoryStore := orascontent.NewMemory()
   409  		memoryStore.Set(evilDesc, buf.Bytes())
   410  		ref := fmt.Sprintf("%s/evil:%s", suite.DockerRegistryHost, tag)
   411  
   412  		config, configDesc, err := orascontent.GenerateConfig(nil)
   413  		suite.Nil(err, "no error generating config")
   414  		memoryStore.Set(configDesc, config)
   415  		testManifest, testManifestDesc, err := orascontent.GenerateManifest(&configDesc, nil, evilDesc)
   416  		suite.Nil(err, "no error creating manifest with evil descriptors")
   417  		err = memoryStore.StoreManifest(ref, testManifestDesc, testManifest)
   418  		suite.Nil(err, "no error pushing manifest with evil descriptors")
   419  		_, err = Copy(newContext(), memoryStore, ref, newResolver(), ref)
   420  		suite.Nil(err, "no error pushing test data")
   421  
   422  		// Step 4: pull malicious tar with oras filestore and ensure error
   423  		tempDir, err := ioutil.TempDir("", "oras_test")
   424  		if err != nil {
   425  			suite.FailNow("error creating temp directory", err)
   426  		}
   427  		defer os.RemoveAll(tempDir)
   428  		store := orascontent.NewFile(tempDir)
   429  		defer store.Close()
   430  		err = store.Load(configDesc, config)
   431  		suite.Nil(err, "no error saving config")
   432  		ref = fmt.Sprintf("%s/evil:%s", suite.DockerRegistryHost, tag)
   433  		_, err = Copy(newContext(), newResolver(), ref, store, ref)
   434  		suite.NotNil(err, "error expected pulling malicious tar")
   435  		suite.Contains(err.Error(),
   436  			expectedError,
   437  			"did not get correct error message",
   438  		)
   439  	}
   440  
   441  	tests := []struct {
   442  		name          string
   443  		headers       []tar.Header
   444  		tag           string
   445  		expectedError string
   446  	}{
   447  		{
   448  			name: "Test symbolic link path traversal",
   449  			headers: []tar.Header{
   450  				{
   451  					Typeflag: tar.TypeDir,
   452  					Name:     "foo/subdir/",
   453  					Mode:     0755,
   454  				},
   455  				{ // Symbolic link to `foo`
   456  					Typeflag: tar.TypeSymlink,
   457  					Name:     "foo/subdir/parent",
   458  					Linkname: "..",
   459  					Mode:     0755,
   460  				},
   461  				{ // Symbolic link to `../etc/passwd`
   462  					Typeflag: tar.TypeSymlink,
   463  					Name:     "foo/subdir/parent/passwd",
   464  					Linkname: "../../etc/passwd",
   465  					Mode:     0644,
   466  				},
   467  				{ // Symbolic link to `../etc`
   468  					Typeflag: tar.TypeSymlink,
   469  					Name:     "foo/subdir/parent/etc",
   470  					Linkname: "../../etc",
   471  					Mode:     0644,
   472  				},
   473  			},
   474  			tag:           "symlink_path",
   475  			expectedError: "no symbolic link allowed",
   476  		},
   477  		{
   478  			name: "Test symbolic link pointing to outside",
   479  			headers: []tar.Header{
   480  				{ // Symbolic link to `/etc/passwd`
   481  					Typeflag: tar.TypeSymlink,
   482  					Name:     "foo/passwd",
   483  					Linkname: "../../../etc/passwd",
   484  					Mode:     0644,
   485  				},
   486  			},
   487  			tag:           "symlink",
   488  			expectedError: "is outside of",
   489  		},
   490  		{
   491  			name: "Test hard link pointing to outside",
   492  			headers: []tar.Header{
   493  				{ // Hard link to `/etc/passwd`
   494  					Typeflag: tar.TypeLink,
   495  					Name:     "foo/passwd",
   496  					Linkname: "../../../etc/passwd",
   497  					Mode:     0644,
   498  				},
   499  			},
   500  			tag:           "hardlink",
   501  			expectedError: "is outside of",
   502  		},
   503  	}
   504  	for _, test := range tests {
   505  		suite.T().Log(test.name)
   506  		testVulnerability(test.headers, test.tag, test.expectedError)
   507  	}
   508  }
   509  
   510  func TestORASTestSuite(t *testing.T) {
   511  	suite.Run(t, new(ORASTestSuite))
   512  }
   513  

View as plain text