index.go

Documentation: helm.sh/helm/v3/pkg/repo

     1  /*
     2  Copyright The Helm Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package repo
    18  
    19  import (
    20  	"bytes"
    21  	"encoding/json"
    22  	"log"
    23  	"os"
    24  	"path"
    25  	"path/filepath"
    26  	"sort"
    27  	"strings"
    28  	"time"
    29  
    30  	"github.com/Masterminds/semver/v3"
    31  	"github.com/pkg/errors"
    32  	"sigs.k8s.io/yaml"
    33  
    34  	"helm.sh/helm/v3/internal/fileutil"
    35  	"helm.sh/helm/v3/internal/urlutil"
    36  	"helm.sh/helm/v3/pkg/chart"
    37  	"helm.sh/helm/v3/pkg/chart/loader"
    38  	"helm.sh/helm/v3/pkg/provenance"
    39  )
    40  
    41  var indexPath = "index.yaml"
    42  
    43  // APIVersionV1 is the v1 API version for index and repository files.
    44  const APIVersionV1 = "v1"
    45  
    46  var (
    47  	// ErrNoAPIVersion indicates that an API version was not specified.
    48  	ErrNoAPIVersion = errors.New("no API version specified")
    49  	// ErrNoChartVersion indicates that a chart with the given version is not found.
    50  	ErrNoChartVersion = errors.New("no chart version found")
    51  	// ErrNoChartName indicates that a chart with the given name is not found.
    52  	ErrNoChartName = errors.New("no chart name found")
    53  	// ErrEmptyIndexYaml indicates that the content of index.yaml is empty.
    54  	ErrEmptyIndexYaml = errors.New("empty index.yaml file")
    55  )
    56  
    57  // ChartVersions is a list of versioned chart references.
    58  // Implements a sorter on Version.
    59  type ChartVersions []*ChartVersion
    60  
    61  // Len returns the length.
    62  func (c ChartVersions) Len() int { return len(c) }
    63  
    64  // Swap swaps the position of two items in the versions slice.
    65  func (c ChartVersions) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
    66  
    67  // Less returns true if the version of entry a is less than the version of entry b.
    68  func (c ChartVersions) Less(a, b int) bool {
    69  	// Failed parse pushes to the back.
    70  	i, err := semver.NewVersion(c[a].Version)
    71  	if err != nil {
    72  		return true
    73  	}
    74  	j, err := semver.NewVersion(c[b].Version)
    75  	if err != nil {
    76  		return false
    77  	}
    78  	return i.LessThan(j)
    79  }
    80  
    81  // IndexFile represents the index file in a chart repository
    82  type IndexFile struct {
    83  	// This is used ONLY for validation against chartmuseum's index files and is discarded after validation.
    84  	ServerInfo map[string]interface{}   `json:"serverInfo,omitempty"`
    85  	APIVersion string                   `json:"apiVersion"`
    86  	Generated  time.Time                `json:"generated"`
    87  	Entries    map[string]ChartVersions `json:"entries"`
    88  	PublicKeys []string                 `json:"publicKeys,omitempty"`
    89  
    90  	// Annotations are additional mappings uninterpreted by Helm. They are made available for
    91  	// other applications to add information to the index file.
    92  	Annotations map[string]string `json:"annotations,omitempty"`
    93  }
    94  
    95  // NewIndexFile initializes an index.
    96  func NewIndexFile() *IndexFile {
    97  	return &IndexFile{
    98  		APIVersion: APIVersionV1,
    99  		Generated:  time.Now(),
   100  		Entries:    map[string]ChartVersions{},
   101  		PublicKeys: []string{},
   102  	}
   103  }
   104  
   105  // LoadIndexFile takes a file at the given path and returns an IndexFile object
   106  func LoadIndexFile(path string) (*IndexFile, error) {
   107  	b, err := os.ReadFile(path)
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	i, err := loadIndex(b, path)
   112  	if err != nil {
   113  		return nil, errors.Wrapf(err, "error loading %s", path)
   114  	}
   115  	return i, nil
   116  }
   117  
   118  // MustAdd adds a file to the index
   119  // This can leave the index in an unsorted state
   120  func (i IndexFile) MustAdd(md *chart.Metadata, filename, baseURL, digest string) error {
   121  	if i.Entries == nil {
   122  		return errors.New("entries not initialized")
   123  	}
   124  
   125  	if md.APIVersion == "" {
   126  		md.APIVersion = chart.APIVersionV1
   127  	}
   128  	if err := md.Validate(); err != nil {
   129  		return errors.Wrapf(err, "validate failed for %s", filename)
   130  	}
   131  
   132  	u := filename
   133  	if baseURL != "" {
   134  		_, file := filepath.Split(filename)
   135  		var err error
   136  		u, err = urlutil.URLJoin(baseURL, file)
   137  		if err != nil {
   138  			u = path.Join(baseURL, file)
   139  		}
   140  	}
   141  	cr := &ChartVersion{
   142  		URLs:     []string{u},
   143  		Metadata: md,
   144  		Digest:   digest,
   145  		Created:  time.Now(),
   146  	}
   147  	ee := i.Entries[md.Name]
   148  	i.Entries[md.Name] = append(ee, cr)
   149  	return nil
   150  }
   151  
   152  // Add adds a file to the index and logs an error.
   153  //
   154  // Deprecated: Use index.MustAdd instead.
   155  func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) {
   156  	if err := i.MustAdd(md, filename, baseURL, digest); err != nil {
   157  		log.Printf("skipping loading invalid entry for chart %q %q from %s: %s", md.Name, md.Version, filename, err)
   158  	}
   159  }
   160  
   161  // Has returns true if the index has an entry for a chart with the given name and exact version.
   162  func (i IndexFile) Has(name, version string) bool {
   163  	_, err := i.Get(name, version)
   164  	return err == nil
   165  }
   166  
   167  // SortEntries sorts the entries by version in descending order.
   168  //
   169  // In canonical form, the individual version records should be sorted so that
   170  // the most recent release for every version is in the 0th slot in the
   171  // Entries.ChartVersions array. That way, tooling can predict the newest
   172  // version without needing to parse SemVers.
   173  func (i IndexFile) SortEntries() {
   174  	for _, versions := range i.Entries {
   175  		sort.Sort(sort.Reverse(versions))
   176  	}
   177  }
   178  
   179  // Get returns the ChartVersion for the given name.
   180  //
   181  // If version is empty, this will return the chart with the latest stable version,
   182  // prerelease versions will be skipped.
   183  func (i IndexFile) Get(name, version string) (*ChartVersion, error) {
   184  	vs, ok := i.Entries[name]
   185  	if !ok {
   186  		return nil, ErrNoChartName
   187  	}
   188  	if len(vs) == 0 {
   189  		return nil, ErrNoChartVersion
   190  	}
   191  
   192  	var constraint *semver.Constraints
   193  	if version == "" {
   194  		constraint, _ = semver.NewConstraint("*")
   195  	} else {
   196  		var err error
   197  		constraint, err = semver.NewConstraint(version)
   198  		if err != nil {
   199  			return nil, err
   200  		}
   201  	}
   202  
   203  	// when customer input exact version, check whether have exact match one first
   204  	if len(version) != 0 {
   205  		for _, ver := range vs {
   206  			if version == ver.Version {
   207  				return ver, nil
   208  			}
   209  		}
   210  	}
   211  
   212  	for _, ver := range vs {
   213  		test, err := semver.NewVersion(ver.Version)
   214  		if err != nil {
   215  			continue
   216  		}
   217  
   218  		if constraint.Check(test) {
   219  			return ver, nil
   220  		}
   221  	}
   222  	return nil, errors.Errorf("no chart version found for %s-%s", name, version)
   223  }
   224  
   225  // WriteFile writes an index file to the given destination path.
   226  //
   227  // The mode on the file is set to 'mode'.
   228  func (i IndexFile) WriteFile(dest string, mode os.FileMode) error {
   229  	b, err := yaml.Marshal(i)
   230  	if err != nil {
   231  		return err
   232  	}
   233  	return fileutil.AtomicWriteFile(dest, bytes.NewReader(b), mode)
   234  }
   235  
   236  // WriteJSONFile writes an index file in JSON format to the given destination
   237  // path.
   238  //
   239  // The mode on the file is set to 'mode'.
   240  func (i IndexFile) WriteJSONFile(dest string, mode os.FileMode) error {
   241  	b, err := json.MarshalIndent(i, "", "  ")
   242  	if err != nil {
   243  		return err
   244  	}
   245  	return fileutil.AtomicWriteFile(dest, bytes.NewReader(b), mode)
   246  }
   247  
   248  // Merge merges the given index file into this index.
   249  //
   250  // This merges by name and version.
   251  //
   252  // If one of the entries in the given index does _not_ already exist, it is added.
   253  // In all other cases, the existing record is preserved.
   254  //
   255  // This can leave the index in an unsorted state
   256  func (i *IndexFile) Merge(f *IndexFile) {
   257  	for _, cvs := range f.Entries {
   258  		for _, cv := range cvs {
   259  			if !i.Has(cv.Name, cv.Version) {
   260  				e := i.Entries[cv.Name]
   261  				i.Entries[cv.Name] = append(e, cv)
   262  			}
   263  		}
   264  	}
   265  }
   266  
   267  // ChartVersion represents a chart entry in the IndexFile
   268  type ChartVersion struct {
   269  	*chart.Metadata
   270  	URLs    []string  `json:"urls"`
   271  	Created time.Time `json:"created,omitempty"`
   272  	Removed bool      `json:"removed,omitempty"`
   273  	Digest  string    `json:"digest,omitempty"`
   274  
   275  	// ChecksumDeprecated is deprecated in Helm 3, and therefore ignored. Helm 3 replaced
   276  	// this with Digest. However, with a strict YAML parser enabled, a field must be
   277  	// present on the struct for backwards compatibility.
   278  	ChecksumDeprecated string `json:"checksum,omitempty"`
   279  
   280  	// EngineDeprecated is deprecated in Helm 3, and therefore ignored. However, with a strict
   281  	// YAML parser enabled, this field must be present.
   282  	EngineDeprecated string `json:"engine,omitempty"`
   283  
   284  	// TillerVersionDeprecated is deprecated in Helm 3, and therefore ignored. However, with a strict
   285  	// YAML parser enabled, this field must be present.
   286  	TillerVersionDeprecated string `json:"tillerVersion,omitempty"`
   287  
   288  	// URLDeprecated is deprecated in Helm 3, superseded by URLs. It is ignored. However,
   289  	// with a strict YAML parser enabled, this must be present on the struct.
   290  	URLDeprecated string `json:"url,omitempty"`
   291  }
   292  
   293  // IndexDirectory reads a (flat) directory and generates an index.
   294  //
   295  // It indexes only charts that have been packaged (*.tgz).
   296  //
   297  // The index returned will be in an unsorted state
   298  func IndexDirectory(dir, baseURL string) (*IndexFile, error) {
   299  	archives, err := filepath.Glob(filepath.Join(dir, "*.tgz"))
   300  	if err != nil {
   301  		return nil, err
   302  	}
   303  	moreArchives, err := filepath.Glob(filepath.Join(dir, "**/*.tgz"))
   304  	if err != nil {
   305  		return nil, err
   306  	}
   307  	archives = append(archives, moreArchives...)
   308  
   309  	index := NewIndexFile()
   310  	for _, arch := range archives {
   311  		fname, err := filepath.Rel(dir, arch)
   312  		if err != nil {
   313  			return index, err
   314  		}
   315  
   316  		var parentDir string
   317  		parentDir, fname = filepath.Split(fname)
   318  		// filepath.Split appends an extra slash to the end of parentDir. We want to strip that out.
   319  		parentDir = strings.TrimSuffix(parentDir, string(os.PathSeparator))
   320  		parentURL, err := urlutil.URLJoin(baseURL, parentDir)
   321  		if err != nil {
   322  			parentURL = path.Join(baseURL, parentDir)
   323  		}
   324  
   325  		c, err := loader.Load(arch)
   326  		if err != nil {
   327  			// Assume this is not a chart.
   328  			continue
   329  		}
   330  		hash, err := provenance.DigestFile(arch)
   331  		if err != nil {
   332  			return index, err
   333  		}
   334  		if err := index.MustAdd(c.Metadata, fname, parentURL, hash); err != nil {
   335  			return index, errors.Wrapf(err, "failed adding to %s to index", fname)
   336  		}
   337  	}
   338  	return index, nil
   339  }
   340  
   341  // loadIndex loads an index file and does minimal validity checking.
   342  //
   343  // The source parameter is only used for logging.
   344  // This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails.
   345  func loadIndex(data []byte, source string) (*IndexFile, error) {
   346  	i := &IndexFile{}
   347  
   348  	if len(data) == 0 {
   349  		return i, ErrEmptyIndexYaml
   350  	}
   351  
   352  	if err := jsonOrYamlUnmarshal(data, i); err != nil {
   353  		return i, err
   354  	}
   355  
   356  	for name, cvs := range i.Entries {
   357  		for idx := len(cvs) - 1; idx >= 0; idx-- {
   358  			if cvs[idx] == nil {
   359  				log.Printf("skipping loading invalid entry for chart %q from %s: empty entry", name, source)
   360  				continue
   361  			}
   362  			// When metadata section missing, initialize with no data
   363  			if cvs[idx].Metadata == nil {
   364  				cvs[idx].Metadata = &chart.Metadata{}
   365  			}
   366  			if cvs[idx].APIVersion == "" {
   367  				cvs[idx].APIVersion = chart.APIVersionV1
   368  			}
   369  			if err := cvs[idx].Validate(); ignoreSkippableChartValidationError(err) != nil {
   370  				log.Printf("skipping loading invalid entry for chart %q %q from %s: %s", name, cvs[idx].Version, source, err)
   371  				cvs = append(cvs[:idx], cvs[idx+1:]...)
   372  			}
   373  		}
   374  	}
   375  	i.SortEntries()
   376  	if i.APIVersion == "" {
   377  		return i, ErrNoAPIVersion
   378  	}
   379  	return i, nil
   380  }
   381  
   382  // jsonOrYamlUnmarshal unmarshals the given byte slice containing JSON or YAML
   383  // into the provided interface.
   384  //
   385  // It automatically detects whether the data is in JSON or YAML format by
   386  // checking its validity as JSON. If the data is valid JSON, it will use the
   387  // `encoding/json` package to unmarshal it. Otherwise, it will use the
   388  // `sigs.k8s.io/yaml` package to unmarshal the YAML data.
   389  func jsonOrYamlUnmarshal(b []byte, i interface{}) error {
   390  	if json.Valid(b) {
   391  		return json.Unmarshal(b, i)
   392  	}
   393  	return yaml.UnmarshalStrict(b, i)
   394  }
   395  
   396  // ignoreSkippableChartValidationError inspect the given error and returns nil if
   397  // the error isn't important for index loading
   398  //
   399  // In particular, charts may introduce validations that don't impact repository indexes
   400  // And repository indexes may be generated by older/non-complient software, which doesn't
   401  // conform to all validations.
   402  func ignoreSkippableChartValidationError(err error) error {
   403  	verr, ok := err.(chart.ValidationError)
   404  	if !ok {
   405  		return err
   406  	}
   407  
   408  	// https://github.com/helm/helm/issues/12748 (JFrog repository strips alias field)
   409  	if strings.HasPrefix(verr.Error(), "validation: more than one dependency with name or alias") {
   410  		return nil
   411  	}
   412  
   413  	return err
   414  }
   415
View as plain text