appconfig.go

Documentation: github.com/palantir/go-githubapp/appconfig

     1  // Copyright 2021 Palantir Technologies, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package appconfig loads repository configuration for GitHub apps. It
    16  // supports loading directly from a file in a repository, loading from remote
    17  // references, and loading an organization-level default. The config itself can
    18  // be in any format.
    19  package appconfig
    20  
    21  import (
    22  	"context"
    23  	"fmt"
    24  	"io/ioutil"
    25  	"net/http"
    26  	"strings"
    27  
    28  	"github.com/google/go-github/v47/github"
    29  	"github.com/pkg/errors"
    30  	"github.com/rs/zerolog"
    31  )
    32  
    33  // RemoteRefParser attempts to parse a RemoteRef from bytes. The parser should
    34  // return nil with a nil error if b does not encode a RemoteRef and nil with a
    35  // non-nil error if b encodes an invalid RemoteRef.
    36  type RemoteRefParser func(path string, b []byte) (*RemoteRef, error)
    37  
    38  // RemoteRef identifies a configuration file in a different repository.
    39  type RemoteRef struct {
    40  	// The repository in "owner/name" format. Required.
    41  	Remote string `yaml:"remote" json:"remote"`
    42  
    43  	// The path to the config file in the repository. If empty, use the first
    44  	// path configured in the loader.
    45  	Path string `yaml:"path" json:"path"`
    46  
    47  	// The reference (branch, tag, or SHA) to read in the repository. If empty,
    48  	// use the default branch of the repository.
    49  	Ref string `yaml:"ref" json:"ref"`
    50  }
    51  
    52  func (r RemoteRef) SplitRemote() (owner, repo string, err error) {
    53  	slash := strings.IndexByte(r.Remote, '/')
    54  	if slash <= 0 || slash >= len(r.Remote)-1 {
    55  		return "", "", errors.Errorf("invalid remote value: %s", r.Remote)
    56  	}
    57  	return r.Remote[:slash], r.Remote[slash+1:], nil
    58  }
    59  
    60  // Config contains unparsed configuration data and metadata about where it was found.
    61  type Config struct {
    62  	Content []byte
    63  
    64  	// Source contains the repository and ref in "owner/name@ref" format. The
    65  	// ref component ("@ref") is optional and may not be present.
    66  	Source   string
    67  	Path     string
    68  	IsRemote bool
    69  }
    70  
    71  // IsUndefined returns true if the Config's content is empty and there is no
    72  // metadata giving a source.
    73  func (c Config) IsUndefined() bool {
    74  	return len(c.Content) == 0 && c.Source == "" && c.Path == ""
    75  }
    76  
    77  // Loader loads configuration for repositories.
    78  type Loader struct {
    79  	paths []string
    80  
    81  	parser       RemoteRefParser
    82  	defaultRepo  string
    83  	defaultPaths []string
    84  }
    85  
    86  // NewLoader creates a Loader that loads configuration from paths.
    87  func NewLoader(paths []string, opts ...Option) *Loader {
    88  	defaultPaths := make([]string, len(paths))
    89  	for i, p := range paths {
    90  		defaultPaths[i] = strings.TrimPrefix(p, ".github/")
    91  	}
    92  
    93  	ld := Loader{
    94  		paths:        paths,
    95  		parser:       YAMLRemoteRefParser,
    96  		defaultRepo:  ".github",
    97  		defaultPaths: defaultPaths,
    98  	}
    99  
   100  	for _, opt := range opts {
   101  		opt(&ld)
   102  	}
   103  
   104  	return &ld
   105  }
   106  
   107  // LoadConfig loads configuration for the repository owner/repo. It first tries
   108  // the Loader's paths in order, following remote references if they exist. If
   109  // no configuration exists at any path in the repository, it tries to load
   110  // default configuration defined by owner for all repositories. If no default
   111  // configuration exists, it returns an undefined Config and a nil error.
   112  //
   113  // If error is non-nil, the Source and Path fields of the returned Config tell
   114  // which file LoadConfig was processing when it encountered the error.
   115  func (ld *Loader) LoadConfig(ctx context.Context, client *github.Client, owner, repo, ref string) (Config, error) {
   116  	logger := zerolog.Ctx(ctx)
   117  
   118  	c := Config{
   119  		Source: fmt.Sprintf("%s/%s@%s", owner, repo, ref),
   120  	}
   121  
   122  	for _, p := range ld.paths {
   123  		c.Path = p
   124  
   125  		logger.Debug().Msgf("Trying configuration at %s in %s", c.Path, c.Source)
   126  		content, exists, err := getFileContents(ctx, client, owner, repo, ref, p)
   127  		if err != nil {
   128  			return c, err
   129  		}
   130  		if !exists {
   131  			continue
   132  		}
   133  
   134  		// if remote refs are enabled, see if the file is a remote reference
   135  		if ld.parser != nil {
   136  			remote, err := ld.parser(p, content)
   137  			if err != nil {
   138  				return c, err
   139  			}
   140  			if remote != nil {
   141  				logger.Debug().Msgf("Found remote configuration at %s in %s", p, c.Source)
   142  				return ld.loadRemoteConfig(ctx, client, *remote, c)
   143  			}
   144  		}
   145  
   146  		// non-remote content found, don't try any other paths
   147  		logger.Debug().Msgf("Found configuration at %s in %s", c.Path, c.Source)
   148  		c.Content = content
   149  		return c, nil
   150  	}
   151  
   152  	// if the repository defined no configuration and org defaults are enabled,
   153  	// try falling back to the defaults
   154  	if ld.defaultRepo != "" && len(ld.defaultPaths) > 0 {
   155  		return ld.loadDefaultConfig(ctx, client, owner)
   156  	}
   157  
   158  	// couldn't find configuration anyhere, so return an empty/undefined one
   159  	return Config{}, nil
   160  }
   161  
   162  func (ld *Loader) loadRemoteConfig(ctx context.Context, client *github.Client, remote RemoteRef, c Config) (Config, error) {
   163  	logger := zerolog.Ctx(ctx)
   164  	notFoundErr := fmt.Errorf("invalid remote reference: file does not exist")
   165  
   166  	owner, repo, err := remote.SplitRemote()
   167  	if err != nil {
   168  		return c, err
   169  	}
   170  
   171  	path := remote.Path
   172  	if path == "" && len(ld.paths) > 0 {
   173  		path = ld.paths[0]
   174  	}
   175  
   176  	// After this point, all errors will be about the remote file, not the
   177  	// local file containing the reference.
   178  	c.Source = fmt.Sprintf("%s/%s", owner, repo)
   179  	c.Path = path
   180  	c.IsRemote = true
   181  
   182  	ref := remote.Ref
   183  	if ref == "" {
   184  		// This is technically not necessary, as passing an empty ref to GitHub
   185  		// uses the default branch. However, callers may expect the Source
   186  		// field in the Config we return to have a non-empty ref.
   187  		r, _, err := client.Repositories.Get(ctx, owner, repo)
   188  		if err != nil {
   189  			if isNotFound(err) {
   190  				return c, notFoundErr
   191  			}
   192  			return c, errors.Wrap(err, "failed to get remote repository")
   193  		}
   194  		ref = r.GetDefaultBranch()
   195  	}
   196  	c.Source = fmt.Sprintf("%s@%s", c.Source, ref)
   197  
   198  	logger.Debug().Msgf("Trying remote configuration at %s in %s", c.Path, c.Source)
   199  	content, exists, err := getFileContents(ctx, client, owner, repo, ref, c.Path)
   200  	if err != nil {
   201  		return c, err
   202  	}
   203  	if !exists {
   204  		// Referencing a remote file that does not exist is an error because
   205  		// this condition is annoying to debug otherwise. From the perspective
   206  		// of a repository, it appears that the application has a configuration
   207  		// file and it is easy to miss that e.g. the ref is wrong.
   208  		return c, notFoundErr
   209  	}
   210  
   211  	c.Content = content
   212  	return c, nil
   213  }
   214  
   215  func (ld *Loader) loadDefaultConfig(ctx context.Context, client *github.Client, owner string) (Config, error) {
   216  	logger := zerolog.Ctx(ctx)
   217  
   218  	r, _, err := client.Repositories.Get(ctx, owner, ld.defaultRepo)
   219  	if err != nil {
   220  		if isNotFound(err) {
   221  			// if the owner has no default repo, return empty/undefined config
   222  			return Config{}, nil
   223  		}
   224  		c := Config{Source: fmt.Sprintf("%s/%s", owner, ld.defaultRepo)}
   225  		return c, errors.Wrap(err, "failed to get default repository")
   226  	}
   227  
   228  	ref := r.GetDefaultBranch()
   229  	c := Config{
   230  		Source: fmt.Sprintf("%s/%s@%s", owner, r.GetName(), ref),
   231  	}
   232  
   233  	for _, p := range ld.defaultPaths {
   234  		c.Path = p
   235  
   236  		logger.Debug().Msgf("Trying default configuration at %s in %s", c.Path, c.Source)
   237  		content, exists, err := getFileContents(ctx, client, owner, r.GetName(), ref, p)
   238  		if err != nil {
   239  			return c, err
   240  		}
   241  		if !exists {
   242  			continue
   243  		}
   244  
   245  		// if remote refs are enabled, see if the file is a remote reference
   246  		if ld.parser != nil {
   247  			remote, err := ld.parser(p, content)
   248  			if err != nil {
   249  				return c, err
   250  			}
   251  			if remote != nil {
   252  				logger.Debug().Msgf("Found remote default configuration at %s in %s", p, c.Source)
   253  				return ld.loadRemoteConfig(ctx, client, *remote, c)
   254  			}
   255  		}
   256  
   257  		// non-remote content found, don't try any other paths
   258  		logger.Debug().Msgf("Found default configuration at %s in %s", c.Path, c.Source)
   259  		c.Content = content
   260  		return c, nil
   261  	}
   262  
   263  	// no default configuration, return an empty/undefined one
   264  	return Config{}, nil
   265  }
   266  
   267  // getFileContents returns the content of the file at path on ref in owner/repo
   268  // if it exists. Returns an empty slice and false if the file does not exist.
   269  func getFileContents(ctx context.Context, client *github.Client, owner, repo, ref, path string) ([]byte, bool, error) {
   270  	file, _, _, err := client.Repositories.GetContents(ctx, owner, repo, path, &github.RepositoryContentGetOptions{
   271  		Ref: ref,
   272  	})
   273  	if err != nil {
   274  		switch {
   275  		case isNotFound(err):
   276  			return nil, false, nil
   277  		case isTooLargeError(err):
   278  			b, err := getLargeFileContents(ctx, client, owner, repo, ref, path)
   279  			return b, true, err
   280  		}
   281  		return nil, false, errors.Wrap(err, "failed to read file")
   282  	}
   283  
   284  	// file will be nil if the path exists but is a directory
   285  	if file == nil {
   286  		return nil, false, nil
   287  	}
   288  
   289  	content, err := file.GetContent()
   290  	if err != nil {
   291  		return nil, true, errors.Wrap(err, "failed to decode file content")
   292  	}
   293  
   294  	return []byte(content), true, nil
   295  }
   296  
   297  // getLargeFileContents is similar to getFileContents, but works for files up
   298  // to 100MB. Unlike getFileContents, it returns an error if the file does not
   299  // exist.
   300  func getLargeFileContents(ctx context.Context, client *github.Client, owner, repo, ref, path string) ([]byte, error) {
   301  	body, res, err := client.Repositories.DownloadContents(ctx, owner, repo, path, &github.RepositoryContentGetOptions{
   302  		Ref: ref,
   303  	})
   304  	if err != nil {
   305  		return nil, errors.Wrap(err, "failed to read file")
   306  	}
   307  	defer func() {
   308  		_ = body.Close()
   309  	}()
   310  
   311  	if res.StatusCode != http.StatusOK {
   312  		return nil, errors.Errorf("failed to read file: unexpected status code %d", res.StatusCode)
   313  	}
   314  
   315  	b, err := ioutil.ReadAll(body)
   316  	if err != nil {
   317  		return nil, errors.Wrap(err, "failed to read file")
   318  	}
   319  	return b, nil
   320  }
   321  
   322  func isNotFound(err error) bool {
   323  	if rerr, ok := err.(*github.ErrorResponse); ok {
   324  		return rerr.Response.StatusCode == http.StatusNotFound
   325  	}
   326  	return false
   327  }
   328  
   329  func isTooLargeError(err error) bool {
   330  	if rerr, ok := err.(*github.ErrorResponse); ok {
   331  		for _, err := range rerr.Errors {
   332  			if err.Code == "too_large" {
   333  				return true
   334  			}
   335  		}
   336  	}
   337  	return false
   338  }
   339
View as plain text