update.go

Documentation: edge-infra.dev/hack/tools/apko-updater-bot/cmd/update

     1  package update
     2  
     3  import (
     4  	"context"
     5  	"crypto/sha1" //nolint:gosec // not used for security. git content hashing
     6  	"errors"
     7  	"fmt"
     8  	"io/fs"
     9  	"os"
    10  	"os/exec"
    11  	"strings"
    12  
    13  	"github.com/google/go-github/v47/github"
    14  
    15  	"edge-infra.dev/hack/tools/apko-updater-bot/internal"
    16  	githubclient "edge-infra.dev/pkg/f8n/devinfra/github-client"
    17  	"edge-infra.dev/pkg/f8n/devinfra/github/ghfs"
    18  	"edge-infra.dev/pkg/lib/build/apko"
    19  	"edge-infra.dev/pkg/lib/cli/rags"
    20  	"edge-infra.dev/pkg/lib/cli/sink"
    21  )
    22  
    23  const (
    24  	apkoUpdateCommitBranchName = "chore/apko-lock-bot-update-files"
    25  	commitMessageTemplate      = "apko.lock.json files updated by the apko-lock-bot:\n"
    26  	prTitle                    = "chore(build): Update rules_apko apko.lock.json files"
    27  )
    28  
    29  func New() *sink.Command {
    30  	var (
    31  		configPath       string
    32  		repository       string
    33  		owner            string
    34  		baseBranch       string
    35  		mergeBranch      string
    36  		commitBranch     string
    37  		apkoLocations    string
    38  		dryRun           bool
    39  		ghPrivateKeyPath string
    40  		ghPrivateKey     string
    41  		ghAppID          int64
    42  		ghInstallationID int64
    43  	)
    44  
    45  	cmd := &sink.Command{
    46  		Use:   "update [flags]",
    47  		Short: "Update the apko.lock.json files based on directories passed in containing apko.yaml and lock file pairs",
    48  		Flags: []*rags.Rag{
    49  			{
    50  				Name:  "config-path",
    51  				Usage: "path to a configuration .yaml file. this configuration overrides all passed flags",
    52  				Value: &rags.String{
    53  					Var: &configPath,
    54  				},
    55  			},
    56  			{
    57  				Name:  "owner",
    58  				Usage: "the owner of the repository in which to perform the PR",
    59  				Value: &rags.String{
    60  					Var:     &owner,
    61  					Default: "ncrvoyix-swt-retail",
    62  				},
    63  			},
    64  			{
    65  				Name:  "repository",
    66  				Usage: "the repository in which to perform the PR",
    67  				Value: &rags.String{
    68  					Var:     &repository,
    69  					Default: "edge-infra",
    70  				},
    71  			},
    72  			{
    73  				Name:  "base-branch",
    74  				Usage: "the base branch to perform the PR against",
    75  				Value: &rags.String{
    76  					Var:     &baseBranch,
    77  					Default: "master",
    78  				},
    79  			},
    80  			{
    81  				Name:  "merge-branch",
    82  				Usage: "name of the branch to perform the PR against",
    83  				Value: &rags.String{
    84  					Var:     &mergeBranch,
    85  					Default: "master",
    86  				},
    87  			},
    88  			{
    89  				Name:  "commit-branch",
    90  				Usage: "the commit branch to merge in",
    91  				Value: &rags.String{
    92  					Var:     &commitBranch,
    93  					Default: apkoUpdateCommitBranchName,
    94  				},
    95  			},
    96  			{
    97  				Name:  "apko-locations",
    98  				Usage: "a comma-separated list of directories that contain apko.yaml and apko.lock.json files",
    99  				Value: &rags.String{Var: &apkoLocations},
   100  			},
   101  			{
   102  				Name:  "dry-run",
   103  				Usage: "dry run the commit and PR",
   104  				Value: &rags.Bool{Var: &dryRun, Default: false},
   105  			},
   106  			{
   107  				Name:  "gh-app-private-key-path",
   108  				Usage: "private key path for the Github app - mutually exclusive with gh-app-private-key",
   109  				Value: &rags.String{Var: &ghPrivateKeyPath},
   110  			},
   111  			{
   112  				Name:  "gh-app-private-key",
   113  				Usage: "the value of the Github app private key - mutually exclusive with gh-app-private-key-path",
   114  				Value: &rags.String{Var: &ghPrivateKey},
   115  			},
   116  			{
   117  				Name:  "gh-app-id",
   118  				Usage: "id of the Github app",
   119  				Value: &rags.Int64{Var: &ghAppID},
   120  			},
   121  			{
   122  				Name:  "gh-installation-id",
   123  				Usage: "id of the Github app",
   124  				Value: &rags.Int64{Var: &ghInstallationID},
   125  			},
   126  		},
   127  		Exec: func(ctx context.Context, r sink.Run) error {
   128  			if configPath != "" {
   129  				r.Log.Info(fmt.Sprintf("loading values from config at path %s", configPath))
   130  				updaterConfig, err := internal.NewConfigFromPath(configPath)
   131  				if err != nil {
   132  					r.Log.Error(err, "error creating config")
   133  					return err
   134  				}
   135  
   136  				owner = updaterConfig.Owner
   137  				repository = updaterConfig.Repository
   138  				baseBranch = updaterConfig.BaseBranch
   139  				mergeBranch = updaterConfig.MergeBranch
   140  				commitBranch = updaterConfig.CommitBranch
   141  				apkoLocations = strings.Join(updaterConfig.ApkoLocations, ",")
   142  				dryRun = updaterConfig.DryRun
   143  				ghPrivateKey = updaterConfig.GHPrivateKey
   144  				ghPrivateKeyPath = updaterConfig.GHPrivateKeyPath
   145  				ghAppID = updaterConfig.GHAppID
   146  				ghInstallationID = updaterConfig.GHInstallationID
   147  			}
   148  
   149  			privateKeyBytes, privateKeyErr := getPrivateKey(ghPrivateKeyPath, ghPrivateKey)
   150  			if privateKeyErr != nil {
   151  				return privateKeyErr
   152  			}
   153  
   154  			repoURL := fmt.Sprintf("https://github.com/%s/%s", owner, repository)
   155  			config := githubclient.Config{
   156  				Repository:     repoURL,
   157  				AppID:          ghAppID,
   158  				InstallationID: ghInstallationID,
   159  				PrivateKey:     privateKeyBytes,
   160  			}
   161  
   162  			client, err := githubclient.NewClient(config)
   163  			if err != nil {
   164  				return err
   165  			}
   166  			if client.Client == nil {
   167  				return fmt.Errorf("the Github client.Client is nil")
   168  			}
   169  
   170  			r.Log.Info("authenticated with Github app")
   171  
   172  			r.Log.Info("creating Github FS for base branch")
   173  			baseFS, err := ghfs.New(ctx, client.Client, owner, repository, baseBranch)
   174  			if err != nil {
   175  				return err
   176  			}
   177  
   178  			r.Log.Info("creating Github FS for commit branch")
   179  			commitBranchFS, commitBranchFSNotFoundErr := ghfs.New(ctx, client.Client, owner, repository, commitBranch)
   180  			if commitBranchFSNotFoundErr != nil {
   181  				if commitBranchFSNotFoundErr == commitBranchFSNotFoundErr.(ghfs.RefNotFoundError) {
   182  					r.Log.Info(fmt.Sprintf("no branch %s available at origin - will create at PR creation time", commitBranch))
   183  				} else {
   184  					return commitBranchFSNotFoundErr
   185  				}
   186  			}
   187  
   188  			var updateFS *ghfs.FS
   189  			var fsBranch string
   190  			if commitBranchFSNotFoundErr == nil {
   191  				r.Log.Info("using the commit branch FS")
   192  				updateFS = commitBranchFS
   193  				fsBranch = commitBranch
   194  			} else {
   195  				r.Log.Info("using the base branch FS")
   196  				updateFS = baseFS
   197  				fsBranch = baseBranch
   198  			}
   199  
   200  			// Get the apko.yamls to lock and check locks for drift
   201  			apkoDirs := strings.Split(apkoLocations, ",")
   202  			apkoPairs := []internal.APKOFilePair{}
   203  
   204  			// Create the temp files
   205  			for _, apkoDir := range apkoDirs {
   206  				r.Log.Info(fmt.Sprintf("reading %s from the FS", apkoDir))
   207  				pair, err := internal.APKOPathToFilePair(updateFS, apkoDir)
   208  				if err != nil && errors.Unwrap(err) == fs.ErrNotExist {
   209  					r.Log.Info(fmt.Sprintf("warning: %s does not exist in the repo - skipping", apkoDir))
   210  					continue
   211  				} else if err != nil {
   212  					return err
   213  				}
   214  
   215  				apkoPairs = append(apkoPairs, pair)
   216  			}
   217  
   218  			// Defer deletion to keep disk tidy
   219  			for _, pair := range apkoPairs {
   220  				defer os.RemoveAll(pair.APKOLocalTempDir)
   221  			}
   222  
   223  			commitBody := commitMessageTemplate
   224  			// Run apko lock on the temp apko.yaml and see what happens
   225  			apkoEntries := []*github.TreeEntry{}
   226  			for _, pair := range apkoPairs {
   227  				apkoEntries, err = loadAPKOTreeEntries(r, pair, apkoEntries)
   228  				if err != nil {
   229  					return err
   230  				}
   231  				commitBody = fmt.Sprintf("%supdated %s\n", commitBody, pair.APKOLockPath)
   232  			}
   233  
   234  			if len(apkoEntries) == 0 {
   235  				r.Log.Info("all apkos for branch are up to date - exiting")
   236  				return nil
   237  			}
   238  
   239  			if dryRun {
   240  				r.Log.Info("dry-run is set, exiting")
   241  				return nil
   242  			}
   243  
   244  			var updateRef *github.Reference
   245  			// If fsBranch == commitRef, get the ref
   246  			switch fsBranch {
   247  			case commitBranch:
   248  				r.Log.Info("getting the upstream in-flight commit branch")
   249  				updateRef, _, err = client.Git.GetRef(ctx, owner, repository, fmt.Sprintf("refs/heads/%s", commitBranch))
   250  				if err != nil {
   251  					return err
   252  				}
   253  			// If fsBranch == baseBranch, need to create ref
   254  			case baseBranch:
   255  				r.Log.Info("creating the commit branch based on the base branch")
   256  				baseRef, _, err := client.Git.GetRef(ctx, owner, repository, fmt.Sprintf("refs/heads/%s", baseBranch))
   257  				if err != nil {
   258  					return err
   259  				}
   260  				newRef := &github.Reference{Ref: github.String(fmt.Sprintf("refs/heads/%s", commitBranch)), Object: &github.GitObject{SHA: baseRef.Object.SHA}}
   261  				updateRef, _, err = client.Git.CreateRef(ctx, owner, repository, newRef)
   262  				if err != nil {
   263  					return err
   264  				}
   265  			default:
   266  				return fmt.Errorf("update branch unknown state")
   267  			}
   268  
   269  			// Push the commit
   270  			r.Log.Info("pushing commit", updateRef.Object.GetSHA())
   271  			err = pushCommit(
   272  				ctx,
   273  				client.Client,
   274  				updateRef,
   275  				apkoEntries,
   276  				owner,
   277  				repository,
   278  				commitBody,
   279  			)
   280  			if err != nil {
   281  				return err
   282  			}
   283  
   284  			prOpts := &github.PullRequestListOptions{Head: fmt.Sprintf("%s:%s", owner, updateRef.GetRef())}
   285  			prList, _, err := client.PullRequests.List(ctx, owner, repository, prOpts)
   286  			if len(prList) == 0 {
   287  				r.Log.Info(fmt.Sprintf("PR for %s does not exist yet - creating", commitBranch))
   288  				pr, err := createPR(ctx, client.Client, owner, repository, mergeBranch, commitBranch, prTitle, commitBody)
   289  				r.Log.Info(fmt.Sprintf("created pr %d", pr.GetNumber()))
   290  				r.Log.Info(pr.GetHTMLURL())
   291  				if err != nil {
   292  					return err
   293  				}
   294  			} else if err != nil {
   295  				r.Log.Error(err, "err from list PRs")
   296  				return err
   297  			} else {
   298  				for _, pr := range prList {
   299  					if pr.Head.GetRef() == updateRef.String() {
   300  						r.Log.Info(fmt.Sprintf("found existing PR #%d for branch %s", pr.GetNumber(), commitBranch))
   301  						r.Log.Info(pr.GetHTMLURL())
   302  					}
   303  				}
   304  			}
   305  
   306  			return nil
   307  		},
   308  	}
   309  	return cmd
   310  }
   311  
   312  // BlobSHA1 calculates the Git SHA1 hash value for the provided content. Git prepends info to the content before taking
   313  // the SHA1 hash which is why you must call this function when working with blob SHA1 values.
   314  func BlobSHA1(content string) string {
   315  	const (
   316  		null   byte = 0
   317  		format      = "blob %d%c%s"
   318  	)
   319  	var blob = fmt.Sprintf(format, len(content), null, content)
   320  	return fmt.Sprintf("%x", sha1.Sum([]byte(blob))) //nolint:gosec // not used for security
   321  }
   322  
   323  // Takes an APKO pair and runs lock on the apko.yaml
   324  // If the lock needs to be updated, the passed pair will be locked, converted into a
   325  // TreeEntry, and the slice of TreeEntries with the new entry will be passed back
   326  // If the lock does not need to be updated, the original entry slice is returned
   327  func loadAPKOTreeEntries(r sink.Run, pair internal.APKOFilePair, entries []*github.TreeEntry) ([]*github.TreeEntry, error) {
   328  	cmd := exec.Command("apko")
   329  	args := []string{"lock", pair.APKOFileLocalPath}
   330  	cmd.Args = append(cmd.Args, args...)
   331  	r.Log.Info(fmt.Sprintf("running apko lock on %s", pair.APKOPath))
   332  	cmd.Stdout = r.Out()
   333  	cmd.Stderr = r.Err()
   334  
   335  	err := cmd.Run()
   336  	if err != nil {
   337  		return nil, err
   338  	}
   339  
   340  	newLockBytes, err := os.ReadFile(fmt.Sprintf("%s/%s", pair.APKOLocalTempDir, apko.APKOLockFileName))
   341  	if err != nil {
   342  		return nil, err
   343  	}
   344  	baseLockSha := BlobSHA1(string(pair.APKOLockFileContent))
   345  	newLockSha := BlobSHA1(string(newLockBytes))
   346  	r.Log.Info(fmt.Sprintf("comparing %s to fresh %s", pair.APKOFileLocalPath, apko.APKOLockFileName))
   347  
   348  	if baseLockSha == newLockSha {
   349  		r.Log.Info(fmt.Sprintf("lock for %s matches - no changes required", pair.APKOPath))
   350  		return entries, nil
   351  	}
   352  
   353  	r.Log.Info(fmt.Sprintf("lock file for %s has drifted - committing to tree", pair.APKOPath))
   354  	r.Log.Info(fmt.Sprintf("origin sha %s did not match new lock sha %s", baseLockSha, newLockSha))
   355  	apkoTreeEntry := &github.TreeEntry{
   356  		Path:    github.String(pair.APKOLockPath),
   357  		Type:    github.String("blob"),
   358  		Mode:    github.String(pair.APKOLockFileMode),
   359  		Content: github.String(string(newLockBytes)),
   360  	}
   361  	entries = append(entries, apkoTreeEntry)
   362  
   363  	return entries, nil
   364  }
   365  
   366  func pushCommit(ctx context.Context, client *github.Client, updateRef *github.Reference, entries []*github.TreeEntry, owner string, repository string, commitBody string) error {
   367  	parent, _, err := client.Repositories.GetCommit(ctx, owner, repository, *updateRef.Object.SHA, nil)
   368  	if err != nil {
   369  		return err
   370  	}
   371  	// Not always populated? Copied from example code
   372  	parent.Commit.SHA = parent.SHA
   373  
   374  	tree, _, err := client.Git.CreateTree(ctx, owner, repository, *updateRef.Object.SHA, entries)
   375  	if err != nil {
   376  		return err
   377  	}
   378  
   379  	// Commit author is pulled from the installation token used during the client creation
   380  	commit := &github.Commit{Message: github.String(commitBody), Tree: tree, Parents: []*github.Commit{parent.Commit}}
   381  
   382  	newCommit, _, err := client.Git.CreateCommit(ctx, owner, repository, commit)
   383  	if err != nil {
   384  		return err
   385  	}
   386  
   387  	updateRef.Object.SHA = newCommit.SHA
   388  	_, _, err = client.Git.UpdateRef(ctx, owner, repository, updateRef, false)
   389  	return err
   390  }
   391  
   392  func createPR(ctx context.Context, client *github.Client, owner string, repo string, mergeBranch string, commitBranch string, prTitle string, prBody string) (*github.PullRequest, error) {
   393  	newPR := &github.NewPullRequest{
   394  		Title:               github.String(prTitle),
   395  		Head:                github.String(commitBranch),
   396  		Base:                github.String(mergeBranch),
   397  		Body:                github.String(prBody),
   398  		MaintainerCanModify: github.Bool(true),
   399  	}
   400  
   401  	pr, _, err := client.PullRequests.Create(ctx, owner, repo, newPR)
   402  	if err != nil {
   403  		return nil, err
   404  	}
   405  	return pr, nil
   406  }
   407  
   408  func getPrivateKey(keyPath string, key string) ([]byte, error) {
   409  	if key != "" && keyPath != "" {
   410  		return nil, fmt.Errorf("gh-app-private-key-path and gh-app-private-key cannot both be set")
   411  	}
   412  
   413  	if key == "" && keyPath == "" {
   414  		return nil, fmt.Errorf("either gh-app-private-key or gh-app-private-key-path must be set for Github app authentication")
   415  	}
   416  
   417  	var ghPrivateKeyPathMissingErr error
   418  	var privateKeyBytes []byte
   419  	if keyPath != "" {
   420  		privateKeyBytes, ghPrivateKeyPathMissingErr = os.ReadFile(keyPath)
   421  		if ghPrivateKeyPathMissingErr != nil {
   422  			return nil, ghPrivateKeyPathMissingErr
   423  		}
   424  	} else if key != "" {
   425  		privateKeyBytes = []byte(key)
   426  	}
   427  
   428  	return privateKeyBytes, nil
   429  }
   430
View as plain text