main.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/scripts/generate-third-party-licenses

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // This program creates two directories in the root directory, THIRD_PARTY_NOTICES and MIRRORED_LIBRARY_SOURCE,
    16  // that contain the licenses of our third-party code and MPL-mandated mirrored library source code.
    17  
    18  package main
    19  
    20  import (
    21  	"fmt"
    22  	"io/ioutil"
    23  	"os"
    24  	"os/exec"
    25  	"path/filepath"
    26  	"strings"
    27  )
    28  
    29  const (
    30  	inputDir                 = "temp-vendor"
    31  	thirdPartyNoticeDir      = "THIRD_PARTY_NOTICES"
    32  	mirroredLibrarySourceDir = "MIRRORED_LIBRARY_SOURCE"
    33  	dirMode                  = 0700
    34  	fileMode                 = 0600
    35  )
    36  
    37  func main() {
    38  	var files []string
    39  	os.RemoveAll(thirdPartyNoticeDir)
    40  	os.RemoveAll(mirroredLibrarySourceDir)
    41  
    42  	// find all the LICENSE files in the vendor directory
    43  	err := filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error {
    44  		if !strings.Contains(path, "LICENSE") && !strings.Contains(path, "LICENCE") {
    45  			return nil
    46  		}
    47  		files = append(files, path)
    48  		return nil
    49  	})
    50  	if err != nil {
    51  		fmt.Printf("error walking vendor directory: %v\n", err)
    52  		os.Exit(1)
    53  	}
    54  
    55  	for _, file := range files {
    56  		licensePath := strings.TrimLeft(file, "temp-vendor/")
    57  		repo, licenseFilename := splitLicensePath(licensePath)
    58  		licenseURL := repoToLicenseURL(repo, licenseFilename)
    59  		fmt.Println(licenseURL)
    60  
    61  		outputFilename := thirdPartyNoticeDir + "/" + licensePath
    62  		outputFileDir := thirdPartyNoticeDir + "/" + repo
    63  		input, err := ioutil.ReadFile(file)
    64  		if err != nil {
    65  			fmt.Println(err)
    66  			os.Exit(1)
    67  		}
    68  		if err := os.MkdirAll(outputFileDir, dirMode); err != nil {
    69  			fmt.Printf("error creating output directory '%v': %v\n", outputFileDir, err)
    70  			os.Exit(1)
    71  		}
    72  
    73  		// copy the license
    74  		if err := ioutil.WriteFile(outputFilename, input, fileMode); err != nil {
    75  			fmt.Println(err)
    76  			os.Exit(1)
    77  		}
    78  
    79  		if licenseRequiresSourceCodeMirroring(input) {
    80  			fmt.Printf("REQUIRES SOURCE MIRRORING: %v\n", file)
    81  			outputSourceDir := mirroredLibrarySourceDir + "/" + repo
    82  
    83  			if err := os.MkdirAll(outputSourceDir, dirMode); err != nil {
    84  				fmt.Printf("error creating output directory '%v': %v\n", outputFileDir, err)
    85  				os.Exit(1)
    86  			}
    87  			// need to remove the actual dir so 'cp' works
    88  			os.Remove(outputSourceDir)
    89  
    90  			sourceDir := "temp-vendor/" + repo
    91  			cmd := exec.Command("cp", "-r", sourceDir, outputSourceDir)
    92  			if output, err := cmd.CombinedOutput(); err != nil {
    93  				fmt.Printf("error copying source code for '%v': %v", sourceDir, string(output))
    94  				os.Exit(1)
    95  			}
    96  		}
    97  	}
    98  }
    99  
   100  func licenseRequiresSourceCodeMirroring(licenseText []byte) bool {
   101  	normalizedText := strings.ToLower(string(licenseText))
   102  	licenseRequiresSourceCodeMirroring := map[string]bool{
   103  		"mozilla public license":                      true,
   104  		"common development and distribution license": true,
   105  		"eclipse public license":                      true,
   106  		"gnu general public license":                  true,
   107  		"lesser general public license":               true,
   108  	}
   109  
   110  	for licenseType := range licenseRequiresSourceCodeMirroring {
   111  		if strings.Contains(normalizedText, licenseType) {
   112  			return true
   113  		}
   114  	}
   115  	return false
   116  }
   117  
   118  func splitLicensePath(path string) (repo string, licenseFilename string) {
   119  	splitPath := strings.Split(path, "/")
   120  	repo = strings.Join(splitPath[:len(splitPath)-1], "/")
   121  	licenseFilename = splitPath[len(splitPath)-1]
   122  	return repo, licenseFilename
   123  }
   124  
   125  func repoToLicenseURL(repo string, licenseFilename string) string {
   126  	if manualLicenseURLMapping[repo] != "" {
   127  		return manualLicenseURLMapping[repo]
   128  	}
   129  	domain, repoRoot, subrepoPath := splitRepo(repo)
   130  
   131  	licensePathInRepo := licenseFilename
   132  	if subrepoPath != "" {
   133  		licensePathInRepo = strings.Join([]string{subrepoPath, licenseFilename}, "/")
   134  	}
   135  
   136  	// TODO: instead of assuming "blob/master", link to the specific SHA we use
   137  	switch domain {
   138  	case "cloud.google.com":
   139  		splitRepoRoot := strings.Split(repoRoot, "/")
   140  		if len(splitRepoRoot) == 2 && splitRepoRoot[0] == "go" {
   141  			return fmt.Sprintf("https://github.com/googleapis/google-cloud-go/blob/master/LICENSE")
   142  		} else {
   143  			panic(fmt.Sprintf("unrecognized repo under cloud.google.com: %v", repoRoot))
   144  		}
   145  	case "sigs.k8s.io":
   146  		return fmt.Sprintf("https://github.com/kubernetes-sigs/%v/blob/master/%v", repoRoot, licensePathInRepo)
   147  	case "github.com":
   148  		return fmt.Sprintf("https://github.com/%v/blob/master/%v", repoRoot, licensePathInRepo)
   149  	case "golang.org":
   150  		if !strings.HasPrefix(repoRoot, "x/") {
   151  			panic(fmt.Sprintf("unhandled domain for repo %v", repo))
   152  		}
   153  		newRepoRoot := strings.TrimLeft(repoRoot, "x/")
   154  		if newRepoRoot == "tools" && strings.Contains(licensePathInRepo, "third_party") {
   155  			// This SHA still contains the licenses for the third_party dir
   156  			return fmt.Sprintf("https://github.com/golang/tools/blob/7414d4c1f71cec71978b1aec0539171a2e42d230/%v", licensePathInRepo)
   157  		} else {
   158  			return fmt.Sprintf("https://github.com/golang/%v/blob/master/%v", newRepoRoot, licensePathInRepo)
   159  		}
   160  	case "k8s.io":
   161  		return fmt.Sprintf("https://github.com/kubernetes/%v/blob/master/%v", repoRoot, licensePathInRepo)
   162  	case "go.uber.org":
   163  		return fmt.Sprintf("https://github.com/uber-go/%v/blob/master/%v", repoRoot, licensePathInRepo)
   164  	case "gopkg.in":
   165  		switch repoRoot {
   166  		case "yaml.v2":
   167  			return fmt.Sprintf("https://github.com/go-yaml/yaml/blob/v2.2.2/%v", licensePathInRepo)
   168  		case "yaml.v3":
   169  			return fmt.Sprintf("https://github.com/go-yaml/yaml/blob/v3/%v", licensePathInRepo)
   170  		default:
   171  			panic(fmt.Sprintf("unhandled domain for repo %v", repo))
   172  		}
   173  	case "go.opencensus.io":
   174  		return fmt.Sprintf("https://github.com/census-instrumentation/opencensus-go/blob/master/%v", licensePathInRepo)
   175  	case "honnef.co":
   176  		return fmt.Sprintf("https://github.com/dominikh/go-tools/blob/master/%v", licensePathInRepo)
   177  	default:
   178  		panic(fmt.Sprintf("unhandled domain for repo %v", repo))
   179  	}
   180  }
   181  
   182  func splitRepo(repo string) (domain string, repoRoot string, subrepoPath string) {
   183  	splitRepo := strings.Split(repo, "/")
   184  	domain = splitRepo[0]
   185  	if len(splitRepo) == 2 {
   186  		repoRoot = splitRepo[1]
   187  	} else if len(splitRepo) > 2 {
   188  		repoRoot = strings.Join(splitRepo[1:3], "/")
   189  	}
   190  	subrepoPath = ""
   191  	if len(splitRepo) > 3 {
   192  		subrepoPath = strings.Join(splitRepo[3:], "/")
   193  	}
   194  	return domain, repoRoot, subrepoPath
   195  }
   196  
   197  var manualLicenseURLMapping = map[string]string{
   198  	"bitbucket.org/creachadair/stringset":                     "https://bitbucket.org/creachadair/stringset/src/master/LICENSE",
   199  	"cloud.google.com/go":                                     "https://github.com/googleapis/google-cloud-go/blob/master/LICENSE",
   200  	"contrib.go.opencensus.io/exporter/prometheus":            "https://github.com/census-ecosystem/opencensus-go-exporter-prometheus/blob/master/LICENSE",
   201  	"go.starlark.net":                                         "https://github.com/google/starlark-go/blob/master/LICENSE",
   202  	"gomodules.xyz/jsonpatch/v2":                              "https://https://github.com/gomodules/jsonpatch/blob/master/LICENSE",
   203  	"google.golang.org/api":                                   "https://github.com/googleapis/google-api-go-client/blob/master/LICENSE",
   204  	"google.golang.org/api/googleapi/internal/uritemplates":   "https://github.com/googleapis/google-api-go-client/blob/master/googleapi/internal/uritemplates/LICENSE",
   205  	"google.golang.org/api/internal/third_party/uritemplates": "https://github.com/googleapis/google-api-go-client/blob/master/internal/third_party/uritemplates/LICENSE",
   206  	"google.golang.org/appengine":                             "https://github.com/golang/appengine/blob/master/LICENSE",
   207  	"google.golang.org/genproto":                              "https://github.com/google/go-genproto/blob/master/LICENSE",
   208  	"google.golang.org/grpc":                                  "https://github.com/grpc/grpc-go/blob/master/LICENSE",
   209  	"google.golang.org/protobuf":                              "https://github.com/protocolbuffers/protobuf-go/blob/master/LICENSE",
   210  	"gopkg.in/fsnotify.v1":                                    "https://github.com/fsnotify/fsnotify/blob/master/LICENSE",
   211  	"gopkg.in/inf.v0":                                         "https://github.com/go-inf/inf/blob/master/LICENSE",
   212  	"gopkg.in/tomb.v1":                                        "https://github.com/go-tomb/tomb/blob/v1/LICENSE",
   213  	"gopkg.in/vmihailenco/msgpack.v4":                         "https://github.com/vmihailenco/msgpack/blob/master/LICENSE",
   214  	"gopkg.in/warnings.v0":                                    "https://github.com/go-warnings/warnings/blob/master/LICENSE",
   215  	"honnef.co/go/tools":                                      "https://github.com/dominikh/go-tools/blob/master/LICENSE",
   216  	"honnef.co/go/tools/lint":                                 "https://github.com/dominikh/go-tools/blob/master/lint/LICENSE",
   217  	"honnef.co/go/tools/ssa":                                  "https://github.com/dominikh/go-tools/blob/master/ssa/LICENSE",
   218  }
   219
View as plain text