#!/usr/bin/env python3 # Copyright 2015 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import datetime import difflib import glob import os import re import sys parser = argparse.ArgumentParser() parser.add_argument( "filenames", help="list of files to check, all files if unspecified", nargs="*" ) rootdir = os.path.dirname(__file__) + "/../../" rootdir = os.path.abspath(rootdir) parser.add_argument("--rootdir", default=rootdir, help="root directory to examine") default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) parser.add_argument( "-v", "--verbose", help="give verbose output regarding why a file does not pass", action="store_true", ) args = parser.parse_args() verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") def get_refs(): refs = {} for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): extension = os.path.basename(path).split(".")[1] with open(path, "r") as ref_file: refs[extension] = ref_file.read().splitlines() return refs def is_generated_file(data, regexs): return regexs["generated"].search(data) def file_passes(filename, refs, regexs): try: with open(filename) as stream: data = stream.read() except OSError as exc: print(f"Unable to open {filename}: {exc}", file=verbose_out) return False # determine if the file is automatically generated generated = is_generated_file(data, regexs) basename = os.path.basename(filename) extension = file_extension(filename) if generated: if extension == "go": extension = "generatego" if extension != "": ref = refs[extension] else: ref = refs[basename] # remove extra content from the top of files if extension in ("go", "generatego"): data, found = regexs["go_build_constraints"].subn("", data, 1) elif extension in ["sh", "py"]: data, found = regexs["shebang"].subn("", data, 1) data = data.splitlines() # if our test file is smaller than the reference it surely fails! if len(ref) > len(data): print( f"File {filename} smaller than reference ({len(data)} < {len(ref)})", file=verbose_out, ) return False # trim our file to the same number of lines as the reference file data = data[: len(ref)] pattern = regexs["year"] for line in data: if pattern.search(line): if generated: print( f"File {filename} has the YEAR field, but it should not be in generated file", file=verbose_out, ) else: print( "File {filename} has the YEAR field, but missing the year of date", file=verbose_out, ) return False if not generated: # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" pattern = regexs["date"] for i, line in enumerate(data): data[i], found = pattern.subn("YEAR", line) if found != 0: break # if we don't match the reference at this point, fail if ref != data: print(f"Header in {filename} does not match reference, diff:", file=verbose_out) if args.verbose: print(file=verbose_out) for line in difflib.unified_diff( ref, data, "reference", filename, lineterm="" ): print(line, file=verbose_out) print(file=verbose_out) return False return True def file_extension(filename): return os.path.splitext(filename)[1].split(".")[-1].lower() skipped_names = [ "third_party", "_output", ".git", "cluster/env.sh", "vendor", "testdata", "test/e2e/generated/bindata.go", "hack/boilerplate/test", "staging/src/k8s.io/kubectl/pkg/generated/bindata.go", ] def normalize_files(files): newfiles = [] for pathname in files: if any(x in pathname for x in skipped_names): continue newfiles.append(pathname) for i, pathname in enumerate(newfiles): if not os.path.isabs(pathname): newfiles[i] = os.path.join(args.rootdir, pathname) return newfiles def get_files(extensions): files = [] if len(args.filenames) > 0: files = args.filenames else: for root, dirs, walkfiles in os.walk(args.rootdir): # don't visit certain dirs. This is just a performance improvement # as we would prune these later in normalize_files(). But doing it # cuts down the amount of filesystem walking we do and cuts down # the size of the file list for dname in skipped_names: if dname in dirs: dirs.remove(dname) for dname in dirs: # dirs that start with __ are ignored if dname.startswith("__"): dirs.remove(dname) for name in walkfiles: pathname = os.path.join(root, name) files.append(pathname) files = normalize_files(files) outfiles = [] for pathname in files: basename = os.path.basename(pathname) extension = file_extension(pathname) if extension in extensions or basename in extensions: outfiles.append(pathname) return outfiles def get_dates(): years = datetime.datetime.now().year return "(%s)" % "|".join(str(year) for year in range(2014, years + 1)) def get_regexs(): regexs = {} # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing regexs["year"] = re.compile("YEAR") # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year # as a regex like: "(2014|2015|2016|2017|2018)"; # company holder names can be anything regexs["date"] = re.compile(get_dates()) # strip the following build constraints/tags: # //go:build # // +build \n\n regexs["go_build_constraints"] = re.compile( r"^(//(go:build| \+build).*\n)+\n", re.MULTILINE ) # strip #!.* from scripts regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) # Search for generated files regexs["generated"] = re.compile(r"^[/*#]+ +.* DO NOT EDIT\.$", re.MULTILINE) return regexs def main(): regexs = get_regexs() refs = get_refs() filenames = get_files(refs) for filename in filenames: if not file_passes(filename, refs, regexs): print(filename) return 0 if __name__ == "__main__": sys.exit(main())