// Copyright 2018 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "encoding/csv" "errors" "flag" "fmt" "io/ioutil" "log" "os" "os/exec" "path/filepath" "strings" "text/template" "time" ) var programName = filepath.Base(os.Args[0]) type substitutions struct { RulesGoDir string } type serverState int const ( asleep serverState = iota awake ) type cleanState int const ( clean cleanState = iota incr ) type benchmark struct { desc string serverState serverState cleanState cleanState incrFile string targets []string result time.Duration } var benchmarks = []benchmark{ { desc: "hello_asleep_clean", serverState: asleep, cleanState: clean, targets: []string{"//:hello"}, }, { desc: "hello_awake_clean", serverState: awake, cleanState: clean, targets: []string{"//:hello"}, }, { desc: "hello_asleep_incr", serverState: asleep, cleanState: incr, incrFile: "hello.go", targets: []string{"//:hello"}, }, { desc: "hello_awake_incr", serverState: awake, cleanState: incr, incrFile: "hello.go", targets: []string{"//:hello"}, }, { desc: "popular_repos_awake_clean", serverState: awake, cleanState: clean, targets: []string{"@io_bazel_rules_go//tests/integration/popular_repos:all"}, }, // TODO: more substantial Kubernetes targets } func main() { log.SetFlags(0) log.SetPrefix(programName + ": ") if err := run(os.Args[1:]); err != nil { log.Fatal(err) } } func run(args []string) error { fs := flag.NewFlagSet(programName, flag.ExitOnError) var rulesGoDir, outPath string fs.StringVar(&rulesGoDir, "rules_go_dir", "", "directory where rules_go is checked out") fs.StringVar(&outPath, "out", "", "csv file to append results to") var keep bool fs.BoolVar(&keep, "keep", false, "if true, the workspace directory won't be deleted at the end") if err := fs.Parse(args); err != nil { return err } if rulesGoDir == "" { return errors.New("-rules_go_dir not set") } if abs, err := filepath.Abs(rulesGoDir); err != nil { return err } else { rulesGoDir = abs } if outPath == "" { return errors.New("-out not set") } if abs, err := filepath.Abs(outPath); err != nil { return err } else { outPath = abs } commit, err := getCommit(rulesGoDir) if err != nil { return err } dir, err := setupWorkspace(rulesGoDir) if err != nil { return err } if !keep { defer cleanupWorkspace(dir) } bazelVersion, err := getBazelVersion() if err != nil { return err } log.Printf("running benchmarks in %s", dir) targetSet := make(map[string]bool) for _, b := range benchmarks { for _, t := range b.targets { targetSet[t] = true } } allTargets := make([]string, 0, len(targetSet)) for t := range targetSet { allTargets = append(allTargets, t) } fetch(allTargets) for i := range benchmarks { b := &benchmarks[i] log.Printf("running benchmark %d/%d: %s", i+1, len(benchmarks), b.desc) if err := runBenchmark(b); err != nil { return fmt.Errorf("error running benchmark %s: %v", b.desc, err) } } log.Printf("writing results to %s", outPath) return recordResults(outPath, time.Now().UTC(), bazelVersion, commit, benchmarks) } func getCommit(rulesGoDir string) (commit string, err error) { wd, err := os.Getwd() if err != nil { return "", err } if err := os.Chdir(rulesGoDir); err != nil { return "", err } defer func() { if cderr := os.Chdir(wd); cderr != nil { if err != nil { err = cderr } } }() out, err := exec.Command("git", "rev-parse", "HEAD").Output() if err != nil { return "", err } outStr := strings.TrimSpace(string(out)) if len(outStr) < 7 { return "", errors.New("git output too short") } return outStr[:7], nil } func setupWorkspace(rulesGoDir string) (workspaceDir string, err error) { workspaceDir, err = ioutil.TempDir("", "bazel_benchmark") if err != nil { return "", err } defer func() { if err != nil { os.RemoveAll(workspaceDir) } }() benchmarkDir := filepath.Join(rulesGoDir, "go", "tools", "bazel_benchmark") files, err := ioutil.ReadDir(benchmarkDir) if err != nil { return "", err } substitutions := substitutions{ RulesGoDir: filepath.Join(benchmarkDir, "..", "..", ".."), } for _, f := range files { name := f.Name() if filepath.Ext(name) != ".in" { continue } srcPath := filepath.Join(benchmarkDir, name) tpl, err := template.ParseFiles(srcPath) if err != nil { return "", err } dstPath := filepath.Join(workspaceDir, name[:len(name)-len(".in")]) out, err := os.Create(dstPath) if err != nil { return "", err } if err := tpl.Execute(out, substitutions); err != nil { out.Close() return "", err } if err := out.Close(); err != nil { return "", err } } if err := os.Chdir(workspaceDir); err != nil { return "", err } return workspaceDir, nil } func cleanupWorkspace(dir string) error { if err := logBazelCommand("clean", "--expunge"); err != nil { return err } return os.RemoveAll(dir) } func getBazelVersion() (string, error) { out, err := exec.Command("bazel", "version").Output() if err != nil { return "", err } prefix := []byte("Build label: ") i := bytes.Index(out, prefix) if i < 0 { return "", errors.New("could not find bazel version in output") } out = out[i+len(prefix):] i = bytes.IndexByte(out, '\n') if i >= 0 { out = out[:i] } return string(out), nil } func fetch(targets []string) error { return logBazelCommand("fetch", targets...) } func runBenchmark(b *benchmark) error { switch b.cleanState { case clean: if err := logBazelCommand("clean"); err != nil { return err } case incr: if err := logBazelCommand("build", b.targets...); err != nil { return err } if b.incrFile == "" { return errors.New("incrFile not set") } data, err := ioutil.ReadFile(b.incrFile) if err != nil { return err } data = bytes.Replace(data, []byte("INCR"), []byte("INCR."), -1) if err := ioutil.WriteFile(b.incrFile, data, 0666); err != nil { return err } } if b.serverState == asleep { if err := logBazelCommand("shutdown"); err != nil { return err } } start := time.Now() if err := logBazelCommand("build", b.targets...); err != nil { return err } b.result = time.Since(start) return nil } func recordResults(outPath string, t time.Time, bazelVersion, commit string, benchmarks []benchmark) (err error) { // TODO(jayconrod): update the header if new columns are added. columnMap, outExists, err := buildColumnMap(outPath, benchmarks) header := buildHeader(columnMap) record := buildRecord(t, bazelVersion, commit, benchmarks, columnMap) defer func() { if err != nil { log.Printf("error writing results: %s: %v", outPath, err) log.Print("data are printed below") log.Print(strings.Join(header, ",")) log.Print(strings.Join(record, ",")) } }() outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) if err != nil { return err } defer func() { if cerr := outFile.Close(); err != nil { err = cerr } }() outCsv := csv.NewWriter(outFile) if !outExists { outCsv.Write(header) } outCsv.Write(record) outCsv.Flush() return outCsv.Error() } func logBazelCommand(command string, args ...string) error { args = append([]string{command}, args...) cmd := exec.Command("bazel", args...) log.Printf("bazel %s\n", strings.Join(args, " ")) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr return cmd.Run() } func buildColumnMap(outPath string, benchmarks []benchmark) (columnMap map[string]int, outExists bool, err error) { columnMap = make(map[string]int) { inFile, oerr := os.Open(outPath) if oerr != nil { goto doneReading } outExists = true defer inFile.Close() inCsv := csv.NewReader(inFile) var header []string header, err = inCsv.Read() if err != nil { goto doneReading } for i, column := range header { columnMap[column] = i } } doneReading: for _, s := range []string{"time", "bazel_version", "commit"} { if _, ok := columnMap[s]; !ok { columnMap[s] = len(columnMap) } } for _, b := range benchmarks { if _, ok := columnMap[b.desc]; !ok { columnMap[b.desc] = len(columnMap) } } return columnMap, outExists, err } func buildHeader(columnMap map[string]int) []string { header := make([]string, len(columnMap)) for name, i := range columnMap { header[i] = name } return header } func buildRecord(t time.Time, bazelVersion, commit string, benchmarks []benchmark, columnMap map[string]int) []string { record := make([]string, len(columnMap)) record[columnMap["time"]] = t.Format("2006-01-02 15:04:05") record[columnMap["bazel_version"]] = bazelVersion record[columnMap["commit"]] = commit for _, b := range benchmarks { record[columnMap[b.desc]] = fmt.Sprintf("%.3f", b.result.Seconds()) } return record }