blob: 89ec5320e05d90acecda06083d4d80276ed976ef [file]
// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Bench is a simple benchmarking tool for Chromium builds.
package main
import (
"bytes"
"encoding/csv"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
)
// main is the entry point of our tool.
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
os.Exit(1)
}
os.Exit(0)
}
// run contains the main logic of the benchmarking tool.
func run() error {
// Ensure we have a $HOME environment variable set, even on platforms that don't have it by
// default, like Windows.
if os.Getenv("HOME") == "" {
homeDir, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("failed to get home directory: %w", err)
}
if err = os.Setenv("HOME", homeDir); err != nil {
return fmt.Errorf("failed to set $HOME environment variable: %w", err)
}
}
if err := parseFlags(); err != nil {
return err
}
// Get the list of commits to benchmark at.
commits, err := getCommits(*revision, *numCommits)
if err != nil {
return err
}
// Make any necessary changes to our environment variables.
if err := setupEnvironment(); err != nil {
return err
}
// Print some information about the active configuration.
fmt.Fprintf(os.Stderr, "\nBenchmark configuration:\n")
fmt.Fprintf(os.Stderr, "- Number of runs: %d\n", *runs)
fmt.Fprintf(os.Stderr, "- Starting revision: %s\n", *revision)
fmt.Fprintf(os.Stderr, "- Number of commits per run: %d\n", len(commits))
if len(commits) > 1 {
fmt.Fprintf(os.Stderr, "- Skip commits made by robots: %t\n", *skipRobots)
}
fmt.Fprintf(os.Stderr, "- Path to depot_tools: %s\n", *depotTools)
fmt.Fprintf(os.Stderr, "- Path to chromium/src: %s\n", *workspaceRoot)
fmt.Fprintf(os.Stderr, "- Path to results file: %s\n", *resultsPath)
fmt.Fprintf(os.Stderr, "- Output directory: %s\n", *outputDir)
fmt.Fprintf(os.Stderr, "- Targets: %s\n", strings.Join(targets, ", "))
fmt.Fprintf(os.Stderr, "- Number of gn threads: %d\n", *gnThreads)
fmt.Fprintf(os.Stderr, "- Always run 'gn gen': %t\n", *alwaysGnGen)
fmt.Fprintf(os.Stderr, "- Always run 'gclient sync': %t\n", *alwaysSync)
if *clean {
fmt.Fprintf(os.Stderr, "- Build mode: Clean builds\n")
} else if *clobber {
fmt.Fprintf(os.Stderr, "- Build mode: Clobber builds\n")
} else {
fmt.Fprintf(os.Stderr, "- Build mode: Incremental builds\n")
}
fmt.Fprintf(os.Stderr, "\nBuild configuration (gn args):\n")
fmt.Fprintf(os.Stderr, "- Using Siso: %t\n", *useSiso)
fmt.Fprintf(os.Stderr, "- Using Remote Execution: %t\n", *useRemoteExec)
fmt.Fprintf(os.Stderr, "- Using reclient: %t\n", *useReclient)
// Print some interesting information about the system.
fmt.Fprintf(os.Stderr, "\nSystem information:\n")
fmt.Fprintf(os.Stderr, "- CPUs: %s\n", cpuInfo())
fmt.Fprintf(os.Stderr, "- Memory: %s\n", memInfo())
fmt.Fprintf(os.Stderr, "\nGetting launch approvals:\n")
// Open the results file for writing.
if *resultsPath == "" {
results = csv.NewWriter(io.Discard)
} else {
fmt.Fprintf(os.Stderr, "- Opening results log ...\n")
f, err := os.OpenFile(*resultsPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
return fmt.Errorf("failed to open results file: %w", err)
}
defer func() {
if err := f.Close(); err != nil {
fmt.Fprintf(os.Stderr, "failed to close results file, data might be incomplete: %v\n", err)
}
}()
results = csv.NewWriter(f)
defer func() {
results.Flush()
if err := results.Error(); err != nil {
fmt.Fprintf(os.Stderr, "failed to flush results to disk, data might be incomplete: %v\n", err)
}
}()
}
// If we need to checkout a different commit, ensure that the workspace is clean and abort
// if it is not. We cannot safely switch between commits in a dirty workspace.
if !benchmarkCurrentCommitOnly() {
fmt.Fprintf(os.Stderr, "- Checking if workspace is clean ...\n")
isClean, err := checkWorkspaceClean()
if err != nil {
return err
}
if !isClean {
return fmt.Errorf("workspace has pending changes, but we need to switch commits - please stash or commit your changes first")
}
}
// Prepare the output directory for the benchmark.
fmt.Fprintf(os.Stderr, "- Preparing the output directory ...\n")
if err = prepareOutputDir(*outputDir); err != nil {
return err
}
// Run the benchmark.
for i := range *runs {
fmt.Fprintf(os.Stderr, "\nBenchmark run %d/%d:\n", i+1, *runs)
for _, commit := range commits {
// Sync to the commit using "gclient sync", if necessary.
if !benchmarkCurrentCommitOnly() {
if err = syncToRevision(commit); err != nil {
return err
}
}
if *clean {
// Clean the output directory using "gn clean".
if err = cleanOutputDir(*outputDir); err != nil {
return err
}
}
// Run "gn gen" to generate the build files, if necessary.
if *alwaysGnGen {
if err = gnGen(*outputDir); err != nil {
return err
}
}
// Build the target using "autoninja".
if err = autoNinja(); err != nil {
return err
}
}
}
return nil
}
// benchmarkCurrentCommitOnly returns whether we should benchmark only the current commit.
// This allows us to skip a few validations and operations and allows the user to benchmark
// even pending changes in a dirty workspace.
func benchmarkCurrentCommitOnly() bool {
return *numCommits == 1 && *revision == "HEAD"
}
// setupEnvironment sets up the necessary environment variables for the benchmark.
func setupEnvironment() error {
// Unset any environment variables that might interfere with the benchmark.
// We will set them ourselves if necessary. Ignore errors, as these variables might not be set.
_ = os.Unsetenv("NINJA_STATUS")
_ = os.Unsetenv("NINJA_SUMMARIZE_BUILD")
_ = os.Unsetenv("RBE_exec_strategy")
_ = os.Unsetenv("SISO_LIMITS")
// Set UPDATE_DEPOT_TOOLS=0 in our environment to avoid updating depot_tools during the
// benchmark. We take care to update it once before the benchmark starts.
if err := os.Setenv("UPDATE_DEPOT_TOOLS", "0"); err != nil {
return fmt.Errorf("failed to set $UPDATE_DEPOT_TOOLS: %w", err)
}
var sisoExperiments, sisoLimits []string
// Set environment variables for the benchmark, if necessary.
if *useReclient {
if *depsCache {
if err := os.Setenv("RBE_enable_deps_cache", "true"); err != nil {
return fmt.Errorf("failed to set $RBE_enable_deps_cache: %w", err)
}
} else {
if err := os.Setenv("RBE_enable_deps_cache", "false"); err != nil {
return fmt.Errorf("failed to set $RBE_enable_deps_cache: %w", err)
}
}
if *racing {
if err := os.Setenv("RBE_exec_strategy", "racing"); err != nil {
return fmt.Errorf("failed to set $RBE_exec_strategy: %w", err)
}
} else if *localFallback {
if err := os.Setenv("RBE_exec_strategy", "remote_local_fallback"); err != nil {
return fmt.Errorf("failed to set $RBE_exec_strategy: %w", err)
}
} else {
if err := os.Setenv("RBE_exec_strategy", "remote"); err != nil {
return fmt.Errorf("failed to set $RBE_exec_strategy: %w", err)
}
}
}
if *useSiso {
if !*depsCache {
sisoExperiments = append(sisoExperiments, "no-fast-deps")
}
if *racing && runtime.GOOS == "darwin" {
// Warn the user that Siso does not use fastlocal by default on a Mac.
fmt.Fprintf(os.Stderr,
"Warning: Racing mode is enabled, but Siso's native RBE client "+
"might not use it by default on a Mac.\n")
fmt.Fprintf(os.Stderr,
"If you want, you can manually override this by setting "+
"$SISO_LIMITS to 'fastlocal=N',\nwhere N is the maximum "+
"number of local jobs you want to run in parallel.\n")
} else if !*racing {
sisoLimits = append(sisoLimits, "fastlocal=0")
}
if !*localFallback {
sisoExperiments = append(sisoExperiments, "no-local-fallback")
}
}
if err := os.Setenv("SISO_EXPERIMENTS", strings.Join(sisoExperiments, ",")); err != nil {
return err
}
if err := os.Setenv("SISO_LIMITS", strings.Join(sisoLimits, ",")); err != nil {
return err
}
return nil
}
// checkWorkspaceClean checks if the workspace is clean and aborts if it is not.
func checkWorkspaceClean() (bool, error) {
// Check if the workspace is clean.
cmd := exec.Command("git", "status", "--porcelain")
cmd.Dir = *workspaceRoot
cmd.Stderr = os.Stderr
out, err := cmd.Output()
if err != nil {
return false, fmt.Errorf("failed to check if workspace is clean: %w", err)
}
if len(out) > 0 {
return false, nil
}
return true, nil
}
// cleanOutputDir runs "gn clean" to clean the output directory.
func cleanOutputDir(outputDir string) error {
fmt.Fprintf(os.Stderr, "- Cleaning the output directory ...\n")
cmd := exec.Command("gn", "clean", outputDir)
cmd.Dir = *workspaceRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to clean the output directory: %w", err)
}
if err := results.Write([]string{"clean", fmt.Sprintf("%f", time.Since(start).Seconds())}); err != nil {
return fmt.Errorf("failed to write results: %w", err)
}
return nil
}
// syncToRevision runs "gclient sync" to sync to a specific commit, if necessary.
func syncToRevision(commit string) error {
currentBranch, err := resolveBranch()
if err != nil {
return err
}
currentCommit, err := resolveCommit("HEAD")
if err != nil {
return err
}
// We need to switch to the commit using "git switch --detach" if either we're
// currently on a branch (to avoid modifying it), or if we're not at the commit
// we want to benchmark.
if currentBranch != "" || currentCommit != commit {
fmt.Fprintf(os.Stderr, "- Switching to commit %s ...\n", commit)
cmd := exec.Command("git", "switch", "--detach", commit)
cmd.Dir = *workspaceRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to `git switch` to commit %q: %w", commit, err)
}
if err := results.Write([]string{"git_switch", fmt.Sprintf("%f", time.Since(start).Seconds())}); err != nil {
return fmt.Errorf("failed to write results: %w", err)
}
} else {
fmt.Fprintf(os.Stderr, "- Already at commit %s, no need to switch.\n", commit)
}
// Sync dependencies and run hooks using `gclient sync` if necessary.
if *alwaysSync || currentCommit != commit {
fmt.Fprintf(os.Stderr, "- Running `gclient sync` ...\n")
cmd := exec.Command(
filepath.Join(*depotTools, "gclient"),
"sync",
"--delete_unversioned_trees",
"--reset")
cmd.Dir = *workspaceRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to checkout commit %q: %w", commit, err)
}
if err := results.Write([]string{"gclient_sync", fmt.Sprintf("%f", time.Since(start).Seconds())}); err != nil {
return fmt.Errorf("failed to write results: %w", err)
}
} else {
fmt.Fprintf(os.Stderr, "- Already at commit %s, no need to run `gclient sync`.\n", commit)
}
return nil
}
// prepareOutputDir prepares the output directory for the benchmark.
func prepareOutputDir(outputDir string) error {
// Ensure that the output directory exists.
if err := os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
// Write the GN args to outputDir/args.gn if they have changed.
gnArgs := []string{
"use_siso = " + strconv.FormatBool(*useSiso),
"use_remoteexec = " + strconv.FormatBool(*useRemoteExec),
"use_reclient = " + strconv.FormatBool(*useReclient),
}
argsFile := filepath.Join(outputDir, "args.gn")
argsContents := []byte(strings.Join(gnArgs, "\n"))
argsChanged, err := writeIfChanged(argsFile, argsContents)
if err != nil {
return fmt.Errorf("failed to write GN args to %q: %w", argsFile, err)
}
buildNinjaExists := true
if _, err = os.Stat(filepath.Join(outputDir, "build.ninja")); err != nil {
if errors.Is(err, os.ErrNotExist) {
buildNinjaExists = false
} else {
return fmt.Errorf("failed to stat build.ninja exists: %w", err)
}
}
if argsChanged || !buildNinjaExists {
return gnGen(outputDir)
}
return nil
}
// gnGen runs "gn gen" to generate the build files.
func gnGen(outputDir string) error {
fmt.Fprintf(os.Stderr, "- Generating Ninja files ...\n")
cmd := exec.Command("gn", "gen", outputDir, "--threads="+strconv.Itoa(*gnThreads))
cmd.Dir = *workspaceRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to generate Ninja files: %w", err)
}
if err := results.Write([]string{"gn_gen", fmt.Sprintf("%f", time.Since(start).Seconds())}); err != nil {
return fmt.Errorf("failed to write results: %w", err)
}
return nil
}
// autoNinja runs "autoninja" to build the target.
func autoNinja() error {
fmt.Fprintf(os.Stderr, "- Building targets ...\n")
autoninjaArgs := []string{
"-C", *outputDir,
}
if *clobber {
autoninjaArgs = append(autoninjaArgs, "-clobber")
}
autoninjaArgs = append(autoninjaArgs, targets...)
autoninjaEnv := os.Environ()
if !*useSiso {
autoninjaEnv = append(autoninjaEnv, "NINJA_STATUS=[%r processes, %f/%t @ %o/s : %es ] ")
}
cmd := exec.Command(filepath.Join(*depotTools, "autoninja"), autoninjaArgs...)
cmd.Dir = *workspaceRoot
cmd.Env = autoninjaEnv
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return fmt.Errorf("build failed: %w", err)
}
if err := results.Write([]string{"build", fmt.Sprintf("%f", time.Since(start).Seconds())}); err != nil {
return fmt.Errorf("failed to write results: %w", err)
}
return nil
}
// writeIfChanged writes the given content to the file, but only if the content has changed.
// It returns true if the file was written, false if it was not written, and an error if the
// operation failed.
func writeIfChanged(filename string, content []byte) (written bool, err error) {
// Read the current content, if any.
current, err := os.ReadFile(filename)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("failed to read file %q: %w", filename, err)
}
// Check if the content has changed.
if bytes.Equal(current, content) {
return false, nil
}
// Write the new content.
if err = os.WriteFile(filename, content, 0644); err != nil {
return true, fmt.Errorf("failed to write file %q: %w", filename, err)
}
return true, nil
}