| #!/usr/bin/python3 |
| # Copyright 2019 the V8 project authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # Runs chromium/src/run_benchmark for a given story and extracts the generated |
| # runtime call stats. |
| |
| import argparse |
| import csv |
| import json |
| import glob |
| import os |
| import pathlib |
| import re |
| import tabulate |
| import shutil |
| import statistics |
| import subprocess |
| import sys |
| import tempfile |
| |
| from callstats_groups import RUNTIME_CALL_STATS_GROUPS |
| |
| |
| JSON_FILE_EXTENSION=".pb_converted.json" |
| |
| def parse_args(): |
| parser = argparse.ArgumentParser( |
| description="Run story and collect runtime call stats.") |
| parser.add_argument("story", metavar="story", nargs=1, help="story to run") |
| parser.add_argument( |
| "--group", |
| dest="group", |
| action="store_true", |
| help="group common stats together into buckets") |
| parser.add_argument( |
| "-r", |
| "--repeats", |
| dest="repeats", |
| metavar="N", |
| action="store", |
| type=int, |
| default=1, |
| help="number of times to run the story") |
| parser.add_argument( |
| "-v", |
| "--verbose", |
| dest="verbose", |
| action="store_true", |
| help="output benchmark runs to stdout") |
| parser.add_argument( |
| "--device", |
| dest="device", |
| action="store", |
| help="device to run the test on. Passed directly to run_benchmark") |
| parser.add_argument( |
| "-d", |
| "--dir", |
| dest="dir", |
| action="store", |
| help=("directory to look for already generated output in. This must " |
| "already exists and it won't re-run the benchmark")) |
| parser.add_argument( |
| "-f", |
| "--format", |
| dest="format", |
| action="store", |
| choices=["csv", "table"], |
| help="output as CSV") |
| parser.add_argument( |
| "-o", |
| "--output", |
| metavar="FILE", |
| dest="out_file", |
| action="store", |
| help="write table to FILE rather stdout") |
| parser.add_argument( |
| "--browser", |
| dest="browser", |
| metavar="BROWSER_TYPE", |
| action="store", |
| default="release", |
| help=("Passed directly to --browser option of run_benchmark. Ignored if " |
| "-executable is used")) |
| parser.add_argument( |
| "-e", |
| "--executable", |
| dest="executable", |
| metavar="EXECUTABLE", |
| action="store", |
| help=("path to executable to run. If not given it will pass '--browser " |
| "release' to run_benchmark")) |
| parser.add_argument( |
| "--chromium-dir", |
| dest="chromium_dir", |
| metavar="DIR", |
| action="store", |
| default=".", |
| help=("path to chromium directory. If not given, the script must be run " |
| "inside the chromium/src directory")) |
| parser.add_argument( |
| "--js-flags", dest="js_flags", action="store", help="flags to pass to v8") |
| parser.add_argument( |
| "--extra-browser-args", |
| dest="browser_args", |
| action="store", |
| help="flags to pass to chrome") |
| parser.add_argument( |
| "--benchmark", |
| dest="benchmark", |
| action="store", |
| default="v8.browsing_desktop", |
| help="benchmark to run") |
| parser.add_argument( |
| "--stdev", |
| dest="stdev", |
| action="store_true", |
| help="adds columns for the standard deviation") |
| parser.add_argument( |
| "--filter", |
| dest="filter", |
| action="append", |
| help="useable with --group to only show buckets specified by filter") |
| parser.add_argument( |
| "--retain", |
| dest="retain", |
| action="store", |
| default="json", |
| choices=["none", "json", "all"], |
| help=("controls artifacts to be retained after run. With none, all files " |
| "are deleted; only the json.gz file is retained for each run; and " |
| "all keep all files")) |
| |
| return parser.parse_args() |
| |
| |
| def process_trace(trace_file): |
| text_string = pathlib.Path(trace_file).read_text() |
| result = json.loads(text_string) |
| |
| output = {} |
| result = result["traceEvents"] |
| for o in result: |
| o = o["args"] |
| if "runtime-call-stats" in o: |
| r = o["runtime-call-stats"] |
| for name in r: |
| count = r[name][0] |
| duration = r[name][1] |
| if name in output: |
| output[name]["count"] += count |
| output[name]["duration"] += duration |
| else: |
| output[name] = {"count": count, "duration": duration} |
| |
| return output |
| |
| |
| def run_benchmark(story, |
| repeats=1, |
| output_dir=".", |
| verbose=False, |
| js_flags=None, |
| browser_args=None, |
| chromium_dir=".", |
| executable=None, |
| benchmark="v8.browsing_desktop", |
| device=None, |
| browser="release"): |
| |
| orig_chromium_dir = chromium_dir |
| xvfb = os.path.join(chromium_dir, "testing", "xvfb.py") |
| if not os.path.isfile(xvfb): |
| chromium_dir = os.path(chromium_dir, "src") |
| xvfb = os.path.join(chromium_dir, "testing", "xvfb.py") |
| if not os.path.isfile(xvfb): |
| print(("chromium_dir does not point to a valid chromium checkout: " + |
| orig_chromium_dir)) |
| sys.exit(1) |
| |
| command = [ |
| xvfb, |
| os.path.join(chromium_dir, "tools", "perf", "run_benchmark"), |
| "run", |
| "--story", |
| story, |
| "--pageset-repeat", |
| str(repeats), |
| "--output-dir", |
| output_dir, |
| "--intermediate-dir", |
| os.path.join(output_dir, "artifacts"), |
| benchmark, |
| ] |
| |
| if executable: |
| command += ["--browser-executable", executable] |
| else: |
| command += ["--browser", browser] |
| |
| if device: |
| command += ["--device", device] |
| if browser_args: |
| command += ["--extra-browser-args", browser_args] |
| if js_flags: |
| command += ["--js-flags", js_flags] |
| |
| if not benchmark.startswith("v8."): |
| # Most benchmarks by default don't collect runtime call stats so enable them |
| # manually. |
| categories = [ |
| "v8", |
| "disabled-by-default-v8.runtime_stats", |
| ] |
| |
| command += ["--extra-chrome-categories", ",".join(categories)] |
| |
| print("Output directory: %s" % output_dir) |
| stdout = "" |
| print(f"Running: {' '.join(command)}\n") |
| proc = subprocess.Popen( |
| command, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| universal_newlines=True) |
| proc.stderr.close() |
| status_matcher = re.compile(r"\[ +(\w+) +\]") |
| for line in iter(proc.stdout.readline, ""): |
| stdout += line |
| match = status_matcher.match(line) |
| if verbose or match: |
| print(line, end="") |
| |
| proc.stdout.close() |
| |
| if proc.wait() != 0: |
| print("\nrun_benchmark failed:") |
| # If verbose then everything has already been printed. |
| if not verbose: |
| print(stdout) |
| sys.exit(1) |
| |
| print("\nrun_benchmark completed") |
| |
| |
| def write_output(f, table, headers, run_count, format="table"): |
| if format == "csv": |
| # strip new lines from CSV output |
| headers = [h.replace("\n", " ") for h in headers] |
| writer = csv.writer(f) |
| writer.writerow(headers) |
| writer.writerows(table) |
| else: |
| # First column is name, and then they alternate between counts and durations |
| summary_count = len(headers) - 2 * run_count - 1 |
| floatfmt = ("",) + (".0f", ".2f") * run_count + (".2f",) * summary_count |
| f.write(tabulate.tabulate(table, headers=headers, floatfmt=floatfmt)) |
| f.write("\n") |
| |
| |
| class Row: |
| |
| def __init__(self, name, run_count): |
| self.name = name |
| self.durations = [0] * run_count |
| self.counts = [0] * run_count |
| self.mean_duration = None |
| self.mean_count = None |
| self.stdev_duration = None |
| self.stdev_count = None |
| |
| def __repr__(self): |
| data_str = ", ".join( |
| str((c, d)) for (c, d) in zip(self.counts, self.durations)) |
| return (f"{self.name}: {data_str}, mean_count: {self.mean_count}, " + |
| f"mean_duration: {self.mean_duration}") |
| |
| def add_data(self, counts, durations): |
| self.counts = counts |
| self.durations = durations |
| |
| def add_data_point(self, run, count, duration): |
| self.counts[run] = count |
| self.durations[run] = duration |
| |
| def prepare(self, stdev=False): |
| if len(self.durations) > 1: |
| self.mean_duration = statistics.mean(self.durations) |
| self.mean_count = statistics.mean(self.counts) |
| if stdev: |
| self.stdev_duration = statistics.stdev(self.durations) |
| self.stdev_count = statistics.stdev(self.counts) |
| |
| def as_list(self): |
| l = [self.name] |
| for (c, d) in zip(self.counts, self.durations): |
| l += [c, d] |
| if self.mean_duration is not None: |
| l += [self.mean_count] |
| if self.stdev_count is not None: |
| l += [self.stdev_count] |
| l += [self.mean_duration] |
| if self.stdev_duration is not None: |
| l += [self.stdev_duration] |
| return l |
| |
| def key(self): |
| if self.mean_duration is not None: |
| return self.mean_duration |
| else: |
| return self.durations[0] |
| |
| |
| class Bucket: |
| |
| def __init__(self, name, run_count): |
| self.name = name |
| self.run_count = run_count |
| self.data = {} |
| self.table = None |
| self.total_row = None |
| |
| def __repr__(self): |
| s = "Bucket: " + self.name + " {\n" |
| if self.table: |
| s += "\n ".join(str(row) for row in self.table) + "\n" |
| elif self.data: |
| s += "\n ".join(str(row) for row in self.data.values()) + "\n" |
| if self.total_row: |
| s += " " + str(self.total_row) + "\n" |
| return s + "}" |
| |
| def add_data_point(self, name, run, count, duration): |
| if name not in self.data: |
| self.data[name] = Row(name, self.run_count) |
| |
| self.data[name].add_data_point(run, count, duration) |
| |
| def prepare(self, stdev=False): |
| if self.data: |
| for row in self.data.values(): |
| row.prepare(stdev) |
| |
| self.table = sorted(self.data.values(), key=Row.key) |
| self.total_row = Row("Total", self.run_count) |
| self.total_row.add_data([ |
| sum(r.counts[i] |
| for r in self.data.values()) |
| for i in range(0, self.run_count) |
| ], [ |
| sum(r.durations[i] |
| for r in self.data.values()) |
| for i in range(0, self.run_count) |
| ]) |
| self.total_row.prepare(stdev) |
| |
| def as_list(self, add_bucket_titles=True, filter=None): |
| t = [] |
| if filter is None or self.name in filter: |
| if add_bucket_titles: |
| t += [["\n"], [self.name]] |
| t += [r.as_list() for r in self.table] |
| t += [self.total_row.as_list()] |
| return t |
| |
| |
| def collect_buckets(story, group=True, repeats=1, output_dir="."): |
| if group: |
| groups = RUNTIME_CALL_STATS_GROUPS |
| else: |
| groups = [] |
| |
| buckets = {} |
| |
| for i in range(0, repeats): |
| story_dir = f"{story.replace(':', '_')}_{i + 1}" |
| trace_dir = os.path.join(output_dir, "artifacts", story_dir, "trace", |
| "traceEvents") |
| |
| # run_benchmark now dumps two files: a .pb.gz file and a .pb_converted.json |
| # file. We only need the latter. |
| trace_file_glob = os.path.join(trace_dir, "*" + JSON_FILE_EXTENSION) |
| trace_files = glob.glob(trace_file_glob) |
| if not trace_files: |
| print("Could not find *%s file in %s" % (JSON_FILE_EXTENSION, trace_dir)) |
| sys.exit(1) |
| if len(trace_files) > 1: |
| print("Expecting one file but got: %s" % trace_files) |
| sys.exit(1) |
| |
| trace_file = trace_files[0] |
| |
| output = process_trace(trace_file) |
| for name in output: |
| bucket_name = "Other" |
| for group in groups: |
| if group[1].match(name): |
| bucket_name = group[0] |
| break |
| |
| value = output[name] |
| if bucket_name not in buckets: |
| bucket = Bucket(bucket_name, repeats) |
| buckets[bucket_name] = bucket |
| else: |
| bucket = buckets[bucket_name] |
| |
| bucket.add_data_point(name, i, value["count"], value["duration"] / 1000.0) |
| return buckets |
| |
| |
| def create_table(buckets, record_bucket_names=True, filter=None): |
| table = [] |
| for bucket in buckets.values(): |
| table += bucket.as_list( |
| add_bucket_titles=record_bucket_names, filter=filter) |
| return table |
| |
| |
| def main(): |
| args = parse_args() |
| story = args.story[0] |
| |
| retain = args.retain |
| if args.dir is not None: |
| output_dir = args.dir |
| if not os.path.isdir(output_dir): |
| print("Specified output directory does not exist: " % output_dir) |
| sys.exit(1) |
| else: |
| output_dir = tempfile.mkdtemp(prefix="runtime_call_stats_") |
| run_benchmark( |
| story, |
| repeats=args.repeats, |
| output_dir=output_dir, |
| verbose=args.verbose, |
| js_flags=args.js_flags, |
| browser_args=args.browser_args, |
| chromium_dir=args.chromium_dir, |
| benchmark=args.benchmark, |
| executable=args.executable, |
| browser=args.browser, |
| device=args.device) |
| |
| try: |
| buckets = collect_buckets( |
| story, group=args.group, repeats=args.repeats, output_dir=output_dir) |
| |
| for b in buckets.values(): |
| b.prepare(args.stdev) |
| |
| table = create_table( |
| buckets, record_bucket_names=args.group, filter=args.filter) |
| |
| headers = [""] + ["Count", "Duration\n(ms)"] * args.repeats |
| if args.repeats > 1: |
| if args.stdev: |
| headers += [ |
| "Count\nMean", "Count\nStdev", "Duration\nMean (ms)", |
| "Duration\nStdev (ms)" |
| ] |
| else: |
| headers += ["Count\nMean", "Duration\nMean (ms)"] |
| |
| if args.out_file: |
| with open(args.out_file, "w", newline="") as f: |
| write_output(f, table, headers, args.repeats, args.format) |
| else: |
| write_output(sys.stdout, table, headers, args.repeats, args.format) |
| finally: |
| if retain == "none": |
| shutil.rmtree(output_dir) |
| elif retain == "json": |
| # Delete all files bottom up except ones ending in JSON_FILE_EXTENSION and |
| # attempt to delete subdirectories (ignoring errors). |
| for dir_name, subdir_list, file_list in os.walk( |
| output_dir, topdown=False): |
| for file_name in file_list: |
| if not file_name.endswith(JSON_FILE_EXTENSION): |
| os.remove(os.path.join(dir_name, file_name)) |
| for subdir in subdir_list: |
| try: |
| os.rmdir(os.path.join(dir_name, subdir)) |
| except OSError: |
| pass |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main()) |
