generate_perf.py

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Initializes all the perf directories.


import optparse
import os
import shutil
import sys


SystemTitles = {
  'android-gn': 'Android Galaxy Nexus',
  'android-nexus4': 'Android Nexus4',
  'android-nexus10': 'Android Nexus10',
  'android-release': 'Chromium Android',
  'chromium-rel-mac5-cg': 'Chromium Mac 10.5 (cg)',
  'chromium-rel-mac5-clang': 'Chromium Mac 10.5 (clang)',
  'chromium-rel-mac6-cg': 'Chromium Mac 10.6 (cg)',
  'chromium-rel-mac6-clang': 'Chromium Mac 10.6 (clang)',
  'chromium-rel-mac7-fyi': 'Chromium Mac 10.7 FYI',
  'chromium-rel-mac7-gpu-intel': 'Chromium Mac 10.7 GPU (Intel)',
  'chromium-rel-win7-dual': 'Chromium Win7',
  'chromium-rel-win7-dual-fyi': 'Chromium Win7 FYI',
  'chromium-rel-win7-gpu-ati': 'Chromium Win7 GPU (ATI)',
  'chromium-rel-win7-gpu-intel': 'Chromium Win7 GPU (Intel)',
  'chromium-rel-win7-gpu-nvidia': 'Chromium Win7 GPU (NVIDIA)',
  'chromium-rel-win7-single': 'Chromium Win7 (single)',
  'chromium-rel-win7-webkit': 'Chromium Win7 (webkit)',
  'chromium-webrtc-rel-7': 'Chromium Win7 (WebRTC)',
  'chromium-webrtc-rel-linux': 'Chromium Linux (WebRTC)',
  'chromium-webrtc-rel-mac': 'Chromium Mac (WebRTC)',
  'chromium-webrtc-rel-xp': 'Chromium WinXP (WebRTC)',
  'chromium-webrtc-stable-tot-rel-linux': 'Chromium Linux (WebRTC stable)',
  'chromium-webrtc-trunk-tot-rel-linux': 'Chromium Linux (WebRTC trunk)',
  'chromium-webrtc-stable-tot-rel-mac': 'Chromium Mac (WebRTC stable)',
  'chromium-webrtc-trunk-tot-rel-mac': 'Chromium Mac (WebRTC trunk)',
  'chromium-webrtc-stable-tot-rel-win': 'Chromium Win (WebRTC stable)',
  'chromium-webrtc-trunk-tot-rel-win': 'Chromium Win (WebRTC trunk)',
  'gpu-fyi-linux-release-ati': 'GPU FYI Linux Release (ATI)',
  'gpu-fyi-linux-release-intel': 'GPU FYI Linux Release (Intel)',
  'gpu-fyi-mac-release-ati': 'GPU FYI Mac Release (ATI)',
  'gpu-fyi-win7-release-ati': 'GPU FYI Win7 Release (ATI)',
  'gpu-fyi-win7-release-intel': 'GPU FYI Win7 Release (Intel)',
  'gpu-fyi-winxp-debug-nvidia': 'GPU FYI WinXP Debug (NVIDIA)',
  'gpu-fyi-winxp-release-nvidia': 'GPU FYI WinXP Release (NVIDIA)',
  'gpu-linux-release-nvidia': 'GPU Linux Release (NVIDIA)',
  'gpu-mac-release-intel': 'GPU Mac Release (Intel)',
  'gpu-webkit-linux-nvidia': 'Webkit GPU Linux (NVIDIA)',
  'gpu-webkit-mac': 'Webkit GPU Mac',
  'gpu-webkit-win7-nvidia': 'Webkit GPU Win7 (NVIDIA)',
  'gpu-win7-rel': 'GPU Win7 Release',
  'gpu-win7-release-nvidia': 'GPU Win7 Release (NVIDIA)',
  'linux-debug': 'Linux Perf (debug)',
  'linux-release': 'Chromium Linux',
  'linux-release-64': 'Chromium Linux 64',
  'linux-release-fyi': 'Chromium Linux FYI',
  'linux-release-hardy': 'Linux Perf',
  'linux-release-lowmem': 'Linux Perf (lowmem)',
  'linux-release-memory': 'Chromium Linux Memory',
  'linux-release-webkit-latest': 'Linux Perf (webkit-latest)',
  'linux-targets': 'Linux Target Builds',
  'lumpy-chrome-perf': 'Lumpy (perf)',
  'mac-debug': 'Mac Perf (debug)',
  'mac-release': 'Mac Perf',
  'mac-release-10.5': 'Mac 10.5 Perf',
  'mac-release-10.5-v8-latest': 'Chromium Mac 10.5 (v8-latest)',
  'mac-release-10.5-webkit-latest': 'Mac 10.5 Perf (webkit-latest)',
  'mac-release-10.6': 'Mac 10.6 Perf',
  'mac-release-10.6-v8-latest': 'Chromium Mac 10.6 (v8-latest)',
  'mac-release-10.6-webkit-latest': 'Mac 10.6 Perf (webkit-latest)',
  'mac-release-memory': 'Chromium Mac Memory',
  'mac-release-old-10.6': 'Old Mac 10.6 Perf',
  'mac-targets': 'Mac Target Builds',
  'nacl-lucid-32-bare-glibc-opt': 'NaCl Lucid32 Bare Glibc Opt',
  'nacl-lucid-32-bare-newlib-opt': 'NaCl Lucid32 Bare Newlib Opt',
  'nacl-lucid-64-bare-glibc-opt': 'NaCl Lucid64 Bare Glibc Opt',
  'nacl-lucid-64-bare-newlib-opt': 'NaCl Lucid64 Bare Newlib Opt',
  'nacl-lucid_64-newlib-arm_qemu-pnacl-spec': 'NaCl Lucid64 Newlib Arm Qemu Pnacl Spec',
  'nacl-lucid_64-newlib-x86_32-pnacl-spec': 'NaCl Lucid64 Newlib x86_32 Pnacl Spec',
  'nacl-lucid_64-newlib-x86_32-spec': 'NaCl Lucid64 Newlib x86_32 Spec',
  'nacl-lucid_64-newlib-x86_64-pnacl-spec': 'NaCl Lucid64 Newlib x86_64 Pnacl Spec',
  'nacl-lucid_64-newlib-x86_64-spec': 'NaCl Lucid64 Newlib x86_64 Spec',
  'nacl-mac10.5-glibc-opt': 'NaCl Mac10.5 Glibc Opt',
  'nacl-mac10.5-newlib-opt': 'NaCl Mac10.5 Newlib Opt',
  'nacl-mac10.6-glibc-opt': 'NaCl Mac10.6 Glibc Opt',
  'nacl-mac10.6-newlib-opt': 'NaCl Mac10.6 Newlib Opt',
  'nacl-mac10.7-glibc-opt': 'NaCl Mac10.7 Glibc Opt',
  'nacl-mac10.7-newlib-opt': 'NaCl Mac10.7 Newlib Opt',
  'nacl-oneiric_32-newlib-arm_hw-pnacl-panda-spec': 'NaCl Oneiric32 Newlib Arm HW Pnacl Spec',
  'nacl-toolchain-linux-pnacl-x86_64': 'NaCl Toolchain Lucid64 Pnacl',
  'nacl-win7-32-bare-glibc-opt': 'NaCl Win7 32bit Bare Glibc Opt',
  'nacl-win7-32-bare-newlib-opt': 'NaCl Win7 32bit Bare Newlib Opt',
  'nacl-win7-64-bare-glibc-opt': 'NaCl Win7 64bit Bare Glibc Opt',
  'nacl-win7-64-bare-newlib-opt': 'NaCl Win7 64bit Bare Newlib Opt',
  'nacl-xp-bare-glibc-opt': 'NaCl XP Bare Glibc Opt',
  'nacl-xp-bare-newlib-opt': 'NaCl XP Bare Newlib Opt',
  'vista-release-dual-core': 'Vista Perf',
  'vista-release-single-core': 'Vista Perf (single)',
  'vista-release-v8-latest': 'Vista Perf (v8-latest)',
  'vista-release-webkit-latest': 'Vista Perf (webkit-latest)',
  'win-release': 'Chrome Win',
  'win-release-chrome-frame': 'Chrome Frame Win',
  'win-targets': 'Windows Target Builds',
  'xp-debug': 'XP Perf (debug)',
  'xp-release': 'Chromium XP',
  'xp-release-dual-core': 'XP Perf',
  'xp-release-memory': 'Chromium XP Memory',
  'xp-release-single-core': 'XP Perf (single)',
  'xp-release-v8-latest': 'XP Perf (v8-latest)',
  'xp-release-webkit-latest': 'XP Perf (webkit-latest)',
  'webrtc-linux-large-tests': 'WebRTC Large Tests Linux',
  'webrtc-mac-large-tests': 'WebRTC Large Tests Mac',
  'webrtc-win-large-tests': 'WebRTC Large Tests Win7',
}

# This is public code and should not contain internal names.
ChromeSystemTitles = {
  'chrome-vista-dual-core': 'Vista Dual Core',
  'chrome-vista-quad-core': 'Vista Quad Core',
}

# These are long-running endure tests that have differently-shaped graphs.
EndureSystemTitles = {
  'endure-linux-dbg': 'Endure Linux Debug',
  'endure-linux-rel': 'Endure Linux Release',
}

TestTitles = {
  'audio_e2e_test': 'WebRTC Audio E2E',
  'audioproc_perf': 'WebRTC Audio Processing Perf',
  'avperf': 'Audio Video Perf',
  'av_perf': 'Audio Video Perf 2',
  'bloat-http': 'Bloat - HTTP',
  'cc_perftests': 'Chrome Compositor Perf',
  'chrome_frame_perf': 'Chrome Frame Perf',
  'control-testControlAttachDetachDOMTree':
    'Control Test: ControlAttachDetachDOMTree',
  'control-testControlAttachDetachDOMTreeWebDriver':
    'Control Test: ControlAttachDetachDOMTreeWebDriver',
  'coverage': 'Code Coverage',
  'database': 'Page Cycler Database',
  'dhtml': 'Page Cycler DHTML',
  'docs-testDocsAlternatelyClickLists':
    'Docs Test: DocsAlternatelyClickLists',
  'docs_wpr-testDocsAlternatelyClickLists':
    'Docs WPR Test: DocsAlternatelyClickLists',
  'dom_perf': 'Dom',
  'dromaeo_domcoreattr': 'Dromaeo DOMCore attr',
  'dromaeo_domcoremodify': 'Dromaeo DOMCore modify',
  'dromaeo_domcorequery': 'Dromaeo DOMCore query',
  'dromaeo_domcoretraverse': 'Dromaeo DOMCore traverse',
  'dromaeo_jslibattrjquery': 'Dromaeo JSLib attr jquery',
  'dromaeo_jslibattrprototype': 'Dromaeo JSLib attr prototype',
  'dromaeo_jslibeventjquery': 'Dromaeo JSLib event jquery',
  'dromaeo_jslibeventprototype': 'Dromaeo JSLib event prototype',
  'dromaeo_jslibmodifyjquery': 'Dromaeo JSLib modify jquery',
  'dromaeo_jslibmodifyprototype': 'Dromaeo JSLib modify prototype',
  'dromaeo_jslibstylejquery': 'Dromaeo JSLib stylej query',
  'dromaeo_jslibstyleprototype': 'Dromaeo JSLib style prototype',
  'dromaeo_jslibtraversejquery': 'Dromaeo JSLib traverse jquery',
  'dromaeo_jslibtraverseprototype': 'Dromaeo JSLib traverse prototype',
  'frame_rate': 'Frame Rate',
  'gmail-testGmailComposeDiscard':
    'Gmail Test: GmailComposeDiscard',
  'gmail-testGmailAlternateThreadlistConversation':
    'Gmail Test: GmailAlternateThreadlistConversation',
  'gmail-testGmailAlternateTwoLabels':
    'Gmail Test: GmailAlternateTwoLabels',
  'gmail-testGmailExpandCollapseConversation':
    'Gmail Test: GmailExpandCollapseConversation',
  'gmail_wpr-testGmailComposeDiscard':
    'Gmail WPR Test: GmailComposeDiscard',
  'gmail_wpr-testGmailAlternateThreadlistConversation':
    'Gmail WPR Test: GmailAlternateThreadlistConversation',
  'gmail_wpr-testGmailAlternateTwoLabels':
    'Gmail WPR Test: GmailAlternateTwoLabels',
  'gmail_wpr-testGmailExpandCollapseConversation':
    'Gmail WPR Test: GmailExpandCollapseConversation',
  'gpu_frame_rate': 'GPU Frame Rate',
  'gpu_latency': 'GPU Latency',
  'gpu_throughput': 'GPU Throughput',
  'idb_perf': 'Other IndexedDB',
  'image_decoding_benchmark': 'Image Decoding Benchmark',
  'indexeddb': 'Page Cycler IndexedDB',
  'indexeddb-testOfflineOnline':
    'IndexedDB Test: OfflineOnline',
  'intl1': 'Page Cycler Intl1',
  'intl2': 'Page Cycler Intl2',
  'isac_fixed_perf': 'WebRTC iSAC Fixed-point Perf',
  'jsgamebench': 'JSGameBench',
  'kraken': 'Kraken',
  'mach_ports': 'Mach Port Usage',
  'memory': 'Memory',
  'memory_benchmark': 'Memory Benchmark',
  'media_perf': 'Media Perf',
  'media_tests_av_perf': 'Media Tests AV Perf',
  'morejs': 'Page Cycler Morejs',
  'moz': 'Page Cycler Moz',
  'moz-http': 'Page Cycler Moz - HTTP',
  'nacl-perf': 'NaCl Perf',
  'new-tab-ui-cold': 'New Tab Cold',
  'new-tab-ui-warm': 'New Tab Warm',
  'octane': 'Octane',
  'page_cycler_2012Q2-netsim': 'Page Cycler 2012Q2 Netsim',
  'plus-testPlusAlternatelyClickStreams':
    'Plus Test: PlusAlternatelyClickStreams',
  'plus_wpr-testPlusAlternatelyClickStreams':
    'Plus WPR Test: PlusAlternatelyClickStreams',
  'pnacl-tools': 'PNaCl Toolchain',
  'pyauto_perf': 'Pyauto Perf',
  'pyauto_webrtc_tests': 'WebRTC PyAuto',
  'pyauto_webrtc_quality_tests': 'WebRTC Quality PyAuto',
  'resource_sizes': 'Resource Sizes',
  'robohornetpro': 'RoboHornet Pro',
  'scrolling_benchmark': 'Scrolling Benchmark',
  'sizes': 'Sizes',
  'spaceport': 'Spaceport',
  'spec2k': 'Spec2K',
  'startup': 'Startup',
  'sunspider': 'SunSpider',
  'sync': 'Sync',
  'tab-switching': 'Tab Switching',
  'targets': 'Target Build Times',
  'v8_benchmark': 'V8 Benchmark',
  'vie_auto_test': 'WebRTC Video Engine Auto Test',
  'webrtc_perf_content_unittests': 'WebRTC content_unittests',
}

BASE_URL = 'http://build.chromium.org/f/chromium'

# Template contents of a config.js file.  One of these gets created in
# each subdirectory for a given perf test, identified with the system
# that it's part of and an appropriate title.
CONFIG_TEMPLATE = """\
var Config = {
  buildslave: '%(system_title)s',
  title: '%(title)s',
  changeLinkPrefix: '%(base_url)s/perf/dashboard/ui/changelog.html?url=/trunk/src&mode=html&range=',
  coverageLinkPrefix: '%(base_url)s/coverage/%(system_dir)s/',
"""
CONFIG_TEMPLATE_END = """};
"""

# Template contents of a chrome_config.js file.  One is create in dashboard
# directory.  This specifies all test names and machines that are running
# the tests.  This is used by Chrome dashboard HTML pages.
CHROME_CONFIG_TEMPLATE = """\
/*
  This file is generated by generate_perf.py.
*/
var ChromeConfig = {
  systemTitles: {
%(system_title)s
  },

  testTitles : {
%(test_title)s
  },
};
"""

# Template contents of a config.js file for endure tests.
ENDURE_CONFIG_TEMPLATE = """\
var Config = {
  buildslave: '%(system_title)s',
  title: '%(title)s',
};
"""

DEFAULT_SYMLINK_LIST = [
  ('details.html',  '../../dashboard/ui/details.html'),
  ('report.html',   '../../dashboard/ui/generic_plotter.html'),
  ('js',            '../../dashboard/ui/js'),
]

CHROME_DEFAULT_SYMLINK_LIST = [
  ('report.html',   '../../dashboard/ui/chrome_report.html'),
  ('js',            '../../dashboard/ui/js'),
]

ENDURE_DEFAULT_SYMLINK_LIST = [
  ('report.html',   '../../dashboard/ui/endure_plotter.html'),
  ('js',            '../../dashboard/ui/endure_js'),
]

# Descriptions for optional detail tabs
DETAIL_TAB_DESC = {
  'view-pages': 'Data',
  'view-coverage': 'Coverage',
}


def WriteJSConfigFile(perf_dir, system_title, test_title, symlink_list):
  contents = CONFIG_TEMPLATE % {
      'base_url': BASE_URL,
      'system_title': system_title,
      'title': test_title,
      'system_dir': os.path.basename(os.path.dirname(perf_dir)),
  }

  # Add detail tabs to config.
  if os.path.basename(perf_dir) in ('coverage'):
    detail_tabs = ['view-coverage']
  elif os.path.basename(perf_dir) in ('sizes', 'targets'):
    detail_tabs = []
  else:
    detail_tabs = ['view-pages']
  contents += "  detailTabs: { 'view-change': 'CL'"
  for tab in detail_tabs:
    contents += ", '%s': '%s'" % (tab, DETAIL_TAB_DESC.get(tab, tab))
  contents += ' }\n'
  contents += CONFIG_TEMPLATE_END

  with open(os.path.join(perf_dir, 'config.js'), 'w') as config_f:
    config_f.write(contents)


def WriteChromeJSConfigFile():
  """Write Chrome JavaScript configuration file."""
  system_title = ''
  sorted_system_titles = sorted(ChromeSystemTitles.items(), key=lambda x: x[0])
  for sys, title in sorted_system_titles:
    system_title += '    \'%s\': \'%s\',\n' % (sys, title)
  system_title = system_title.rstrip()

  test_title = ''
  sorted_test_titles = sorted(TestTitles.items(), key=lambda x: x[0])
  for test, title in sorted_test_titles:
    test_title += '    \'%s\': \'%s\',\n' % (test, title)
  test_title = test_title.rstrip()

  contents = CHROME_CONFIG_TEMPLATE % {
      'system_title': system_title,
      'test_title': test_title,
  }
  with open(os.path.join('dashboard', 'chrome_config.js'), 'w') as config_f:
    config_f.write(contents)


def WriteEndureJSConfigFile(perf_dir, system_title, test_title, symlink_list):
  contents = ENDURE_CONFIG_TEMPLATE % {
      'system_title': system_title,
      'title': test_title,
  }

  with open(os.path.join(perf_dir, 'config.js'), 'w') as config_f:
    config_f.write(contents)


def TestInit(perf_dir, system_title, test_title, symlink_list, endure=False):
  for slink, target in symlink_list:
    slink = os.path.join(perf_dir, slink)
    # Remove the old symlinks first.  Catch exceptions on the
    # assumption that this is the first time and the symlink
    # doesn't exist.  If file already exist, delete them and
    # recreate the symlink.  If it's due to some other problem,
    # the symlink creation afterwards will fail for us.
    try:
      os.unlink(slink)
    except EnvironmentError:
      pass
    try:
      os.symlink(target, slink)
    except OSError:
      RemovePath(slink)
      os.symlink(target, slink)

  if endure:
    WriteEndureJSConfigFile(perf_dir, system_title, test_title, symlink_list)
  else:
    WriteJSConfigFile(perf_dir, system_title, test_title, symlink_list)


def RemovePath(path):
  """Remove the given path (file or dir)."""
  if os.path.isdir(path):
    shutil.rmtree(path, ignore_errors=True)
    return
  try:
    os.remove(path)
  except OSError:
    pass


def main():
  parser = optparse.OptionParser()
  parser.add_option(
      '-C', '--chrome', action='store_true',
      help='Initializes perf directories for Chrome.')
  parser.add_option(
      '-E', '--endure', action='store_true',
      help='Initializes perf directories for endure tests.')
  (options, args) = parser.parse_args()

  # Find the script's directory and cd there.
  main_dir = os.path.dirname(os.path.abspath(__file__))
  os.chdir(main_dir)

  if options.chrome:
    system_titles = ChromeSystemTitles
    symlink_list = CHROME_DEFAULT_SYMLINK_LIST
    WriteChromeJSConfigFile()
  elif options.endure:
    system_titles = EndureSystemTitles
    symlink_list = ENDURE_DEFAULT_SYMLINK_LIST
  else:
    system_titles = SystemTitles
    symlink_list = DEFAULT_SYMLINK_LIST

  # Find all of the system directories.
  for system_dir in os.listdir('.'):
    # Skip the entry if it's not a directory or should be ignored.
    if not os.path.isdir(system_dir) or system_dir not in system_titles:
      continue
    os.chmod(system_dir, 0755)
    system_title = system_titles[system_dir]

    # Find all of the test directories.
    for test_dir in os.listdir(system_dir):
      # Skip the entry if it's not in our mapping of test titles.
      if test_dir not in TestTitles:
        continue
      test_title = TestTitles[test_dir]
      perf_dir = os.path.join(system_dir, test_dir)
      os.chmod(perf_dir, 0755)
      TestInit(perf_dir, system_title, test_title, symlink_list, options.endure)
  return 0


if __name__ == '__main__':
  sys.exit(main())