google/webpagereplay_utils.py - chromium/src/tools/python - Git at Google

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """A class to help start/stop a Web Page Replay Server.

 The page cycler tests use this module to run Web Page Replay
 (see tools/build/scripts/slave/runtest.py).

 If run from the command-line, the module will launch Web Page Replay
 and the specified test:

    ./webpagereplay_utils.py --help  # list options
    ./webpagereplay_utils.py 2012Q2  # run a WPR-enabled test
 """

 import logging
 import optparse
 import os
 import shutil
 import signal
 import subprocess
 import sys
 import tempfile
 import time
 import urllib


 USAGE = '%s [options] CHROME_EXE TEST_NAME' % os.path.basename(sys.argv[0])
 USER_DATA_DIR = '{TEMP}/webpagereplay_utils-chrome'

 # The port numbers must match those in chrome/test/perf/page_cycler_test.cc.
 HTTP_PORT = 8080
 HTTPS_PORT = 8413


 class ReplayError(Exception):
   """Catch-all exception for the module."""
   pass

 class ReplayNotFoundError(Exception):
   pass

 class ReplayNotStartedError(Exception):
   pass


 class ReplayLauncher(object):
   LOG_FILE = 'log.txt'

   def __init__(self, replay_dir, archive_path, log_dir, replay_options=None):
     """Initialize ReplayLauncher.

     Args:
       replay_dir: directory that has replay.py and related modules.
       archive_path: either a directory that contains WPR archives or,
           a path to a specific WPR archive.
       log_dir: where to write log.txt.
       replay_options: a list of options strings to forward to replay.py.
     """
     self.replay_dir = replay_dir
     self.archive_path = archive_path
     self.log_dir = log_dir
     self.replay_options = replay_options if replay_options else []

     self.log_name = os.path.join(self.log_dir, self.LOG_FILE)
     self.log_fh = None
     self.proxy_process = None

     self.wpr_py = os.path.join(self.replay_dir, 'replay.py')
     if not os.path.exists(self.wpr_py):
       raise ReplayNotFoundError('Path does not exist: %s' % self.wpr_py)
     self.wpr_options = [
         '--port', str(HTTP_PORT),
         '--ssl_port', str(HTTPS_PORT),
         '--use_closest_match',
         # TODO(slamm): Add traffic shaping (requires root):
         # '--net', 'fios',
         ]
     self.wpr_options.extend(self.replay_options)

   def _OpenLogFile(self):
     if not os.path.exists(self.log_dir):
       os.makedirs(self.log_dir)
     return open(self.log_name, 'w')

   def StartServer(self):
     cmd_line = [self.wpr_py]
     cmd_line.extend(self.wpr_options)
     # TODO(slamm): Support choosing archive on-the-fly.
     cmd_line.append(self.archive_path)
     self.log_fh = self._OpenLogFile()
     logging.debug('Starting Web-Page-Replay: %s', cmd_line)
     self.proxy_process = subprocess.Popen(
       cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
     if not self.IsStarted():
       raise ReplayNotStartedError(
           'Web Page Replay failed to start. See the log file: ' + self.log_name)

   def IsStarted(self):
     """Checks to see if the server is up and running."""
     for _ in range(5):
       if self.proxy_process.poll() is not None:
         # The process has exited.
         break
       try:
         up_url = '%s://localhost:%s/web-page-replay-generate-200'
         http_up_url = up_url % ('http', HTTP_PORT)
         https_up_url = up_url % ('https', HTTPS_PORT)
         if (200 == urllib.urlopen(http_up_url, None, {}).getcode() and
             200 == urllib.urlopen(https_up_url, None, {}).getcode()):
           return True
       except IOError:
         time.sleep(1)
     return False

   def StopServer(self):
     if self.proxy_process:
       logging.debug('Stopping Web-Page-Replay')
       # Use a SIGINT here so that it can do graceful cleanup.
       # Otherwise, we will leave subprocesses hanging.
       self.proxy_process.send_signal(signal.SIGINT)
       self.proxy_process.wait()
     if self.log_fh:
       self.log_fh.close()


 class ChromiumPaths(object):
   """Collect all the path handling together."""
   PATHS = {
       'archives':   'src/data/page_cycler/webpagereplay',
       '.wpr':       'src/data/page_cycler/webpagereplay/{TEST_NAME}.wpr',
       '.wpr_alt':   'src/tools/page_cycler/webpagereplay/tests/{TEST_NAME}.wpr',
       'start.html': 'src/tools/page_cycler/webpagereplay/start.html',
       'extension':  'src/tools/page_cycler/webpagereplay/extension',
       'logs':       'src/webpagereplay_logs/{TEST_EXE_NAME}',
       'replay':     'tools/build/third_party/webpagereplay',
       }

   def __init__(self, **replacements):
     """Initialize ChromiumPaths.

     Args:
       replacements: a dict of format replacements for PATHS such as
           {'TEST_NAME': '2012Q2', 'TEST_EXE_NAME': 'performance_ui_tests'}.
     """
     module_dir = os.path.dirname(__file__)
     self.base_dir = os.path.abspath(os.path.join(
         module_dir, '..', '..', '..', '..'))
     self.replacements = replacements

   def __getitem__(self, key):
     path_parts = [x.format(**self.replacements)
                   for x in self.PATHS[key].split('/')]
     return os.path.join(self.base_dir, *path_parts)


 def LaunchChromium(chrome_exe, chromium_paths, test_name,
                    is_dns_forwarded, use_auto):
   """Launch chromium to run WPR-backed page cycler tests.

   These options need to be kept in sync with
   src/chrome/test/perf/page_cycler_test.cc.
   """
   REPLAY_HOST='127.0.0.1'
   user_data_dir = USER_DATA_DIR.format(**{'TEMP': tempfile.gettempdir()})
   chromium_args = [
       chrome_exe,
       '--load-extension=%s' % chromium_paths['extension'],
       '--testing-fixed-http-port=%s' % HTTP_PORT,
       '--testing-fixed-https-port=%s' % HTTPS_PORT,
       '--disable-background-networking',
       '--enable-experimental-extension-apis',
       '--enable-file-cookies',
       '--enable-logging',
       '--log-level=0',
       '--enable-stats-table',
       '--enable-benchmarking',
       '--ignore-certificate-errors',
       '--metrics-recording-only',
       '--activate-on-launch',
       '--no-first-run',
       '--no-proxy-server',
       '--user-data-dir=%s' % user_data_dir,
       '--window-size=1280,1024',
       ]
   if not is_dns_forwarded:
     chromium_args.append('--host-resolver-rules=MAP * %s' % REPLAY_HOST)
   start_url = 'file://%s?test=%s' % (chromium_paths['start.html'], test_name)
   if use_auto:
     start_url += '&auto=1'
   chromium_args.append(start_url)
   if os.path.exists(user_data_dir):
     shutil.rmtree(user_data_dir)
   os.makedirs(user_data_dir)
   try:
     logging.debug('Starting Chrome: %s', chromium_args)
     retval = subprocess.call(chromium_args)
   finally:
     shutil.rmtree(user_data_dir)


 def main():
   log_level = logging.DEBUG
   logging.basicConfig(level=log_level,
                       format='%(asctime)s %(filename)s:%(lineno)-3d'
                              ' %(levelname)s %(message)s',
                       datefmt='%y%m%d %H:%M:%S')

   option_parser = optparse.OptionParser(usage=USAGE)
   option_parser.add_option(
       '', '--auto', action='store_true', default=False,
       help='Start test automatically.')
   option_parser.add_option(
       '', '--replay-dir', default=None,
       help='Run replay from this directory instead of tools/build/third_party.')
   replay_group = optparse.OptionGroup(option_parser,
       'Options for replay.py', 'These options are passed through to replay.py.')
   replay_group.add_option(
       '', '--record', action='store_true', default=False,
       help='Record a new WPR archive.')
   replay_group.add_option(  # use default that does not require sudo
       '', '--dns_forwarding', default=False, action='store_true',
       help='Forward DNS requests to the local replay server.')
   option_parser.add_option_group(replay_group)
   options, args = option_parser.parse_args()
   if len(args) != 2:
     option_parser.error('Need CHROME_EXE and TEST_NAME.')
     return 1
   chrome_exe, test_name = args

   if not os.path.exists(chrome_exe):
     print >>sys.stderr, 'Chrome path does not exist:', chrome_exe
     return 1

   chromium_paths = ChromiumPaths(
       TEST_NAME=test_name,
       TEST_EXE_NAME='webpagereplay_utils')
   if os.path.exists(chromium_paths['archives']):
     archive_path = chromium_paths['.wpr']
   else:
     archive_path = chromium_paths['.wpr_alt']
   if not os.path.exists(archive_path) and not options.record:
     print >>sys.stderr, 'Archive does not exist:', archive_path
     return 1

   replay_options = []
   if options.record:
     replay_options.append('--record')
   if not options.dns_forwarding:
     replay_options.append('--no-dns_forwarding')

   if options.replay_dir:
     replay_dir = options.replay_dir
   else:
     replay_dir = chromium_paths['replay']
   wpr = ReplayLauncher(replay_dir, archive_path,
                        chromium_paths['logs'], replay_options)
   try:
     wpr.StartServer()
     LaunchChromium(chrome_exe, chromium_paths, test_name,
                    options.dns_forwarding, options.auto)
   finally:
     wpr.StopServer()
   return 0

 if '__main__' == __name__:
   sys.exit(main())
	#!/usr/bin/env python
	# Copyright (c) 2012 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""A class to help start/stop a Web Page Replay Server.

	The page cycler tests use this module to run Web Page Replay
	(see tools/build/scripts/slave/runtest.py).

	If run from the command-line, the module will launch Web Page Replay
	and the specified test:

	./webpagereplay_utils.py --help # list options
	./webpagereplay_utils.py 2012Q2 # run a WPR-enabled test
	"""

	import logging
	import optparse
	import os
	import shutil
	import signal
	import subprocess
	import sys
	import tempfile
	import time
	import urllib


	USAGE = '%s [options] CHROME_EXE TEST_NAME' % os.path.basename(sys.argv[0])
	USER_DATA_DIR = '{TEMP}/webpagereplay_utils-chrome'

	# The port numbers must match those in chrome/test/perf/page_cycler_test.cc.
	HTTP_PORT = 8080
	HTTPS_PORT = 8413


	class ReplayError(Exception):
	"""Catch-all exception for the module."""
	pass

	class ReplayNotFoundError(Exception):
	pass

	class ReplayNotStartedError(Exception):
	pass


	class ReplayLauncher(object):
	LOG_FILE = 'log.txt'

	def __init__(self, replay_dir, archive_path, log_dir, replay_options=None):
	"""Initialize ReplayLauncher.

	Args:
	replay_dir: directory that has replay.py and related modules.
	archive_path: either a directory that contains WPR archives or,
	a path to a specific WPR archive.
	log_dir: where to write log.txt.
	replay_options: a list of options strings to forward to replay.py.
	"""
	self.replay_dir = replay_dir
	self.archive_path = archive_path
	self.log_dir = log_dir
	self.replay_options = replay_options if replay_options else []

	self.log_name = os.path.join(self.log_dir, self.LOG_FILE)
	self.log_fh = None
	self.proxy_process = None

	self.wpr_py = os.path.join(self.replay_dir, 'replay.py')
	if not os.path.exists(self.wpr_py):
	raise ReplayNotFoundError('Path does not exist: %s' % self.wpr_py)
	self.wpr_options = [
	'--port', str(HTTP_PORT),
	'--ssl_port', str(HTTPS_PORT),
	'--use_closest_match',
	# TODO(slamm): Add traffic shaping (requires root):
	# '--net', 'fios',
	]
	self.wpr_options.extend(self.replay_options)

	def _OpenLogFile(self):
	if not os.path.exists(self.log_dir):
	os.makedirs(self.log_dir)
	return open(self.log_name, 'w')

	def StartServer(self):
	cmd_line = [self.wpr_py]
	cmd_line.extend(self.wpr_options)
	# TODO(slamm): Support choosing archive on-the-fly.
	cmd_line.append(self.archive_path)
	self.log_fh = self._OpenLogFile()
	logging.debug('Starting Web-Page-Replay: %s', cmd_line)
	self.proxy_process = subprocess.Popen(
	cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
	if not self.IsStarted():
	raise ReplayNotStartedError(
	'Web Page Replay failed to start. See the log file: ' + self.log_name)

	def IsStarted(self):
	"""Checks to see if the server is up and running."""
	for _ in range(5):
	if self.proxy_process.poll() is not None:
	# The process has exited.
	break
	try:
	up_url = '%s://localhost:%s/web-page-replay-generate-200'
	http_up_url = up_url % ('http', HTTP_PORT)
	https_up_url = up_url % ('https', HTTPS_PORT)
	if (200 == urllib.urlopen(http_up_url, None, {}).getcode() and
	200 == urllib.urlopen(https_up_url, None, {}).getcode()):
	return True
	except IOError:
	time.sleep(1)
	return False

	def StopServer(self):
	if self.proxy_process:
	logging.debug('Stopping Web-Page-Replay')
	# Use a SIGINT here so that it can do graceful cleanup.
	# Otherwise, we will leave subprocesses hanging.
	self.proxy_process.send_signal(signal.SIGINT)
	self.proxy_process.wait()
	if self.log_fh:
	self.log_fh.close()


	class ChromiumPaths(object):
	"""Collect all the path handling together."""
	PATHS = {
	'archives': 'src/data/page_cycler/webpagereplay',
	'.wpr': 'src/data/page_cycler/webpagereplay/{TEST_NAME}.wpr',
	'.wpr_alt': 'src/tools/page_cycler/webpagereplay/tests/{TEST_NAME}.wpr',
	'start.html': 'src/tools/page_cycler/webpagereplay/start.html',
	'extension': 'src/tools/page_cycler/webpagereplay/extension',
	'logs': 'src/webpagereplay_logs/{TEST_EXE_NAME}',
	'replay': 'tools/build/third_party/webpagereplay',
	}

	def __init__(self, **replacements):
	"""Initialize ChromiumPaths.

	Args:
	replacements: a dict of format replacements for PATHS such as
	{'TEST_NAME': '2012Q2', 'TEST_EXE_NAME': 'performance_ui_tests'}.
	"""
	module_dir = os.path.dirname(__file__)
	self.base_dir = os.path.abspath(os.path.join(
	module_dir, '..', '..', '..', '..'))
	self.replacements = replacements

	def __getitem__(self, key):
	path_parts = [x.format(**self.replacements)
	for x in self.PATHS[key].split('/')]
	return os.path.join(self.base_dir, *path_parts)


	def LaunchChromium(chrome_exe, chromium_paths, test_name,
	is_dns_forwarded, use_auto):
	"""Launch chromium to run WPR-backed page cycler tests.

	These options need to be kept in sync with
	src/chrome/test/perf/page_cycler_test.cc.
	"""
	REPLAY_HOST='127.0.0.1'
	user_data_dir = USER_DATA_DIR.format(**{'TEMP': tempfile.gettempdir()})
	chromium_args = [
	chrome_exe,
	'--load-extension=%s' % chromium_paths['extension'],
	'--testing-fixed-http-port=%s' % HTTP_PORT,
	'--testing-fixed-https-port=%s' % HTTPS_PORT,
	'--disable-background-networking',
	'--enable-experimental-extension-apis',
	'--enable-file-cookies',
	'--enable-logging',
	'--log-level=0',
	'--enable-stats-table',
	'--enable-benchmarking',
	'--ignore-certificate-errors',
	'--metrics-recording-only',
	'--activate-on-launch',
	'--no-first-run',
	'--no-proxy-server',
	'--user-data-dir=%s' % user_data_dir,
	'--window-size=1280,1024',
	]
	if not is_dns_forwarded:
	chromium_args.append('--host-resolver-rules=MAP * %s' % REPLAY_HOST)
	start_url = 'file://%s?test=%s' % (chromium_paths['start.html'], test_name)
	if use_auto:
	start_url += '&auto=1'
	chromium_args.append(start_url)
	if os.path.exists(user_data_dir):
	shutil.rmtree(user_data_dir)
	os.makedirs(user_data_dir)
	try:
	logging.debug('Starting Chrome: %s', chromium_args)
	retval = subprocess.call(chromium_args)
	finally:
	shutil.rmtree(user_data_dir)


	def main():
	log_level = logging.DEBUG
	logging.basicConfig(level=log_level,
	format='%(asctime)s %(filename)s:%(lineno)-3d'
	' %(levelname)s %(message)s',
	datefmt='%y%m%d %H:%M:%S')

	option_parser = optparse.OptionParser(usage=USAGE)
	option_parser.add_option(
	'', '--auto', action='store_true', default=False,
	help='Start test automatically.')
	option_parser.add_option(
	'', '--replay-dir', default=None,
	help='Run replay from this directory instead of tools/build/third_party.')
	replay_group = optparse.OptionGroup(option_parser,
	'Options for replay.py', 'These options are passed through to replay.py.')
	replay_group.add_option(
	'', '--record', action='store_true', default=False,
	help='Record a new WPR archive.')
	replay_group.add_option( # use default that does not require sudo
	'', '--dns_forwarding', default=False, action='store_true',
	help='Forward DNS requests to the local replay server.')
	option_parser.add_option_group(replay_group)
	options, args = option_parser.parse_args()
	if len(args) != 2:
	option_parser.error('Need CHROME_EXE and TEST_NAME.')
	return 1
	chrome_exe, test_name = args

	if not os.path.exists(chrome_exe):
	print >>sys.stderr, 'Chrome path does not exist:', chrome_exe
	return 1

	chromium_paths = ChromiumPaths(
	TEST_NAME=test_name,
	TEST_EXE_NAME='webpagereplay_utils')
	if os.path.exists(chromium_paths['archives']):
	archive_path = chromium_paths['.wpr']
	else:
	archive_path = chromium_paths['.wpr_alt']
	if not os.path.exists(archive_path) and not options.record:
	print >>sys.stderr, 'Archive does not exist:', archive_path
	return 1

	replay_options = []
	if options.record:
	replay_options.append('--record')
	if not options.dns_forwarding:
	replay_options.append('--no-dns_forwarding')

	if options.replay_dir:
	replay_dir = options.replay_dir
	else:
	replay_dir = chromium_paths['replay']
	wpr = ReplayLauncher(replay_dir, archive_path,
	chromium_paths['logs'], replay_options)
	try:
	wpr.StartServer()
	LaunchChromium(chrome_exe, chromium_paths, test_name,
	options.dns_forwarding, options.auto)
	finally:
	wpr.StopServer()
	return 0

	if '__main__' == __name__:
	sys.exit(main())