tools/run-swarmed.py - chromium/src - Git at Google

 #!/usr/bin/env python

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Runs a Fuchsia gtest-based test on Swarming, optionally many times,
 collecting the output of the runs into a directory. Useful for flake checking,
 and faster than using trybots by avoiding repeated bot_update, compile, archive,
 etc. and allowing greater parallelism.

 To use, run in a new shell (it blocks until all Swarming jobs complete):

   tools/run-swarmed.py -t content_unittests --out-dir=out/fuch

 The logs of the runs will be stored in results/ (or specify a results directory
 with --results=some_dir). You can then do something like `grep -L SUCCESS
 results/*` to find the tests that failed or otherwise process the log files.
 """

 import argparse
 import multiprocessing
 import os
 import shutil
 import subprocess
 import sys


 INTERNAL_ERROR_EXIT_CODE = -1000


 def _Spawn(args):
   """Triggers a swarming job. The arguments passed are:
   - The index of the job;
   - The command line arguments object;
   - The hash of the isolate job used to trigger.

   The return value is passed to a collect-style map() and consists of:
   - The index of the job;
   - The json file created by triggering and used to collect results;
   - The command line arguments object.
   """
   index, args, isolated_hash = args
   json_file = os.path.join(args.results, '%d.json' % index)
   trigger_args = [
       'tools/swarming_client/swarming.py', 'trigger',
       '-S', 'https://chromium-swarm.appspot.com',
       '-I', 'https://isolateserver.appspot.com',
       '-d', 'pool', args.pool,
       '-s', isolated_hash,
       '--dump-json', json_file,
   ]
   if args.target_os == 'fuchsia':
     trigger_args += [
       '-d', 'os', 'Linux',
       '-d', 'kvm', '1',
       '-d', 'gpu', 'none',
       '-d', 'cpu', args.arch,
     ]
   elif args.target_os == 'win':
     trigger_args += [ '-d', 'os', 'Windows' ]
   elif args.target_os == 'android':
     # The canonical version numbers are stored in the infra repository here:
     # build/scripts/slave/recipe_modules/swarming/api.py
     cpython_version = 'version:2.7.14.chromium14'
     vpython_version = 'git_revision:96f81e737868d43124b4661cf1c325296ca04944'
     cpython_pkg = (
         '.swarming_module:infra/python/cpython/${platform}:' +
         cpython_version)
     vpython_native_pkg = (
         '.swarming_module:infra/tools/luci/vpython-native/${platform}:' +
         vpython_version)
     vpython_pkg = (
         '.swarming_module:infra/tools/luci/vpython/${platform}:' +
         vpython_version)
     trigger_args += [
         '-d', 'os', 'Android',
         '-d', 'device_os', args.device_os,
         '--cipd-package', cpython_pkg,
         '--cipd-package', vpython_native_pkg,
         '--cipd-package', vpython_pkg,
         '--env-prefix', 'PATH', '.swarming_module',
         '--env-prefix', 'PATH', '.swarming_module/bin',
         '--env-prefix', 'VPYTHON_VIRTUALENV_ROOT',
         '.swarming_module_cache/vpython',
     ]
   trigger_args += [
       '--',
       '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json',
       '--system-log-file=${ISOLATED_OUTDIR}/system_log']
   if args.gtest_filter:
     trigger_args.append('--gtest_filter=' + args.gtest_filter)
   elif args.target_os == 'fuchsia':
     filter_file = \
         'testing/buildbot/filters/fuchsia.' + args.test_name + '.filter'
     if os.path.isfile(filter_file):
       trigger_args.append('--test-launcher-filter-file=../../' + filter_file)
   with open(os.devnull, 'w') as nul:
     subprocess.check_call(trigger_args, stdout=nul)
   return (index, json_file, args)


 def _Collect(spawn_result):
   index, json_file, args = spawn_result
   p = subprocess.Popen([
     'tools/swarming_client/swarming.py', 'collect',
     '-S', 'https://chromium-swarm.appspot.com',
     '--json', json_file,
     '--task-output-stdout=console'],
     stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
   stdout = p.communicate()[0]
   if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
     exit_code = INTERNAL_ERROR_EXIT_CODE
     file_suffix = '.INTERNAL_ERROR'
   else:
     exit_code = p.returncode
     file_suffix = '' if exit_code == 0 else '.FAILED'
   filename = '%d%s.stdout.txt' % (index, file_suffix)
   with open(os.path.join(args.results, filename), 'w') as f:
     f.write(stdout)
   return exit_code


 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('-C', '--out-dir', default='out/fuch',
                       help='Build directory.')
   parser.add_argument('--target-os', default='detect', help='gn target_os')
   parser.add_argument('--test-name', '-t', required=True,
                       help='Name of test to run.')
   parser.add_argument('--arch', '-a', default='detect',
                       help='CPU architecture of the test binary.')
   parser.add_argument('--copies', '-n', type=int, default=1,
                       help='Number of copies to spawn.')
   parser.add_argument('--device-os', default='M',
                       help='Run tests on the given version of Android.')
   parser.add_argument('--pool', default='Chrome',
                       help='Use the given swarming pool.')
   parser.add_argument('--results', '-r', default='results',
                       help='Directory in which to store results.')
   parser.add_argument('--gtest_filter',
                       help='Use the given gtest_filter, rather than the '
                            'default filter file, if any.')

   args = parser.parse_args()

   if args.target_os == 'detect':
     with open(os.path.join(args.out_dir, 'args.gn')) as f:
       gn_args = {}
       for l in f:
         l = l.split('#')[0].strip()
         if not l: continue
         k, v = map(str.strip, l.split('=', 1))
         gn_args[k] = v
     if 'target_os' in gn_args:
       args.target_os = gn_args['target_os'].strip('"')
     else:
       args.target_os = { 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' }[
                            sys.platform]

   # Determine the CPU architecture of the test binary, if not specified.
   if args.arch == 'detect' and args.target_os == 'fuchsia':
     executable_info = subprocess.check_output(
         ['file', os.path.join(args.out_dir, args.test_name)])
     if 'ARM aarch64' in executable_info:
       args.arch = 'arm64',
     else:
       args.arch = 'x86-64'

   subprocess.check_call(
       ['tools/mb/mb.py', 'isolate', '//' + args.out_dir, args.test_name])

   print 'If you get authentication errors, follow:'
   print '  https://www.chromium.org/developers/testing/isolated-testing/for-swes#TOC-Login-on-the-services'

   print 'Uploading to isolate server, this can take a while...'
   archive_output = subprocess.check_output(
       ['tools/swarming_client/isolate.py', 'archive',
        '-I', 'https://isolateserver.appspot.com',
        '-i', os.path.join(args.out_dir, args.test_name + '.isolate'),
        '-s', os.path.join(args.out_dir, args.test_name + '.isolated')])
   isolated_hash = archive_output.split()[0]

   if os.path.isdir(args.results):
     shutil.rmtree(args.results)
   os.makedirs(args.results)

   try:
     print 'Triggering %d tasks...' % args.copies
     pool = multiprocessing.Pool()
     spawn_args = map(lambda i: (i, args, isolated_hash), range(args.copies))
     spawn_results = pool.imap_unordered(_Spawn, spawn_args)

     exit_codes = []
     collect_results = pool.imap_unordered(_Collect, spawn_results)
     for result in collect_results:
       exit_codes.append(result)
       successes = sum(1 for x in exit_codes if x == 0)
       errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
       failures = len(exit_codes) - successes - errors
       clear_to_eol = '\033[K'
       print('\r[%d/%d] collected: '
             '%d successes, %d failures, %d bot errors...%s' % (len(exit_codes),
                 args.copies, successes, failures, errors, clear_to_eol)),
       sys.stdout.flush()

     print
     print 'Results logs collected into', os.path.abspath(args.results) + '.'
   finally:
     pool.close()
     pool.join()
   return 0


 if __name__ == '__main__':
   sys.exit(main())
	#!/usr/bin/env python

	# Copyright 2017 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Runs a Fuchsia gtest-based test on Swarming, optionally many times,
	collecting the output of the runs into a directory. Useful for flake checking,
	and faster than using trybots by avoiding repeated bot_update, compile, archive,
	etc. and allowing greater parallelism.

	To use, run in a new shell (it blocks until all Swarming jobs complete):

	tools/run-swarmed.py -t content_unittests --out-dir=out/fuch

	The logs of the runs will be stored in results/ (or specify a results directory
	with --results=some_dir). You can then do something like `grep -L SUCCESS
	results/*` to find the tests that failed or otherwise process the log files.
	"""

	import argparse
	import multiprocessing
	import os
	import shutil
	import subprocess
	import sys


	INTERNAL_ERROR_EXIT_CODE = -1000


	def _Spawn(args):
	"""Triggers a swarming job. The arguments passed are:
	- The index of the job;
	- The command line arguments object;
	- The hash of the isolate job used to trigger.

	The return value is passed to a collect-style map() and consists of:
	- The index of the job;
	- The json file created by triggering and used to collect results;
	- The command line arguments object.
	"""
	index, args, isolated_hash = args
	json_file = os.path.join(args.results, '%d.json' % index)
	trigger_args = [
	'tools/swarming_client/swarming.py', 'trigger',
	'-S', 'https://chromium-swarm.appspot.com',
	'-I', 'https://isolateserver.appspot.com',
	'-d', 'pool', args.pool,
	'-s', isolated_hash,
	'--dump-json', json_file,
	]
	if args.target_os == 'fuchsia':
	trigger_args += [
	'-d', 'os', 'Linux',
	'-d', 'kvm', '1',
	'-d', 'gpu', 'none',
	'-d', 'cpu', args.arch,
	]
	elif args.target_os == 'win':
	trigger_args += [ '-d', 'os', 'Windows' ]
	elif args.target_os == 'android':
	# The canonical version numbers are stored in the infra repository here:
	# build/scripts/slave/recipe_modules/swarming/api.py
	cpython_version = 'version:2.7.14.chromium14'
	vpython_version = 'git_revision:96f81e737868d43124b4661cf1c325296ca04944'
	cpython_pkg = (
	'.swarming_module:infra/python/cpython/${platform}:' +
	cpython_version)
	vpython_native_pkg = (
	'.swarming_module:infra/tools/luci/vpython-native/${platform}:' +
	vpython_version)
	vpython_pkg = (
	'.swarming_module:infra/tools/luci/vpython/${platform}:' +
	vpython_version)
	trigger_args += [
	'-d', 'os', 'Android',
	'-d', 'device_os', args.device_os,
	'--cipd-package', cpython_pkg,
	'--cipd-package', vpython_native_pkg,
	'--cipd-package', vpython_pkg,
	'--env-prefix', 'PATH', '.swarming_module',
	'--env-prefix', 'PATH', '.swarming_module/bin',
	'--env-prefix', 'VPYTHON_VIRTUALENV_ROOT',
	'.swarming_module_cache/vpython',
	]
	trigger_args += [
	'--',
	'--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json',
	'--system-log-file=${ISOLATED_OUTDIR}/system_log']
	if args.gtest_filter:
	trigger_args.append('--gtest_filter=' + args.gtest_filter)
	elif args.target_os == 'fuchsia':
	filter_file = \
	'testing/buildbot/filters/fuchsia.' + args.test_name + '.filter'
	if os.path.isfile(filter_file):
	trigger_args.append('--test-launcher-filter-file=../../' + filter_file)
	with open(os.devnull, 'w') as nul:
	subprocess.check_call(trigger_args, stdout=nul)
	return (index, json_file, args)


	def _Collect(spawn_result):
	index, json_file, args = spawn_result
	p = subprocess.Popen([
	'tools/swarming_client/swarming.py', 'collect',
	'-S', 'https://chromium-swarm.appspot.com',
	'--json', json_file,
	'--task-output-stdout=console'],
	stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	stdout = p.communicate()[0]
	if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
	exit_code = INTERNAL_ERROR_EXIT_CODE
	file_suffix = '.INTERNAL_ERROR'
	else:
	exit_code = p.returncode
	file_suffix = '' if exit_code == 0 else '.FAILED'
	filename = '%d%s.stdout.txt' % (index, file_suffix)
	with open(os.path.join(args.results, filename), 'w') as f:
	f.write(stdout)
	return exit_code


	def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('-C', '--out-dir', default='out/fuch',
	help='Build directory.')
	parser.add_argument('--target-os', default='detect', help='gn target_os')
	parser.add_argument('--test-name', '-t', required=True,
	help='Name of test to run.')
	parser.add_argument('--arch', '-a', default='detect',
	help='CPU architecture of the test binary.')
	parser.add_argument('--copies', '-n', type=int, default=1,
	help='Number of copies to spawn.')
	parser.add_argument('--device-os', default='M',
	help='Run tests on the given version of Android.')
	parser.add_argument('--pool', default='Chrome',
	help='Use the given swarming pool.')
	parser.add_argument('--results', '-r', default='results',
	help='Directory in which to store results.')
	parser.add_argument('--gtest_filter',
	help='Use the given gtest_filter, rather than the '
	'default filter file, if any.')

	args = parser.parse_args()

	if args.target_os == 'detect':
	with open(os.path.join(args.out_dir, 'args.gn')) as f:
	gn_args = {}
	for l in f:
	l = l.split('#')[0].strip()
	if not l: continue
	k, v = map(str.strip, l.split('=', 1))
	gn_args[k] = v
	if 'target_os' in gn_args:
	args.target_os = gn_args['target_os'].strip('"')
	else:
	args.target_os = { 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' }[
	sys.platform]

	# Determine the CPU architecture of the test binary, if not specified.
	if args.arch == 'detect' and args.target_os == 'fuchsia':
	executable_info = subprocess.check_output(
	['file', os.path.join(args.out_dir, args.test_name)])
	if 'ARM aarch64' in executable_info:
	args.arch = 'arm64',
	else:
	args.arch = 'x86-64'

	subprocess.check_call(
	['tools/mb/mb.py', 'isolate', '//' + args.out_dir, args.test_name])

	print 'If you get authentication errors, follow:'
	print ' https://www.chromium.org/developers/testing/isolated-testing/for-swes#TOC-Login-on-the-services'

	print 'Uploading to isolate server, this can take a while...'
	archive_output = subprocess.check_output(
	['tools/swarming_client/isolate.py', 'archive',
	'-I', 'https://isolateserver.appspot.com',
	'-i', os.path.join(args.out_dir, args.test_name + '.isolate'),
	'-s', os.path.join(args.out_dir, args.test_name + '.isolated')])
	isolated_hash = archive_output.split()[0]

	if os.path.isdir(args.results):
	shutil.rmtree(args.results)
	os.makedirs(args.results)

	try:
	print 'Triggering %d tasks...' % args.copies
	pool = multiprocessing.Pool()
	spawn_args = map(lambda i: (i, args, isolated_hash), range(args.copies))
	spawn_results = pool.imap_unordered(_Spawn, spawn_args)

	exit_codes = []
	collect_results = pool.imap_unordered(_Collect, spawn_results)
	for result in collect_results:
	exit_codes.append(result)
	successes = sum(1 for x in exit_codes if x == 0)
	errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
	failures = len(exit_codes) - successes - errors
	clear_to_eol = '\033[K'
	print('\r[%d/%d] collected: '
	'%d successes, %d failures, %d bot errors...%s' % (len(exit_codes),
	args.copies, successes, failures, errors, clear_to_eol)),
	sys.stdout.flush()

	print
	print 'Results logs collected into', os.path.abspath(args.results) + '.'
	finally:
	pool.close()
	pool.join()
	return 0


	if __name__ == '__main__':
	sys.exit(main())