recipes/recipe_modules/presubmit/api.py - chromium/tools/depot_tools.git - Git at Google

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 from recipe_engine import recipe_api

 from PB.recipe_engine import result as result_pb2
 from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2

 # 8 minutes seems like a reasonable upper bound on presubmit timings.
 # According to event mon data we have, it seems like anything longer than
 # this is a bug, and should just instant fail.
 _DEFAULT_TIMEOUT_S = 480


 class PresubmitApi(recipe_api.RecipeApi):

   def __init__(self, properties, **kwargs):
     super(PresubmitApi, self).__init__(**kwargs)

     self._runhooks = properties.runhooks
     self._timeout_s = properties.timeout_s or _DEFAULT_TIMEOUT_S

   @property
   def presubmit_support_path(self):
     return self.repo_resource('presubmit_support.py')

   def __call__(self, *args, **kwargs):
     """Returns a presubmit step."""

     kwargs['venv'] = True
     name = kwargs.pop('name', 'presubmit')
     with self.m.depot_tools.on_path():
       presubmit_args = list(args) + [
           '--json_output', self.m.json.output(),
       ]
       if self.m.resultdb.enabled:
         kwargs['wrapper'] = ('rdb', 'stream', '--')
       step_data = self.m.python(
           name, self.presubmit_support_path, presubmit_args, **kwargs)
       output = step_data.json.output or {}
       if self.m.step.active_result.retcode != 0:
         return output

       # Run with vpython3 directly
       del (kwargs['venv'])
       presubmit_args = list(args) + [
           '--json_output',
           self.m.json.output(),
       ]
       step_data = self.m.step(name + " py3",
                               ['vpython3', self.presubmit_support_path] +
                               presubmit_args, **kwargs)
       output2 = step_data.json.output or {}

       # combine outputs
       for key in output:
         if key in output2:
           output[key] += output2[key]
           del (output2[key])
       for key in output2:
         output[key] = output2[key]
       return output

   def prepare(self, root_solution_revision=None):
     """Sets up a presubmit run.

     This includes:
       - setting up the checkout w/ bot_update
       - locally committing the applied patch
       - running hooks, if requested

     This expects the gclient configuration to already have been set.

     Args:
       root_solution_revision: revision of the root solution

     Returns:
       the StepResult from the bot_update step.
     """
     # Set up the root solution revision by either passing the revision
     # to this function or adding it to the input properties.
     root_solution_revision = (
         root_solution_revision or
         self.m.properties.get('root_solution_revision'))

     # Expect callers to have already set up their gclient configuration.
     bot_update_step = self.m.bot_update.ensure_checkout(
         timeout=3600, no_fetch_tags=True,
         root_solution_revision=root_solution_revision)

     relative_root = self.m.gclient.get_gerrit_patch_root().rstrip('/')

     abs_root = self.m.context.cwd.join(relative_root)
     with self.m.context(cwd=abs_root):
       # TODO(unowned): Consider either:
       #  - extracting user name & email address from the issue, or
       #  - using a dedicated and clearly nonexistent name/email address
       self.m.git('-c', 'user.email=commit-bot@chromium.org',
               '-c', 'user.name=The Commit Bot',
               'commit', '-a', '-m', 'Committed patch',
               name='commit-git-patch', infra_step=False)

     if self._runhooks:
       with self.m.context(cwd=self.m.path['checkout']):
         self.m.gclient.runhooks()

     return bot_update_step

   def execute(self, bot_update_step, skip_owners=False):
     """Runs presubmit and sets summary markdown if applicable.

     Args:
       * bot_update_step: the StepResult from a previously executed bot_update step.
       * skip_owners: a boolean indicating whether Owners checks should be skipped.

     Returns:
       a RawResult object, suitable for being returned from RunSteps.
     """
     relative_root = self.m.gclient.get_gerrit_patch_root().rstrip('/')
     abs_root = self.m.context.cwd.join(relative_root)
     got_revision_properties = self.m.bot_update.get_project_revision_properties(
         # Replace path.sep with '/', since most recipes are written assuming '/'
         # as the delimiter. This breaks on windows otherwise.
         relative_root.replace(self.m.path.sep, '/'), self.m.gclient.c)
     upstream = bot_update_step.json.output['properties'].get(
         got_revision_properties[0])

     presubmit_args = [
       '--issue', self.m.tryserver.gerrit_change.change,
       '--patchset', self.m.tryserver.gerrit_change.patchset,
       '--gerrit_url', 'https://%s' % self.m.tryserver.gerrit_change.host,
       '--gerrit_project', self.m.tryserver.gerrit_change.project,
       '--gerrit_branch', self.m.tryserver.gerrit_change_target_ref,
       '--gerrit_fetch',
     ]
     if self.m.cq.active and self.m.cq.run_mode == self.m.cq.DRY_RUN:
       presubmit_args.append('--dry_run')

     presubmit_args.extend([
       '--root', abs_root,
       '--commit',
       '--verbose', '--verbose',
       '--skip_canned', 'CheckTreeIsOpen',
       '--upstream', upstream,  # '' if not in bot_update mode.
     ])

     if skip_owners:
       presubmit_args.extend([
         '--skip_canned', 'CheckOwners'
       ])

     raw_result = result_pb2.RawResult()
     step_json = self(
         *presubmit_args,
         timeout=self._timeout_s,
         # ok_ret='any' causes all exceptions to be ignored in this step
         ok_ret='any')
     # Set recipe result values
     if step_json:
       raw_result.summary_markdown = _createSummaryMarkdown(step_json)

     retcode = self.m.step.active_result.retcode
     if retcode == 0:
       raw_result.status = common_pb2.SUCCESS
       return raw_result

     self.m.step.active_result.presentation.status = 'FAILURE'
     if self.m.step.active_result.exc_result.had_timeout:
       raw_result.status = common_pb2.FAILURE
       raw_result.summary_markdown += (
           '\n\nTimeout occurred during presubmit step.')
     elif retcode == 1:
       raw_result.status = common_pb2.FAILURE
       self.m.tryserver.set_test_failure_tryjob_result()
     else:
       raw_result.status = common_pb2.INFRA_FAILURE
       self.m.tryserver.set_invalid_test_results_tryjob_result()
     # Handle unexpected errors not caught by json output
     if raw_result.summary_markdown == '':
       raw_result.status = common_pb2.INFRA_FAILURE
       raw_result.summary_markdown = (
         'Something unexpected occurred'
         ' while running presubmit checks.'
         ' Please [file a bug](https://bugs.chromium.org'
         '/p/chromium/issues/entry?components='
         'Infra%3EClient%3EChrome&status=Untriaged)'
       )
     return raw_result


 def _limitSize(message_list, char_limit=450):
   """Returns a list of strings within a certain character length.

   Args:
      * message_list (List[str]) - The message to truncate as a list
        of lines (without line endings).
   """
   hint = ('**The complete output can be'
           ' found at the bottom of the presubmit stdout.**')
   char_count = 0
   for index, message in enumerate(message_list):
     char_count += len(message)
     if char_count > char_limit:
       if index == 0:
         # Show at minimum part of the first error message
         first_message = message_list[index].splitlines()
         return ['\n'.join(
           _limitSize(first_message)
           )
         ]
       total_errors = len(message_list)
       # If code is being cropped, the closing code tag will
       # get removed, so add it back before the hint.
       code_tag = '```'
       message_list[index - 1] = '\n'.join((message_list[index - 1], code_tag))
       oversized_msg = ('\n**Error size > %d chars, '
       'there are %d more error(s) (%d total)**') % (
         char_limit, total_errors - index, total_errors
       )
       return message_list[:index] + [oversized_msg, hint]
   return message_list


 def _createSummaryMarkdown(step_json):
   """Returns a string with data on errors, warnings, and notifications.

   Extracts the number of errors, warnings and notifications
   from the dictionary(step_json).

   Then it lists all the errors line by line.

   Args:
       * step_json = {
         'errors': [
           {
             'message': string,
             'long_text': string,
             'items: [string],
             'fatal': boolean
           }
         ],
         'notifications': [
           {
             'message': string,
             'long_text': string,
             'items: [string],
             'fatal': boolean
           }
         ],
         'warnings': [
           {
             'message': string,
             'long_text': string,
             'items: [string],
             'fatal': boolean
           }
         ]
       }
   """
   errors = step_json['errors']
   warning_count = len(step_json['warnings'])
   notif_count = len(step_json['notifications'])
   description = (
     '#### There are %d error(s), %d warning(s),'
     ' and %d notifications(s). Here are the errors:') % (
       len(errors), warning_count, notif_count
   )
   error_messages = []

   for error in errors:
     error_messages.append(
       '**ERROR**\n```\n%s\n%s\n```' % (
       error['message'], error['long_text'])
     )

   error_messages = _limitSize(error_messages)
   # Description is not counted in the total message size.
   # It is inserted afterward to ensure it is the first message seen.
   error_messages.insert(0, description)
   if warning_count or notif_count:
     error_messages.append(
       ('#### To see notifications and warnings,'
       ' look at the stdout of the presubmit step.')
     )
   return '\n\n'.join(error_messages)
	# Copyright 2016 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	from recipe_engine import recipe_api

	from PB.recipe_engine import result as result_pb2
	from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2

	# 8 minutes seems like a reasonable upper bound on presubmit timings.
	# According to event mon data we have, it seems like anything longer than
	# this is a bug, and should just instant fail.
	_DEFAULT_TIMEOUT_S = 480


	class PresubmitApi(recipe_api.RecipeApi):

	def __init__(self, properties, **kwargs):
	super(PresubmitApi, self).__init__(**kwargs)

	self._runhooks = properties.runhooks
	self._timeout_s = properties.timeout_s or _DEFAULT_TIMEOUT_S

	@property
	def presubmit_support_path(self):
	return self.repo_resource('presubmit_support.py')

	def __call__(self, args, *kwargs):
	"""Returns a presubmit step."""

	kwargs['venv'] = True
	name = kwargs.pop('name', 'presubmit')
	with self.m.depot_tools.on_path():
	presubmit_args = list(args) + [
	'--json_output', self.m.json.output(),
	]
	if self.m.resultdb.enabled:
	kwargs['wrapper'] = ('rdb', 'stream', '--')
	step_data = self.m.python(
	name, self.presubmit_support_path, presubmit_args, **kwargs)
	output = step_data.json.output or {}
	if self.m.step.active_result.retcode != 0:
	return output

	# Run with vpython3 directly
	del (kwargs['venv'])
	presubmit_args = list(args) + [
	'--json_output',
	self.m.json.output(),
	]
	step_data = self.m.step(name + " py3",
	['vpython3', self.presubmit_support_path] +
	presubmit_args, **kwargs)
	output2 = step_data.json.output or {}

	# combine outputs
	for key in output:
	if key in output2:
	output[key] += output2[key]
	del (output2[key])
	for key in output2:
	output[key] = output2[key]
	return output

	def prepare(self, root_solution_revision=None):
	"""Sets up a presubmit run.

	This includes:
	- setting up the checkout w/ bot_update
	- locally committing the applied patch
	- running hooks, if requested

	This expects the gclient configuration to already have been set.

	Args:
	root_solution_revision: revision of the root solution

	Returns:
	the StepResult from the bot_update step.
	"""
	# Set up the root solution revision by either passing the revision
	# to this function or adding it to the input properties.
	root_solution_revision = (
	root_solution_revision or
	self.m.properties.get('root_solution_revision'))

	# Expect callers to have already set up their gclient configuration.
	bot_update_step = self.m.bot_update.ensure_checkout(
	timeout=3600, no_fetch_tags=True,
	root_solution_revision=root_solution_revision)

	relative_root = self.m.gclient.get_gerrit_patch_root().rstrip('/')

	abs_root = self.m.context.cwd.join(relative_root)
	with self.m.context(cwd=abs_root):
	# TODO(unowned): Consider either:
	# - extracting user name & email address from the issue, or
	# - using a dedicated and clearly nonexistent name/email address
	self.m.git('-c', 'user.email=commit-bot@chromium.org',
	'-c', 'user.name=The Commit Bot',
	'commit', '-a', '-m', 'Committed patch',
	name='commit-git-patch', infra_step=False)

	if self._runhooks:
	with self.m.context(cwd=self.m.path['checkout']):
	self.m.gclient.runhooks()

	return bot_update_step

	def execute(self, bot_update_step, skip_owners=False):
	"""Runs presubmit and sets summary markdown if applicable.

	Args:
	* bot_update_step: the StepResult from a previously executed bot_update step.
	* skip_owners: a boolean indicating whether Owners checks should be skipped.

	Returns:
	a RawResult object, suitable for being returned from RunSteps.
	"""
	relative_root = self.m.gclient.get_gerrit_patch_root().rstrip('/')
	abs_root = self.m.context.cwd.join(relative_root)
	got_revision_properties = self.m.bot_update.get_project_revision_properties(
	# Replace path.sep with '/', since most recipes are written assuming '/'
	# as the delimiter. This breaks on windows otherwise.
	relative_root.replace(self.m.path.sep, '/'), self.m.gclient.c)
	upstream = bot_update_step.json.output['properties'].get(
	got_revision_properties[0])

	presubmit_args = [
	'--issue', self.m.tryserver.gerrit_change.change,
	'--patchset', self.m.tryserver.gerrit_change.patchset,
	'--gerrit_url', 'https://%s' % self.m.tryserver.gerrit_change.host,
	'--gerrit_project', self.m.tryserver.gerrit_change.project,
	'--gerrit_branch', self.m.tryserver.gerrit_change_target_ref,
	'--gerrit_fetch',
	]
	if self.m.cq.active and self.m.cq.run_mode == self.m.cq.DRY_RUN:
	presubmit_args.append('--dry_run')

	presubmit_args.extend([
	'--root', abs_root,
	'--commit',
	'--verbose', '--verbose',
	'--skip_canned', 'CheckTreeIsOpen',
	'--upstream', upstream, # '' if not in bot_update mode.
	])

	if skip_owners:
	presubmit_args.extend([
	'--skip_canned', 'CheckOwners'
	])

	raw_result = result_pb2.RawResult()
	step_json = self(
	*presubmit_args,
	timeout=self._timeout_s,
	# ok_ret='any' causes all exceptions to be ignored in this step
	ok_ret='any')
	# Set recipe result values
	if step_json:
	raw_result.summary_markdown = _createSummaryMarkdown(step_json)

	retcode = self.m.step.active_result.retcode
	if retcode == 0:
	raw_result.status = common_pb2.SUCCESS
	return raw_result

	self.m.step.active_result.presentation.status = 'FAILURE'
	if self.m.step.active_result.exc_result.had_timeout:
	raw_result.status = common_pb2.FAILURE
	raw_result.summary_markdown += (
	'\n\nTimeout occurred during presubmit step.')
	elif retcode == 1:
	raw_result.status = common_pb2.FAILURE
	self.m.tryserver.set_test_failure_tryjob_result()
	else:
	raw_result.status = common_pb2.INFRA_FAILURE
	self.m.tryserver.set_invalid_test_results_tryjob_result()
	# Handle unexpected errors not caught by json output
	if raw_result.summary_markdown == '':
	raw_result.status = common_pb2.INFRA_FAILURE
	raw_result.summary_markdown = (
	'Something unexpected occurred'
	' while running presubmit checks.'
	' Please [file a bug](https://bugs.chromium.org'
	'/p/chromium/issues/entry?components='
	'Infra%3EClient%3EChrome&status=Untriaged)'
	)
	return raw_result


	def _limitSize(message_list, char_limit=450):
	"""Returns a list of strings within a certain character length.

	Args:
	* message_list (List[str]) - The message to truncate as a list
	of lines (without line endings).
	"""
	hint = ('**The complete output can be'
	' found at the bottom of the presubmit stdout.**')
	char_count = 0
	for index, message in enumerate(message_list):
	char_count += len(message)
	if char_count > char_limit:
	if index == 0:
	# Show at minimum part of the first error message
	first_message = message_list[index].splitlines()
	return ['\n'.join(
	_limitSize(first_message)
	)
	]
	total_errors = len(message_list)
	# If code is being cropped, the closing code tag will
	# get removed, so add it back before the hint.
	code_tag = '```'
	message_list[index - 1] = '\n'.join((message_list[index - 1], code_tag))
	oversized_msg = ('\n**Error size > %d chars, '
	'there are %d more error(s) (%d total)**') % (
	char_limit, total_errors - index, total_errors
	)
	return message_list[:index] + [oversized_msg, hint]
	return message_list


	def _createSummaryMarkdown(step_json):
	"""Returns a string with data on errors, warnings, and notifications.

	Extracts the number of errors, warnings and notifications
	from the dictionary(step_json).

	Then it lists all the errors line by line.

	Args:
	* step_json = {
	'errors': [
	{
	'message': string,
	'long_text': string,
	'items: [string],
	'fatal': boolean
	}
	],
	'notifications': [
	{
	'message': string,
	'long_text': string,
	'items: [string],
	'fatal': boolean
	}
	],
	'warnings': [
	{
	'message': string,
	'long_text': string,
	'items: [string],
	'fatal': boolean
	}
	]
	}
	"""
	errors = step_json['errors']
	warning_count = len(step_json['warnings'])
	notif_count = len(step_json['notifications'])
	description = (
	'#### There are %d error(s), %d warning(s),'
	' and %d notifications(s). Here are the errors:') % (
	len(errors), warning_count, notif_count
	)
	error_messages = []

	for error in errors:
	error_messages.append(
	'ERROR\n```\n%s\n%s\n```' % (
	error['message'], error['long_text'])
	)

	error_messages = _limitSize(error_messages)
	# Description is not counted in the total message size.
	# It is inserted afterward to ensure it is the first message seen.
	error_messages.insert(0, description)
	if warning_count or notif_count:
	error_messages.append(
	('#### To see notifications and warnings,'
	' look at the stdout of the presubmit step.')
	)
	return '\n\n'.join(error_messages)