scripts/slave/recipe_modules/test_utils/tests/run_tests_with_patch.py

# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

DEPS = [
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/step',

    'chromium_swarming',
    'chromium_tests',
    'test_utils',
]

from recipe_engine.recipe_api import Property
from recipe_engine import post_process

PROPERTIES = {
  'has_valid_results': Property(default=True),
  'retry_failed_shards': Property(default=None),
  'per_suffix_failures': Property(default=None),
  'per_suffix_valid': Property(default=None),
}

def RunSteps(api, has_valid_results, retry_failed_shards, per_suffix_failures,
             per_suffix_valid):
  test_kwargs = {
      'has_valid_results': has_valid_results,
  }
  run_tests_kwargs = {}

  if retry_failed_shards:
    run_tests_kwargs['retry_failed_shards'] = retry_failed_shards
    test_kwargs['runs_on_swarming'] = True
    test_kwargs['per_suffix_failures'] = per_suffix_failures
    test_kwargs['per_suffix_valid'] = per_suffix_valid

  tests = [
      api.chromium_tests.steps.MockTest(name='test', api=api, **test_kwargs),
      api.chromium_tests.steps.MockTest(name='test2', api=api),
  ]

  invalid, failing = api.test_utils.run_tests_with_patch(
      api, tests, **run_tests_kwargs)
  if invalid:
    api.step('%s invalid' % ','.join(sorted(t.name for t in invalid)), None)
  else:
    api.step('NONE invalid', None)

  if failing:
    failed_tests = []
    for t in failing:
      _, failed = t.with_patch_failures_including_retry()
      tup = (t.name,)
      if failed:
        tup = tup + ('#'.join(sorted(failed)),)
      failed_tests.append(':'.join(tup))

    failed_tests_s = ','.join(failed_tests)

    api.step('%s failing' % failed_tests_s, None)
  else:
    api.step('NONE failing', None)


def GenTests(api):
  # TODO(martiniss): Rewrite these tests to use assertions in RunSteps.
  yield (
      api.test('success') +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'NONE invalid') +
      api.post_process(post_process.MustRun, 'NONE failing') +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('invalid_results') +
      api.properties(has_valid_results=False) +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'test invalid') +
      api.post_process(post_process.MustRun, 'test failing') +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('retry_shards_retry_succeeds') +
      api.properties(retry_failed_shards=True, per_suffix_failures={
          'with patch': set(['testA']),
      }) +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'NONE invalid') +
      api.post_process(post_process.MustRun, 'NONE failing') +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('retry_shards_retry_still_fails') +
      api.properties(retry_failed_shards=True, per_suffix_failures={
          'with patch': set(['testA']),
          'retry shards with patch': set(['testA']),
      }) +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'NONE invalid') +
      api.post_process(post_process.MustRun, 'test:testA failing') +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('retry_shards_retry_subset_fails') +
      api.properties(retry_failed_shards=True, per_suffix_failures={
          'with patch': set(['testA', 'testB']),
          'retry shards with patch': set(['testB', 'testC']),
      }) +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'NONE invalid') +
      api.post_process(post_process.MustRun, 'test:testB failing') +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('retry_shards_invalid_then_valid') +
      api.properties(retry_failed_shards=True, per_suffix_failures={
          'with patch': set(['testA', 'testB']),
      }, per_suffix_valid={
          'with patch': False,
          'retry shards with patch': True,
      }) +
      api.post_process(post_process.MustRun, 'test (with patch)') +
      api.post_process(post_process.MustRun, 'test2 (with patch)') +
      api.post_process(post_process.MustRun, 'NONE invalid') +
      api.post_process(post_process.MustRun, 'NONE failing') +
      api.post_process(post_process.DropExpectation)
  )