#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unit tests for verification/try_server.py."""

import json
import logging
import optparse
import os
import re
import StringIO
import sys
import time
import unittest
import urllib

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(ROOT_DIR, '..'))

# In tests/
import mocks  # pylint: disable=W0403

from testing_support import auto_stub

# In root
import buildbot_json
from verification import base
from verification import try_server

# pylint: disable=W0212
SUCCESS = buildbot_json.SUCCESS
WARNINGS = buildbot_json.WARNINGS
FAILURE = buildbot_json.FAILURE
SKIPPED = buildbot_json.SKIPPED
EXCEPTION = buildbot_json.EXCEPTION


class FakeTryServer(auto_stub.SimpleMock):
  """Stateful try server mock.

  Includes calls to send try jobs TryChange() and visible results from HTTP
  requests.
  """
  def __init__(self, unit_test):
    super(FakeTryServer, self).__init__(unit_test)

    # Try server immutable properties.
    self.steps = ['update', 'compile', 'test1', 'test2']
    self.server_url = 'http://foo/bar'

    # State of the try server.
    self._builds = { 'linux': [], 'mac': [] }
    self.pending_builds = {
      'linux': [],
      'mac': []
    }
    # Default mocks.
    self.unit_test.mock(urllib, 'urlopen', self._mockurlopen)
    self.unit_test.mock(try_server.trychange, 'TryChange', self.TryChangeMock)

  def TryChangeMock(self, cmd, _change, swallow_exception):
    """Mocks trychange.py."""
    self.assertEqual(swallow_exception, True)
    parser = optparse.OptionParser()
    parser.add_option('--bot', action='append')
    parser.add_option('--clobber', action='store_true', default=False)
    parser.add_option('--email')
    parser.add_option('--issue', type='int')
    parser.add_option('--name')
    parser.add_option('--no_search', action='store_true')
    parser.add_option('--patchset', type='int')
    parser.add_option('--revision')
    parser.add_option('--rietveld_url')
    parser.add_option('--user')
    options, args = parser.parse_args(cmd)
    self.assertEqual(options.email, 'user1@example.com')
    self.assertEqual(options.issue, 42)
    self.assertEqual(options.no_search, True)
    self.assertEqual(options.patchset, 23)
    self.assertEqual(
        options.rietveld_url,
        '%s/download/issue42_23.diff' % self.unit_test.context.rietveld.url)
    self.assertEqual(options.user, 'user1')
    self.assertEqual(args, ['extra_flags'])
    bot = ', '.join(options.bot)
    call = 'trychange b={%s} c=%s r=%s' % (
        bot, options.clobber, options.revision)
    if options.name != '42-23':
      call += ' n=%s' % options.name
    self.calls.append(call)
    logging.debug(self.calls[-1])

  def _mockurlopen_internal(self, sub_url):
    """Returns data to be encoded before returning."""
    # sub_url is str on python <= 2.6 and unicode for >= 2.7
    self.unit_test.assertTrue(isinstance(sub_url, basestring))
    expected_url = self.server_url + '/json/'
    self.unit_test.assertTrue(sub_url.startswith(expected_url))
    self.calls.append(sub_url[len(expected_url):])
    baseurl = '^%s/json/' % re.escape(self.server_url)
    match = re.match(baseurl + r'builders/(\w+)/builds/\?(.+)$', sub_url)
    if match:
      data = {}
      for query in match.group(2).split('&'):
        m = re.match(r'select=(\w+)', query)
        self.unit_test.assertTrue(m, (match.group(2), query))
        build = int(m.group(1))
        data[str(build)] = self._builds[match.group(1)][build]
      return data

    match = re.match(baseurl + r'builders/(\w+)/builds/_all$', sub_url)
    if match:
      # Data is not stored exactly as the try server serves it.
      data = {}
      for i, build in enumerate(self._builds[match.group(1)]):
        data[str(i)] = build
      return data

    match = re.match(baseurl + r'builders/\?(.+)$', sub_url)
    if match:
      data = {}
      for query in match.group(1).split('&'):
        m = re.match(r'select=(\w+)', query)
        self.unit_test.assertTrue(m, (match.group(1), query))
        builder = m.group(1)
        data[builder] = {
          'cachedBuilds': range(len(self._builds[builder])),
          'pendingBuilds': len(self.pending_builds.get(builder, [])),
        }
      return data

    match = re.match(baseurl + r'builders/(\w+)/pendingBuilds$', sub_url)
    if match:
      return self.pending_builds[match.group(1)]

  def _mockurlopen(self, sub_url):
    """Mocks urllib.urlopen() and JSON encode + StringIO buffers."""
    sub_url = re.match(r'^(.+)[\&\?]filter=1$', sub_url).group(1)
    data = self._mockurlopen_internal(sub_url)
    self.unit_test.assertNotEquals(None, data, sub_url)
    #logging.debug('_mockurlopen(%s) -> %s' % (sub_url, data))
    return StringIO.StringIO(json.dumps(data))

  def add_build(self, builder, revision, reason, step_results):
    """Add a build to a builder."""
    self.assertEqual(len(self.steps), len(step_results))
    assert isinstance(revision, (str, int))
    data = {
      'reason': reason or self.unit_test.pending.pending_name(),
      'sourceStamp': {
        'revision': 'sol@%s' % revision,
        'hasPatch': True,
      },
      'steps': [],
      'blame': ['user1@example.com'],
      'number': 0,
      'slave': 'foo',
    }
    result = max(step_results)
    if result in (SUCCESS, WARNINGS) and step_results[-1] is None:
      result = None
    for i, step in enumerate(self.steps):
      data['steps'].append({
        'name': step,
        'results': [step_results[i]],
      })
    data['results'] = [result]
    self._builds[builder].append(data)

  def set_build_result(self, builder, result):
    """Override the build result for every steps to |result|."""
    for step in self._builds[builder][-1]['steps']:
      step['results'] = [result]
    self._builds[builder][-1]['results'] = [result]


class TryServerSvnTest(mocks.TestCase):
  def setUp(self):
    super(TryServerSvnTest, self).setUp()
    self.email = 'user1@example.com'
    self.user = 'user1'
    self.timestamp = [1.]
    # Mocks http://chromium-status.appspot.com/lkgr
    self.lkgr = 123
    self.try_server = FakeTryServer(self)
    self.mock(time, 'time', lambda: self.timestamp[-1])

    try_server.TryRunnerSvn.update_latency = 0
    self.builders_and_tests = {
      'linux': ['test1', 'test2'],
      'mac': ['test1', 'test2'],
    }
    self.try_runner = try_server.TryRunnerSvn(
        self.context,
        self.try_server.server_url,
        self.email,
        self.builders_and_tests,
        ['ignored_step'],
        'sol',
        ['extra_flags',],
        lambda: self.lkgr,
        )
    self.pending.revision = 123

  def tearDown(self):
    try:
      if not self.has_failed():
        self.try_server.check_calls([])
    finally:
      super(TryServerSvnTest, self).tearDown()

  def get_verif(self):
    return self.pending.verifications[self.try_runner.name]

  def assertPending(
      self, state, nb_jobs, error_message,
      linux_build=1,
      mac_build=1,
      linux_state=None,
      mac_state=None,
      linux_clobber=False,
      mac_clobber=False,
      linux_rev=123,
      mac_rev=123,
      linux_sent=1,
      mac_sent=1,
      linux_name='42-23',
      mac_name='42-23'):
    if linux_state is None:
      linux_state = state
    if mac_state is None:
      mac_state = state
    self.assertEqual([self.try_runner.name], self.pending.verifications.keys())
    self.assertEqual(error_message, self.get_verif().error_message)
    self.assertEqual(nb_jobs, len(self.get_verif().try_jobs))
    self.assertEqual(
        len(self.builders_and_tests), len(self.get_verif().try_jobs))
    self.assertEqual(linux_name, self.get_verif().try_jobs[0].name)
    self.assertEqual('linux', self.get_verif().try_jobs[0].builder)
    self.assertEqual(linux_rev, self.get_verif().try_jobs[0].revision)
    self.assertEqual(linux_sent, self.get_verif().try_jobs[0].sent)
    self.assertEqual(linux_clobber, self.get_verif().try_jobs[0].clobber)
    self.assertEqual(linux_build, self.get_verif().try_jobs[0].build)
    self.assertEqual(linux_state, self.get_verif().try_jobs[0].get_state())
    if len(self.builders_and_tests) > 1:
      self.assertEqual(mac_name, self.get_verif().try_jobs[1].name)
      self.assertEqual('mac', self.get_verif().try_jobs[1].builder)
      self.assertEqual(mac_rev, self.get_verif().try_jobs[1].revision)
      self.assertEqual(mac_sent, self.get_verif().try_jobs[1].sent)
      self.assertEqual(mac_clobber, self.get_verif().try_jobs[1].clobber)
      self.assertEqual(mac_build, self.get_verif().try_jobs[1].build)
      self.assertEqual(mac_state, self.get_verif().try_jobs[1].get_state())
    self.assertEqual(state, self.get_verif().get_state())

  def testVoid(self):
    self.assertEqual(self.pending.verifications.keys(), [])

  def testVerificationVoid(self):
    self.try_runner.verify(self.pending)
    self.assertPending(base.PROCESSING, 2, None, linux_build=None,
        mac_build=None)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.context.status.check_names(['try server'] * 2)

  def testVoidUpdate(self):
    self.try_runner.update_status([])

  def test_steps_quality(self):
    self.assertEqual(None, try_server.steps_quality([]))
    self.assertEqual(True, try_server.steps_quality([True, None]))
    self.assertEqual(False, try_server.steps_quality([True, None, False]))

  def testStepQualityNone(self):
    self.try_runner.status.builders['linux'].builds.cache()
    self.assertEqual(
        (None, 0),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))
    self.try_server.check_calls(['builders/linux/builds/_all'])

  def testStepQualityGood(self):
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, None, None, None])
    self.try_runner.status.builders['linux'].builds.cache()
    self.try_server.check_calls(['builders/linux/builds/_all'])
    self.assertEqual(
        ([True, None, None, None], 1),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))
    self.try_server.set_build_result('linux', SUCCESS)
    self.try_runner.status.builders['linux'].builds.refresh()
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))
    self.try_server.check_calls(['builders/linux/builds/_all'])

  def testStepQualityBad(self):
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS])
    self.try_runner.status.builders['linux'].builds.cache()
    # Also test that FakeTryServer.add_build() is implemented correctly.
    self.assertEqual(
        ([True, True, False, True], 1),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))
    self.try_server.check_calls(['builders/linux/builds/_all'])

  def testStepQualityBadIncomplete(self):
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, None])
    self.try_runner.status.builders['linux'].builds.cache()
    # Also test that FakeTryServer.add_build() is implemented correctly.
    self.assertEqual(
        ([True, True, False, None], 1),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))
    self.try_server.check_calls(['builders/linux/builds/_all'])

  def testStepQualityGoodAndBad(self):
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_server.add_build('linux', 123, None, [FAILURE, None, None, None])
    self.try_runner.status.builders['linux'].builds.cache()
    self.try_server.check_calls(['builders/linux/builds/_all'])
    self.assertEqual(
        ([True] * 4, 2),
        self.try_runner.step_db.revision_quality_builder_steps('linux', 123))

  def testQualityAutomatic(self):
    self.try_runner.verify(self.pending)
    self.assertEqual(
        (None, 0),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.try_server.add_build(
        'linux', 123, 'georges tried stuff',
        [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_runner.update_status([self.pending])
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.try_server.check_calls(
        [ 'trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123',
          'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    self.context.status.check_names(['try server'] * 2)

  def testQualityManual(self):
    self.try_server.add_build(
        'linux', 123, 'georges tried stuff',
        [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_runner.status.builders['linux'].builds.cache()
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.try_server.check_calls(['builders/linux/builds/_all'])

  def _simple(self, status_linux, status_mac=None, error_msg=None):
    """status_linux affects test1, status_mac affects test2."""
    def is_failure(status):
      return status in (FAILURE, EXCEPTION)

    self.assertEqual(
        bool(is_failure(status_linux) or is_failure(status_mac)),
        bool(error_msg))
    if status_mac is None:
      status_mac = status_linux
    self.lkgr = 12
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_server.add_build(
        'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])

    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, status_linux, SUCCESS])
    self.try_server.add_build(
        'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, status_mac])
    self.try_runner.update_status([self.pending])

    if is_failure(status_linux):
      self.assertEqual(123, self.try_runner.get_lkgr('linux'))
      self.assertEqual(
          123, self.try_runner.step_db.last_good_revision_builder('linux'))
    else:
      self.assertEqual(123, self.try_runner.get_lkgr('linux'))
      self.assertEqual(
          123, self.try_runner.step_db.last_good_revision_builder('linux'))
    if is_failure(status_mac):
      self.assertEqual(123, self.try_runner.get_lkgr('mac'))
      self.assertEqual(
          123, self.try_runner.step_db.last_good_revision_builder('mac'))
    else:
      self.assertEqual(123, self.try_runner.get_lkgr('mac'))
      self.assertEqual(
          123, self.try_runner.step_db.last_good_revision_builder('mac'))

    if error_msg:
      # Can't test failure without testing automatic retry mechanism.
      expected = (
          [ 'builders/?select=linux&select=mac',
            'builders/linux/builds/_all', 'builders/mac/builds/_all'])

      if is_failure(status_linux):
        expected.append(
            'trychange b={linux:test1} c=False r=sol@123 n=42-23 (retry)')
        linux_build = None
        linux_state = base.PROCESSING
        linux_name = '42-23 (retry)'
      else:
        linux_build = 1
        linux_state = base.SUCCEEDED
        linux_name = '42-23'

      if is_failure(status_mac):
        expected.append(
            'trychange b={mac:test2} c=False r=sol@123 n=42-23 (retry)')
        mac_build = None
        mac_state = base.PROCESSING
        mac_name = '42-23 (retry)'
      else:
        mac_build = 1
        mac_state = base.SUCCEEDED
        mac_name = '42-23'

      self.assertPending(
          base.PROCESSING, 2, None, linux_build=linux_build,
          mac_build=mac_build, linux_state=linux_state, mac_state=mac_state,
          linux_name=linux_name, mac_name=mac_name)
      self.try_server.check_calls(expected)

      if is_failure(status_linux):
        self.try_server.add_build(
            'linux', 123, linux_name, [SUCCESS, SUCCESS, status_linux, SUCCESS])
      if is_failure(status_mac):
        self.try_server.add_build(
            'mac', 123, mac_name, [SUCCESS, SUCCESS, SUCCESS, status_mac])

      self.try_runner.update_status([self.pending])
      if is_failure(status_linux):
        linux_state = base.FAILED
        linux_build = 2
      else:
        linux_build = 1
      if is_failure(status_mac):
        mac_state = base.FAILED
        mac_build = 2
      else:
        mac_build = 1
      self.assertPending(
          base.FAILED, 2, error_msg,
          linux_build=linux_build, mac_build=mac_build,
          linux_state=linux_state, mac_state=mac_state,
          linux_name=linux_name, mac_name=mac_name)

      if is_failure(status_linux) and is_failure(status_mac):
        self.try_server.check_calls(
            [ 'builders/?select=linux&select=mac',
              'builders/linux/builds/_all', 'builders/mac/builds/_all'])
        self.context.checkout.check_calls(
            [ 'prepare(123)',
              'apply_patch(%r)' % self.context.rietveld.patchsets[-2],
              'prepare(123)',
              'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
      elif is_failure(status_linux):
        self.try_server.check_calls(
            ['builders/?select=linux', 'builders/linux/builds/_all'])
        self.context.checkout.check_calls(
            [ 'prepare(123)',
              'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
      else:
        self.try_server.check_calls(
            ['builders/?select=mac', 'builders/mac/builds/_all'])
        self.context.checkout.check_calls(
            [ 'prepare(123)',
              'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
    else:
      self.assertPending(base.SUCCEEDED, 2, None)
      self.try_server.check_calls(
          [ 'builders/?select=linux&select=mac',
            'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    count = 6 + 3 * (
        int(is_failure(status_linux)) + int(is_failure(status_mac)))
    self.context.status.check_names(['try server'] * count)

  def testImmediateSuccess(self):
    self._simple(SUCCESS)

  def testImmediateWarnings(self):
    self._simple(WARNINGS)

  def testImmediateSkipped(self):
    self._simple(SKIPPED)

  def second_fail_msg(
      self, clname, step2, step1, builder, number, is_clobber=False):
    extra = ''
    if is_clobber:
      extra = ' (clobber build)'
    return (
        u'Try job failure for %s on %s for step "%s"%s.\n'
        u'It\'s a second try, previously, step "%s" failed.\n'
        u'%s/buildstatus?builder=%s&number=%s\n') % (
            clname, builder, step2, extra, step1, self.try_server.server_url,
            builder, number)
  def testImmediateFailureLinux(self):
    self._simple(
        FAILURE, SUCCESS,
        self.second_fail_msg('42-23 (retry)', 'test1', 'test1', 'linux', 2))

  def testImmediateFailureMac(self):
    self._simple(
        SUCCESS, FAILURE,
        self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2))

  def testImmediateDoubleFailure(self):
    self._simple(
        FAILURE, FAILURE,
        self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2))

  def testImmediateException(self):
    self._simple(
        SUCCESS, EXCEPTION,
        self.second_fail_msg('42-23 (retry)', 'test2', 'test2', 'mac', 2))

  def testSuccess(self):
    self.lkgr = 2
    # Normal workflow with incremental success.
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])

    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 2, None, linux_build=None, mac_build=None)
    self.try_server.check_calls(
        ['builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])

    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, None, None, None])
    self.try_runner.update_status([self.pending])
    self.assertPending(base.PROCESSING, 2, None, linux_build=0, mac_build=None)
    self.try_server.check_calls(
        ['builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])

    self.try_server.add_build(
        'mac', 123, None, [SUCCESS, None, None, None])
    self.try_runner.update_status([self.pending])
    self.assertPending(base.PROCESSING, 2, None, linux_build=0, mac_build=0)
    self.try_server.check_calls(
        ['builders/?select=mac',
          'builders/mac/builds/_all', 'builders/linux/builds/?select=0'])

    # This one will be cached since it's now immutable.
    self.try_server.set_build_result('mac', SUCCESS)
    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 2, None, linux_build=0, mac_build=0,
        mac_state=base.SUCCEEDED)
    self.try_server.check_calls(
        ['builders/linux/builds/?select=0', 'builders/mac/builds/?select=0'])

    self.try_server.set_build_result('linux', SUCCESS)
    self.try_runner.update_status([self.pending])
    self.assertPending(base.SUCCEEDED, 2, None, linux_build=0, mac_build=0)
    self.assertEqual(
        123, self.try_runner.step_db.last_good_revision_builder('linux'))
    self.assertEqual(
        123, self.try_runner.step_db.last_good_revision_builder('mac'))
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'mac', 123))
    self.try_server.check_calls(['builders/linux/builds/?select=0'])
    self.context.status.check_names(['try server'] * 6)

  def testIgnorePreviousJobs(self):
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])

    self.try_runner.update_status([self.pending])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])

    self.try_server.add_build('linux', 12, None, [None, None, None, None])
    self.try_server.add_build('mac', 12, None, [None, None, None, None])
    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 2, None, linux_build=None,
        mac_build=None)
    self.try_server.check_calls(
        ['builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])

    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_server.add_build(
        'mac', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_runner.update_status([self.pending])
    self.assertPending(base.SUCCEEDED, 2, None)
    self.assertEqual(
        123, self.try_runner.step_db.last_good_revision_builder('linux'))
    self.assertEqual(
        123, self.try_runner.step_db.last_good_revision_builder('mac'))
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.assertEqual(
        ([True] * 4, 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'mac', 123))
    self.try_server.check_calls(
        ['builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    self.context.status.check_names(['try server'] * 6)

  def testNames(self):
    job = try_server.TryJob(
        builder='builder', revision=123, tests=['test1'], clobber=False)
    self.assertEqual(None, job.name)

  def testLostJob(self):
    # Test that a job is automatically retried if it was never started up. It
    # does happen.
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])

    # Keep a copy of the try jobs to compare later.
    self.try_runner.update_status([self.pending])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    self.assertPending(
        base.PROCESSING, 2, None, mac_sent=self.timestamp[-1],
        linux_build=None, mac_build=None)

    # lost_try_job_delay + 2 seconds later.
    # linux is pending, mac is lost.
    self.try_server.pending_builds['linux'] = [
      {
        'reason': '42-23',
      }
    ]
    self.timestamp.append(self.try_runner.lost_try_job_delay + 2)
    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 2, None, mac_sent=self.timestamp[-1],
        linux_build=None, mac_build=None,
        mac_name='42-23 (previous was lost)')
    self.try_server.check_calls(
        # Look if there is pending build on each builder.
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all',
          'builders/linux/pendingBuilds',
          # Retry only mac.
          'trychange b={mac:test1,test2} c=False r=sol@123 n=42-23 (previous '
          'was lost)'])

    # linux job was completed.
    self.try_server.pending_builds['linux'] = []
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 2, None, mac_sent=self.timestamp[1],
        linux_build=0, mac_build=None,
        linux_state=base.SUCCEEDED,
        mac_name='42-23 (previous was lost)')
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])

    # 2 * (lost_try_job_delay + 2) seconds later, mac job started and completed.
    self.timestamp.append(2 * (self.try_runner.lost_try_job_delay + 2))
    self.try_server.add_build(
        'mac', 123, '42-23 (previous was lost)',
        [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.SUCCEEDED, 2, None, mac_sent=self.timestamp[1],
        linux_build=0, mac_build=0,
        mac_name='42-23 (previous was lost)')
    self.try_server.check_calls(
        ['builders/?select=mac', 'builders/mac/builds/_all'])

    self.try_runner.update_status([self.pending])
    self.context.checkout.check_calls(
        [ 'prepare(123)',
          'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
    self.context.status.check_names(['try server'] * 7)

  def testFailedStepRetryLkgr(self):
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS])

    self.lkgr = 122
    self.try_runner.update_status([self.pending])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all',
          # Only the failed test is retried, on lkgr revision.
          'trychange b={linux:test1} c=False r=sol@122 n=42-23 (retry)'])
    self.assertEqual(['test1'], self.get_verif().try_jobs[0].failed_steps)
    self.context.checkout.check_calls(
        [ 'prepare(122)',
          'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
    self.context.status.check_names(['try server'] * 5)

  def testFailedUpdate(self):
    # It must not retry a failed update.
    # Add succeededing builds, this sets quality to True, which disable retry
    # mechanism.
    self.try_server.add_build(
        'linux', 123, 'georges tried stuff',
        [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.try_server.add_build(
        'mac', 123, 'georges tried stuff', [SUCCESS, SUCCESS, SUCCESS, SUCCESS])
    self.lkgr = 123

    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.try_server.add_build(
        'linux', 123, None, [FAILURE, None, None, None])
    self.try_runner.update_status([self.pending])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    self.assertEqual('linux', self.get_verif().try_jobs[0].builder)
    self.assertEqual(['update'], self.get_verif().try_jobs[0].failed_steps)
    self.assertPending(
        base.FAILED, 2,
        (u'Try job failure for 42-23 on linux for step '
        u'"update".\n%s/buildstatus?builder=linux&number=1\n\n'
        u'Step "update" is always a major failure.\n'
        u'Look at the try server FAQ for more details.') %
            self.try_server.server_url,
        mac_build=None,
        mac_state=base.PROCESSING)
    self.context.status.check_names(['try server'] * 4)

  def testFailedCompileRetryClobber(self):
    # It must retry once a non-clobber compile.
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, FAILURE, None, None])
    self.lkgr = 122
    self.try_runner.update_status([self.pending])
    self.context.checkout.check_calls(
        [ 'prepare(122)',
          'apply_patch(%r)' % self.context.rietveld.patchsets[-1]])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all',
          # Retries at lkgr.
          'trychange b={linux:test1,test2} c=True r=sol@122 n=42-23 (retry)'])
    self.assertEqual(['compile'], self.get_verif().try_jobs[0].failed_steps)
    self.assertPending(
        base.PROCESSING, 2, None, linux_rev=122,
        linux_clobber=True, linux_build=None, mac_build=None,
        linux_name='42-23 (retry)')

    self.try_server.add_build(
        'linux', 122, '42-23 (retry)', [SUCCESS, FAILURE, None, None])
    self.try_runner.update_status([self.pending])
    self.try_server.check_calls(
        [ 'builders/?select=linux&select=mac',
          'builders/linux/builds/_all', 'builders/mac/builds/_all'])
    self.assertEqual(['compile'], self.get_verif().try_jobs[0].failed_steps)
    self.assertPending(
        base.FAILED, 2,
        self.second_fail_msg('42-23 (retry)', 'compile', 'compile', 'linux', 1,
            True),
        linux_rev=122,
        linux_clobber=True,
        mac_build=None,
        mac_state=base.PROCESSING,
        linux_name='42-23 (retry)')
    self.context.status.check_names(['try server'] * 7)

  def testTooManyRetries(self):
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    job = self.pending.verifications[self.try_runner.name].try_jobs[0]
    self.try_runner._send_jobs(
        self.pending, [job], False, {job.builder:job.tests}, 'foo')
    self.try_server.check_calls(
        [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo'])
    job = self.pending.verifications[self.try_runner.name].try_jobs[0]
    self.try_runner._send_jobs(
        self.pending, [job], False, {job.builder:job.tests}, 'foo')
    self.try_server.check_calls(
        [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo'])
    job = self.pending.verifications[self.try_runner.name].try_jobs[0]
    self.try_runner._send_jobs(
        self.pending, [job], False, {job.builder:job.tests}, 'foo')
    self.try_server.check_calls(
        [ 'trychange b={linux:test1,test2} c=False r=sol@123 n=foo'])
    job = self.pending.verifications[self.try_runner.name].try_jobs[0]
    try:
      self.try_runner._send_jobs(
          self.pending, [job], False, {job.builder:job.tests}, 'foo')
      self.fail()
    except base.DiscardPending:
      pass
    self.context.status.check_names(['try server'] * 5)

  def testNoTry(self):
    self.pending.description += '\nNOTRY=true'
    self.try_runner.verify(self.pending)
    self.assertEqual(
        base.SUCCEEDED,
        self.pending.verifications[self.try_runner.name].get_state())

  def testNoTryWrong(self):
    self.pending.description += '\nNOTRY=true2'
    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2, mac:test1,test2} c=False r=sol@123'])
    self.context.status.check_names(['try server'] * 2)
    self.assertEqual(
        base.PROCESSING,
        self.pending.verifications[self.try_runner.name].get_state())

  def testSuccessIgnoredFailure(self):
    # Simplify testing code by removing mac.
    del self.try_runner.builders_and_tests['mac']
    # Add fake failing ignored step.
    self.try_server.steps = [
        'update', 'compile', 'ignored_step', 'test1', 'test2']

    self.try_runner.verify(self.pending)
    self.try_server.check_calls(
        ['trychange b={linux:test1,test2} c=False r=sol@123'])

    self.try_runner.update_status([self.pending])
    self.assertPending(
        base.PROCESSING, 1, None, linux_build=None, mac_build=None)
    self.try_server.check_calls(
        ['builders/?select=linux', 'builders/linux/builds/_all'])

    self.try_server.add_build(
        'linux', 123, None, [SUCCESS, SUCCESS, FAILURE, SUCCESS, SUCCESS])
    self.try_runner.update_status([self.pending])
    self.assertPending(base.SUCCEEDED, 1, None, linux_build=0, mac_build=None)
    self.try_server.check_calls(
        ['builders/?select=linux', 'builders/linux/builds/_all'])

    self.try_server.set_build_result('linux', SUCCESS)
    self.try_runner.update_status([self.pending])
    self.assertPending(base.SUCCEEDED, 1, None, linux_build=0, mac_build=0)
    # TODO(maruel): Fix, since StepDb doesn't know about ignored steps.
    self.assertEqual(
        None, self.try_runner.step_db.last_good_revision_builder('linux'))
    self.assertEqual(
        ([True, True, False, True, True], 1),
        self.try_runner.step_db.revision_quality_builder_steps(
          'linux', 123))
    self.context.status.check_names(['try server'] * 3)


if __name__ == '__main__':
  logging.basicConfig(
      level=[logging.WARNING, logging.INFO, logging.DEBUG][
        min(sys.argv.count('-v'), 2)],
      format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s')
  unittest.main()