# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from collections import defaultdict
import copy
from datetime import datetime

from common import acl
from common import constants
from common.waterfall import failure_type
from gae_libs import token
from gae_libs.http import auth_util
from gae_libs.handlers.base_handler import BaseHandler, Permission
from handlers import handlers_util
from handlers import result_status
from handlers.result_status import NO_TRY_JOB_REASON_MAP
from libs import analysis_status
from model import result_status as analysis_result_status
from model import suspected_cl_status
from model.base_build_model import BaseBuildModel
from model.result_status import RESULT_STATUS_TO_DESCRIPTION
from model.suspected_cl_confidence import SuspectedCLConfidence
from model.suspected_cl_status import CL_STATUS_TO_DESCRIPTION
from model.wf_analysis import WfAnalysis
from model.wf_suspected_cl import WfSuspectedCL
from model.wf_try_job import WfTryJob
from waterfall import build_failure_analysis_pipelines
from waterfall import build_util
from waterfall import buildbot
from waterfall import suspected_cl_util
from waterfall import waterfall_config

NON_SWARMING = object()

_ANALYSIS_CL_STATUS_MAP = {
    analysis_result_status.FOUND_CORRECT: suspected_cl_status.CORRECT,
    analysis_result_status.FOUND_INCORRECT: suspected_cl_status.INCORRECT
}


def _FormatDatetime(dt):
  if not dt:
    return None
  else:
    return dt.strftime('%Y-%m-%d %H:%M:%S UTC')


def _GetCLDict(analysis, cl_info):
  if not cl_info:
    return {}

  cl_keys = suspected_cl_util.GetCLInfo(cl_info)
  repo_name = cl_keys[0]
  revision = cl_keys[1]
  for cl in analysis.suspected_cls:
    if cl['repo_name'] == repo_name and cl['revision'] == revision:
      return cl
  return {}


def _GetTriageHistory(analysis):
  if (not auth_util.IsCurrentUserAdmin() or not analysis.completed or
      not analysis.triage_history):
    return None

  triage_history = []
  for triage_record in analysis.triage_history:
    triage_history.append({
        'triage_time':
            _FormatDatetime(
                datetime.utcfromtimestamp(triage_record['triage_timestamp'])),
        'user_name':
            triage_record['user_name'],
        'triaged_cl':
            _GetCLDict(analysis, triage_record.get('triaged_cl')),
        'result_status': (
            RESULT_STATUS_TO_DESCRIPTION.get(
                triage_record.get('result_status')) or
            CL_STATUS_TO_DESCRIPTION.get(triage_record.get('cl_status'))),
        'version':
            triage_record.get('version'),
    })

  return triage_history


def _GetOrganizedAnalysisResultBySuspectedCL(master_name, builder_name,
                                             analysis_result):
  """Group tests it they have the same suspected CLs."""
  organized_results = defaultdict(list)

  if not analysis_result:
    return organized_results

  for step_failure in analysis_result.get('failures', []):
    step_name = step_failure['step_name']
    supported = step_failure.get('supported', True)
    step_revisions_index = {}
    organized_suspected_cls = organized_results[step_name]
    is_flaky = step_failure.get('flaky', False)

    if not step_failure.get('tests'):
      # Non swarming, just group the whole step together.
      shared_result = {
          'first_failure':
              step_failure['first_failure'],
          'last_pass':
              step_failure.get('last_pass'),
          'first_failure_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  step_failure['first_failure']),
          'last_pass_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  step_failure['last_pass']),
          'supported':
              supported,
          'tests': [],
          'suspected_cls':
              step_failure['suspected_cls'],
          'flaky':
              is_flaky,
      }
      organized_suspected_cls.append(shared_result)
      continue

    # Swarming tests.
    for index, cl in enumerate(step_failure['suspected_cls']):
      step_revisions_index[cl['revision']] = index

    # Groups tests by suspected CLs' revision.
    # Keys are the indices of each test in the test list.
    # Format is as below:
    # {
    #     1: {
    #         'tests': ['test1', 'test2'],
    #         'revisions': ['rev1'],
    #         'suspected_cls': [
    #             # suspected cl info for rev1 at step level.
    #         ]
    #     },
    #     3: {
    #         'tests': ['test3'],
    #         'revisions': ['rev3', 'rev2'],
    #         'suspected_cls': [
    #             # suspected cl info for rev2, rev3 at step level.
    #         ]
    #     }
    # }
    tests_group = defaultdict(list)
    for index, test in enumerate(step_failure['tests']):
      # Get all revisions for this test and check if there is
      # any other test has the same culprit(represented by revision) set and
      # are flaky or reliable at the same time.
      test_name = test['test_name']
      is_flaky = test.get('flaky', False)
      found_group = False
      revisions = set()
      for cl in test['suspected_cls']:
        revisions.add(cl['revision'])
      for group in tests_group.values():
        # Found tests that have the same culprit(represented by revision),
        # add current test to group.
        if revisions == set(group['revisions']) and is_flaky == group['flaky']:
          group['tests'].append(test_name)
          found_group = True
          break
      if not found_group:
        # First test with that revision set, add a new group.
        group_suspected_cls = []
        for revision in revisions:
          group_suspected_cls.append(
              step_failure['suspected_cls'][step_revisions_index[revision]])
        tests_group[index] = {
            'tests': [test_name],
            'revisions': list(revisions),
            'suspected_cls': group_suspected_cls,
            'flaky': is_flaky,
        }

    for index, group in tests_group.iteritems():
      # Reorganize heuristic results by culprits.
      test_result = step_failure['tests'][index]
      shared_result = {
          'first_failure':
              test_result['first_failure'],
          'last_pass':
              test_result.get('last_pass'),
          'first_failure_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  test_result['first_failure']),
          'last_pass_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  test_result['last_pass']),
          'supported':
              supported,
          'tests':
              group['tests'],
          'suspected_cls':
              group['suspected_cls'],
          'flaky':
              group['flaky'],
      }
      organized_suspected_cls.append(shared_result)

  return organized_results


def _AttachBuildIdToSuspectedCLs(master_name, builder_name, suspected_cls):
  for cl in suspected_cls:
    cl['build_id'] = buildbot.GetBuildId(master_name, builder_name,
                                         cl.get('build_number'))
  return suspected_cls


def _GetAnalysisResultWithTryJobInfo(show_debug_info, organized_results,
                                     master_name, builder_name, build_number):
  """Reorganizes analysis result and try job result by step_name and culprit.

  Returns:
    update_result (dict): A dict of classified results.

    The format for those dicts are as below:
    {
        # A dict of results that contains both
        # heuristic analysis results and try job results.
        'reliable_failures': {
            'step1': {
                'results': [
                    {
                        'try_job':{
                            'ref_name': 'step1',
                            'try_job_key': 'm/b/119',
                            'status': analysis_status.COMPLETED,
                            'try_job_url': 'url/121',
                            'try_job_build_number': 121,
                        },
                        'heuristic_analysis': {
                            'suspected_cls': [
                                {
                                    'build_number': 98,
                                    'repo_name': 'chromium',
                                    'revision': 'r98_1',
                                    'commit_position': None,
                                    'url': None,
                                    'score': 5,
                                    'hints': {
                                        'added f98.cc (and it was in log)': 5,
                                    },
                                }
                            ]
                        }
                        'tests': ['test1', 'test2'],
                        'first_failure': 98,
                        'last_pass': 97,
                        'supported': True
                    }
                ]
            }
        },
        # A dict of result for flaky tests.
        'flaky_failures': {...},
        # A dict of results for all the other conditions,
        # such as non-swarming tests or swarming rerun failed.
        'unclassified_failures': {...}
    }
  """
  updated_results = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))

  if not organized_results:
    return updated_results

  try_job_info = handlers_util.GetAllTryJobResults(
      master_name, builder_name, build_number, show_debug_info)

  if not try_job_info:
    return updated_results

  for step_name, try_jobs in try_job_info.iteritems():
    try_jobs = try_jobs['try_jobs']
    step_heuristic_results = organized_results[step_name]
    step_updated_results = updated_results[step_name]['results']

    # Finds out try job result index and heuristic result index for each test.
    test_result_map = defaultdict(lambda: defaultdict(int))

    for index, try_job in enumerate(try_jobs):
      if not try_job.get('tests'):  # Compile or non-swarming.
        test_result_map[NON_SWARMING]['try_job_index'] = index
        continue
      for test_name in try_job['tests']:
        test_result_map[test_name]['try_job_index'] = index

    for index, heuristic_result in enumerate(step_heuristic_results):
      if not heuristic_result.get('tests'):  # Compile or non-swarming.
        test_result_map[NON_SWARMING]['heuristic_index'] = index
        continue
      for test_name in heuristic_result['tests']:
        test_result_map[test_name]['heuristic_index'] = index

    # Group tests based on indices.
    indices_test_map = defaultdict(list)
    for test_name, indices in test_result_map.iteritems():
      indices_test_map[(indices['try_job_index'],
                        indices['heuristic_index'])].append(test_name)

    for (try_job_index, heuristic_index), tests in indices_test_map.iteritems():
      try_job_result = try_jobs[try_job_index]
      heuristic_result = step_heuristic_results[heuristic_index]

      final_result = {
          'try_job':
              try_job_result,
          'heuristic_analysis': {
              'suspected_cls':
                  _AttachBuildIdToSuspectedCLs(
                      master_name, builder_name,
                      heuristic_result['suspected_cls']),
          },
          'tests':
              tests if tests != [NON_SWARMING] else [],
          'first_failure':
              heuristic_result['first_failure'],
          'last_pass':
              heuristic_result['last_pass'],
          'first_failure_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  heuristic_result['first_failure']),
          'last_pass_build_id':
              buildbot.GetBuildId(master_name, builder_name,
                                  heuristic_result['last_pass']),
          'supported':
              heuristic_result['supported']
      }

      if (('status' not in try_job_result or
           try_job_result['status'] in NO_TRY_JOB_REASON_MAP.values()) or
          try_job_result.get('can_force')):
        # There is no try job info but only heuristic result.
        try_job_result['status'] = try_job_result.get('status',
                                                      result_status.UNKNOWN)
        step_updated_results['unclassified_failures'].append(final_result)
      elif (try_job_result['status'] == result_status.FLAKY or
            bool(heuristic_result.get('flaky'))):
        step_updated_results['flaky_failures'].append(final_result)
      else:
        step_updated_results['reliable_failures'].append(final_result)

  return updated_results


def _PrepareTryJobDataForCompileFailure(analysis):
  try_job_data = {}
  if not (analysis.failure_result_map and  # pragma: no branch.
          constants.COMPILE_STEP_NAME in analysis.failure_result_map):
    return try_job_data  # pragma: no cover.

  referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(
      analysis.failure_result_map[constants.COMPILE_STEP_NAME])
  try_job = WfTryJob.Get(*referred_build_keys)
  if not try_job or not try_job.compile_results:
    return try_job_data  # pragma: no cover.
  result = try_job.compile_results[-1]

  try_job_data['status'] = analysis_status.STATUS_TO_DESCRIPTION.get(
      try_job.status, 'unknown').lower()
  try_job_data['url'] = result.get('url')
  try_job_data['completed'] = try_job.completed
  try_job_data['failed'] = try_job.failed
  try_job_data['culprit'] = result.get('culprit',
                                       {}).get(constants.COMPILE_STEP_NAME)

  return try_job_data


def _PopulateHeuristicDataForCompileFailure(analysis, data):
  if analysis.result:  # pragma: no branch.
    compile_failure = None
    for failure in analysis.result.get('failures', []):
      if failure['step_name'] == constants.COMPILE_STEP_NAME:
        compile_failure = failure
    if compile_failure:  # pragma: no branch.
      data['first_failure'] = compile_failure['first_failure']
      data['last_pass'] = compile_failure['last_pass']
      data['first_failure_build_id'] = buildbot.GetBuildId(
          analysis.master_name, analysis.builder_name,
          compile_failure['first_failure'])
      data['last_pass_build_id'] = buildbot.GetBuildId(
          analysis.master_name, analysis.builder_name,
          compile_failure['last_pass'])
      data['suspected_cls_by_heuristic'] = _AttachBuildIdToSuspectedCLs(
          analysis.master_name, analysis.builder_name,
          compile_failure['suspected_cls'])


def _GetAllSuspectedCLsAndCheckStatus(master_name, builder_name, build_number,
                                      analysis):
  if not analysis or not analysis.suspected_cls:
    return []
  build_key = BaseBuildModel.CreateBuildKey(master_name, builder_name,
                                            build_number)
  suspected_cls = copy.deepcopy(analysis.suspected_cls)

  confidences = SuspectedCLConfidence.Get()

  for cl in suspected_cls:
    cl['status'] = _ANALYSIS_CL_STATUS_MAP.get(analysis.result_status, None)
    cl['confidence'] = None

    suspected_cl = WfSuspectedCL.Get(cl['repo_name'], cl['revision'])
    if not suspected_cl:
      continue

    build = suspected_cl.builds.get(build_key)
    if build:
      cl['status'] = build['status']
      cl['confidence'] = '%d%%' % (
          suspected_cl_util.GetSuspectedCLConfidenceScore(confidences, build) or
          0)

  return suspected_cls


class BuildFailure(BaseHandler):
  PERMISSION_LEVEL = Permission.ANYONE

  def _PrepareCommonDataForFailure(self, analysis):
    return {
        'master_name':
            analysis.master_name,
        'builder_name':
            analysis.builder_name,
        'build_number':
            analysis.build_number,
        'build_id':
            buildbot.GetBuildId(analysis.master_name, analysis.builder_name,
                                analysis.build_number),
        'pipeline_status_path':
            analysis.pipeline_status_path,
        'show_debug_info':
            self._ShowDebugInfo(),
        'analysis_request_time':
            _FormatDatetime(analysis.request_time),
        'analysis_start_time':
            _FormatDatetime(analysis.start_time),
        'analysis_end_time':
            _FormatDatetime(analysis.end_time),
        'analysis_duration':
            analysis.duration,
        'analysis_update_time':
            _FormatDatetime(analysis.updated_time),
        'analysis_completed':
            analysis.completed,
        'analysis_failed':
            analysis.failed,
        'analysis_correct':
            analysis.correct,
        'analysis_is_duplicate':
            analysis.is_duplicate,
        'triage_history':
            _GetTriageHistory(analysis),
        'show_admin_controls':
            auth_util.IsCurrentUserAdmin(),
        'triage_reference_analysis_master_name':
            analysis.triage_reference_analysis_master_name,
        'triage_reference_analysis_builder_name':
            analysis.triage_reference_analysis_builder_name,
        'triage_reference_analysis_build_number':
            analysis.triage_reference_analysis_build_number
    }

  def _PrepareDataForCompileFailure(self, analysis, data):
    data['infra_exception'] = analysis.failure_type == failure_type.INFRA
    # Check result from heuristic analysis.
    _PopulateHeuristicDataForCompileFailure(analysis, data)
    # Check result from try job.
    data['try_job'] = _PrepareTryJobDataForCompileFailure(analysis)

  def _PrepareDataForTestFailures(self,
                                  analysis,
                                  build_info,
                                  data,
                                  show_debug_info=False):

    data['status_message_map'] = result_status.STATUS_MESSAGE_MAP

    organized_results = _GetOrganizedAnalysisResultBySuspectedCL(
        analysis.master_name, analysis.builder_name, analysis.result)
    analysis_result = _GetAnalysisResultWithTryJobInfo(
        show_debug_info, organized_results, *build_info)

    data['analysis_result'] = analysis_result
    return data

  def HandleGet(self):
    """Renders analysis result of the failure.

    If the final analysis result is available, set cache-control to 1 day to
    avoid overload by unnecessary and frequent query from clients; otherwise
    set cache-control to 5 seconds to allow repeated query.
    """
    url = self.request.get('url').strip()
    build_info = buildbot.ParseBuildUrl(url)
    if not build_info:
      return BaseHandler.CreateError(
          'Url "%s" is not pointing to a build.' % url, 404)
    master_name, builder_name, build_number = build_info

    analysis = WfAnalysis.Get(master_name, builder_name, build_number)

    if not analysis:
      if self.request.get('redirect') == '1':
        return BaseHandler.CreateError(
            'No permission to schedule a new analysis.', 401)
      else:
        return BaseHandler.CreateError(
            'Please schedule analyses on home page instead.', 400)

    data = self._PrepareCommonDataForFailure(analysis)
    data['suspected_cls'] = _GetAllSuspectedCLsAndCheckStatus(
        master_name, builder_name, build_number, analysis)

    # TODO(crbug.com/702444): Do not assume failure_type.INFRA implies a
    # compile failure. Either use a special template, or choose the appropriate
    # one based on the type of job (compile/test).
    if analysis.failure_type == failure_type.COMPILE or (
        analysis.failure_type == failure_type.INFRA):
      self._PrepareDataForCompileFailure(analysis, data)
      return {'template': 'waterfall/compile_failure.html', 'data': data}
    else:
      self._PrepareDataForTestFailures(analysis, build_info, data,
                                       self._ShowDebugInfo())
      return {'template': 'waterfall/test_failure.html', 'data': data}

  @token.VerifyXSRFToken()
  def HandlePost(self):
    """Triggers an analysis on demand and redirects to the result page."""
    url = self.request.get('url').strip()

    is_admin = auth_util.IsCurrentUserAdmin()
    user_email = auth_util.GetUserEmail()
    if not acl.CanTriggerNewAnalysis(user_email, is_admin):
      # No permission to schedule a new analysis.
      return self.CreateRedirect('/waterfall/failure?redirect=1&url=%s' % url)

    build_info = buildbot.ParseBuildUrl(url)
    if not build_info:
      return BaseHandler.CreateError(
          'Url "%s" is not pointing to a build.' % url, 404)
    master_name, builder_name, build_number = build_info

    analysis = None
    if not (waterfall_config.MasterIsSupported(master_name) or
            auth_util.IsCurrentUserAdmin()):
      # If the build failure was already analyzed, just show it to the user.
      analysis = WfAnalysis.Get(master_name, builder_name, build_number)
      if not analysis:
        return BaseHandler.CreateError(
            'Master "%s" is not supported yet.' % master_name, 501)

    if not analysis:
      # Only allow admin to force a re-run and set the build_completed.
      force = is_admin and self.request.get('force') == '1'

      build = build_util.GetBuildInfo(master_name, builder_name, build_number)
      if not build:
        return BaseHandler.CreateError(
            'Can\'t get information about build "%s/%s/%s".' %
            (master_name, builder_name, build_number), 501)

      if not build.completed and force:
        return BaseHandler.CreateError(
            'Can\'t force a rerun for an incomplete build "%s/%s/%s".' %
            (master_name, builder_name, build_number), 501)

      build_failure_analysis_pipelines.ScheduleAnalysisIfNeeded(
          master_name,
          builder_name,
          build_number,
          build_completed=build.completed,
          force=force,
          queue_name=constants.WATERFALL_ANALYSIS_QUEUE)

    return self.CreateRedirect('/waterfall/failure?redirect=1&url=%s' % url)