franky/batch_runner.py

#!/usr/bin/env python
#
# Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs Franky test suites with different parameters."""

from __future__ import absolute_import
from __future__ import print_function
import argparse
import csv
import datetime
import json
import logging
import os

from appium_util import appium_driver_util
import device_info
import download_testcases_csv
from reports import generate_report
from reports import html_report
import runner
from test_util import test_common
from utilities import gsutil
from utilities import util
from six.moves import zip


# Format of csv batch_runner file:
# col1, col2, col3, col4, ...., colN
# val1, val2, val3, val4, ...., valN
# where val1..valN are strings that store values,
# for boolean keys if a value is empty it means that it is enabled
# to disable flag just need to us `#DISABLED#`.
_FIELD_DISABLE_FLAG = '#DISABLED#'
_FRANKY_WEBVIEW_GS_PATH = 'gs://franky-reports/android/webview'


def _ParseArgs(passed_args=None):
  """Parse batch parameters.

  Args:
    passed_args: A list of arguments.

  Returns:
    A list of parameters.
  """
  arg_parser = argparse.ArgumentParser(
      description='Runs batch file of Franky test suites.')
  arg_parser.add_argument(
      '-b',
      '--batch-file',
      action='append',
      default=None,
      help='Test suite file(s) to run; can be a glob. Multiple arguments '
      'add up. Default are all files matching "*<channel>*.csv" in CWD.'
      'File type is determined by extension: .csv is CSV, .txt is text proto.')
  arg_parser.add_argument(
      '-w',
      '--workbook',
      default=None,
      help='Download the specified workbook as a CSV file. To download multiple'
      'workbooks use comma as delimiter')
  arg_parser.add_argument(
      '--extra-args',
      default=None,
      help='@-separated args that will pass to runner.py and '
           'may override values from a workbook if they are the same. '
           'E.g. "@--appium-port@8080" or "--appium-port@8080"')
  arg_parser.add_argument(
      '--batch-report-folder',
      default=None,
      required=True,
      help='The full path to the directory where batch report will store. '
      'Required parameter.')
  arg_parser.add_argument(
      '-id', '--device-id',
      help='The device ID/serial number to run test on.')
  arg_parser.add_argument(
      '--no-batch-report-upload',
      action='store_true',
      default=False,
      help='Upload or not batch report to GS-folder.')
  arg_parser.add_argument(
      '-c',
      '--channel',
      default='Canary',
      help='A channel to run tests it against. Default is Canary.')

  args = arg_parser.parse_args(passed_args)

  if not (args.batch_file or args.workbook):
    raise test_common.ArgumentError(
        '--batch-file or --workbook has to be specified!')

  if args.workbook:
    spreadsheet_data = [el.strip() for el in args.workbook.split(',')]
    if len(spreadsheet_data) < 2:
      raise test_common.ArgumentError(' '.join([
          '--workbook has to have at least 2 comma separated parameters:',
          '"spreadsheet_key, workbook_id"']))
    args.batch_report_folder = os.path.join(
        args.batch_report_folder,
        '%s_batch_%s' % (BatchReportPrefix(args.device_id, args.channel),
                         str(datetime.datetime.now().isoformat())))
    # Make entire path to batch report folder
    util.CreateEntirePathFolder(args.batch_report_folder)
    args.batch_file = download_testcases_csv.DownloadCSVFromGoogleSheet(
        args, spreadsheet_data[0], set(spreadsheet_data[1:]),
        folder=args.batch_report_folder
    )

  return args


def ParseSuiteParameters(file_path):
  """Parsing batch csv file.

  Args:
    file_path: A path to csv file.

  Returns:
    List of list with Franky parameters.
  """
  lst = []
  with open(file_path) as csv_file:
    csv_reader = csv.reader(csv_file)
    lst = [row for row in csv_reader]
  commands = []
  column_names = lst.pop(0)
  for fields in lst:
    command = []
    for column, field in zip(column_names, fields):
      if field == _FIELD_DISABLE_FLAG:
        continue
      # For channel and platform need to specify values only.
      if column not in ('channel', 'platform'):
        command.append(column)
      if field:
        command.append(field)
    commands.append(command)
  return commands


def GenerateBatchReport(batch_report_path, file_name):
  """Generates batch report based json files located in batch_report_path.

  Args:
    batch_report_path: A path to folder with JSON reports.
    file_name: A batch report name.

  Returns:
    batch report file path.
  """
  if not batch_report_path:
    logging.error('Batch report path can not be empty!')
    return
  data = {}
  # Looking and parse json files located in batch_report_path.
  for f in util.GetFilesFromFolder(batch_report_path):
    with open(f) as json_file:
      data[f] = json.load(json_file)

  if not data:
    logging.error('There is no generated report!')
  for json_file in data:
    generate_report.main([json_file])
  full_file_name = os.path.join(batch_report_path, file_name)
  franky_html_report = html_report.HTMLReport(full_file_name,
                                              'Batch runner report')
  franky_html_report.WriteBatchReport(list(data.values()))
  return full_file_name


def UploadBatchReport(batch_report, gs_report_name):
  """Uploads batch report to GS folder based on webview.

  Args:
    batch_report: A path to html batch report.
    gs_report_name: A name of batch report in GS.
  """
  gs_file = '%s/%s___%s.html' % (
      _FRANKY_WEBVIEW_GS_PATH,
      gs_report_name,
      datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'))
  gsutil.CopyHTMLReportToCloudStorage(batch_report, gs_file)


def BatchReportPrefix(device_id, channel):
  """Generate batch report name.

  Args:
    device_id: A device ID.
    channel: A channel used to run tests.

  Returns:
     Batch report name in format
      webview-package_channel__SDK__abi__brand__board__build.
  """
  system_webview_info = device_info.SystemWebviewInfo(device_id)
  return '%s__%s__%s__%s__%s__%s__%s' % (
      system_webview_info['webview_provider_version'],
      channel,
      device_info.GetAndroidAPILevel(device_id),
      device_info.GetAndroidCPUInformation(device_id),
      device_info.GetAndroidProductBrand(device_id),
      device_info.GetAndroidProductBoard(device_id),
      device_info.GetAndroidProductBuildId(device_id))


def main(passed_args=None):
  batch_args = _ParseArgs(passed_args)
  try:
    for params in ParseSuiteParameters(batch_args.batch_file[0]):
      # Update -r/--report-format param to json or regenerate to html report.
      params.extend(['-r', 'json'])
      if batch_args.extra_args:
        # Appends param ignoring empty values.
        params.extend([arg for arg in batch_args.extra_args.split('@') if arg])
      if batch_args.device_id:
        params.extend(['-id', batch_args.device_id])
      params.extend(['--report-folder', batch_args.batch_report_folder])
      params.extend([batch_args.channel, appium_driver_util.ANDROID])
      print('Run tests with params: %s' % params)
      try:
        runner.main(params)
      except Exception as ex:
        print('Test-suite with [%s] params failed: %s' % (params, ex))
  finally:
    batchreport_prefix = BatchReportPrefix(batch_args.device_id,
                                           batch_args.channel)
    batch_report = GenerateBatchReport(
        batch_args.batch_report_folder,
        batchreport_prefix + '_batch_report.html')
    if not batch_args.no_batch_report_upload:
      UploadBatchReport(batch_report, batchreport_prefix)


if __name__ == '__main__':
  main()