| #!/usr/bin/env vpython |
| # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """upload goma related logs.""" |
| |
| import argparse |
| import collections |
| import json |
| import os |
| import sys |
| |
| from slave import goma_utils |
| |
| GOMA_LOGS_PROJECT = 'goma-logs' |
| |
| |
| def GetBigQueryClient(service_account_json): |
| """Returns bigquery client for goma logs. |
| |
| Args: |
| service_account_json: a JSON file for BigQuery service account. |
| """ |
| if not service_account_json: |
| return None |
| |
| # TODO(yyanagisawa): remove following debug print. |
| # (crbug.com/822042) |
| |
| for env in ('PYTHONPATH', 'VPYTHON_CLEAR_PYTHONPATH'): |
| print >> sys.stderr, '%s=%s' % (env, os.environ.get(env, '<not set>')) |
| print >> sys.stderr, 'sys.path=%s' % sys.path |
| |
| # TODO(yyanagisawa): move this back to the top line when flakiness solved |
| # (crbug.com/822042) |
| try: |
| from google.cloud import bigquery |
| from google.oauth2 import service_account |
| except ImportError as e: |
| print >> sys.stderr, e |
| for p in sys.path: |
| print 'path=%s' % p |
| for root, dirs, files in os.walk(p): |
| print >> sys.stderr, ' root=%s, dirs=%s, files=%s' % (root, dirs, files) |
| print |
| raise |
| |
| creds = service_account.Credentials.from_service_account_file( |
| service_account_json) |
| return bigquery.client.Client(project=GOMA_LOGS_PROJECT, credentials=creds) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='Upload goma related logs') |
| parser.add_argument('--upload-compiler-proxy-info', |
| action='store_true', |
| help='If set, the script will upload the latest ' |
| 'compiler_proxy.INFO.') |
| parser.add_argument('--log-url-json-file', |
| help='If set, the script will write url of uploaded ' |
| 'log visualizer.') |
| parser.add_argument('--ninja-log-outdir', |
| metavar='DIR', |
| help='Directory that has .ninja_log file.') |
| parser.add_argument('--ninja-log-compiler', |
| metavar='COMPILER', |
| help='compiler name used for the build.') |
| # TODO(shinyak): Remove this when everyone is using --ninja-log-command-file. |
| parser.add_argument('--ninja-log-command', |
| metavar='COMMAND', |
| help='command line options of the build.') |
| parser.add_argument('--ninja-log-command-file', |
| metavar='FILE', |
| help='command line options of the build, which is ' |
| 'written in the file. this option is preferred to ' |
| '--ninja-log-command.') |
| parser.add_argument('--build-exit-status', |
| type=int, |
| metavar='EXIT_STATUS', |
| help='build command exit status.') |
| parser.add_argument('--goma-stats-file', |
| metavar='FILENAME', |
| help='Filename of a GomaStats binary protobuf. ' |
| 'If empty or non-existing file, it will report error ' |
| 'to chrome infra monitoring system.') |
| parser.add_argument('--goma-counterz-file', |
| help='Filename of a CounterzStats binary protobuf. ' |
| 'If empty or non-existing file, it will report error ' |
| 'to chrome infra monitoring system.') |
| parser.add_argument('--goma-crash-report-id-file', |
| metavar='FILENAME', |
| help='Filename that has a crash report id.') |
| |
| parser.add_argument('--json-status', |
| metavar='JSON', |
| help='path of json file generated from' |
| ' ./goma_ctl.py jsonstatus') |
| parser.add_argument('--skip-sendgomatsmon', action='store_true', |
| help='Represent whether send jsonstatus' |
| ' and goma or compile.py exit_status log to TsMon.' |
| ' This option is used when no need to send goma status' |
| ' to monitoring server.') |
| |
| parser.add_argument('--gsutil-py-path', |
| help='Specify path to gsutil.py script in depot_tools.') |
| |
| # Arguments set to os.environ |
| parser.add_argument('--buildbot-buildername', |
| default='unknown', |
| help='buildbot buildername') |
| parser.add_argument('--buildbot-mastername', |
| default='unknown', |
| help='buildbot mastername') |
| parser.add_argument('--buildbot-slavename', |
| default='unknown', |
| help='buildbot slavename') |
| |
| # For CompileEvents. |
| parser.add_argument('--build-id', default=0, type=long, |
| help='unique ID of the current build') |
| parser.add_argument('--build-step-name', default='', |
| help='step name of the current build') |
| parser.add_argument('--bigquery-service-account-json', default='', |
| metavar='FILENAME', |
| help='Service account json for BigQuery') |
| |
| # Builder ID. |
| parser.add_argument('--builder-id-json', default='', |
| metavar='FILENAME', |
| help='path to Builder ID json file') |
| |
| # From Runtime API. |
| parser.add_argument('--is-luci', |
| action='store_true', |
| help='True if this runs on LUCI') |
| parser.add_argument('--is-experimental', |
| action='store_true', |
| help='True if experimental') |
| |
| args = parser.parse_args() |
| tsmon_counters = [] |
| |
| override_gsutil = None |
| if args.gsutil_py_path: |
| # Needs to add '--', otherwise gsutil options will be passed to gsutil.py. |
| override_gsutil = ['python', args.gsutil_py_path, '--'] |
| |
| viewer_urls = {} |
| |
| builder_id = {} |
| if args.builder_id_json: |
| with open(args.builder_id_json) as f: |
| builder_id = json.load(f) |
| |
| if args.upload_compiler_proxy_info: |
| viewer_url = goma_utils.UploadGomaCompilerProxyInfo( |
| builder=args.buildbot_buildername, |
| master=args.buildbot_mastername, |
| slave=args.buildbot_slavename, |
| builder_id=builder_id, |
| is_luci=args.is_luci, |
| is_experimental=args.is_experimental, |
| override_gsutil=override_gsutil |
| ) |
| if viewer_url is not None: |
| viewer_urls['compiler_proxy_log'] = viewer_url |
| |
| if args.ninja_log_command_file: |
| # TODO(shinyak): Assuming file exists. |
| with open(args.ninja_log_command_file, 'r') as f: |
| ninja_log_command = f.read() |
| else: |
| ninja_log_command = args.ninja_log_command |
| |
| if args.ninja_log_outdir: |
| viewer_url = goma_utils.UploadNinjaLog( |
| outdir=args.ninja_log_outdir, |
| compiler=args.ninja_log_compiler, |
| command=ninja_log_command, |
| exit_status=args.build_exit_status, |
| build_id=args.build_id, |
| step_name=args.build_step_name, |
| override_gsutil=override_gsutil |
| ) |
| if viewer_url is not None: |
| viewer_urls['ninja_log'] = viewer_url |
| |
| if args.log_url_json_file: |
| with open(args.log_url_json_file, 'w') as f: |
| f.write(json.dumps(viewer_urls)) |
| |
| bqclient = None |
| # TODO(yyanagisawa): remove move this when flakiness is solved |
| # (crbug.com/822042) |
| # Without Build ID, we do not use BigQuery. Let me minimize the |
| # points that cause crbug.com/822042. |
| if args.build_id: |
| bqclient = GetBigQueryClient(args.bigquery_service_account_json) |
| |
| if args.goma_stats_file and args.build_id and bqclient: |
| # TODO(yyanagisawa): move this back to the top line when flakiness is solved |
| # (crbug.com/822042) |
| from slave import goma_bq_utils |
| goma_bq_utils.SendCompileEvent(args.goma_stats_file, |
| args.goma_counterz_file, |
| args.json_status, |
| args.build_exit_status, |
| args.goma_crash_report_id_file, |
| args.build_id, |
| args.build_step_name, |
| bqclient) |
| |
| if args.goma_stats_file: |
| counter = goma_utils.MakeGomaExitStatusCounter( |
| args.goma_stats_file, |
| goma_crash_report=args.goma_crash_report_id_file, |
| builder=args.buildbot_buildername, |
| master=args.buildbot_mastername, |
| slave=args.buildbot_slavename, |
| builder_id=builder_id, |
| is_luci=args.is_luci) |
| if counter: |
| tsmon_counters.append(counter) |
| |
| if not args.skip_sendgomatsmon: |
| # In the case of goma_start is failed, |
| # we want log to investigate failed reason. |
| # So, let me send some logs instead of |
| # error in parse_args() using required option. |
| assert args.json_status is not None and os.path.exists(args.json_status) |
| counter = goma_utils.MakeGomaStatusCounter( |
| args.json_status, |
| args.build_exit_status, |
| builder=args.buildbot_buildername, |
| master=args.buildbot_mastername, |
| slave=args.buildbot_slavename, |
| builder_id=builder_id, |
| is_luci=args.is_luci) |
| if counter: |
| tsmon_counters.append(counter) |
| |
| counter = goma_utils.MakeGomaFailureReasonCounter( |
| args.json_status, |
| args.build_exit_status, |
| builder=args.buildbot_buildername, |
| master=args.buildbot_mastername, |
| slave=args.buildbot_slavename, |
| builder_id=builder_id, |
| is_luci=args.is_luci) |
| if counter: |
| tsmon_counters.append(counter) |
| |
| if tsmon_counters: |
| goma_utils.SendCountersToTsMon(tsmon_counters) |
| |
| return 0 |
| |
| |
| if '__main__' == __name__: |
| sys.exit(main()) |