blob: 80176d2c2caeb341c3808accee73001de34461d1 [file] [log] [blame]
# Copyright (c) 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Functions specific to upload goma related info to BigQuery.
import json
import os
from infra_libs import bigquery
from slave.goma import compile_events_pb2
from slave import goma_utils
# The Google BigQuery dataset and table for CompileEvent.
COMPILE_EVENTS_DATASET = 'client_events'
COMPILE_EVENTS_TABLE = 'compile_events'
def SetStepInfo(json_file, exit_status, step_info):
Set StepInfo for the given exit status.
json_file: " jsonstatus" file.
exit_status: exit status of a compile step.
step_info: an instance of compile_events_pb2.StepInfo to be filled.
with open(json_file) as f:
json_statuses = json.load(f)
if not json_statuses:
print('no json status is recorded in %s' % json_file)
if len(json_statuses.get('notice', [])) != 1:
print('unknown json statuses style: %s' % json_statuses)
json_status = json_statuses['notice'][0]
if json_status['version'] != 1:
print('unknown version: %s' % json_status)
infra_status = json_status.get('infra_status')
status = compile_events_pb2.StepInfo.SUCCESS
goma_failure_reason = compile_events_pb2.StepInfo.GOMA_OK
if infra_status is None:
goma_failure_reason = compile_events_pb2.StepInfo.GOMA_SETUP_FAILURE
elif infra_status.get('ping_status_code', 200) != 200:
goma_failure_reason = compile_events_pb2.StepInfo.GOMA_PING_FAILURE
elif infra_status.get('num_user_error', 0) > 0:
goma_failure_reason = compile_events_pb2.StepInfo.GOMA_BUILD_ERROR
if exit_status is None:
if goma_failure_reason == compile_events_pb2.StepInfo.GOMA_OK:
# Maybe some failure on goma set up?
goma_failure_reason = compile_events_pb2.StepInfo.GOMA_UNKNOWN_FAILURE
elif exit_status != 0:
status = compile_events_pb2.StepInfo.FAILURE
if goma_failure_reason != compile_events_pb2.StepInfo.GOMA_OK:
status = compile_events_pb2.StepInfo.EXCEPTION
step_info.status = status
step_info.goma_failure_reason = goma_failure_reason
except Exception as ex:
print('error while making goma status counter for ts_mon: jons_file=%s: %s'
% (json_file, ex))
def SendCompileEvent(goma_stats_file, goma_counterz_file,
json_file, exit_status,
goma_crash_report, build_id, step_name,
"""Insert CompileEvent to BigQuery table.
goma_stats_file: a file that has binary representation of GomaStats.
goma_counterz_file: a file that has binary representation of CounterzStats.
json_file: " jsonstatus" file.
exit_status: exit status of a compile step.
goma_crash_report: a file that has compiler_proxy crash report id
if it crashed.
build_id: Build ID string.
step_name: a name of a compile step.
bqclient: an instance of BigQuery client.
event = compile_events_pb2.CompileEvent()
event.build_id = build_id
event.step_name = step_name or ''
SetStepInfo(json_file, exit_status, event.step_info)
event.exit_status = compile_events_pb2.CompileEvent.DIED_WITH_UNKOWN_REASON
if goma_stats_file and os.path.exists(goma_stats_file):
with open(goma_stats_file, 'rb') as f:
event.exit_status = compile_events_pb2.CompileEvent.OK
if goma_counterz_file and os.path.exists(goma_counterz_file):
with open(goma_counterz_file, 'rb') as f:
if goma_crash_report and os.path.exists(goma_crash_report):
with open(goma_crash_report) as f:
event.crash_id =
event.exit_status = compile_events_pb2.CompileEvent.CRASHED
if goma_utils.IsCompilerProxyKilledByFatalError():
event.exit_status = compile_events_pb2.CompileEvent.DIED_WITH_LOG_FATAL
# BQ uploader.
print 'Uploaded CompileEvent to BQ %s.%s' % (
except Exception, inst: # safety net
print('failed to send CompileEvent to BQ: %s' % inst)