scripts/slave/build_scan_db.py

#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Contains configuration and setup of a build scan database.

The database is a versioned JSON file built of NamedTuples.
"""

import collections
import json
import logging
import optparse
import os
import sys

from common import chromium_utils
from slave import gatekeeper_ng_config


DATA_DIR = os.path.dirname(os.path.abspath(__file__))


# Bump each time there is an incompatible change in build_db.
BUILD_DB_VERSION = 5


_BuildDB = collections.namedtuple('BuildDB', [
    'build_db_version',  # An int representing the build_db version.
    'masters',   # {mastername: {buildername: {buildnumber: BuildDBBuild}}}}
    'sections',  # {section_hash: human_readable_json_of_gatekeeper_section}
    'aux',       # Dictionary to keep auxiliary information such as triggered
                 # revisions.
])


_BuildDBBuild = collections.namedtuple('BuildDBBuild', [
    'finished',  # True if the build has finished, False otherwise.
    'succeeded',  # True if finished and would have not closed the tree.
    'triggered',  # {section: [steps which triggered the section]}
])


class JsonNode(object):
  """Allows for serialization of NamedTuples to JSON."""
  def _asdict(self):  # pylint: disable=R0201
    return {}

  # TODO(stip): recursively encode child nodes.
  def asJson(self):
    nodes_to_encode = [(k, v) for k, v in self._asdict().iteritems()
                       if hasattr(v, 'asJson')]
    standard_nodes = [(k, v) for k, v in self._asdict().iteritems()
                      if not hasattr(v, 'asJson')]
    newly_encoded_nodes = [(k, v.asJson()) for k, v in nodes_to_encode]

    return dict(standard_nodes + newly_encoded_nodes)


class BuildDB(_BuildDB, JsonNode):
  pass


class BuildDBBuild(_BuildDBBuild, JsonNode):
  pass


class BadConf(Exception):
  pass


def gen_db(**kwargs):
  """Helper function to generate a default database."""
  defaults = [
      ('build_db_version', BUILD_DB_VERSION),
      ('masters', {}),
      ('sections', {}),
      ('aux', {}),
  ]

  for key, default in defaults:
    kwargs.setdefault(key, default)

  return BuildDB(**kwargs)

def gen_build(**kwargs):
  """Helper function to generate a default build."""
  defaults = [('finished', False),
              ('succeeded', False),
              ('triggered', {})]

  for key, default in defaults:
    kwargs.setdefault(key, default)

  return BuildDBBuild(**kwargs)


def load_from_json(f):
  """Load a build from a JSON stream."""
  json_build_db = json.load(f)

  if json_build_db.get('build_db_version') != BUILD_DB_VERSION:
    raise BadConf('file is an older db version: %r (expecting %d)' % (
        json_build_db.get('build_db_version'), BUILD_DB_VERSION))

  masters = json_build_db.get('masters', {})
  # Convert build dicts into BuildDBBuilds.
  build_db = gen_db()
  for mastername, master in masters.iteritems():
    build_db.masters.setdefault(mastername, {})
    for buildername, builder in master.iteritems():
      build_db.masters[mastername].setdefault(buildername, {})
      for buildnumber, build in builder.iteritems():
        # Note that buildnumber is forced to be an int here, and
        # we use * instead of ** -- until the serializer is recursive,
        # BuildDBBuild will be written as a value list (tuple).
        build_db.masters[mastername][buildername][
            int(buildnumber)] = BuildDBBuild(*build)

  if 'aux' in json_build_db:
    build_db.aux.update(json_build_db['aux'])

  return build_db


def get_build_db(filename):
  """Open the build_db file.

  filename: the filename of the build db.
  """
  build_db = gen_db()

  if os.path.isfile(filename):
    print 'loading build_db from', filename
    try:
      with open(filename) as f:
        build_db = load_from_json(f)
    except BadConf as e:
      new_fn = '%s.old' % filename
      logging.warn('error loading %s: %s, moving to %s' % (
          filename, e, new_fn))
      chromium_utils.MoveFile(filename, new_fn)

  return build_db


def convert_db_to_json(build_db_data, gatekeeper_config, f):
  """Converts build_db to a format suitable for JSON encoding and writes it."""
  # Remove all but the last finished build.
  for builders in build_db_data.masters.values():
    for builder in builders:
      unfinished = [(k, v) for k, v in builders[builder].iteritems()
                    if not v.finished]

      finished = [(k, v) for k, v in builders[builder].iteritems()
                  if v.finished]

      builders[builder] = dict(unfinished)

      if finished:
        max_finished = max(finished, key=lambda x: x[0])
        builders[builder][max_finished[0]] = max_finished[1]

  build_db = gen_db(masters=build_db_data.masters, aux=build_db_data.aux)

  # Output the gatekeeper sections we're operating with, so a human reading the
  # file can debug issues. This is discarded by the parser in get_build_db.
  used_sections = set([])
  for masters in build_db_data.masters.values():
    for builder in masters.values():
      used_sections |= set(t for b in builder.values() for t in b.triggered)

  for master in gatekeeper_config.values():
    for section in master:
      section_hash = gatekeeper_ng_config.gatekeeper_section_hash(section)
      if section_hash in used_sections:
        build_db.sections[section_hash] = section

  json.dump(build_db.asJson(), f, cls=gatekeeper_ng_config.SetEncoder,
            sort_keys=True)


def save_build_db(build_db_data, gatekeeper_config, filename):
  """Save the build_db file.

  build_db: dictionary to jsonize and store as build_db.
  gatekeeper_config: the gatekeeper config used for this pass.
  filename: the filename of the build db.
  """
  print 'saving build_db to', filename
  with open(filename, 'wb') as f:
    convert_db_to_json(build_db_data, gatekeeper_config, f)


def main():
  prog_desc = 'Parses the build_db and outputs to stdout.'
  usage = '%prog [options]'
  parser = optparse.OptionParser(usage=(usage + '\n\n' + prog_desc))
  parser.add_option('--json', default=os.path.join(DATA_DIR, 'gatekeeper.json'),
                    help='location of gatekeeper configuration file')
  parser.add_option('--build-db', default='build_db.json',
                    help='records the build status information for builders')
  options, _ = parser.parse_args()

  build_db = get_build_db(options.build_db)
  gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json)

  convert_db_to_json(build_db, gatekeeper_config, sys.stdout)
  print

  return 0


if __name__ == '__main__':
  sys.exit(main())