tests/masters_util.py

# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Buildbot master utility functions.
"""

import json
import errno
import logging
import os
import sys
import time

BUILD_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(BUILD_DIR, 'scripts'))

from tools import mastermap
from common import find_depot_tools  # pylint: disable=W0611
import subprocess2


def sublists(superlist, n):
  """Breaks a list into list of sublists, each of length no more than n."""
  result = []
  for cut in range(0, len(superlist), n):
    result.append(superlist[cut:cut + n])
  return result


def pid_exists(pid):
  """Returns True if there is a process in the system with given |pid|."""
  try:
    os.kill(pid, 0)
  except OSError as error:
    if error.errno == errno.EPERM:
      return True
    elif error.errno == errno.ESRCH:
      return False
    raise
  return True


def is_master_alive(master, path):
  """Reads master's *.pid file and checks for corresponding PID in the system.
  If there is no such process, removes stale *.pid file and returns False.

  Returns:
    True - *.pid file exists and corresponding process is running.
    False - *.pid file doesn't exist or there is no such process.
  """
  pid_path = os.path.join(path, 'twistd.pid')
  contents = None
  try:
    with open(pid_path) as f:
      contents = f.read()
    if pid_exists(int(contents.strip())):
      return True
    logging.warning('Ghost twistd.pid for %s, removing it', master)
  except IOError as error:
    if error.errno == errno.ENOENT:
      return False
    raise
  except ValueError:
    logging.warning('Corrupted twistd.pid for %s, removing it: %r',
                    master, contents)
  remove_file(pid_path)
  return False


def remove_file(path):
  """Deletes file at given |path| if it exists. Does nothing if it's not there
  or can not be deleted."""
  try:
    os.remove(path)
  except OSError:
    pass


def start_master(master, path, dry_run=False):
  """Asynchronously starts the |master| at given |path|.
  If |dry_run| is True, will start the master in a limited mode suitable only
  for integration testing purposes.

  Returns:
    True - the master was successfully started.
    False - the master failed to start, details are in the log.
  """
  try:
    env = os.environ.copy()

    # Note that this is a development master.
    env['BUILDBOT_MASTER_IS_DEV'] = '1'

    if dry_run:
      # Ask ChromiumGitPoller not to pull git repos.
      env['NO_REVISION_AUDIT'] = '0'
      env['POLLER_DRY_RUN'] = '1'
    subprocess2.check_output(
        ['make', 'start'], timeout=120, cwd=path, env=env,
        stderr=subprocess2.STDOUT)
  except subprocess2.CalledProcessError as e:
    logging.error('Error: cannot start %s' % master)
    print e
    return False
  return True


def stop_master(master, path, force=False):
  """Issues 'stop' command and waits for master to terminate. If |force| is True
  will try to kill master process if it fails to terminate in time by itself.

  Returns:
    True - master was stopped, killed or wasn't running.
    False - master is still running.
  """
  if terminate_master(master, path, 'stop', timeout=10):
    return True
  if not force:
    logging.warning('Master %s failed to stop in time', master)
    return False
  logging.warning('Master %s failed to stop in time, killing it', master)
  if terminate_master(master, path, 'kill', timeout=2):
    return True
  logging.warning('Master %s is still running', master)
  return False


def terminate_master(master, path, command, timeout=10):
  """Executes 'make |command|' and waits for master to stop running or until
  |timeout| seconds pass.

  Returns:
    True - the master was terminated or wasn't running.
    False - the command failed, or master failed to terminate in time.
  """
  if not is_master_alive(master, path):
    return True
  try:
    env = os.environ.copy()
    env['NO_REVISION_AUDIT'] = '0'
    subprocess2.check_output(
        ['make', command], timeout=5, cwd=path, env=env,
        stderr=subprocess2.STDOUT)
  except subprocess2.CalledProcessError as e:
    if not is_master_alive(master, path):
      return True
    logging.warning('Master %s was not terminated: \'make %s\' failed: %s',
                    master, command, e)
    return False
  return wait_for_termination(master, path, timeout=timeout)


def wait_for_termination(master, path, timeout=10):
  """Waits for master to finish running and cleans up pid file.
  Waits for at most |timeout| seconds.

  Returns:
    True - master has stopped or wasn't running.
    False - master failed to terminate in time.
  """
  started = time.time()
  while True:
    now = time.time()
    if now > started + timeout:
      break
    if not is_master_alive(master, path):
      logging.info('Master %s stopped in %.1f sec.', master, now - started)
      return True
    time.sleep(0.1)
  return False


def search_for_exceptions(path):
  """Looks in twistd.log for an exception.

  Returns True if an exception is found.
  """
  twistd_log = os.path.join(path, 'twistd.log')
  with open(twistd_log) as f:
    lines = f.readlines()
    stripped_lines = [l.strip() for l in lines]
    try:
      i = stripped_lines.index('--- <exception caught here> ---')
      # Found an exception at line 'i'!  Now find line 'j', the number
      # of lines from 'i' where there's a blank line.  If we cannot find
      # a blank line, then we will show up to 10 lines from i.
      try:
        j = stripped_lines[i:-1].index('')
      except ValueError:
        j = 10
      # Print from either 15 lines back from i or the start of the log
      # text to j lines after i.
      return ''.join(lines[max(i-15, 0):i+j])
    except ValueError:
      pass
  return False


def json_probe(sensitive, allports):
  """Looks through the port range and finds a master listening.
  sensitive: Indicates whether partial success should be reported.

  Returns (port, name) or None.
  """
  procs = {}
  for ports in sublists(allports, 30):
    for port in ports:
      # urllib2 does not play nicely with threading. Using curl lets us avoid
      # the GIL.
      procs[port] = subprocess2.Popen(
          ['curl', '-fs', '-m2', 'http://localhost:%d/json/project' % port],
          stdin=subprocess2.VOID,
          stdout=subprocess2.PIPE,
          stderr=subprocess2.VOID)
    for port in ports:
      stdout, _ = procs[port].communicate()
      if procs[port].returncode != 0:
        continue
      try:
        data = json.loads(stdout) or {}
        if not data or (not 'projectName' in data and not 'title' in data):
          logging.debug('Didn\'t get valid data from port %d' % port)
          if sensitive:
            return (port, None)
          continue
        name = data.get('projectName', data.get('title'))
        return (port, name)
      except ValueError:
        logging.warning('Didn\'t get valid data from port %d' % port)
        # presume this is some other type of server
        #  E.g. X20 on a dev workstation.
        continue

  return None


def wait_for_start(master, name, path, ports):
  """Waits for ~30s for the masters to open its web server."""
  logging.info("Waiting for master %s on ports %s" % (name, ports))
  for i in range(300):
    result = json_probe(False, ports)
    if result is None:
      exception = search_for_exceptions(path)
      if exception:
        return exception
      time.sleep(0.1)
      continue
    port, got_name = result # pylint: disable=unpacking-non-sequence
    if got_name != name:
      return 'Wrong %s name, expected %s, got %s on port %d' % (
          master, name, got_name, port)
    logging.info("Found master %s on port %s, iteration %d" % (name, port, i))
    # The server is now answering /json requests. Check that the log file
    # doesn't have any other exceptions just in case there was some other
    # unexpected error.
    return search_for_exceptions(path)

  return 'Didn\'t find open port for %s' % master


def check_for_no_masters():
  ports = range(8000, 8099) + range(8200, 8299) + range(9000, 9099)
  ports = [x for x in ports if x not in mastermap.PORT_BLACKLIST]
  result = json_probe(True, ports)
  if result is None:
    return True
  if result[1] is None:
    logging.error('Something is listening on port %d' % result[0])
    return False
  logging.error('Found unexpected master %s on port %d' %
                (result[1], result[0]))
  return False