scripts/slave/slave_utils.py

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Functions specific to build slaves, shared by several buildbot scripts.
"""

import datetime
import glob
import os
import re
import shutil
import sys
import tempfile
import time

from common import chromium_utils
from slave.bootstrap import ImportMasterConfigs # pylint: disable=W0611
from common.chromium_utils import GetActiveMaster # pylint: disable=W0611

# These codes used to distinguish true errors from script warnings.
ERROR_EXIT_CODE = 1
WARNING_EXIT_CODE = 88


# Local errors.
class PageHeapError(Exception):
  pass


# Cache the path to gflags.exe.
_gflags_exe = None


def SubversionExe():
  # TODO(pamg): move this into platform_utils to support Mac and Linux.
  if chromium_utils.IsWindows():
    return 'svn.bat'  # Find it in the user's path.
  elif chromium_utils.IsLinux() or chromium_utils.IsMac():
    return 'svn'  # Find it in the user's path.
  else:
    raise NotImplementedError(
        'Platform "%s" is not currently supported.' % sys.platform)


def GitExe():
  return 'git.bat' if chromium_utils.IsWindows() else 'git'


def SubversionCat(wc_dir):
  """Output the content of specified files or URLs in SVN.
  """
  try:
    return chromium_utils.GetCommandOutput([SubversionExe(), 'cat',
                                            wc_dir])
  except chromium_utils.ExternalError:
    return None


class NotGitWorkingCopy(Exception): pass
class NotSVNWorkingCopy(Exception): pass
class NotAnyWorkingCopy(Exception): pass
class InvalidSVNRevision(Exception): pass


def ScrapeSVNInfoRevision(wc_dir, regexp):
  """Runs 'svn info' on a working copy and applies the supplied regex and
  returns the matched group as an int.
  regexp can be either a compiled regex or a string regex.
  throws NotSVNWorkingCopy if wc_dir is not in a working copy.
  throws InvalidSVNRevision if matched group is not alphanumeric.
  """
  if isinstance(regexp, (str, unicode)):
    regexp = re.compile(regexp)
  retval, svn_info = chromium_utils.GetStatusOutput([SubversionExe(), 'info',
                                                    wc_dir])
  if retval or 'is not a working copy' in svn_info:
    raise NotSVNWorkingCopy(wc_dir)
  match = regexp.search(svn_info)
  if not match or not match.groups():
    raise InvalidSVNRevision(
        '%s did not match in svn info %s.' % (regexp.pattern, svn_info))
  text = match.group(1)
  if text.isalnum():
    return int(text)
  else:
    raise InvalidSVNRevision(text)


def SubversionRevision(wc_dir):
  """Finds the last svn revision of a working copy and returns it as an int."""
  return ScrapeSVNInfoRevision(wc_dir, r'(?s).*Revision: (\d+).*')


def SubversionLastChangedRevision(wc_dir_or_file):
  """Finds the last changed svn revision of a fs path returns it as an int."""
  return ScrapeSVNInfoRevision(wc_dir_or_file,
                               r'(?s).*Last Changed Rev: (\d+).*')


def GitHash(wc_dir):
  """Finds the current commit hash of the wc_dir."""
  retval, text = chromium_utils.GetStatusOutput(
      [GitExe(), 'rev-parse', 'HEAD'], cwd=wc_dir)
  if retval or 'fatal: Not a git repository' in text:
    raise NotGitWorkingCopy(wc_dir)
  return text.strip()


def GetHashOrRevision(wc_dir):
  """Gets the svn revision or git hash of wc_dir as a string. Throws
  NotAnyWorkingCopy if neither are appropriate."""
  try:
    return str(SubversionRevision(wc_dir))
  except NotSVNWorkingCopy:
    pass
  try:
    return GitHash(wc_dir)
  except NotGitWorkingCopy:
    pass
  raise NotAnyWorkingCopy(wc_dir)


def GitOrSubversion(wc_dir):
  """Returns the VCS for the given directory.

  Returns:
    'svn' if the directory is a valid svn repo
    'git' if the directory is a valid git repo root
    None otherwise
  """
  ret, out = chromium_utils.GetStatusOutput([SubversionExe(), 'info', wc_dir])
  if not ret and 'is not a working copy' not in out:
    return 'svn'

  ret, out = chromium_utils.GetStatusOutput(
      [GitExe(), 'rev-parse', '--is-inside-work-tree'], cwd=wc_dir)
  if not ret and 'fatal: Not a git repository' not in out:
    return 'git'

  return None


def GetBuildRevisions(src_dir, webkit_dir=None, revision_dir=None):
  """Parses build revisions out of the provided directories.

  Args:
    src_dir: The source directory to be used to check the revision in.
    webkit_dir: Optional WebKit directory, relative to src_dir.
    revision_dir: If provided, this dir will be used for the build revision
      instead of the mandatory src_dir.

  Returns a tuple of the build revision and (optional) WebKit revision.
  NOTICE: These revisions are strings, since they can be both Subversion numbers
  and Git hashes.
  """
  abs_src_dir = os.path.abspath(src_dir)
  webkit_revision = None
  if webkit_dir:
    webkit_dir = os.path.join(abs_src_dir, webkit_dir)
    webkit_revision = GetHashOrRevision(webkit_dir)

  if revision_dir:
    revision_dir = os.path.join(abs_src_dir, revision_dir)
    build_revision = GetHashOrRevision(revision_dir)
  else:
    build_revision = GetHashOrRevision(src_dir)
  return (build_revision, webkit_revision)


def GetZipFileNames(
    mastername, buildnumber, parent_buildnumber, build_revision,
    webkit_revision=None, extract=False, use_try_buildnumber=True):
  base_name = 'full-build-%s' % chromium_utils.PlatformName()

  if 'try' in mastername and use_try_buildnumber:
    if extract:
      if not parent_buildnumber:
        raise Exception('missing parent_buildnumber')
      version_suffix = '_%s' % parent_buildnumber
    else:
      version_suffix = '_%s' % buildnumber
  elif webkit_revision:
    version_suffix = '_wk%s_%s' % (webkit_revision, build_revision)
  else:
    version_suffix = '_%s' % build_revision

  return base_name, version_suffix


def SlaveBuildName(chrome_dir):
  """Extracts the build name of this slave (e.g., 'chrome-release') from the
  leaf subdir of its build directory.
  """
  return os.path.basename(SlaveBaseDir(chrome_dir))


def SlaveBaseDir(chrome_dir):
  """Finds the full path to the build slave's base directory (e.g.
  'c:/b/chrome/chrome-release').  This is assumed to be the parent of the
  shallowest 'build' directory in the chrome_dir path.

  Raises chromium_utils.PathNotFound if there is no such directory.
  """
  result = ''
  prev_dir = ''
  curr_dir = chrome_dir
  while prev_dir != curr_dir:
    (parent, leaf) = os.path.split(curr_dir)
    if leaf == 'build':
      # Remember this one and keep looking for something shallower.
      result = parent
    if leaf == 'slave':
      # We are too deep, stop now.
      break
    prev_dir = curr_dir
    curr_dir = parent
  if not result:
    raise chromium_utils.PathNotFound('Unable to find slave base dir above %s' %
                                      chrome_dir)
  return result


def GetStagingDir(start_dir):
  """Creates a chrome_staging dir in the starting directory. and returns its
  full path.
  """
  start_dir = os.path.abspath(start_dir)
  staging_dir = os.path.join(SlaveBaseDir(start_dir), 'chrome_staging')
  chromium_utils.MaybeMakeDirectory(staging_dir)
  return staging_dir


def SetPageHeap(chrome_dir, exe, enable):
  """Enables or disables page-heap checking in the given executable, depending
  on the 'enable' parameter.  gflags_exe should be the full path to gflags.exe.
  """
  global _gflags_exe
  if _gflags_exe is None:
    _gflags_exe = chromium_utils.FindUpward(chrome_dir,
                                            'tools', 'memory', 'gflags.exe')
  command = [_gflags_exe]
  if enable:
    command.extend(['/p', '/enable', exe, '/full'])
  else:
    command.extend(['/p', '/disable', exe])
  result = chromium_utils.RunCommand(command)
  if result:
    description = {True: 'enable', False: 'disable'}
    raise PageHeapError('Unable to %s page heap for %s.' %
                        (description[enable], exe))


def LongSleep(secs):
  """A sleep utility for long durations that avoids appearing hung.

  Sleeps for the specified duration.  Prints output periodically so as not to
  look hung in order to avoid being timed out.  Since this function is meant
  for long durations, it assumes that the caller does not care about losing a
  small amount of precision.

  Args:
    secs: The time to sleep, in seconds.
  """
  secs_per_iteration = 60
  time_slept = 0

  # Make sure we are dealing with an integral duration, since this function is
  # meant for long-lived sleeps we don't mind losing floating point precision.
  secs = int(round(secs))

  remainder = secs % secs_per_iteration
  if remainder > 0:
    time.sleep(remainder)
    time_slept += remainder
    sys.stdout.write('.')
    sys.stdout.flush()

  while time_slept < secs:
    time.sleep(secs_per_iteration)
    time_slept += secs_per_iteration
    sys.stdout.write('.')
    sys.stdout.flush()

  sys.stdout.write('\n')


def RunPythonCommandInBuildDir(build_dir, target, command_line_args,
                               server_dir=None, filter_obj=None):
  if sys.platform == 'win32':
    python_exe = 'python.exe'
  else:
    os.environ['PYTHONPATH'] = (chromium_utils.FindUpward(build_dir, 'tools',
                                                          'python')
                                + ':' +os.environ.get('PYTHONPATH', ''))
    python_exe = 'python'

  command = [python_exe] + command_line_args
  return chromium_utils.RunCommand(command, filter_obj=filter_obj)


class RunCommandCaptureFilter(object):
  lines = []

  def FilterLine(self, in_line):
    self.lines.append(in_line)
    return None

  def FilterDone(self, last_bits):
    self.lines.append(last_bits)
    return None


def GypFlagIsOn(options, flag):
  value = GetGypFlag(options, flag, False)
  # The values we understand as Off are False and a text zero.
  if value is False or value == '0':
    return False
  return True


def GetGypFlag(options, flag, default=None):
  gclient = options.factory_properties.get('gclient_env', {})
  defines = gclient.get('GYP_DEFINES', '')
  gypflags = dict([(a, c if b == '=' else True) for (a, b, c) in
                   [x.partition('=') for x in defines.split(' ')]])
  if flag not in gypflags:
    return default
  return gypflags[flag]


def GSUtilSetup():
  # Get the path to the gsutil script.
  gsutil = os.path.join(os.path.dirname(__file__), 'gsutil')
  gsutil = os.path.normpath(gsutil)
  if chromium_utils.IsWindows():
    gsutil += '.bat'

  # Get the path to the boto file containing the password.
  boto_file = os.path.join(os.path.dirname(__file__), '..', '..', 'site_config',
                           '.boto')

  # Make sure gsutil uses this boto file if it exists.
  if os.path.exists(boto_file):
    os.environ['AWS_CREDENTIAL_FILE'] = boto_file
    os.environ['BOTO_CONFIG'] = boto_file
  return gsutil


def GSUtilGetMetadataField(name, provider_prefix=None):
  """Returns: (str) the metadata field to use with Google Storage

  The Google Storage specification for metadata can be found at:
  https://developers.google.com/storage/docs/gsutil/addlhelp/WorkingWithObjectMetadata
  """
  # Already contains custom provider prefix
  if name.lower().startswith('x-'):
    return name

  # See if it's innately supported by Google Storage
  if name in (
      'Cache-Control',
      'Content-Disposition',
      'Content-Encoding',
      'Content-Language',
      'Content-MD5',
      'Content-Type',
  ):
    return name

  # Add provider prefix
  if not provider_prefix:
    provider_prefix = 'x-goog-meta'
  return '%s-%s' % (provider_prefix, name)


def GSUtilCopy(source, dest, mimetype=None, gs_acl=None, cache_control=None,
               metadata=None, override_gsutil=None):
  """Copy a file to Google Storage.

  Runs the following command:
    gsutil -h Content-Type:<mimetype> \
           -h Cache-Control:<cache_control> \
        cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path>

  Args:
    source: the source URI
    dest: the destination URI
    mimetype: optional value to add as a Content-Type header
    gs_acl: optional value to add as a canned-acl
    cache_control: optional value to set Cache-Control header
    metadata: (dict) A dictionary of string key/value metadata entries to set
        (see `gsutil cp' '-h' option)
    override_gsutil (list): optional argv to run gsutil
  Returns:
    The status code returned from running the generated gsutil command.
  """

  if not source.startswith('gs://') and not source.startswith('file://'):
    source = 'file://' + source
  if not dest.startswith('gs://') and not dest.startswith('file://'):
    dest = 'file://' + dest
  # The setup also sets up some env variables - for now always run that.
  gsutil = GSUtilSetup()
  # Run the gsutil command. gsutil internally calls command_wrapper, which
  # will try to run the command 10 times if it fails.
  command = list(override_gsutil or [gsutil])

  if not metadata:
    metadata = {}
  if mimetype:
    metadata['Content-Type'] = mimetype
  if cache_control:
    metadata['Cache-Control'] = cache_control
  for k, v in sorted(metadata.iteritems(), key=lambda (k, _): k):
    field = GSUtilGetMetadataField(k)
    param = (field) if v is None else ('%s:%s' % (field, v))
    command += ['-h', param]
  command.extend(['cp'])
  if gs_acl:
    command.extend(['-a', gs_acl])
  command.extend([source, dest])
  return chromium_utils.RunCommand(command)


def GSUtilCopyFile(filename, gs_base, subdir=None, mimetype=None, gs_acl=None,
                   cache_control=None, metadata=None, override_gsutil=None):
  """Copy a file to Google Storage.

  Runs the following command:
    gsutil -h Content-Type:<mimetype> \
        -h Cache-Control:<cache_control> \
        cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path>

  Args:
    filename: the file to upload
    gs_base: the bucket to upload the file to
    subdir: optional subdirectory withing the bucket
    mimetype: optional value to add as a Content-Type header
    gs_acl: optional value to add as a canned-acl
    override_gsutil (list): optional argv to run gsutil
  Returns:
    The status code returned from running the generated gsutil command.
  """

  source = 'file://' + filename
  dest = gs_base
  if subdir:
    # HACK(nsylvain): We can't use normpath here because it will break the
    # slashes on Windows.
    if subdir == '..':
      dest = os.path.dirname(gs_base)
    else:
      dest = '/'.join([gs_base, subdir])
  dest = '/'.join([dest, os.path.basename(filename)])
  return GSUtilCopy(source, dest, mimetype, gs_acl, cache_control,
                    metadata=metadata, override_gsutil=override_gsutil)


def GSUtilCopyDir(src_dir, gs_base, dest_dir=None, gs_acl=None,
                  cache_control=None):
  """Upload the directory and its contents to Google Storage."""

  if os.path.isfile(src_dir):
    assert os.path.isdir(src_dir), '%s must be a directory' % src_dir

  gsutil = GSUtilSetup()
  command = [gsutil, '-m']
  if cache_control:
    command.extend(['-h', 'Cache-Control:%s' % cache_control])
  command.extend(['cp', '-R'])
  if gs_acl:
    command.extend(['-a', gs_acl])
  if dest_dir:
    command.extend([src_dir, gs_base + '/' + dest_dir])
  else:
    command.extend([src_dir, gs_base])
  return chromium_utils.RunCommand(command)

def GSUtilDownloadFile(src, dst):
  """Copy a file from Google Storage."""
  gsutil = GSUtilSetup()

  # Run the gsutil command. gsutil internally calls command_wrapper, which
  # will try to run the command 10 times if it fails.
  command = [gsutil]
  command.extend(['cp', src, dst])
  return chromium_utils.RunCommand(command)


def GSUtilMoveFile(source, dest, gs_acl=None):
  """Move a file on Google Storage."""

  gsutil = GSUtilSetup()

  # Run the gsutil command. gsutil internally calls command_wrapper, which
  # will try to run the command 10 times if it fails.
  command = [gsutil]
  command.extend(['mv', source, dest])
  status = chromium_utils.RunCommand(command)

  if status:
    return status

  if gs_acl:
    command = [gsutil]
    command.extend(['setacl', gs_acl, dest])
    status = chromium_utils.RunCommand(command)

  return status


def GSUtilDeleteFile(filename):
  """Delete a file on Google Storage."""

  gsutil = GSUtilSetup()

  # Run the gsutil command. gsutil internally calls command_wrapper, which
  # will try to run the command 10 times if it fails.
  command = [gsutil]
  command.extend(['rm', filename])
  return chromium_utils.RunCommand(command)


# Python doesn't support the type of variable scope in nested methods needed
# to avoid the global output variable.  This variable should only ever be used
# by GSUtilListBucket.
command_output = ''


def GSUtilListBucket(gs_base, args):
  """List the contents of a Google Storage bucket."""

  gsutil = GSUtilSetup()

  # Run the gsutil command. gsutil internally calls command_wrapper, which
  # will try to run the command 10 times if it fails.
  global command_output
  command_output = ''

  def GatherOutput(line):
    global command_output
    command_output += line + '\n'
  command = [gsutil, 'ls'] + args + [gs_base]
  status = chromium_utils.RunCommand(command, parser_func=GatherOutput)
  return (status, command_output)


def LogAndRemoveFiles(temp_dir, regex_pattern):
  """Removes files in |temp_dir| that match |regex_pattern|.
  This function prints out the name of each directory or filename before
  it deletes the file from disk."""
  regex = re.compile(regex_pattern)
  if not os.path.isdir(temp_dir):
    return
  for dir_item in os.listdir(temp_dir):
    if regex.search(dir_item):
      full_path = os.path.join(temp_dir, dir_item)
      print 'Removing leaked temp item: %s' % full_path
      try:
        if os.path.islink(full_path) or os.path.isfile(full_path):
          os.remove(full_path)
        elif os.path.isdir(full_path):
          chromium_utils.RemoveDirectory(full_path)
        else:
          print 'Temp item wasn\'t a file or directory?'
      except OSError, e:
        print >> sys.stderr, e
        # Don't fail.


def RemoveOldSnapshots(desktop):
  """Removes ChromiumSnapshot files more than one day old. Such snapshots are
  created when certain tests timeout (e.g., Chrome Frame integration tests)."""
  # Compute the file prefix of a snapshot created one day ago.
  yesterday = datetime.datetime.now() - datetime.timedelta(1)
  old_snapshot = yesterday.strftime('ChromiumSnapshot%Y%m%d%H%M%S')
  # Collect snapshots at least as old as that one created a day ago.
  to_delete = []
  for snapshot in glob.iglob(os.path.join(desktop, 'ChromiumSnapshot*.png')):
    if os.path.basename(snapshot) < old_snapshot:
      to_delete.append(snapshot)
  # Delete the collected snapshots.
  for snapshot in to_delete:
    print 'Removing old snapshot: %s' % snapshot
    try:
      os.remove(snapshot)
    except OSError, e:
      print >> sys.stderr, e


def RemoveChromeDesktopFiles():
  """Removes Chrome files (i.e. shortcuts) from the desktop of the current user.
  This does nothing if called on a non-Windows platform."""
  if chromium_utils.IsWindows():
    desktop_path = os.environ['USERPROFILE']
    desktop_path = os.path.join(desktop_path, 'Desktop')
    LogAndRemoveFiles(desktop_path, r'^(Chromium|chrome) \(.+\)?\.lnk$')
    RemoveOldSnapshots(desktop_path)


def RemoveJumpListFiles():
  """Removes the files storing jump list history.
  This does nothing if called on a non-Windows platform."""
  if chromium_utils.IsWindows():
    custom_destination_path = os.path.join(os.environ['USERPROFILE'],
                                           'AppData',
                                           'Roaming',
                                           'Microsoft',
                                           'Windows',
                                           'Recent',
                                           'CustomDestinations')
    LogAndRemoveFiles(custom_destination_path, '.+')


def RemoveTempDirContents():
  """Obliterate the entire contents of the temporary directory, excluding
  paths in sys.argv.
  """
  temp_dir = os.path.abspath(tempfile.gettempdir())
  print 'Removing contents of %s' % temp_dir

  print '  Inspecting args for files to skip'
  whitelist = set()
  for i in sys.argv:
    try:
      if '=' in i:
        i = i.split('=')[1]
      low = os.path.abspath(i.lower())
      if low.startswith(temp_dir.lower()):
        whitelist.add(low)
    except TypeError:
      # If the argument is too long, windows will freak out and pop a TypeError.
      pass
  if whitelist:
    print '  Whitelisting:'
    for w in whitelist:
      print '    %r' % w

  start_time = time.time()
  for root, dirs, files in os.walk(temp_dir):
    for f in files:
      p = os.path.join(root, f)
      if p.lower() not in whitelist:
        try:
          os.remove(p)
        except OSError:
          pass
      else:
        print '  Keeping file %r (whitelisted)' % p
    for d in dirs[:]:
      p = os.path.join(root, d)
      if p.lower() not in whitelist:
        try:
          # TODO(iannucci): Make this deal with whitelisted items which are
          # inside of |d|

          # chromium_utils.RemoveDirectory gives access denied error when called
          # in this loop.
          shutil.rmtree(p, ignore_errors=True)
          # Remove it so that os.walk() doesn't try to recurse into
          # a non-existing directory.
          dirs.remove(d)
        except OSError:
          pass
      else:
        print '  Keeping dir %r (whitelisted)' % p
  print '   Removing temp contents took %.1f s' % (time.time() - start_time)


def RemoveChromeTemporaryFiles():
  """A large hammer to nuke what could be leaked files from unittests or
  files left from a unittest that crashed, was killed, etc."""
  # NOTE: print out what is cleaned up so the bots don't timeout if
  # there is a lot to cleanup and also se we see the leaks in the
  # build logs.
  # At some point a leading dot got added, support with and without it.
  kLogRegex = r'^\.?(com\.google\.Chrome|org\.chromium)\.'
  if chromium_utils.IsWindows():
    RemoveTempDirContents()
    RemoveChromeDesktopFiles()
    RemoveJumpListFiles()
  elif chromium_utils.IsLinux():
    LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex)
    LogAndRemoveFiles('/dev/shm', kLogRegex)
  elif chromium_utils.IsMac():
    nstempdir_path = '/usr/local/libexec/nstempdir'
    if os.path.exists(nstempdir_path):
      ns_temp_dir = chromium_utils.GetCommandOutput([nstempdir_path]).strip()
      if ns_temp_dir:
        LogAndRemoveFiles(ns_temp_dir, kLogRegex)
    for i in ('Chromium', 'Google Chrome'):
      # Remove dumps.
      crash_path = '%s/Library/Application Support/%s/Crash Reports' % (
          os.environ['HOME'], i)
      LogAndRemoveFiles(crash_path, r'^.+\.dmp$')
  else:
    raise NotImplementedError(
        'Platform "%s" is not currently supported.' % sys.platform)


def WriteLogLines(logname, lines, perf=None):
  logname = logname.rstrip()
  lines = [line.rstrip() for line in lines]
  for line in lines:
    print '@@@STEP_LOG_LINE@%s@%s@@@' % (logname, line)
  if perf:
    perf = perf.rstrip()
    print '@@@STEP_LOG_END_PERF@%s@%s@@@' % (logname, perf)
  else:
    print '@@@STEP_LOG_END@%s@@@' % logname


def ZipAndUpload(bucket, archive, *targets):
  """Uploads a zipped archive to the specified Google Storage bucket.

  Args:
    bucket: Google Storage bucket to upload to.
    archive: Name of the .zip archive.
    *targets: List of targets that should be included in the archive.

  Returns:
    Path to the uploaded archive on Google Storage.
  """
  local_archive = os.path.join(tempfile.mkdtemp(archive), archive)
  zip_cmd = [
    'zip',
    '-9',
    '--filesync',
    '--recurse-paths',
    '--symlinks',
    local_archive,
  ]
  zip_cmd.extend(targets)

  chromium_utils.RunCommand(zip_cmd)
  GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive))
  return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive)