scripts/tools/remove_old_builddirs.py

#!/usr/bin/env python
# Copyright (c) 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import os
import shutil
import sys
import time


# Unless otherwise noted, all ages are in seconds and all sizes are in bytes.


def DAYS(secs):
  return float(secs) / (60 * 60 * 24)


def gen_masters_dirs(build_dir):
  msdir = os.path.join(build_dir, 'masters')
  return [os.path.join(msdir, x) for x in os.listdir(msdir)
          if x.startswith('master') and os.path.isdir(os.path.join(msdir, x))]


def is_builder_dir(bdir):
  if not os.path.isdir(bdir):
    return False
  # It seems buildbot puts this file into every builder directory.
  if not os.path.exists(os.path.join(bdir, 'builder')):
    return False
  if not os.path.isfile(os.path.join(bdir, 'builder')):
    return False
  return True


def get_change_time(f):
  return os.stat(f).st_mtime


def get_age(f):
  return time.time() - get_change_time(f)


def get_size(f):
  return os.stat(f).st_size


def get_builders(mdir):
  for d in os.listdir(mdir):
    bdir = os.path.join(mdir, d)
    if is_builder_dir(bdir):
      yield bdir


def find_builders(build_dir):
  for mdir in gen_masters_dirs(build_dir):
    for bdir in get_builders(mdir):
      yield bdir


def get_dir_size(d):
  s = 0
  for p in os.listdir(d):
    f = os.path.join(d, p)
    if os.path.isdir(f):
      s += get_dir_size(f)
    else:
      s += get_size(f)
  return s


def MiB(size):
  return float(size) / (1024**2)


def preformat_build_dir(bdir, size=None, age=None):
  builder = os.path.basename(bdir)
  master = os.path.basename(os.path.dirname(bdir))
  if size is None:
    size = get_dir_size(bdir)
  if age is None:
    age = get_age(bdir)
  return (master, builder, '%.0f days' % DAYS(age), '%.1f MiB' % MiB(size))


def preformat_build_logs(bdir, logs, size):
  builder = os.path.basename(bdir)
  master = os.path.basename(os.path.dirname(bdir))
  build_min, build_max = min(logs), max(logs)
  builds_range = 'builds ~ %6i(%4.f days) .. %6i(%4.0f days)' % (
      build_min, DAYS(get_logs_age(logs[build_min])),
      build_max, DAYS(get_logs_age(logs[build_max])))
  return (master, builder, builds_range, '%.1f MiB' % MiB(size))


def print_table(table):
  max_lens = [0] * len(table[0])
  for row in table:
    for i, cell in enumerate(row):
      max_lens[i] = max(len(cell), max_lens[i])
  fmt = '\t'.join('%' + str(l) + 's' for l in max_lens)
  for row in table:
    print fmt % row


def get_logs_age(logs):
  """Returns age of the most recent logfile."""
  return min(map(get_age, logs))


def get_old_logs(bdir, min_age_days):
  """Return tuple ({build_number: logfiles}, number of logs, total size)."""
  # Delete either all files from a build OR none.
  # Hence, first scan to get all log files.
  build_logs = {}
  for f in os.listdir(bdir):
    if f == 'builder':  # Skip this special file.
      continue
    try:
      build_number = int(f.split('-')[0])
    except (ValueError, TypeError, IndexError):
      continue
    logfile = os.path.join(bdir, f)
    build_logs.setdefault(build_number, []).append(logfile)
  # Second scan by buildnumber, modifying logs dict inside loop.
  count, size = 0, 0
  for build_number, logfiles in list(build_logs.iteritems()):
    if DAYS(get_logs_age(logfiles)) < min_age_days:
      build_logs.pop(build_number)
      continue
    count += len(logfiles)
    size += sum(map(get_size, logfiles))
  return build_logs, count, size


def delete_old_logs(opts):
  index = {}  # sort_key => build_number => list of logs files.
  total_size, total_count = 0, 0
  for bdir in find_builders(opts.build_dir):
    age = get_age(bdir)
    if DAYS(age) > opts.min_age_days:
      print ('WARNING: builder is %.0f DAYS old, '
             'consider deleting it completely instead.' % DAYS(age))
      if not opts.force:
        raise Exception('maybe first do: $ %s --min-age-days %i ?' % (
                         os.path.basename(__file__), opts.min_age_days))
    build_logs, count, size = get_old_logs(bdir, opts.min_age_days)
    if not build_logs:
      assert count == 0 and size == 0
      continue
    index[(size, count, bdir)] = build_logs
    total_size += size
    total_count += count

  table = []
  for k in sorted(index, reverse=True):
    size, _, bdir = k
    table.append(preformat_build_logs(bdir, index[k], size))
  print_table(table)

  metrics = '%i logfiles weighing %0.f MiB' % (total_count, MiB(total_size))
  if opts.force:
    print 'deleting these logs files (%s)' % metrics
  else:
    prompt = 'do you want to delete these logs (%s)?' % metrics
    if raw_input(prompt).strip().lower() not in ('y', 'yes'):
      print 'aborting.'
      return 0

  for size, count, bdir in sorted(index, reverse=True):
    print 'cleaning %5i logfiles weighing %5.0f MiB in %s...' % (
          count, MiB(size), bdir)
    if opts.dry_run:
      continue
    for logfiles in index[(size, count, bdir)].itervalues():
      for logfile in logfiles:
        assert DAYS(get_age(logfile)) > opts.min_age_days
        os.remove(logfile)
  print 'done%s.' % (' (dry run)' if opts.dry_run else '')


def main(args):
  parser = argparse.ArgumentParser()
  parser.add_argument(
      'build_dir',
      help='path to either build/ or build_internal/')
  parser.add_argument(
      '--just-logs', default=False, action='store_true',
      help=('delete individual build log files instead of builder directories. '
            'WARNING: this will change the mod time of a builder dir, thus '
            'making it younger to this script than it actually was. It\'s '
            'recommended that you *first* prune old builddirs completely, '
            'and then delete individual logs.'))
  parser.add_argument(
      '--min-age-days', type=int,
      help='min age in days to be considered for deletion')
  parser.add_argument(
      '--min-size-mib', type=int,
      help='min size in MiB to be considered for deletion')
  parser.add_argument(
      '-d', '--dry-run', action='store_true', default=False,
      help='do not actually delete dirs')
  parser.add_argument(
      '-f', '--force', action='store_true', default=False,
      help=('do not ask for confirmation. '
           'This isn\'t allowed if min age is less than 1 month.'))
  opts = parser.parse_args(args)
  if opts.force and opts.min_age_days < 30:
    parser.error('safety feature: '
                 '--force can\'t be used to delete too recent log files.')
  if opts.just_logs:
    if opts.min_size_mib:
      parser.error('--min-size-mib can\'t be used with --just-logs')
    return delete_old_logs(opts)
  else:
    return delete_old_builders(opts)


def delete_old_builders(opts):
  # TODO(tandrii): move above main after review
  index = {}
  total_size = 0
  for bdir in find_builders(opts.build_dir):
    age = get_age(bdir)
    if DAYS(age) < opts.min_age_days:
      continue
    size = get_dir_size(bdir)
    if MiB(size) < opts.min_size_mib:
      continue
    total_size += size
    index[(-size, -age, bdir)] = preformat_build_dir(bdir, size, age)
  print_table([index[k] for k in sorted(index)])

  total_size_str = 'total size: %0.f MiB' % MiB(total_size)
  if opts.force:
    print 'deleting these builders dirs (%s)' % total_size_str
  else:
    prompt = 'do you want to delete these builder dirs (%s)?' % total_size_str
    if raw_input(prompt).strip().lower() not in ('y', 'yes'):
      print 'aborting.'
      return 0

  for key in index:
    bdir = key[-1]
    print 'removing %s...' % bdir
    if not opts.dry_run:
      shutil.rmtree(bdir)
  print 'done%s.' % (' (dry run)' if opts.dry_run else '')


if __name__ == '__main__':
  main(sys.argv[1:])