blob: c1f7bd18b878097e28f39c1555b87b2108980288 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Reduce the amount of data in ../test/data/*.json"""
import json
import logging
import optparse
import os
import re
import sys
class Filterer(object):
def __init__(self):
self._deleted_builds = {}
self.max_cached_builds = 10
self._allowed_builders = ('linux_clang', 'linux', 'linux_touch')
def reset(self):
self._deleted_builds = {}
def reduce_data(self, data):
"""Reduces the amount of data sent from a server to simplify testing and the
amount of test data stored.
for line in data[:]:
original_url, original_response = line
# Unpack and filter the response.
new_url, new_response = self.filter_response(
original_url, json.loads(original_response))
if not new_url or new_response is None:
# Repack the string.
line[1] = json.dumps(new_response, separators=(',',':'))'%s length: %d -> %d' % (
new_url, len(original_response), len(line[1])))
if len(line[1]) > 20000:
return data
def filter_response(self, url, response):
"""Trims a single request.
|response| must be decoded json. Decoded json will be returned.
# Builders
match = re.match('.+/json/builders/(\w+)(|\?filter=1)$', url)
if match:
return url, self._filter_builder(, response)
match = re.match('.+/json/builders(|\?filter=1)$', url)
if match:
for builder in response.keys():
value = self._filter_builder(builder, response[builder])
if value is None:
del response[builder]
response[builder] = value
return url, response
# Pending
match = re.match('.+/json/builders/(\w+)/pendingBuilds(|\?filter=1)$', url)
if match:
assert in self._allowed_builders
return url, self._filter_pending(response)
# Builds
match = re.match('.+/json/builders/(\w+)/builds/_all(|\?filter=1)$', url)
if match:
builder =
assert builder in self._allowed_builders, url
keys = response.keys()
keys = [int(k) for k in keys if int(k) not in self._deleted(builder)]
keys = sorted(keys)[:self.max_cached_builds]
response = dict((k, v) for k, v in response.iteritems() if int(k) in keys)
return url, response
match = re.match('.+/json/builders/(\w+)/builds/\?(.+?)(|\&filter=1)$', url)
if match:
assert in self._allowed_builders
# Ignore the query, reconstruct it from what it kept.
for build in response.keys():
value = self._filter_build(response[build])
if value is None:
del response[build]
response[build] = value
if not response:
return None, None
url = '%s?%s' % (
url.split('?', 1)[0],
'&'.join('select=%s' % b for b in sorted(response)))
return url, response
match = re.match('.+/json/builders/(\w+)/builds/(\d+)(|\?filter=1)$', url)
if match:
return url, self._filter_build(response)
# Slaves
match = re.match('.+/json/slaves/([^/]+?)(|\?filter=1)$', url)
if match:
return url, self._filter_slave(, response)
match = re.match('.+/json/slaves(|\?filter=1)$', url)
if match:
for slave in response.keys():
value = self._filter_slave(slave, response[slave])
if value is None:
del response[slave]
response[slave] = value
return url, response
# Project
match = re.match('.+/json/project(|\?filter=1)$', url)
if match:
return url, response
assert False, url
def _filter_pending(pending):
"""Trim pendingBuilds."""
return pending[:2]
def _filter_builder(self, builder, response):
"""Trims a builder.
Reduces the number of cached builds.
# TODO(maruel): Reduce the number of slaves.
if builder not in self._allowed_builders:
return None
builds_kept = response['cachedBuilds'][-self.max_cached_builds:]
builds_discarded = response['cachedBuilds'][:-self.max_cached_builds]
assert len(builds_kept) <= self.max_cached_builds
assert (
len(builds_kept) + len(builds_discarded) ==
if builds_discarded:
assert min(builds_kept) > max(builds_discarded)
response['cachedBuilds'] = builds_kept
self._deleted(builder).union(int(b) for b in builds_discarded)
if response.get('currentBuilds'):
response['currentBuilds'] = [
build for build in response['currentBuilds']
if int(build) not in self._deleted(builder)
if response.get('pendingBuilds', 0) > 2:
response['pendingBuilds'] = 2
return response
def _filter_build(self, response):
"""Trims a build."""
if response['builderName'] not in self._allowed_builders:
return None
if int(response['number']) in self._deleted(response['builderName']):
return None
# TODO(maruel): Fix StatusJson to not push that much logs data.
if 'logs' in response:
del response['logs']
if response.get('currentStep') and response['currentStep'].get('logs'):
del response['currentStep']['logs']
for step in response['steps']:
if 'logs' in step:
del step['logs']
return response
def _filter_slave(self, _slave, response):
"""Trims a slave."""
if response.get('builders'):
for builder in response['builders'].keys():
if not builder in self._allowed_builders:
del response['builders'][builder]
if not response['builders']:
return None
if response.get('builderName'):
if not response['builderName'] in self._allowed_builders:
return None
if response.get('runningBuilds'):
for i, build in enumerate(response['runningBuilds'][:]):
value = self._filter_build(build)
if value is None:
response['runningBuilds'][i] = value
return response
def _deleted(self, builder):
return self._deleted_builds.get(builder, set())
def main():
parser = optparse.OptionParser(
parser.add_option('-v', '--verbose', action='count', default=0)
parser.add_option('-d', '--dry-run', action='store_true')
options, args = parser.parse_args()
if args:
parser.error('Unsupported args: %s' % args)
level=[logging.WARNING, logging.INFO, logging.DEBUG][
min(2, options.verbose)])
datadir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')
for filename in os.listdir(datadir):
if not filename.endswith('.json') or filename.endswith('_expected.json'):
filepath = os.path.join(datadir, filename)
print 'Processing %s' % filename
data = json.load(open(filepath))
data = Filterer().reduce_data(data)
if not options.dry_run:
json.dump(data, open(filepath, 'w'), separators=(',',':'))
return 0
if __name__ == '__main__':