blob: 7083799cd3ebcc4328a76026c8c972eaaefc287c [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Remove the build metadata embedded in the artifacts of a build."""
import json
import multiprocessing
import optparse
import os
import platform
import Queue
import shutil
import subprocess
import sys
import tempfile
import threading
import zipfile
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
def build_bitness(build_dir):
# This function checks whether the target (not host) word size is 64-bits.
# Since 64-bit hosts can cross-compile 32-bit binaries, check the GN args to
# see what CPU we're targetting.
args_gn = os.path.join(build_dir, '')
with open(args_gn) as f:
for line in f:
decommented = line.split('#', 1)[0]
key_and_value = decommented.split('=', 1)
if len(key_and_value) != 2:
key = key_and_value[0].strip()
value = key_and_value[1].strip()
if key == 'target_cpu':
if value.endswith('64'):
return 64
return 32
# If we don't find anything, or if there is no GN args file, default to the
# host architecture.
if platform.machine().endswith('64'):
return 64
return 32
def get_files_to_clean(build_dir, recursive=False):
"""Get the list of files to clean."""
allowed = frozenset(
('', '.app', '.dylib', '.nexe', '.so'))
non_x_ok_exts = frozenset(('.isolated', '.jar'))
min_timestamp = 0
if os.path.exists(os.path.join(build_dir, '')):
min_timestamp = os.path.getmtime(os.path.join(build_dir, ''))
def check(f):
if not os.path.isfile(f) or os.path.basename(f).startswith('.'):
return False
if os.path.getmtime(os.path.join(build_dir, f)) < min_timestamp:
return False
ext = os.path.splitext(f)[1]
return (ext in non_x_ok_exts) or (ext in allowed and os.access(f, os.X_OK))
ret_files = set()
for root, dirs, files in os.walk(build_dir):
if not recursive:
dirs[:] = [d for d in dirs if d.endswith(('_apk', ''))]
for f in (f for f in files if check(os.path.join(root, f))):
ret_files.add(os.path.relpath(os.path.join(root, f), build_dir))
return ret_files
def remove_zip_timestamps(filename):
"""Remove the timestamps embedded in a zip archive."""
sys.stdout.write('Processing: %s\n' % os.path.basename(filename))
with zipfile.ZipFile(filename, 'r') as zf:
# Creates a temporary file.
out_file, out_filename = tempfile.mkstemp(prefix='remote_apk_timestamp')
with zipfile.ZipFile(out_filename, 'w') as zf_o:
# Copy the data from the original file to the new one.
for info in zf.infolist():
# Overwrite the timestamp with a constant value.
info.date_time = (1980, 1, 1, 0, 0, 0)
# Remove the original file and replace it by the modified one.
shutil.move(out_filename, filename)
if os.path.isfile(out_filename):
def remove_metadata_worker(file_queue, failed_queue, build_dir, bitness):
"""Worker thread for the remove_metadata function."""
while True:
f = file_queue.get()
if f.endswith('.jar'):
remove_zip_timestamps(os.path.join(build_dir, f))
def remove_metadata(build_dir, recursive, bitness):
"""Remove the build metadata from the artifacts of a build."""
with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f:
blacklist = frozenset(json.load(f))
files = Queue.Queue()
for f in get_files_to_clean(build_dir, recursive) - blacklist:
failed_files = Queue.Queue()
for _ in xrange(multiprocessing.cpu_count()):
worker = threading.Thread(target=remove_metadata_worker,
worker.daemon = True
if not failed_files.empty():
print >> sys.stderr, 'Failed for the following files:'
failed_files_list = []
while not failed_files.empty():
print >> sys.stderr, '\n'.join(' ' + i for i in sorted(failed_files_list))
return 1
return 0
def main():
parser = optparse.OptionParser(usage='%prog [options]')
# TODO(sebmarchand): Add support for reading the list of artifact from a
# .isolated file.
parser.add_option('--build-dir', help='The build directory.')
parser.add_option('-r', '--recursive', action='store_true', default=False,
help='Indicates if the script should be recursive.')
options, _ = parser.parse_args()
if not options.build_dir:
parser.error('--build-dir is required')
bitness = build_bitness(options.build_dir)
return remove_metadata(options.build_dir, options.recursive, bitness)
if __name__ == '__main__':