Tools/Scripts/make-dist

#!/usr/bin/env python3
# Copyright (C) 2014 Igalia S.L.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

from __future__ import print_function
from contextlib import closing

import argparse
import errno
import multiprocessing
import os
import re
import shutil
import subprocess
import tarfile


def enum(**enums):
    return type('Enum', (), enums)


class Rule(object):
    Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3)

    def __init__(self, type, pattern):
        self.type = type
        self.original_pattern = pattern
        self.pattern = re.compile(pattern)

    def test(self, file):
        if not(self.pattern.search(file)):
            return Rule.Result.NO_MATCH
        return self.type


class Ruleset(object):
    _global_rules = None

    def __init__(self):
        # By default, accept all files.
        self.rules = [Rule(Rule.Result.INCLUDE, '.*')]

    @classmethod
    def global_rules(cls):
        if not cls._global_rules:
            cls._global_rules = Ruleset()
        return cls._global_rules

    @classmethod
    def add_global_rule(cls, rule):
        cls.global_rules().add_rule(rule)

    def add_rule(self, rule):
        self.rules.append(rule)

    def passes(self, file):
        allowed = False
        for rule in self.rules:
            result = rule.test(file)
            if result == Rule.Result.NO_MATCH:
                continue
            allowed = Rule.Result.INCLUDE == result
        return allowed


class File(object):
    def __init__(self, source_root, tarball_root):
        self.source_root = source_root
        self.tarball_root = tarball_root

    def should_skip_file(self, path):
        # Do not skip files explicitly added from the manifest.
        return False

    def get_files(self):
        yield (self.source_root, self.tarball_root)


class Directory(object):
    def __init__(self, source_root, tarball_root):
        self.source_root = source_root
        self.tarball_root = tarball_root
        self.rules = Ruleset()

        self.files_in_version_control = self.list_files_in_version_control()

    def add_rule(self, rule):
        self.rules.add_rule(rule)

    def get_tarball_path(self, filename):
        return filename.replace(self.source_root, self.tarball_root, 1)

    def list_files_in_version_control(self):
        # FIXME: Only git is supported for now.
        p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE, text="ascii")
        out = p.communicate()[0]
        if not out:
            return []
        return frozenset(out.splitlines())

    def should_skip_file(self, path):
        return path not in self.files_in_version_control

    def get_files(self):
        for root, dirs, files in os.walk(self.source_root):

            def passes_all_rules(entry):
                return Ruleset.global_rules().passes(entry) and self.rules.passes(entry)

            to_keep = list(filter(passes_all_rules, dirs))
            del dirs[:]
            dirs.extend(to_keep)

            for file in files:
                file = os.path.join(root, file)
                if not passes_all_rules(file):
                    continue
                yield (file, self.get_tarball_path(file))


class Manifest(object):
    def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'):
        self.current_directory = None
        self.directories = []
        self.tarball_root = tarball_root
        self.source_root = source_root
        self.build_root = build_root

        # Normalize the tarball root so that it starts and ends with a slash.
        if not self.tarball_root.endswith('/'):
            self.tarball_root = self.tarball_root + '/'
        if not self.tarball_root.startswith('/'):
            self.tarball_root = '/' + self.tarball_root

        with open(manifest_filename, 'r') as file:
            for line in file.readlines():
                self.process_line(line)

    def add_rule(self, rule):
        if self.current_directory is not None:
            self.current_directory.add_rule(rule)
        else:
            Ruleset.add_global_rule(rule)

    def add_directory(self, directory):
        self.current_directory = directory
        self.directories.append(directory)

    def get_full_source_path(self, source_path):
        if not os.path.exists(source_path):
            source_path = os.path.join(self.source_root, source_path)
        if not os.path.exists(source_path):
            raise Exception('Could not find directory %s' % source_path)
        return source_path

    def get_full_tarball_path(self, path):
        return self.tarball_root + path

    def get_source_and_tarball_paths_from_parts(self, parts):
        full_source_path = self.get_full_source_path(parts[1])
        if len(parts) > 2:
            full_tarball_path = self.get_full_tarball_path(parts[2])
        else:
            full_tarball_path = self.get_full_tarball_path(parts[1])
        return (full_source_path, full_tarball_path)

    def process_line(self, line):
        parts = line.split()
        if not parts:
            return
        if parts[0].startswith("#"):
            return

        if parts[0] == "directory" and len(parts) > 1:
            self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts)))
        elif parts[0] == "file" and len(parts) > 1:
            self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts)))
        elif parts[0] == "exclude" and len(parts) > 1:
            self.add_rule(Rule(Rule.Result.EXCLUDE, parts[1]))
        elif parts[0] == "include" and len(parts) > 1:
            self.add_rule(Rule(Rule.Result.INCLUDE, parts[1]))
        else:
            raise Exception('Line does not begin with a correct rule:\n\t' + line)

    def should_skip_file(self, directory, filename):
        # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir.
        if filename.startswith(self.build_root):
            return False

        return directory.should_skip_file(filename)

    def get_files(self):
        for directory in self.directories:
            for file_tuple in directory.get_files():
                if self.should_skip_file(directory, file_tuple[0]):
                    continue
                yield file_tuple

    def create_tarfile(self, output):
        count = 0
        for file_tuple in self.get_files():
            count = count + 1

        with closing(tarfile.open(output, 'w')) as tarball:
            for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1):
                print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r')
                tarball.add(file_path, tarball_path)
        print("Wrote {0}".format(output).ljust(40))


class Distcheck(object):
    BUILD_DIRECTORY_NAME = "_build"
    INSTALL_DIRECTORY_NAME = "_install"

    def __init__(self, source_root, build_root):
        self.source_root = source_root
        self.build_root = build_root

    def extract_tarball(self, tarball_path):
        with closing(tarfile.open(tarball_path, 'r')) as tarball:
            tarball.extractall(self.build_root)

    def configure(self, dist_dir, build_dir, install_dir, port, build_unified):
        def create_dir(directory, directory_type):
            try:
                os.mkdir(directory)
            except OSError as e:
                if e.errno != errno.EEXIST or not os.path.isdir(directory):
                    raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e)))

        create_dir(build_dir, "build")
        create_dir(install_dir, "install")

        command = ['cmake', '-GNinja', '-DPORT=%s' % port, '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', dist_dir]
        if not build_unified:
            command.append('-DENABLE_UNIFIED_BUILDS=OFF')
        subprocess.check_call(command, cwd=build_dir)

    def build(self, build_dir):
        command = ['cmake', '--build', '.']
        make_args = os.getenv('MAKE_ARGS')
        if make_args:
            command.append('--')
            command.extend(make_args.split(' '))
        else:
            command.extend(['-j', '%d' % multiprocessing.cpu_count()])
        subprocess.check_call(command, cwd=build_dir)

    def check_symbols(self, build_dir):
        check_bss = os.path.join(self.source_root, 'Tools', 'Scripts', 'check-for-global-bss-symbols-in-webkitgtk-libs')
        libjsc = os.path.join(build_dir, 'lib', 'libjavascriptcoregtk-6.0.so')
        libwk = os.path.join(build_dir, 'lib', 'libwebkitgtk-6.0.so')
        subprocess.check_call([check_bss, libjsc, libwk])

        check_version_script = os.path.join(self.source_root, 'Tools', 'Scripts', 'check-for-invalid-symbols-in-version-script')
        version_script = os.path.join(os.path.dirname(build_dir), 'Source', 'WebKit', 'webkitglib-symbols.map')
        subprocess.check_call([check_version_script, version_script, libwk])

    def install(self, build_dir):
        subprocess.check_call(['cmake', '--install', '.'], cwd=build_dir)

    def clean(self, dist_dir):
        shutil.rmtree(dist_dir)

    def check(self, tarball, port, build_unified):
        tarball_name, ext = os.path.splitext(os.path.basename(tarball))
        dist_dir = os.path.join(self.build_root, tarball_name)
        build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME)
        install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME)

        self.extract_tarball(tarball)
        self.configure(dist_dir, build_dir, install_dir, port, build_unified)
        self.build(build_dir)
        if port == 'GTK':
            self.check_symbols(build_dir)
        self.install(build_dir)
        self.clean(dist_dir)

if __name__ == "__main__":
    class FilePathAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, self.dest, os.path.abspath(values))

    def ensure_version_if_possible(arguments):
        if arguments.version is not None:
            return

        pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit/webkitgtk-6.0.pc")
        if os.path.isfile(pkgconfig_file):
            p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE, text="ascii")
            version = p.communicate()[0]
            if version:
                arguments.version = version.rstrip('\n')


    def get_tarball_root_and_output_filename_from_arguments(arguments):
        tarball_root = arguments.tarball_name
        if arguments.version is not None:
            tarball_root += '-' + arguments.version

        output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar")
        return tarball_root, output_filename

    parser = argparse.ArgumentParser(description='Build a distribution bundle.')
    parser.add_argument('-c', '--check', action='store_true',
                        help='Check the tarball')
    parser.add_argument('-u', '--unified', action='store_true',
                        help='Build unified when checking the tarball')
    parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(),
                        help='The top-level directory of the source distribution. ' + \
                              'Directory for relative paths. Defaults to current directory.')
    parser.add_argument('--version', type=str, default=None,
                        help='The version of the tarball to generate')
    parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(),
                        help='The top-level path of directory of the build root. ' + \
                              'By default is the current directory.')
    parser.add_argument('-t', '--tarball-name', type=str, default='webkitgtk',
                        help='Base name of tarball. Defaults to "webkitgtk".')
    parser.add_argument('-p', '--port', type=str, default='GTK',
                        help='Port to be built in tarball check. Defaults to "GTK".')
    parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.')

    arguments = parser.parse_args()

    # Paths in the manifest are relative to the source directory, and this script assumes that
    # current working directory is the source directory, so change the current working directory
    # to be the source directory.
    os.chdir(arguments.source_dir)

    ensure_version_if_possible(arguments)
    tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments)

    manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root)
    manifest.create_tarfile(output_filename)

    if arguments.check:
        Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename, arguments.port, arguments.unified)