| #!/usr/bin/env python3 | |
| # | |
| # Copyright (C) 2018, 2020, 2024 Igalia S.L. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are met: | |
| # | |
| # 1. Redistributions of source code must retain the above copyright notice, this | |
| # list of conditions and the following disclaimer. | |
| # 2. Redistributions in binary form must reproduce the above copyright notice, | |
| # this list of conditions and the following disclaimer in the documentation | |
| # and/or other materials provided with the distribution. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND | |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | |
| # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR | |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES | |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND | |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| import argparse | |
| import base64 | |
| import datetime | |
| import hashlib | |
| import json | |
| import logging | |
| import os | |
| from pathlib import PurePath | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import tempfile | |
| import zipfile | |
| top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..')) | |
| sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'jhbuild')) | |
| sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'Scripts', 'webkitpy')) | |
| from webkitpy.port.linux_container_sdk_utils import maybe_enter_webkit_container_sdk, maybe_use_container_sdk_root_dir | |
| maybe_enter_webkit_container_sdk() | |
| maybe_use_container_sdk_root_dir() | |
| import jhbuildutils | |
| from binary_bundling.ldd import SharedObjectResolver | |
| from binary_bundling.bundle import BinaryBundler | |
| INSTALL_DEPS_SCRIPT_TEMPLATE = """\ | |
| #!/bin/bash | |
| set -eu -o pipefail | |
| REQUIREDPACKAGES="%(packages_needed)s" | |
| if ! which apt-get >/dev/null; then | |
| echo "This script only supports apt-get based distributions like Debian or Ubuntu." | |
| exit 1 | |
| fi | |
| # Calling dpkg-query is slow, so call it only once and cache the results | |
| TMPCHECKPACKAGES="$(mktemp)" | |
| dpkg-query --show --showformat='${binary:Package} ${db:Status-Status}\\n' > "${TMPCHECKPACKAGES}" | |
| TOINSTALL="" | |
| for PACKAGE in ${REQUIREDPACKAGES}; do | |
| if ! grep -qxF "${PACKAGE} installed" "${TMPCHECKPACKAGES}"; then | |
| TOINSTALL="${TOINSTALL} ${PACKAGE}" | |
| fi | |
| done | |
| rm -f "${TMPCHECKPACKAGES}" | |
| if [[ -z "${TOINSTALL}" ]]; then | |
| echo "All required dependencies are already installed" | |
| else | |
| echo "Need to install the following extra packages: ${TOINSTALL}" | |
| [[ ${#} -gt 0 ]] && [[ "${1}" == "--printonly" ]] && exit 0 | |
| SUDO="" | |
| [[ ${UID} -ne 0 ]] && SUDO="sudo" | |
| AUTOINSTALL="" | |
| if [[ ${#} -gt 0 ]] && [[ "${1}" == "--autoinstall" ]]; then | |
| AUTOINSTALL="-y" | |
| export DEBIAN_FRONTEND="noninteractive" | |
| [[ ${UID} -ne 0 ]] && SUDO="sudo --preserve-env=DEBIAN_FRONTEND" | |
| ${SUDO} apt-get update | |
| fi | |
| set -x | |
| ${SUDO} apt-get install --no-install-recommends ${AUTOINSTALL} ${TOINSTALL} | |
| fi | |
| """ | |
| FONT_CONF_FILE = """\ | |
| <?xml version="1.0"?> | |
| <!DOCTYPE fontconfig SYSTEM "fonts.dtd"> | |
| <fontconfig> | |
| <its:rules xmlns:its="http://www.w3.org/2005/11/its" version="1.0"> | |
| <its:translateRule translate="no" selector="/fontconfig/*[not(self::description)]"/> | |
| </its:rules> | |
| <description>Default configuration file</description> | |
| <!-- Font directory list, relative to XDG_DATA_HOME --> | |
| <dir prefix="xdg">fonts/fonts</dir> | |
| <!-- Load system customization files, relative to previous dir --> | |
| <include ignore_missing="yes">conf.d</include> | |
| <!-- Font cache directory list, relative to XDG_CACHE_HOME --> | |
| <cachedir prefix="xdg">fontconfig</cachedir> | |
| <!-- Include also the system font config if is on the standard dir --> | |
| <include ignore_missing="yes">/etc/fonts/fonts.conf</include> | |
| <!-- Accept deprecated 'mono' alias, replacing it with 'monospace' --> | |
| <match target="pattern"> | |
| <test qual="any" name="family"> | |
| <string>mono</string> | |
| </test> | |
| <edit name="family" mode="assign" binding="same"> | |
| <string>monospace</string> | |
| </edit> | |
| </match> | |
| <!-- Accept alternate 'sans serif' spelling, replacing it with 'sans-serif' --> | |
| <match target="pattern"> | |
| <test qual="any" name="family"> | |
| <string>sans serif</string> | |
| </test> | |
| <edit name="family" mode="assign" binding="same"> | |
| <string>sans-serif</string> | |
| </edit> | |
| </match> | |
| <!-- Accept deprecated 'sans' alias, replacing it with 'sans-serif' --> | |
| <match target="pattern"> | |
| <test qual="any" name="family"> | |
| <string>sans</string> | |
| </test> | |
| <edit name="family" mode="assign" binding="same"> | |
| <string>sans-serif</string> | |
| </edit> | |
| </match> | |
| </fontconfig> | |
| """ | |
| _log = logging.getLogger(__name__) | |
| LOG_MESSAGE = 25 | |
| class Archiver(object): | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, type, v, tb): | |
| return self._archive.close() | |
| class TarArchiver(Archiver): | |
| def __init__(self, path, compresslevel): | |
| self._archive = tarfile.open(path, 'w:xz', preset=compresslevel) | |
| def add_file(self, system_path, zip_path): | |
| return self._archive.add(system_path, zip_path) | |
| class ZipArchiver(Archiver): | |
| def __init__(self, path, compresslevel): | |
| self._archive = zipfile.ZipFile(path, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=compresslevel) | |
| def add_file(self, system_path, zip_path): | |
| if os.path.islink(system_path): | |
| symlink_zip_info = zipfile.ZipInfo(zip_path) | |
| symlink_zip_info.create_system = 3 # Unix (for symlink support) | |
| symlink_zip_info.external_attr = 0xA1ED0000 # Zip softlink magic number | |
| return self._archive.writestr(symlink_zip_info, os.readlink(system_path)) | |
| return self._archive.write(system_path, zip_path) | |
| class BundleCreator(object): | |
| def __init__(self, configuration, platform, bundle_type, syslibs, ldd, should_strip_objects, compression_type, compression_level, destination = None, revision = None, builder_name = None): | |
| self._configuration = configuration | |
| self._platform = platform.lower() | |
| self._revision = revision | |
| self._bundle_binaries = ['jsc', 'MiniBrowser'] if bundle_type == 'all' else [ bundle_type ] | |
| self._bundle_type = bundle_type | |
| self._buildername = builder_name | |
| self._syslibs = syslibs | |
| self._shared_object_resolver = SharedObjectResolver(ldd) | |
| self._should_strip_objects = should_strip_objects | |
| self._compression_type = compression_type | |
| self._compression_level = int(compression_level) | |
| self._tmpdir = None | |
| self._wrappers = [] | |
| self._port_binary_preffix = 'WebKit' if self._platform == 'gtk' else 'WPE' | |
| wk_build_path = os.environ['WEBKIT_OUTPUTDIR'] if 'WEBKIT_OUTPUTDIR' in os.environ else \ | |
| os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'WebKitBuild')) | |
| self._buildpath = os.path.join(wk_build_path, self._platform.upper(), self._configuration.capitalize()) | |
| self._dlopenwrap_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'webkitpy', 'binary_bundling', 'dlopenwrap')) | |
| if not os.path.isdir(self._dlopenwrap_dir): | |
| raise RuntimeError("Can't find the dlopenwrap directory at: %s" % self._dlopenwrap_dir) | |
| default_bundle_name = bundle_type + '_' + self._platform + '_' + self._configuration + '.' + self._compression_type | |
| if destination and os.path.isdir(destination): | |
| self._bundle_file_path = os.path.join(destination, default_bundle_name) | |
| else: | |
| self._bundle_file_path = os.path.join(wk_build_path, default_bundle_name) | |
| def _create_tempdir(self, basedir = None): | |
| if basedir is not None: | |
| if not os.path.isdir(basedir): | |
| raise ValueError('%s is not a directory' % basedir) | |
| return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(basedir), 'tmp')) | |
| return tempfile.mkdtemp() | |
| def _run_cmd_and_get_output(self, command): | |
| _log.debug("EXEC %s" % command) | |
| command_process = subprocess.Popen(command, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, | |
| encoding='utf-8') | |
| stdout, stderr = command_process.communicate() | |
| return command_process.returncode, stdout, stderr | |
| def _get_osprettyname(self): | |
| with open('/etc/os-release', 'r') as osrelease_handle: | |
| for line in osrelease_handle.readlines(): | |
| if line.startswith('PRETTY_NAME='): | |
| return line.split('=')[1].strip().strip('"') | |
| return None | |
| def _generate_readme(self): | |
| _log.info('Generate README.txt file') | |
| readme_file = os.path.join(self._tmpdir, 'README.txt') | |
| with open(readme_file, 'w') as readme_handle: | |
| readme_handle.write('Bundle details:\n') | |
| readme_handle.write(' - WebKit Platform: %s\n' % self._platform.upper()) | |
| readme_handle.write(' - Configuration: %s\n' % self._configuration.capitalize()) | |
| if self._revision: | |
| readme_handle.write(' - WebKit Revision: %s\n' % self._revision) | |
| readme_handle.write(' - Bundle type: %s\n' % self._bundle_type) | |
| if self._buildername: | |
| readme_handle.write(' - Builder name: %s\n' % self._buildername) | |
| readme_handle.write(' - Builder date: %s\n' % datetime.datetime.now().isoformat()) | |
| readme_handle.write(' - Builder OS: %s\n' % self._get_osprettyname()) | |
| if self._syslibs == 'generate-install-script': | |
| readme_handle.write('\nRequired dependencies:\n') | |
| readme_handle.write(' - This bundle depends on several system libraries that are assumed to be installed.\n') | |
| readme_handle.write(' - To ensure all the required libraries are installed execute the script: install-dependencies.sh\n') | |
| readme_handle.write(' - You can pass the flag "--autoinstall" to this script to automatically install the dependencies if needed.\n') | |
| readme_handle.write('\nRun instructions:\n') | |
| wrapper = 'wrapper' if len(self._wrappers) == 1 else 'wrappers' | |
| readme_handle.write(' - Execute the %s in this directory:\n' % wrapper) | |
| for wrapper in self._wrappers: | |
| readme_handle.write(' * %s\n' % wrapper) | |
| readme_handle.write('\nNotes:\n') | |
| readme_handle.write(' - Do not execute any binary from the "bin" subdir as it will not run directly. Use the %s detailed above.\n' % wrapper) | |
| readme_handle.write(' - This bundle was generated with the script "Tools/Scripts/generate-bundle" available on the WebKit repository.\n') | |
| readme_handle.write(' - Please report bugs at https://bugs.webkit.org (Product: WebKit and Component: WebKitGTK or WPEWebKit).\n') | |
| return True | |
| def _generate_wrapper(self, interpreter, binary_to_wrap, skip_wrapper_if_missing_binary=False): | |
| variables = dict() | |
| mydir = self._bundler.VAR_MYDIR | |
| lib_dir = os.path.join(self._bundler.destination_dir(), 'lib') | |
| sys_lib_dir = os.path.join(self._bundler.destination_dir(), 'sys/lib') | |
| bin_dir = os.path.join(self._bundler.destination_dir(), 'bin') | |
| share_dir = os.path.join(self._bundler.destination_dir(), 'share') | |
| sys_share_dir = os.path.join(self._bundler.destination_dir(), 'sys/share') | |
| binary_to_wrap_path = os.path.join(bin_dir, binary_to_wrap) | |
| if not (os.path.isfile(binary_to_wrap_path) and os.access(binary_to_wrap_path, os.X_OK)): | |
| if skip_wrapper_if_missing_binary: | |
| return | |
| raise RuntimeError('Unable to find binary to wrap for %s' % binary_to_wrap) | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'gio')): | |
| gio_var = 'GIO_MODULE_DIR' if self._syslibs == 'bundle-all' else 'GIO_EXTRA_MODULES' | |
| variables[gio_var] = '${%s}/sys/lib/gio' % mydir | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'gst')): | |
| gst_var = 'GST_PLUGIN_SYSTEM_PATH_1_0' if self._syslibs == 'bundle-all' else 'GST_PLUGIN_PATH_1_0' | |
| variables[gst_var] = '${%s}/sys/lib/gst' % mydir | |
| variables['GST_REGISTRY_1_0'] = '${%s}/sys/lib/gst/gstreamer-1.0.registry' % mydir | |
| if os.path.isfile(os.path.join(bin_dir, 'gst-plugin-scanner')): | |
| variables['GST_PLUGIN_SCANNER'] = '${%s}/bin/gst-plugin-scanner' % mydir | |
| # pipewire plugins may be needed by gstreamer-pipewire | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'pipewire', 'spa')): | |
| variables['SPA_PLUGIN_DIR'] = '${%s}/sys/lib/pipewire/spa' % mydir | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'pipewire', 'modules')): | |
| variables['PIPEWIRE_MODULE_DIR'] = '${%s}/sys/lib/pipewire/modules' % mydir | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'dri')): | |
| variables['LIBGL_DRIVERS_PATH'] = '${%s}/sys/lib/dri' % mydir | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'glvnd/egl_vendor.d')): | |
| variables['__EGL_VENDOR_LIBRARY_DIRS'] = '${%s}/sys/lib/glvnd/egl_vendor.d' % mydir | |
| if os.path.isdir(os.path.join(sys_share_dir)): | |
| assert(self._syslibs == 'bundle-all') | |
| variables['XDG_DATA_HOME'] = '${%s}/sys/share' % mydir | |
| variables['XDG_DATA_DIRS'] = '${%s}/sys/share' % mydir | |
| if os.path.isdir(os.path.join(sys_share_dir, 'fonts')): | |
| variables['FONTCONFIG_PATH'] = '${%s}/sys/share/fonts/config' % mydir | |
| if os.path.isdir(os.path.join(sys_lib_dir, 'gtk')): | |
| variables['GTK_PATH'] = '${%s}/sys/lib/gtk' % mydir | |
| if binary_to_wrap != 'jsc': | |
| variables['WEBKIT_EXEC_PATH'] = '${%s}/bin' % mydir | |
| variables['WEBKIT_INJECTED_BUNDLE_PATH'] = '${%s}/lib' % mydir | |
| if os.path.isfile(os.path.join(share_dir, 'inspector.gresource')): | |
| variables['WEBKIT_INSPECTOR_RESOURCES_PATH'] = '${%s}/share' % mydir | |
| if self._syslibs == 'bundle-all': | |
| # with bundle-all we can't execute system bwrap, we can only execute our own binaries. | |
| variables['WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS'] = '1' | |
| variables['PATH'] = '${%s}/bin' % mydir | |
| # Use C wrapper for bundle-all to avoid executing the shell under that env. | |
| if self._syslibs == 'bundle-all': | |
| self._bundler.generate_and_build_static_cwrapper(interpreter, binary_to_wrap, variables) | |
| else: | |
| self._bundler.generate_wrapper_script(interpreter, binary_to_wrap, variables) | |
| self._wrappers.append(binary_to_wrap) | |
| def _generate_install_deps_script(self, system_packages_needed): | |
| if not system_packages_needed: | |
| return | |
| if 'MiniBrowser' in self._bundle_binaries: | |
| # Add some extra packages that are needed but the script can't automatically detect | |
| for extra_needed_pkg in ['ca-certificates', 'shared-mime-info']: | |
| system_packages_needed.add(extra_needed_pkg) | |
| # And remove some packages that may be detected due to indirect deps (gstreamer/gio) but are really not needed | |
| for not_needed_pkg in ['dconf-gsettings-backend', 'gvfs', 'pitivi', 'gstreamer1.0-convolver-pulseeffects', 'gstreamer1.0-x', | |
| 'gstreamer1.0-adapter-pulseeffects', 'gstreamer1.0-autogain-pulseeffects', 'gstreamer1.0-alsa', | |
| 'gstreamer1.0-clutter-3.0', 'gstreamer1.0-crystalizer-pulseeffects', 'gstreamer1.0-gtk3', 'gstreamer1.0-nice']: | |
| if not_needed_pkg in system_packages_needed: | |
| system_packages_needed.remove(not_needed_pkg) | |
| # Sometimes the package is identified with an arch suffix, but not always | |
| not_needed_pkg_arch = not_needed_pkg + ':amd64' | |
| if not_needed_pkg_arch in system_packages_needed: | |
| system_packages_needed.remove(not_needed_pkg_arch) | |
| installdeps_file = os.path.join(self._tmpdir, 'install-dependencies.sh') | |
| with open(installdeps_file, 'w') as installdeps_handle: | |
| installdeps_handle.write(INSTALL_DEPS_SCRIPT_TEMPLATE % {'packages_needed' : ' '.join(system_packages_needed)} ) | |
| os.chmod(installdeps_file, 0o755) | |
| def _remove_tempdir(self): | |
| if not self._tmpdir: | |
| return | |
| if os.path.isdir(self._tmpdir): | |
| shutil.rmtree(self._tmpdir) | |
| def _touch_reset_mtime(self): | |
| _log.info('Update timestamps (touch)') | |
| now = datetime.datetime.now().timestamp() | |
| for root, dirs, files in os.walk(self._tmpdir): | |
| for name in files: | |
| os.utime(os.path.join(root, name), times=(now, now)) | |
| for name in dirs: | |
| os.utime(os.path.join(root, name), times=(now, now)) | |
| def create(self): | |
| self._tmpdir = self._create_tempdir(self._buildpath) | |
| self._bundler = BinaryBundler(self._tmpdir, self._buildpath) | |
| if os.path.isfile(self._bundle_file_path): | |
| _log.info('Removing previous bundle %s' % self._bundle_file_path) | |
| os.remove(self._bundle_file_path) | |
| for bundle_binary in self._bundle_binaries: | |
| self._create_bundle(bundle_binary) | |
| self._generate_readme() | |
| # Potentially avoids errors related to ZIP not able to store mtimes older than 1980 | |
| self._touch_reset_mtime() | |
| if self._compression_type == 'zip': | |
| archiver = ZipArchiver(self._bundle_file_path, self._compression_level) | |
| elif self._compression_type == 'tar.xz': | |
| archiver = TarArchiver(self._bundle_file_path, self._compression_level) | |
| else: | |
| raise NotImplementedError('Support for compression type %s not implemented' % self._compression_type) | |
| self._create_archive(archiver) | |
| self._remove_tempdir() | |
| if not os.path.isfile(self._bundle_file_path): | |
| raise RuntimeError('Unable to create the file %s' % self._bundle_file_path) | |
| _log.log(LOG_MESSAGE, 'Bundle file created at: %s' % self._bundle_file_path) | |
| return self._bundle_file_path | |
| def _get_webkit_binaries(self): | |
| webkit_binaries = [] | |
| bin_dir = os.path.join(self._buildpath, 'bin') | |
| for entry in os.listdir(bin_dir): | |
| if entry.startswith(self._port_binary_preffix) and (entry.endswith('Process') or entry.endswith('Driver')): | |
| binary = os.path.join(bin_dir, entry) | |
| if os.path.isfile(binary) and os.access(binary, os.X_OK): | |
| webkit_binaries.append(binary) | |
| if len(webkit_binaries) < 2: | |
| raise RuntimeError('Could not find required WebKit Process binaries. Check if you are passing the right platform value.') | |
| return webkit_binaries | |
| def _get_webkit_lib(self, name): | |
| libkeyname = name.lower() | |
| lib_dir = os.path.join(self._buildpath, 'lib') | |
| bundle_lib = None | |
| for entry in os.listdir(lib_dir): | |
| if entry.endswith('.so') and libkeyname in entry.lower() and 'test' not in entry.lower(): | |
| assert(bundle_lib == None) | |
| bundle_lib = os.path.join(lib_dir, entry) | |
| break | |
| assert(bundle_lib) | |
| return bundle_lib | |
| def _create_archive(self, archiver): | |
| _log.info('Create archive') | |
| with archiver: | |
| for dirname, subdirs, files in os.walk(self._tmpdir): | |
| for filename in files: | |
| system_file_path = os.path.join(dirname, filename) | |
| zip_file_path = system_file_path.replace(self._tmpdir, '', 1).lstrip('/') | |
| archiver.add_file(system_file_path, zip_file_path) | |
| def _get_system_package_name(self, object): | |
| if not shutil.which('dpkg'): | |
| raise RuntimeError('Adding system dependencies only supported for dpkg-based distros. Try passing --syslibs=bundle-all') | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['dpkg', '-S', object]) | |
| if retcode != 0: | |
| # Give a second-try with the realpath of the object. | |
| # This fixes issue on Ubuntu-20.04 that has a /lib symlink to /usr/lib | |
| # and objects point to /lib, but dpkg only recognizes the files on /usr/lib | |
| object_realpath = os.path.realpath(object) | |
| if object_realpath != object: | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['dpkg', '-S', object_realpath]) | |
| if retcode != 0: | |
| # Package not found | |
| return None | |
| package = stdout.split(' ')[0].rstrip(':') | |
| _log.info('Add dependency on system package [%s]: %s' %(package, object)) | |
| return package | |
| def _get_pkg_config_var(self, module, var_name, assert_value=True): | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['pkg-config', '--variable=%s' % var_name, module]) | |
| if retcode != 0: | |
| raise RuntimeError('The pkg-config command returned status %d' % retcode) | |
| var_value = stdout.strip() | |
| if assert_value: | |
| if var_name.endswith('dir') and not os.path.isdir(var_value): | |
| raise RuntimeError('The pkg-config entry for %s is not a directory: %s' % (var_name, var_value)) | |
| if len(var_value) == 0: | |
| raise RuntimeError('The pkg-config entry for %s has zero lenght' % var_value) | |
| return var_value | |
| def _list_files_directory(self, directory, list_inside_subdirs=False, filter_suffix=None): | |
| found_files = [] | |
| for entry in os.listdir(directory): | |
| entry_path = os.path.join(directory, entry) | |
| if os.path.isdir(entry_path): | |
| if list_inside_subdirs: | |
| found_files.extend(self._list_files_directory(entry_path, list_inside_subdirs, filter_suffix)) | |
| elif os.path.isfile(entry_path): | |
| if filter_suffix is None or entry_path.endswith(filter_suffix): | |
| found_files.append(entry_path) | |
| if len(found_files) == 0: | |
| raise RunTimeError('Can not find any matching file at directory %s' % directory) | |
| return found_files | |
| def _get_gio_modules(self): | |
| gio_module_dir = self._get_pkg_config_var('gio-2.0', 'giomoduledir') | |
| return self._list_files_directory(gio_module_dir, filter_suffix='.so') | |
| def _get_gtk_modules(self, subdir): | |
| gtk_lib_dir = self._get_pkg_config_var('gtk4', 'libdir') | |
| gtk_binary_version = self._get_pkg_config_var('gtk4', 'gtk_binary_version') | |
| gtk_module_dir = os.path.join(gtk_lib_dir, 'gtk-4.0', gtk_binary_version, subdir) | |
| if not os.path.isdir(gtk_module_dir): | |
| return [] # gtk4 ships now the modules statically linked in the lib: https://gitlab.gnome.org/GNOME/gtk/-/merge_requests/8522 | |
| return self._list_files_directory(gtk_module_dir, filter_suffix='.so') | |
| def _get_gstreamer_modules(self): | |
| gstreamer_plugins_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginsdir') | |
| gstreamer_plugin_list = self._list_files_directory(gstreamer_plugins_dir, filter_suffix='.so') | |
| # Discard the Gstreamer plugins that are not needed and that increase the final bundle | |
| # size significantly (either because their own size or because they pull extra libraries) | |
| gst_plugin_ignore_list = [ '1394', 'a52dec', 'aasink', 'aja', 'amrnb', 'amrwbdec', 'cacasink', 'cdparanoia', 'dc1394', 'de265', 'directfb', 'dv', | |
| 'dvdlpcmdec', 'flite', 'fluidsynthmidi', 'ges', 'gtk', 'gtkwayland', 'lc3', 'ldac', 'libvisual', 'modplug', 'msdk', 'musepack', | |
| 'neonhttpsrc', 'nvcodec', 'openaptx', 'opencv', 'openmpt', 'openni2', 'python', 'qml6', 'qmlgl', 'qroverlay', 'rtspclientsink', | |
| 'sbc', 'shout2', 'spandsp', 'taglib', 'uvch264', 'voaacenc', 'voamrwbenc', 'waylandsink', 'wildmidi', 'zbar', 'zxing', | |
| 'codec2json', 'camerabin', 'ximagesrc', 'openexr', 'nle', 'ladspa', 'openal'] | |
| return [p for p in gstreamer_plugin_list if os.path.basename(p).removeprefix('libgst').removesuffix('.so') not in gst_plugin_ignore_list] | |
| def _get_gstreamer_plugin_scanner(self): | |
| def plugin_scanner_found(plugin_scanner_path): | |
| return (os.path.isfile(plugin_scanner_path) and os.access(plugin_scanner_path, os.X_OK)) | |
| gstreamer_plugin_scanner_dir = gio_module_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginscannerdir', assert_value=False) | |
| gstreamer_plugin_scanner = os.path.join(gstreamer_plugin_scanner_dir, 'gst-plugin-scanner') | |
| if not plugin_scanner_found(gstreamer_plugin_scanner): | |
| # pluginscannerdir pkg-config variable is available only on recent versions of gst | |
| # give it a second chance with some known directories on debian/ubuntu and fedora/centos | |
| lib_dir = self._get_pkg_config_var('gstreamer-1.0', 'libdir') | |
| plugins_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginsdir') | |
| for gstreamer_plugin_scanner_dir in [ os.path.join(lib_dir, 'gstreamer1.0/gstreamer-1.0'), plugins_dir, '/usr/libexec/gstreamer-1.0']: | |
| gstreamer_plugin_scanner = os.path.join(gstreamer_plugin_scanner_dir, 'gst-plugin-scanner') | |
| if plugin_scanner_found(gstreamer_plugin_scanner): | |
| break | |
| if not plugin_scanner_found(gstreamer_plugin_scanner): | |
| raise RuntimeError('Could not find gst-plugin-scanner binary at path "%s".' % gstreamer_plugin_scanner) | |
| return gstreamer_plugin_scanner | |
| def _get_libc_libraries(self): | |
| libc_library_names = [ 'libanl', 'libBrokenLocale', 'libc', 'libcrypt', 'libdl', 'libgcc_s', 'libm', | |
| 'libmvec', 'libnsl', 'libpthread', 'libresolv', 'librt', 'libthread_db', 'libutil'] | |
| libc_libraries = [] | |
| # libc does not have pkg-config support, use gcc to find the dir for the libraries | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['gcc', '-print-file-name=libc.so.6']) | |
| if retcode != 0: | |
| raise RuntimeError('The gcc command returned status %d' % retcode) | |
| lib_dir = os.path.dirname(os.path.realpath(stdout.strip())) | |
| if not os.path.isdir(lib_dir): | |
| raise RuntimeError('The value returned by gcc is not a directory: %s' % lib_dir) | |
| for entry in os.listdir(lib_dir): | |
| if any(entry.startswith(l+'.so.') for l in libc_library_names) or (entry.startswith('libnss_') and '.so.' in entry): | |
| libc_libraries.append(os.path.join(lib_dir, entry)) | |
| for libc_library_prefix in libc_library_names: | |
| if not any(os.path.basename(l).startswith(libc_library_prefix+'.so.') for l in libc_libraries): | |
| raise RuntimeError('Unable to find libc library %s' % libc_library_prefix) | |
| if not any(os.path.basename(l).startswith('libnss_') for l in libc_libraries): | |
| raise RuntimeError('Unable to find libc libnss_ libraries') | |
| return libc_libraries | |
| def _get_mesa_libraries(self): | |
| mesa_library_names = ['libglapi', 'libEGL', 'libGL', 'libGLESv2', 'libGLX', 'libdrm', 'libgbm', 'libMesaOpenCL'] | |
| mesa_libraries = [] | |
| try: | |
| lib_dir = self._get_pkg_config_var('dri', 'libdir') | |
| except RuntimeError: | |
| lib_dir = os.path.dirname(self._get_pkg_config_var('dri', 'dridriverdir')) | |
| if os.path.isdir(lib_dir): | |
| for entry in os.listdir(lib_dir): | |
| if '.so.' in entry: | |
| entry_libname = entry.split('.so.', 1)[0] | |
| # Mesa libraries are either named as "libNAME.so.1" or have a suffix like "libNAME_mesa.so.0" or "libNAME_indirect.so.0" | |
| # Hypothetical future suffixes are unknown so we match any library named "libNAME" or any library that starts with "libNAME_" | |
| if any((entry_libname == l or entry_libname.startswith(l+'_')) for l in mesa_library_names): | |
| mesa_libraries.append(os.path.join(lib_dir, entry)) | |
| _log.debug('Found the following mesa libraries: %s' % ', '.join(mesa_libraries)) | |
| for required_mesa_library_prefix in ['libGL', 'libGLESv2']: | |
| if not any(os.path.basename(l).startswith(required_mesa_library_prefix+'.so.') for l in mesa_libraries): | |
| raise RuntimeError('Unable to find mesa library with prefix %s' % required_mesa_library_prefix) | |
| return mesa_libraries | |
| def _get_mesa_dri_drivers(self): | |
| dridriver_dir = self._get_pkg_config_var('dri', 'dridriverdir') | |
| return self._list_files_directory(dridriver_dir, filter_suffix='.so') | |
| def _get_gdk_pixbuf_loaders(self): | |
| loaders_dir = self._get_pkg_config_var('gdk-pixbuf-2.0', 'gdk_pixbuf_moduledir') | |
| return self._list_files_directory(loaders_dir, filter_suffix='.so') | |
| def _get_gdk_query_loaders_binary(self): | |
| gdk_pixbuf_query_loader_value = self._get_pkg_config_var('gdk-pixbuf-2.0', 'gdk_pixbuf_query_loaders') | |
| gdk_pixbuf_libdir = self._get_pkg_config_var('gdk-pixbuf-2.0', 'libdir') | |
| # Some distros (like Debian) don't ship the binary on the declared path on pkg-config, but inside $libdir/gdk-pixbuf-2.0 | |
| for gdk_pixbuf_binary in [ gdk_pixbuf_query_loader_value, os.path.join(gdk_pixbuf_libdir, 'gdk-pixbuf-2.0/gdk-pixbuf-query-loaders') ]: | |
| if os.path.isfile(gdk_pixbuf_binary) and os.access(gdk_pixbuf_binary, os.X_OK): | |
| return gdk_pixbuf_binary | |
| raise RuntimeError('Unable to find a valid path to gdk-pixbuf-query-loaders binary') | |
| def _get_pipewire_spa_basedir_and_plugins(self): | |
| spa_plugin_dir = self._get_pkg_config_var('libspa-0.2', 'plugindir') | |
| pipewire_spa_plugins = self._list_files_directory(spa_plugin_dir, list_inside_subdirs=True, filter_suffix='.so') | |
| return spa_plugin_dir, pipewire_spa_plugins | |
| def _get_pipewire_basedir_and_modules(self): | |
| pipewire_module_dir = self._get_pkg_config_var('libpipewire-0.3', 'moduledir') | |
| pipewire_modules = self._list_files_directory(pipewire_module_dir, list_inside_subdirs=True, filter_suffix='.so') | |
| return pipewire_module_dir, pipewire_modules | |
| def _get_sparkle_cdm_plugins(self): | |
| sparkle_plugins = [] | |
| lib_dir = self._get_pkg_config_var('glib-2.0', 'libdir') | |
| sparkle_dir = os.path.join(lib_dir, 'sparkle-cdm') | |
| if os.path.isdir(sparkle_dir): | |
| sparkle_plugins = self._list_files_directory(sparkle_dir, list_inside_subdirs=True, filter_suffix='.so') | |
| return sparkle_plugins | |
| def _add_object_or_get_sysdep(self, object, object_type, binary_interpreter_relativepath=None): | |
| provided_by_system_package = None | |
| if self._syslibs == 'bundle-all': | |
| self._bundler.copy_and_maybe_strip_patchelf(object, type=object_type, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True, patchelf_setinterpreter_relativepath=binary_interpreter_relativepath) | |
| else: | |
| provided_by_system_package = self._get_system_package_name(object) | |
| if not provided_by_system_package: | |
| self._bundler.copy_and_maybe_strip_patchelf(object, type=object_type, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=False) | |
| return provided_by_system_package | |
| def _ensure_wpe_backend_symlink(self): | |
| # WPE/WPERenderer dlopens this library without a version suffix, | |
| # so we need to ensure there is a proper symlink | |
| bundle_lib_dir = os.path.join(self._tmpdir, 'lib') | |
| wpe_backend_soname = 'libWPEBackend-fdo-1.0.so' | |
| previous_dir = os.getcwd() | |
| for entry in os.listdir(bundle_lib_dir): | |
| if entry.startswith(wpe_backend_soname + '.'): | |
| os.chdir(bundle_lib_dir) | |
| if not os.path.exists(wpe_backend_soname): | |
| os.symlink(entry, wpe_backend_soname) | |
| os.chdir(previous_dir) | |
| break | |
| if not os.path.isfile(os.path.join(bundle_lib_dir, wpe_backend_soname)): | |
| raise RuntimeError('Unable to create libWPEBackend-fdo-1.0.so symlink') | |
| def _dlopenwrap_make(self, action=None): | |
| dlopenwrap_path = os.path.join(self._dlopenwrap_dir, 'dlopenwrap.so') | |
| previous_cwd = os.getcwd() | |
| os.chdir(self._dlopenwrap_dir) | |
| if action == 'build': | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['make']) | |
| if (retcode != 0) or (not os.path.isfile(dlopenwrap_path)): | |
| raise RuntimeError('Error building dlopenwrap.so: The make command returned %d status with this output: %s %s' % (retcode, stdout, stderr)) | |
| elif action == 'clean': | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['make', 'clean']) | |
| if (retcode != 0) or os.path.isfile(dlopenwrap_path): | |
| raise RuntimeError('Error cleaning dlopenwrap.so: The make command returned %d status with this output: %s %s' % (retcode, stdout, stderr)) | |
| else: | |
| raise NotImplementedError('Unknown action %s' % action) | |
| os.chdir(previous_cwd) | |
| return dlopenwrap_path | |
| def _discover_egl_mesa_config_file(self): | |
| egl_libdir = self._get_pkg_config_var('egl', 'libdir') | |
| # Not known way of getting the directory dynamically, assume it is there. | |
| icd_egl_mesa_config_candidates = ['/usr/share/glvnd/egl_vendor.d/50_mesa.json', | |
| os.path.join(egl_libdir, 'GL/glvnd/egl_vendor.d/50_mesa.json'), | |
| os.path.join(egl_libdir, 'glvnd/egl_vendor.d/50_mesa.json')] | |
| for icd_egl_mesa_config_candidate in icd_egl_mesa_config_candidates: | |
| if os.path.isfile(icd_egl_mesa_config_candidate): | |
| return icd_egl_mesa_config_candidate | |
| raise RuntimeError('Could not find mesa EGL ICD config') | |
| def _get_mesa_egl_icd_driver_path(self, icd_config): | |
| with open(icd_config, 'r') as f: | |
| json_data = json.load(f) | |
| egl_icd_library_name = json_data['ICD']['library_path'] | |
| if os.path.isabs(egl_icd_library_name): | |
| raise NotImplementedError('Path rewritting needs to be implemented to handle ICD config files with abspath') | |
| egl_libdir = self._get_pkg_config_var('egl', 'libdir') | |
| egl_icd_library_path = os.path.join(egl_libdir, egl_icd_library_name) | |
| if not os.path.isfile(egl_icd_library_path): | |
| raise RuntimeError('Unable to find the EGL ICD library %s' % egl_icd_library_path) | |
| return egl_icd_library_path | |
| def _get_total_size_dir(self, directory): | |
| size = 0 | |
| for root, dirs, files in os.walk(directory): | |
| for file in files: | |
| file_path = os.path.join(root, file) | |
| if os.path.isfile(file_path): | |
| size += os.path.getsize(file_path) | |
| return size | |
| def _generate_font_conf(self, data_directory): | |
| font_conf_path = os.path.join(data_directory, 'fonts.conf') | |
| with open(font_conf_path, 'w') as font_conf_handle: | |
| font_conf_handle.write(FONT_CONF_FILE) | |
| def _generate_tls_cafile(self, data_directory): | |
| cafile_path = os.path.join(data_directory, 'bundle-ca-certificates.pem') | |
| if shutil.which('trust'): | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['trust', 'extract', '--format=pem-bundle', cafile_path]) | |
| else: | |
| _log.warning('trust: command not found, please install p11-kit.') | |
| retcode = 127 | |
| # trust fails inside flatpak due to https://gitlab.com/freedesktop-sdk/freedesktop-sdk/-/issues/904 | |
| # fall-back to trying to manually copy the bundle | |
| if (retcode != 0) or (not os.path.isfile(cafile_path)) or (os.path.getsize(cafile_path) < 100): | |
| _log.warning('Failed to get system CA bundle with the trust command. Trying to fall-back to manual copy.') | |
| if os.path.isfile(cafile_path): | |
| os.unlink(cafile_path) | |
| for candidate_cafile in ['/etc/ssl/certs/ca-certificates.crt', '/etc/ssl/cert.pem', '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem']: | |
| if os.path.isfile(candidate_cafile): | |
| shutil.copy(candidate_cafile, cafile_path) | |
| break | |
| if (not os.path.isfile(cafile_path)) or (os.path.getsize(cafile_path) < 100): | |
| raise RuntimeError('Failed to generate or copy the system TLS CAFile with the certificates.') | |
| def _get_libs_for_object_and_and_copy_interpreter_if_needed(self, object, previous_interpreter): | |
| libraries, current_interpreter = self._shared_object_resolver.get_libs_and_interpreter(object) | |
| if current_interpreter is None: | |
| raise RuntimeError('Could not determine interpreter for binary %s' % object) | |
| if previous_interpreter is None: | |
| if self._syslibs == 'bundle-all': | |
| self._bundler.copy_and_maybe_strip_patchelf(current_interpreter, type='interpreter', strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True) | |
| elif previous_interpreter != current_interpreter: | |
| raise RuntimeError('Detected binaries with different interpreters: %s != %s' %(copied_interpreter, interpreter)) | |
| return libraries, current_interpreter | |
| def _create_bundle(self, bundle_binary): | |
| main_binary_path = os.path.join(self._buildpath, 'bin', bundle_binary) | |
| if not os.path.isfile(main_binary_path) or not os.access(main_binary_path, os.X_OK): | |
| raise ValueError('Cannot find binary for %s at %s' % (bundle_binary, main_binary_path) ) | |
| interpreter = None | |
| needs_to_create_wpe_backend_symlink = False | |
| gio_modules = [] | |
| gtk_print_modules = [] | |
| gtk_im_modules = [] | |
| gstreamer_modules = [] | |
| mesa_dri_drivers = [] | |
| pipewire_spa_plugins = [] | |
| pipewire_modules = [] | |
| gdk_pixbuf_loaders = [] | |
| egl_icd_mesa_config_file = None | |
| libraries_checked = set() | |
| system_packages_needed = set() | |
| objects_to_copy = [ main_binary_path ] | |
| # We only want to separate the libraries in lib and sys/lib for MiniBrowser and 'all' bundles | |
| self._bundler.set_use_sys_lib_directory('MiniBrowser' in self._bundle_binaries) | |
| if bundle_binary == 'MiniBrowser': | |
| # bunddle WebKit libraries | |
| objects_to_copy.extend(self._get_webkit_binaries()) | |
| objects_to_copy.append(self._get_webkit_lib('InjectedBundle')) | |
| if self._platform == 'wpe': | |
| webinspector_gresource = os.path.join(self._buildpath, 'share', 'inspector.gresource') | |
| self._bundler.copy(webinspector_gresource, subdir='share') | |
| # Bundle extra system related libraries | |
| gio_modules = self._get_gio_modules() | |
| objects_to_copy.extend(gio_modules) | |
| gstreamer_modules = self._get_gstreamer_modules() | |
| objects_to_copy.extend(gstreamer_modules) | |
| # graphics drivers | |
| mesa_dri_drivers = self._get_mesa_dri_drivers() | |
| objects_to_copy.extend(mesa_dri_drivers) | |
| objects_to_copy.extend(self._get_mesa_libraries()) | |
| # Sparkle CDM | |
| objects_to_copy.extend(self._get_sparkle_cdm_plugins()) | |
| if self._syslibs == 'bundle-all': | |
| egl_icd_mesa_config_file = self._discover_egl_mesa_config_file() | |
| objects_to_copy.append(self._get_mesa_egl_icd_driver_path(egl_icd_mesa_config_file)) | |
| # need to ship a copy of gst pluging scanner | |
| objects_to_copy.append(self._get_gstreamer_plugin_scanner()) | |
| # pipewire | |
| pipewire_spa_plugin_basedir, pipewire_spa_plugins = self._get_pipewire_spa_basedir_and_plugins() | |
| objects_to_copy.extend(pipewire_spa_plugins) | |
| pipewire_modules_basedir, pipewire_modules = self._get_pipewire_basedir_and_modules() | |
| objects_to_copy.extend(pipewire_modules) | |
| # system extra libraries | |
| objects_to_copy.extend(self._get_libc_libraries()) | |
| # gtk modules | |
| if self._platform == 'gtk': | |
| gtk_print_modules = self._get_gtk_modules('printbackends') | |
| objects_to_copy.extend(gtk_print_modules) | |
| gdk_pixbuf_loaders = self._get_gdk_pixbuf_loaders() | |
| objects_to_copy.extend(gdk_pixbuf_loaders) | |
| objects_to_copy.append(self._get_gdk_query_loaders_binary()) | |
| # our dlopenwrap lib | |
| objects_to_copy.append(self._dlopenwrap_make('build')) | |
| for object in objects_to_copy: | |
| system_package = None | |
| if object in gio_modules: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/gio') | |
| elif object in gstreamer_modules: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/gst') | |
| elif object in gtk_print_modules: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/gtk/printbackends') | |
| elif object in gtk_im_modules: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/gtk/immodules') | |
| elif object in mesa_dri_drivers: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/dri') | |
| elif object in gdk_pixbuf_loaders: | |
| system_package = self._add_object_or_get_sysdep(object, 'lib/gdk-pixbuf/loaders') | |
| elif object in pipewire_spa_plugins and self._syslibs == 'bundle-all': | |
| # This is a level-2 indirect dependency (gstreamer -> gst-pipewire -> pipewire-spa-plugin) | |
| # For the case of system_packages we rely on the distro dependencies betwen packages | |
| # But the bundle-all case we need a special case because this dep is not automatically detected (dlopen of plugins) | |
| pipewire_spa_plugin_dest_subdir = object.replace(pipewire_spa_plugin_basedir, '', 1) | |
| pipewire_spa_plugin_dest_subdir = os.path.dirname(pipewire_spa_plugin_dest_subdir.lstrip('/')) | |
| pipewire_spa_plugin_dest_subdir = os.path.join('lib/pipewire/spa', pipewire_spa_plugin_dest_subdir).rstrip('/') | |
| self._bundler.copy_and_maybe_strip_patchelf(object, type=pipewire_spa_plugin_dest_subdir, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True) | |
| elif object in pipewire_modules and self._syslibs == 'bundle-all': | |
| # Same than above | |
| # workaround broken pipewire modules found in flatpak SDK that depend on missing libs | |
| if os.path.basename(object) in ['libpipewire-module-filter-chain-sofa.so', 'libpipewire-module-rtp-session.so']: | |
| continue | |
| pipewire_modules_dest_subdir = object.replace(pipewire_modules_basedir, '', 1) | |
| pipewire_modules_dest_subdir = os.path.dirname(pipewire_modules_dest_subdir.lstrip('/')) | |
| pipewire_modules_dest_subdir = os.path.join('lib/pipewire/modules', pipewire_modules_dest_subdir).rstrip('/') | |
| self._bundler.copy_and_maybe_strip_patchelf(object, type=pipewire_modules_dest_subdir, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True) | |
| elif '.so' in object: | |
| if 'libwpebackend' in object.lower(): | |
| needs_to_create_wpe_backend_symlink = True | |
| system_package = self._add_object_or_get_sysdep(object, 'lib') | |
| else: | |
| _, interpreter = self._get_libs_for_object_and_and_copy_interpreter_if_needed(object, interpreter) | |
| system_package = self._add_object_or_get_sysdep(object, 'bin', interpreter) | |
| # There is no need to examine the libraries linked with objects coming from a system package, | |
| # because system packages already declare dependencies between them. | |
| # However, if we are running with self._syslibs == 'bundle-all' then system_package will be None, | |
| # and everything will be examined and bundled as we don't account for system packages in that case. | |
| if system_package: | |
| system_packages_needed.add(system_package) | |
| else: | |
| # get_libs_for_object gets all libraries recursively with 'ldd' for the object. | |
| libraries, interpreter = self._get_libs_for_object_and_and_copy_interpreter_if_needed(object, interpreter) | |
| for library in libraries: | |
| if library in libraries_checked: | |
| _log.debug('Skip already checked [lib]: %s' % library) | |
| continue | |
| libraries_checked.add(library) | |
| system_package = self._add_object_or_get_sysdep(library, 'lib') | |
| if system_package: | |
| system_packages_needed.add(system_package) | |
| elif 'libwpebackend' in library.lower(): | |
| needs_to_create_wpe_backend_symlink = True | |
| if needs_to_create_wpe_backend_symlink: | |
| self._ensure_wpe_backend_symlink() | |
| # Now copy data files to share dir (only needed when bunding all for MiniBrowser). | |
| # We assume that the system uses standard paths at /usr/share and /etc for this resources | |
| # Every path should be checked and if some one is not found, then it will raise an error. | |
| if bundle_binary == 'MiniBrowser' and self._syslibs == 'bundle-all': | |
| target_sys_share_dir = os.path.join(self._tmpdir, 'sys/share') | |
| # Copy glvnd EGL json files | |
| _log.info('Copy EGL config files') | |
| egl_icd_configdir_dest = os.path.join(self._tmpdir, 'sys/lib/glvnd/egl_vendor.d') | |
| if not os.path.isdir(egl_icd_configdir_dest): | |
| os.makedirs(egl_icd_configdir_dest) | |
| shutil.copy(egl_icd_mesa_config_file, egl_icd_configdir_dest) | |
| # Copy glib-2 compiled schemas | |
| _log.info('Copy Glib compiled schemas') | |
| schemas_target_dir = os.path.join(target_sys_share_dir, 'glib-2.0/schemas') | |
| os.makedirs(schemas_target_dir) | |
| compiled_schemas_file = '/usr/share/glib-2.0/schemas/gschemas.compiled' | |
| if not os.path.isfile(compiled_schemas_file): | |
| raise NotImplementedError("Can't find the gschemas.compiled file at the standard location in this system") | |
| shutil.copy(compiled_schemas_file, schemas_target_dir) | |
| _log.info('Copy shared mime info cache') | |
| # Same for shared-mime-info cache | |
| mime_target_dir = os.path.join(target_sys_share_dir, 'mime') | |
| os.makedirs(mime_target_dir) | |
| mime_cache = '/usr/share/mime/mime.cache' | |
| if not os.path.isfile(mime_cache): | |
| raise NotImplementedError("Can't find the mime.cache file from shared-mime-info at the standard location in this system") | |
| shutil.copy(mime_cache, mime_target_dir) | |
| _log.info('Generate fontconfig config') | |
| # Fontconfig fonts and config | |
| target_font_config_dir = os.path.join(target_sys_share_dir, 'fonts/config') | |
| target_font_data_dir = os.path.join(target_sys_share_dir, 'fonts/fonts') | |
| os.makedirs(target_font_config_dir) | |
| # We keep assuming standard locations for this files on the target system | |
| system_font_config_dir = '/etc/fonts/conf.d' | |
| system_font_data_dir = '/usr/share/fonts' | |
| if not (os.path.isdir(system_font_config_dir) and os.path.isdir(system_font_data_dir)): | |
| raise NotImplementedError("Can't find the fontconfig files at the standard location in this system") | |
| shutil.copytree(system_font_config_dir, os.path.join(target_font_config_dir, os.path.basename(system_font_config_dir))) | |
| self._generate_font_conf(target_font_config_dir) | |
| _log.info('Copy fonts') | |
| # We copy every font, so this may copy lot of data if you have lot of fonts installed. | |
| # So print a warning in that ase | |
| system_font_data_size = self._get_total_size_dir(system_font_data_dir) | |
| if system_font_data_size > 104857600: # 100 MB | |
| _log.warning('Copying %d MB of font data from the system at /usr/share/fonts. This should be optimized.' % int(system_font_data_size / (1024*1024))) | |
| shutil.copytree(system_font_data_dir, target_font_data_dir) | |
| # Copy XKB data | |
| _log.info('Copy XKB data') | |
| system_xkb_data = '/usr/share/X11/xkb' | |
| if not (os.path.isdir(system_xkb_data)): | |
| raise NotImplementedError("Can't find the XKB data files at the standard location in this system") | |
| target_xkb_data = os.path.join(target_sys_share_dir, 'xkb') | |
| shutil.copytree(system_xkb_data, target_xkb_data) | |
| # Flatpak configures gnutls to use the p11-kit trust store and then uses p11-kit to access the host SSL certificates. | |
| # - It starts a p11 kit server on the hosts and bind-mounting a socket that later is used from inside the sandbox via p11-kit-client.so | |
| # See: https://github.com/p11-glue/p11-kit/issues/68 | |
| # Starting a p11-kit server is problematic for the use case of the bundle, and also will only work if the host is configured to use p11-kit | |
| # So we take a different approach here. | |
| # 1. We generate a bundle-ca-certificates.pem file with all the certs of the system were the bundle was generated and we ship this file | |
| # 2. At runtime we look for the system TLS ca-bundle.pem file and we use it. If we can't find it then we default to using the one shipped inside the bundle. | |
| # We pass the path to the TLS ca-bundle.pem file using the env var WEBKIT_TLS_CAFILE_PEM | |
| # The host p11-kit setup may still work if it is using the same paths for loading the p11-kit modules, but otherwise it will fail to load the modules. | |
| # That is ok, since we don't need anymore a working p11-kit setup. If it works it will be used, but if not we have a fallback. | |
| # To debug this is useful to export P11_KIT_DEBUG=all | |
| # This approach also allows this bundle script go generate a working bundle when gnutls is not configured to use p11-kit. | |
| _log.info('Generate TLS CA certificates file') | |
| certs_target_dir = os.path.join(target_sys_share_dir, 'certs') | |
| os.makedirs(certs_target_dir) | |
| self._generate_tls_cafile(certs_target_dir) | |
| if self._platform == 'gtk': | |
| _log.info('Copy basic GTK icons.') | |
| icons_target_dir = os.path.join(target_sys_share_dir, 'icons') | |
| os.makedirs(icons_target_dir) | |
| gtk_icon_basedir = '/usr/share/icons' | |
| gtk_target_icon_dir = os.path.join(icons_target_dir, 'hicolor') | |
| gtk_icon_dirs_copied = 0 | |
| for gtk_icon_theme in ['Adwaita', 'hicolor', 'gnome']: | |
| gtk_source_icon_dir = os.path.join(gtk_icon_basedir, gtk_icon_theme) | |
| if os.path.isdir(gtk_source_icon_dir): | |
| gtk_icon_dirs_copied += 1 | |
| shutil.copytree(gtk_source_icon_dir, gtk_target_icon_dir, ignore_dangling_symlinks=True, dirs_exist_ok=True) | |
| else: | |
| _log.warning(f"Can't find the GTK icon theme at path '{gtk_source_icon_dir}' in this system. Skipping it. This may cause missing icons in MiniBrowser.") | |
| if not gtk_icon_dirs_copied: | |
| raise NotImplementedError('No GTK icon theme dirs were found in this system.') | |
| retcode, stdout, stderr = self._run_cmd_and_get_output(['gtk4-update-icon-cache', '-f', '-t', gtk_target_icon_dir]) | |
| if retcode != 0: | |
| raise RuntimeError("The command to update gtk icons cache returned non-zero status: %d.\n\tstdout: %s\n\tstderr: %s" %(retcode, stdout, stderr)) | |
| # Finally generate the wrappers. | |
| self._generate_wrapper(interpreter, bundle_binary) | |
| if bundle_binary == 'MiniBrowser': | |
| # Generate the wrapper for WebDriver only if it was built (-DENABLE_WEBDRIVER=ON) | |
| self._generate_wrapper(interpreter, self._port_binary_preffix + 'WebDriver', skip_wrapper_if_missing_binary=True) | |
| self._generate_install_deps_script(system_packages_needed) | |
| # Clean remaining stuff | |
| if bundle_binary == 'MiniBrowser' and self._syslibs == 'bundle-all': | |
| self._dlopenwrap_make('clean') | |
| def configure_logging(selected_log_level='info'): | |
| class LogHandler(logging.StreamHandler): | |
| def __init__(self, stream): | |
| super().__init__(stream) | |
| def format(self, record): | |
| if record.levelno > LOG_MESSAGE: | |
| return '%s: %s' % (record.levelname, record.getMessage()) | |
| return record.getMessage() | |
| logging.addLevelName(LOG_MESSAGE, 'MESSAGE') | |
| if selected_log_level == 'debug': | |
| log_level = logging.DEBUG | |
| elif selected_log_level == 'info': | |
| log_level = logging.INFO | |
| elif selected_log_level == 'quiet': | |
| log_level = logging.NOTSET | |
| elif selected_log_level == 'minimal': | |
| log_level = logging.getLevelName(LOG_MESSAGE) | |
| handler = LogHandler(sys.stdout) | |
| logger = logging.getLogger() | |
| logger.addHandler(handler) | |
| logger.setLevel(log_level) | |
| return handler | |
| def main(): | |
| parser = argparse.ArgumentParser('usage: %prog [options]') | |
| configuration = parser.add_mutually_exclusive_group(required=True) | |
| configuration.add_argument('--debug', action='store_const', const='debug', dest='configuration') | |
| configuration.add_argument('--release', action='store_const', const='release', dest='configuration') | |
| parser.add_argument('--platform', dest='platform', choices=['gtk', 'wpe'], required=True, | |
| help='The WebKit port to generate the bundle') | |
| parser.add_argument('--bundle', dest='bundle_binary', choices=['jsc', 'MiniBrowser', 'all'], required=True, | |
| help='Select what main binary should be included in the bundle') | |
| parser.add_argument('--syslibs', dest='syslibs', choices=['bundle-all', 'generate-install-script'], default='generate-install-script', | |
| help='If value is "bundle-all", the bundle will include _all_ the system libraries instead of a install-dependencies script.\n' | |
| 'If value is "generate-install-script", the system libraries will not be bundled and a install-dependencies script will be generated for this distribution.') | |
| parser.add_argument('--ldd', dest='ldd', default='ldd', help='Use alternative ldd (useful for non-native binaries') | |
| parser.add_argument('--compression', dest='compression_type', choices=['zip', 'tar.xz'], default='zip') | |
| parser.add_argument('--compression-level', dest='compression_level', choices=[str(l) for l in range(1, 10)], default=6) | |
| parser.add_argument('--destination', action='store', dest='destination', | |
| help='Optional path were to store the bundle') | |
| parser.add_argument('--no-strip', action='store_true', dest='no_strip', | |
| help='Do not strip the binaries and libraries inside the bundle') | |
| parser.add_argument('--log-level', dest='log_level', choices=['quiet', 'minimal', 'info', 'debug'], default='info') | |
| parser.add_argument('--revision', action='store', dest='webkit_version') | |
| parser.add_argument('--builder-name', action='store', dest='builder_name') | |
| options = parser.parse_args() | |
| jhbuildutils.enter_jhbuild_environment_if_available(options.platform) | |
| configure_logging(options.log_level) | |
| bundle_creator = BundleCreator(options.configuration, options.platform, options.bundle_binary, options.syslibs, options.ldd, | |
| not options.no_strip, options.compression_type, options.compression_level, options.destination, options.webkit_version, options.builder_name) | |
| bundle_file_path = bundle_creator.create() | |
| return 0 | |
| if __name__ == '__main__': | |
| sys.exit(main()) |