WebKitGTK: rework how the minibrowser is downloaded on the CI (#25263)
* Until now, when running the WebKitGTK tests on the taskcluster CI
a pre-running hook was triggerred that installed some custom Debian packages
for the stable channel and unpacked a tarball for the nightly channel.
The tarball from the nightly channel required to install a lot of extra
(uneeeded) dependencies (it was not optimized) slowing down the CI tests.
* We recently re-worked how the pre-built products are generated, now we
provide built products for stable, beta and nightly using the same bundle
type: a zip file that contains just webkitgtk with the minimum libraries
needed for it and is built for the target operating system where it will
run. Currently we support Ubuntu LTS and LTS-1. TC CI runs in Ubuntu LTS.
* This patch add supports for the command "wpt --install-browser" for product
webkitgtk_minibrowser and uses this command to install the browser on the CI.
The built product downloaded not longer requires using a specific path and
is unpacked in the local directory.
* This also means end-users can also install webkitgtk via this way and use it.
But currently it will only work for them if they run Ubuntu LTS.
We may add support for more distros in the future.
* Finally this commit also schedules weekly webkitgtk_minibrowser runs for the
beta channel on the CI.
diff --git a/.taskcluster.yml b/.taskcluster.yml
index b1a8162..c5e3a68 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -7,7 +7,7 @@
run_task:
$if: 'tasks_for == "github-push"'
then:
- $if: 'event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/chrome_nightly", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly", "refs/heads/triggers/webkitgtk_minibrowser_stable", "refs/heads/triggers/webkitgtk_minibrowser_nightly", "refs/heads/triggers/servo_nightly"]'
+ $if: 'event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/chrome_nightly", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly", "refs/heads/triggers/webkitgtk_minibrowser_stable", "refs/heads/triggers/webkitgtk_minibrowser_beta", "refs/heads/triggers/webkitgtk_minibrowser_nightly", "refs/heads/triggers/servo_nightly"]'
then: true
else: false
else:
diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py
index 3a3b0d8..69bf72f 100755
--- a/tools/ci/run_tc.py
+++ b/tools/ci/run_tc.py
@@ -43,16 +43,10 @@
import sys
import tarfile
import tempfile
-import time
import zipfile
-from socket import error as SocketError # NOQA: N812
-import errno
-try:
- from urllib2 import urlopen
-except ImportError:
- # Python 3 case
- from urllib.request import urlopen
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from wpt.utils import get_download_to_descriptor
root = os.path.abspath(
os.path.join(os.path.dirname(__file__),
@@ -153,79 +147,12 @@
dest = os.path.join("/tmp", deb_archive)
deb_url = "https://dl.google.com/linux/direct/%s" % deb_archive
with open(dest, "w") as f:
- download_url_to_descriptor(f, deb_url)
+ get_download_to_descriptor(f, deb_url)
run(["sudo", "apt-get", "-qqy", "update"])
run(["sudo", "gdebi", "-qn", "/tmp/%s" % deb_archive])
-def install_webkitgtk_from_apt_repository(channel):
- # Configure webkitgtk.org/debian repository for $channel and pin it with maximum priority
- run(["sudo", "apt-key", "adv", "--fetch-keys", "https://webkitgtk.org/debian/apt.key"])
- with open("/tmp/webkitgtk.list", "w") as f:
- f.write("deb [arch=amd64] https://webkitgtk.org/apt bionic-wpt-webkit-updates %s\n" % channel)
- run(["sudo", "mv", "/tmp/webkitgtk.list", "/etc/apt/sources.list.d/"])
- with open("/tmp/99webkitgtk", "w") as f:
- f.write("Package: *\nPin: origin webkitgtk.org\nPin-Priority: 1999\n")
- run(["sudo", "mv", "/tmp/99webkitgtk", "/etc/apt/preferences.d/"])
- # Install webkit2gtk from the webkitgtk.org/apt repository for $channel
- run(["sudo", "apt-get", "-qqy", "update"])
- run(["sudo", "apt-get", "-qqy", "upgrade"])
- run(["sudo", "apt-get", "-qqy", "-t", "bionic-wpt-webkit-updates", "install", "webkit2gtk-driver"])
-
-
-def download_url_to_descriptor(fd, url, max_retries=5):
- """Download an URL in chunks and saves it to a file descriptor (truncating it)
- It doesn't close the descriptor, but flushes it on success.
- It retries the download in case of ECONNRESET up to max_retries."""
- if max_retries < 1:
- max_retries = 1
- wait = 2
- for current_retry in range(1, max_retries+1):
- try:
- print("INFO: Downloading %s Try %d/%d" % (url, current_retry, max_retries))
- resp = urlopen(url)
- # We may come here in a retry, ensure to truncate fd before start writing.
- fd.seek(0)
- fd.truncate(0)
- while True:
- chunk = resp.read(16*1024)
- if not chunk:
- break # Download finished
- fd.write(chunk)
- fd.flush()
- # Success
- return
- except SocketError as e:
- if current_retry < max_retries and e.errno == errno.ECONNRESET:
- # Retry
- print("ERROR: Connection reset by peer. Retrying after %ds..." % wait)
- time.sleep(wait)
- wait *= 2
- else:
- # Maximum retries or unknown error
- raise
-
-
-def install_webkitgtk_from_tarball_bundle(channel):
- with tempfile.NamedTemporaryFile(suffix=".tar.xz") as temp_tarball:
- download_url = "https://webkitgtk.org/built-products/nightly/webkitgtk-nightly-build-last.tar.xz"
- download_url_to_descriptor(temp_tarball, download_url)
- run(["sudo", "tar", "xfa", temp_tarball.name, "-C", "/"])
- # Install dependencies
- run(["sudo", "apt-get", "-qqy", "update"])
- run(["sudo", "/opt/webkitgtk/nightly/install-dependencies"])
-
-
-def install_webkitgtk(channel):
- if channel in ("experimental", "dev", "nightly"):
- install_webkitgtk_from_tarball_bundle(channel)
- elif channel in ("beta", "stable"):
- install_webkitgtk_from_apt_repository(channel)
- else:
- raise ValueError("Unrecognized release channel: %s" % channel)
-
-
def start_xvfb():
start(["sudo", "Xvfb", os.environ["DISPLAY"], "-screen", "0",
"%sx%sx%s" % (os.environ["SCREEN_WIDTH"],
@@ -270,7 +197,7 @@
base_url = task_url(artifact["task"])
if artifact["task"] not in artifact_list_by_task:
with tempfile.TemporaryFile() as f:
- download_url_to_descriptor(f, base_url + "/artifacts")
+ get_download_to_descriptor(f, base_url + "/artifacts")
f.seek(0)
artifacts_data = json.load(f)
artifact_list_by_task[artifact["task"]] = artifacts_data
@@ -290,7 +217,7 @@
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open(dest_path, "wb") as f:
- download_url_to_descriptor(f, url)
+ get_download_to_descriptor(f, url)
if artifact.get("extract"):
unpack(dest_path)
@@ -327,9 +254,6 @@
# later in taskcluster-run.py.
if args.channel != "nightly":
install_chrome(args.channel)
- elif "webkitgtk_minibrowser" in args.browser:
- assert args.channel is not None
- install_webkitgtk(args.channel)
if args.xvfb:
start_xvfb()
@@ -436,7 +360,7 @@
return None
with tempfile.TemporaryFile() as f:
- download_url_to_descriptor(f, task_url(task_id))
+ get_download_to_descriptor(f, task_url(task_id))
f.seek(0)
task_data = json.load(f)
event_data = task_data.get("extra", {}).get("github_event")
diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index 245ee7f..a4bd72a 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -20,6 +20,8 @@
return ["--install-browser", "--processes=12"]
if product == "chrome" and channel == "nightly":
return ["--install-browser", "--install-webdriver"]
+ if product == "webkitgtk_minibrowser":
+ return ["--install-browser"]
return []
diff --git a/tools/ci/tc/tasks/test.yml b/tools/ci/tc/tasks/test.yml
index 1b5c7ac..78891bf 100644
--- a/tools/ci/tc/tasks/test.yml
+++ b/tools/ci/tc/tasks/test.yml
@@ -207,6 +207,12 @@
- trigger-weekly
- trigger-push
- vars:
+ browser: webkitgtk_minibrowser
+ channel: beta
+ use:
+ - trigger-weekly
+ - trigger-push
+ - vars:
browser: servo
channel: nightly
use:
diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py
index 35f60d1..44e19fe 100644
--- a/tools/wpt/browser.py
+++ b/tools/wpt/browser.py
@@ -12,7 +12,7 @@
from six.moves.urllib.parse import urlsplit
import requests
-from .utils import call, get, rmtree, untar, unzip
+from .utils import call, get, rmtree, untar, unzip, get_download_to_descriptor, sha256sum
uname = platform.uname()
@@ -110,7 +110,7 @@
return NotImplemented
@abstractmethod
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
"""Find the binary of the WebDriver."""
return NotImplemented
@@ -297,7 +297,7 @@
return None
return path
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("geckodriver")
def get_version_and_channel(self, binary):
@@ -506,7 +506,7 @@
def find_binary(self, venv_path=None, channel=None):
return self.apk_path
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
raise NotImplementedError
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -656,7 +656,7 @@
self.logger.warning("Unable to find the browser binary.")
return None
- def find_webdriver(self, channel=None, browser_binary=None):
+ def find_webdriver(self, venv_path=None, channel=None, browser_binary=None):
return find_executable("chromedriver")
def webdriver_supports_browser(self, webdriver_binary, browser_binary):
@@ -824,7 +824,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("chromedriver")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -931,7 +931,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
raise NotImplementedError
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -986,7 +986,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("operadriver")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1063,7 +1063,7 @@
return find_executable("Microsoft Edge Canary", os.pathsep.join(macpaths))
return binary
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("msedgedriver")
def webdriver_supports_browser(self, webdriver_binary, browser_binary):
@@ -1178,7 +1178,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("MicrosoftWebDriver")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1212,7 +1212,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("IEDriverServer.exe")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1240,7 +1240,7 @@
def find_binary(self, venv_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
path = None
if channel == "preview":
path = "/Applications/Safari Technology Preview.app/Contents/MacOS"
@@ -1332,7 +1332,7 @@
path = find_executable("servo")
return path
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return None
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1365,7 +1365,7 @@
def find_binary(self, venev_path=None, channel=None):
raise NotImplementedError
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
raise NotImplementedError
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1390,7 +1390,7 @@
def find_binary(self, venv_path=None, channel=None):
return None
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return None
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
@@ -1402,7 +1402,101 @@
class WebKitGTKMiniBrowser(WebKit):
+
+ def _get_osidversion(self):
+ with open('/etc/os-release', 'r') as osrelease_handle:
+ for line in osrelease_handle.readlines():
+ if line.startswith('ID='):
+ os_id = line.split('=')[1].strip().strip('"')
+ if line.startswith('VERSION_ID='):
+ version_id = line.split('=')[1].strip().strip('"')
+ assert(os_id)
+ assert(version_id)
+ osidversion = os_id + '-' + version_id
+ assert(' ' not in osidversion)
+ assert(len(osidversion) > 3)
+ return osidversion.capitalize()
+
+
+ def download(self, dest=None, channel=None, rename=None):
+ base_dowload_uri = "https://webkitgtk.org/built-products/"
+ base_download_dir = base_dowload_uri + "x86_64/release/" + channel + "/" + self._get_osidversion() + "/MiniBrowser/"
+ try:
+ response = get(base_download_dir + "LAST-IS")
+ except requests.exceptions.HTTPError as e:
+ if e.response.status_code == 404:
+ raise RuntimeError("Can't find a WebKitGTK MiniBrowser %s bundle for %s at %s"
+ % (channel, self._get_osidversion(), base_dowload_uri))
+ raise
+
+ bundle_filename = response.text.strip()
+ bundle_url = base_download_dir + bundle_filename
+
+ if dest is None:
+ dest = self._get_dest(None, channel)
+ bundle_file_path = os.path.join(dest, bundle_filename)
+
+ self.logger.info("Downloading WebKitGTK MiniBrowser bundle from %s" % bundle_url)
+ with open(bundle_file_path, "w+b") as f:
+ get_download_to_descriptor(f, bundle_url)
+
+ bundle_filename_no_ext, _ = os.path.splitext(bundle_filename)
+ bundle_hash_url = base_download_dir + bundle_filename_no_ext + ".sha256sum"
+ bundle_expected_hash = get(bundle_hash_url).text.strip().split(" ")[0]
+ bundle_computed_hash = sha256sum(bundle_file_path)
+
+ if bundle_expected_hash != bundle_computed_hash:
+ self.logger.error("Calculated SHA256 hash is %s but was expecting %s" % (bundle_computed_hash,bundle_expected_hash))
+ raise RuntimeError("The WebKitGTK MiniBrowser bundle at %s has incorrect SHA256 hash." % bundle_file_path)
+ return bundle_file_path
+
+ def install(self, dest=None, channel=None, prompt=True):
+ dest = self._get_dest(dest, channel)
+ bundle_path = self.download(dest, channel)
+ bundle_uncompress_directory = os.path.join(dest, "webkitgtk_minibrowser")
+
+ # Clean it from previous runs
+ if os.path.exists(bundle_uncompress_directory):
+ rmtree(bundle_uncompress_directory)
+ os.mkdir(bundle_uncompress_directory)
+
+ with open(bundle_path, "rb") as f:
+ unzip(f, bundle_uncompress_directory)
+
+ install_dep_script = os.path.join(bundle_uncompress_directory, "install-dependencies.sh")
+ if os.path.isfile(install_dep_script):
+ self.logger.info("Executing install-dependencies.sh script from bundle.")
+ install_dep_cmd = [install_dep_script]
+ if not prompt:
+ install_dep_cmd.append("--autoinstall")
+ # use subprocess.check_call() directly to display unbuffered stdout/stderr in real-time.
+ subprocess.check_call(install_dep_cmd)
+
+ minibrowser_path = os.path.join(bundle_uncompress_directory, "MiniBrowser")
+ if not os.path.isfile(minibrowser_path):
+ raise RuntimeError("Can't find a MiniBrowser binary at %s" % minibrowser_path)
+
+ os.remove(bundle_path)
+ install_ok_file = os.path.join(bundle_uncompress_directory, ".installation-ok")
+ open(install_ok_file, "w").close() # touch
+ self.logger.info("WebKitGTK MiniBrowser bundle for channel %s installed." % channel)
+ return minibrowser_path
+
+ def _find_executable_in_channel_bundle(self, binary, venv_path=None, channel=None):
+ if venv_path:
+ venv_base_path = self._get_dest(venv_path, channel)
+ bundle_dir = os.path.join(venv_base_path, "webkitgtk_minibrowser")
+ install_ok_file = os.path.join(bundle_dir, ".installation-ok")
+ if os.path.isfile(install_ok_file):
+ return find_executable(binary, bundle_dir)
+ return None
+
+
def find_binary(self, venv_path=None, channel=None):
+ minibrowser_path = self._find_executable_in_channel_bundle("MiniBrowser", venv_path, channel)
+ if minibrowser_path:
+ return minibrowser_path
+
libexecpaths = ["/usr/libexec/webkit2gtk-4.0"] # Fedora path
triplet = "x86_64-linux-gnu"
# Try to use GCC to detect this machine triplet
@@ -1414,15 +1508,13 @@
pass
# Add Debian/Ubuntu path
libexecpaths.append("/usr/lib/%s/webkit2gtk-4.0" % triplet)
- if channel == "nightly":
- libexecpaths.append("/opt/webkitgtk/nightly")
return find_executable("MiniBrowser", os.pathsep.join(libexecpaths))
- def find_webdriver(self, channel=None):
- path = os.environ['PATH']
- if channel == "nightly":
- path = "%s:%s" % (path, "/opt/webkitgtk/nightly")
- return find_executable("WebKitWebDriver", path)
+ def find_webdriver(self, venv_path=None, channel=None):
+ webdriver_path = self._find_executable_in_channel_bundle("WebKitWebDriver", venv_path, channel)
+ if not webdriver_path:
+ webdriver_path = find_executable("WebKitWebDriver")
+ return webdriver_path
def version(self, binary=None, webdriver_binary=None):
if binary is None:
@@ -1456,7 +1548,7 @@
def find_binary(self, venv_path=None, channel=None):
return find_executable("epiphany")
- def find_webdriver(self, channel=None):
+ def find_webdriver(self, venv_path=None, channel=None):
return find_executable("WebKitWebDriver")
def install_webdriver(self, dest=None, channel=None, browser_binary=None):
diff --git a/tools/wpt/install.py b/tools/wpt/install.py
index 8177022..638065a 100644
--- a/tools/wpt/install.py
+++ b/tools/wpt/install.py
@@ -7,7 +7,8 @@
'chrome_android': 'dev',
'edgechromium': 'dev',
'safari': 'preview',
- 'servo': 'nightly'
+ 'servo': 'nightly',
+ 'webkitgtk_minibrowser': 'nightly'
}
channel_by_name = {
diff --git a/tools/wpt/run.py b/tools/wpt/run.py
index f289e09..89f0336 100644
--- a/tools/wpt/run.py
+++ b/tools/wpt/run.py
@@ -662,18 +662,19 @@
browser_cls = browser.WebKitGTKMiniBrowser
def install(self, channel=None):
- raise NotImplementedError
+ if self.prompt_install(self.name):
+ return self.browser.install(self.venv.path, channel, self.prompt)
def setup_kwargs(self, kwargs):
if kwargs["binary"] is None:
- binary = self.browser.find_binary(channel=kwargs["browser_channel"])
+ binary = self.browser.find_binary(venv_path=self.venv.path, channel=kwargs["browser_channel"])
if binary is None:
raise WptrunError("Unable to find MiniBrowser binary")
kwargs["binary"] = binary
if kwargs["webdriver_binary"] is None:
- webdriver_binary = self.browser.find_webdriver(channel=kwargs["browser_channel"])
+ webdriver_binary = self.browser.find_webdriver(venv_path=self.venv.path, channel=kwargs["browser_channel"])
if webdriver_binary is None:
raise WptrunError("Unable to find WebKitWebDriver in PATH")
diff --git a/tools/wpt/utils.py b/tools/wpt/utils.py
index 6556ff4..61dcda5 100644
--- a/tools/wpt/utils.py
+++ b/tools/wpt/utils.py
@@ -5,8 +5,11 @@
import stat
import subprocess
import tarfile
+import time
import zipfile
from io import BytesIO
+from socket import error as SocketError # NOQA: N812
+from six.moves.urllib.request import urlopen
MYPY = False
if MYPY:
@@ -100,6 +103,41 @@
return resp
+def get_download_to_descriptor(fd, url, max_retries=5):
+ """Download an URL in chunks and saves it to a file descriptor (truncating it)
+ It doesn't close the descriptor, but flushes it on success.
+ It retries the download in case of ECONNRESET up to max_retries.
+ This function is meant to download big files directly to the disk without
+ caching the whole file in memory.
+ """
+ if max_retries < 1:
+ max_retries = 1
+ wait = 2
+ for current_retry in range(1, max_retries+1):
+ try:
+ logger.info("Downloading %s Try %d/%d" % (url, current_retry, max_retries))
+ resp = urlopen(url)
+ # We may come here in a retry, ensure to truncate fd before start writing.
+ fd.seek(0)
+ fd.truncate(0)
+ while True:
+ chunk = resp.read(16*1024)
+ if not chunk:
+ break # Download finished
+ fd.write(chunk)
+ fd.flush()
+ # Success
+ return
+ except SocketError as e:
+ if current_retry < max_retries and e.errno == errno.ECONNRESET:
+ # Retry
+ logger.error("Connection reset by peer. Retrying after %ds..." % wait)
+ time.sleep(wait)
+ wait *= 2
+ else:
+ # Maximum retries or unknown error
+ raise
+
def rmtree(path):
# This works around two issues:
# 1. Cannot delete read-only files owned by us (e.g. files extracted from tarballs)
@@ -114,3 +152,13 @@
raise
return shutil.rmtree(path, onerror=handle_remove_readonly)
+
+
+def sha256sum(file_path):
+ """Computes the SHA256 hash sum of a file"""
+ from hashlib import sha256
+ hash = sha256()
+ with open(file_path, 'rb') as f:
+ for chunk in iter(lambda: f.read(4096), b''):
+ hash.update(chunk)
+ return hash.hexdigest()
