[wptrunner] Speed up `wptserve` startup (#41897)
`serve` constructs a list of routes and their handlers and passes them
to each server, which runs in a subprocess. However, when `start()`ing
each subprocess [0], `multiprocessing` implicitly pickles and writes all
handlers to an underlying OS pipe for transport [1]. This can block each
`start()` for a long time (~0.5s) because `StringHandler`s hold file
contents that, collectively, can easily exceed the pipe's initial size
[2].
The worst offenders are:
* /_pdf_js/pdf.worker.js: 1.7 MB
* /_pdf_js/pdf.js: 0.7 MB
* /resources/testdriver.js: 53 kB
Change the static handlers to lazily read their files so that only the
filename needs to be transported over the pipe. Unfortunately,
`/resources/testdriver.js` needs a bespoke handler because it's composed
of two files concatenated together that are not templates. Empirically,
this reduces the time until the tests start running from ~10s to ~5s.
See also: https://crbug.com/1479850
[0]: https://github.com/web-platform-tests/wpt/blob/a4d75218/tools/serve/serve.py#L629
[1]: https://github.com/python/cpython/blob/3.11/Lib/multiprocessing/popen_spawn_posix.py#L62
[2]: https://man7.org/linux/man-pages/man7/pipe.7.html says 64 KiB for Linux.
diff --git a/tools/wptrunner/wptrunner/environment.py b/tools/wptrunner/wptrunner/environment.py
index 9e217ed..24c88aa 100644
--- a/tools/wptrunner/wptrunner/environment.py
+++ b/tools/wptrunner/wptrunner/environment.py
@@ -247,13 +247,7 @@
route_builder.add_static(path, format_args, content_type, route,
headers=headers)
- data = b""
- with open(os.path.join(repo_root, "resources", "testdriver.js"), "rb") as fp:
- data += fp.read()
- with open(os.path.join(here, "testdriver-extra.js"), "rb") as fp:
- data += fp.read()
- route_builder.add_handler("GET", "/resources/testdriver.js",
- StringHandler(data, "text/javascript"))
+ route_builder.add_handler("GET", "/resources/testdriver.js", TestdriverLoader())
for url_base, paths in self.test_paths.items():
if url_base == "/":
@@ -311,6 +305,27 @@
return failed, pending
+class TestdriverLoader:
+ """A special static handler for serving `/resources/testdriver.js`.
+
+ This handler lazily reads `testdriver{,-extra}.js` so that wptrunner doesn't
+ need to pass the entire file contents to child `wptserve` processes, which
+ can slow `wptserve` startup by several seconds (crbug.com/1479850).
+ """
+ def __init__(self):
+ self._handler = None
+
+ def __call__(self, request, response):
+ if not self._handler:
+ data = b""
+ with open(os.path.join(repo_root, "resources", "testdriver.js"), "rb") as fp:
+ data += fp.read()
+ with open(os.path.join(here, "testdriver-extra.js"), "rb") as fp:
+ data += fp.read()
+ self._handler = StringHandler(data, "text/javascript")
+ return self._handler(request, response)
+
+
def wait_for_service(logger, host, port, timeout=60, server_process=None):
"""Waits until network service given as a tuple of (host, port) becomes
available, `timeout` duration is reached, or the `server_process` exits at
diff --git a/tools/wptserve/wptserve/handlers.py b/tools/wptserve/wptserve/handlers.py
index 00ba067..6d79230 100644
--- a/tools/wptserve/wptserve/handlers.py
+++ b/tools/wptserve/wptserve/handlers.py
@@ -491,7 +491,7 @@
return rv
-class StaticHandler(StringHandler):
+class StaticHandler:
def __init__(self, path, format_args, content_type, **headers):
"""Handler that reads a file from a path and substitutes some fixed data
@@ -501,10 +501,26 @@
:param format_args: Dictionary of values to substitute into the template file
:param content_type: Content type header to server the response with
:param headers: List of headers to send with responses"""
+ self._path = path
+ self._format_args = format_args
+ self._content_type = content_type
+ self._headers = headers
+ self._handler = None
- with open(path) as f:
- data = f.read()
- if format_args:
- data = data % format_args
+ def __getnewargs_ex__(self):
+ # Do not pickle `self._handler`, which can be arbitrarily large.
+ args = self._path, self._format_args, self._content_type
+ return args, self._headers
- return super().__init__(data, content_type, **headers)
+ def __call__(self, request, response):
+ # Load the static file contents lazily so that this handler can be
+ # pickled and sent to child processes efficiently. Transporting file
+ # contents across processes can slow `wptserve` startup by several
+ # seconds (crbug.com/1479850).
+ if not self._handler:
+ with open(self._path) as f:
+ data = f.read()
+ if self._format_args:
+ data = data % self._format_args
+ self._handler = StringHandler(data, self._content_type, **self._headers)
+ return self._handler(request, response)
