Remove the 'http_lock' and 'file_lock' code from webkitpy.
We used to allow running run-webkit-tests concurrently and they would
coordinate amongst themselves when running the http tests using file
locks. This was used by the Qt port on their buildbots, but it's not
clear that this was needed anywhere else.
This patch removes all of that code, and deletes the generic file lock
module we had as it was now only being used when parsing ASAN logs,
which is only needed on mac and linux (where we can just use flock).
This patch also moves starting and stopping the servers from the
layout_test_runner to the manager, where we can wait to stop the servers
until after retrying any tests. This keeps us from having to start the
servers twice if we need to retry an HTTP test. This will speed up
HTTP retries a tiny bit, but slow down the overall run-webkit-tests
execution a bit in the normal case (no HTTP retries). This tradeoff
seems worth it in order to simplify the logic overall.
R=abarth@chromium.org, ojan@chromium.org
BUG=252035
Review URL: https://chromiumcodereview.appspot.com/17320009
git-svn-id: svn://svn.chromium.org/blink/trunk@152975 bbb929c8-8fbe-4397-9dbb-9b2b20218538
diff --git a/Tools/Scripts/webkitpy/common/system/file_lock.py b/Tools/Scripts/webkitpy/common/system/file_lock.py
deleted file mode 100644
index 3ca8b3c..0000000
--- a/Tools/Scripts/webkitpy/common/system/file_lock.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""This class helps to lock files exclusively across processes."""
-
-import logging
-import os
-import sys
-import time
-
-
-_log = logging.getLogger(__name__)
-
-
-class FileLock(object):
-
- def __init__(self, lock_file_path, max_wait_time_sec=20):
- self._lock_file_path = lock_file_path
- self._lock_file_descriptor = None
- self._max_wait_time_sec = max_wait_time_sec
-
- def _create_lock(self):
- if sys.platform == 'win32':
- import msvcrt
- msvcrt.locking(self._lock_file_descriptor, msvcrt.LK_NBLCK, 32)
- else:
- import fcntl
- fcntl.flock(self._lock_file_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
-
- def _remove_lock(self):
- if sys.platform == 'win32':
- import msvcrt
- msvcrt.locking(self._lock_file_descriptor, msvcrt.LK_UNLCK, 32)
- else:
- import fcntl
- fcntl.flock(self._lock_file_descriptor, fcntl.LOCK_UN)
-
- def acquire_lock(self):
- self._lock_file_descriptor = os.open(self._lock_file_path, os.O_TRUNC | os.O_CREAT)
- start_time = time.time()
- while True:
- try:
- self._create_lock()
- return True
- except IOError:
- if time.time() - start_time > self._max_wait_time_sec:
- _log.debug("File locking failed: %s" % str(sys.exc_info()))
- os.close(self._lock_file_descriptor)
- self._lock_file_descriptor = None
- return False
- # There's no compelling reason to spin hard here, so sleep for a bit.
- time.sleep(0.01)
-
- def release_lock(self):
- try:
- if self._lock_file_descriptor:
- self._remove_lock()
- os.close(self._lock_file_descriptor)
- self._lock_file_descriptor = None
- os.unlink(self._lock_file_path)
- except (IOError, OSError):
- _log.debug("Warning in release lock: %s" % str(sys.exc_info()))
diff --git a/Tools/Scripts/webkitpy/common/system/file_lock_integrationtest.py b/Tools/Scripts/webkitpy/common/system/file_lock_integrationtest.py
deleted file mode 100644
index 7b1b426..0000000
--- a/Tools/Scripts/webkitpy/common/system/file_lock_integrationtest.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import os
-import tempfile
-import unittest2 as unittest
-
-from webkitpy.common.system.file_lock import FileLock
-
-
-class FileLockTest(unittest.TestCase):
-
- def setUp(self):
- self._lock_name = "TestWebKit" + str(os.getpid()) + ".lock"
- self._lock_path = os.path.join(tempfile.gettempdir(), self._lock_name)
- self._file_lock1 = FileLock(self._lock_path, 0.1)
- self._file_lock2 = FileLock(self._lock_path, 0.1)
-
- def tearDown(self):
- self._file_lock1.release_lock()
- self._file_lock2.release_lock()
-
- def test_lock_lifecycle(self):
- # Create the lock.
- self._file_lock1.acquire_lock()
- self.assertTrue(os.path.exists(self._lock_path))
-
- # Try to lock again.
- self.assertFalse(self._file_lock2.acquire_lock())
-
- # Release the lock.
- self._file_lock1.release_lock()
- self.assertFalse(os.path.exists(self._lock_path))
-
- def test_stuck_lock(self):
- open(self._lock_path, 'w').close()
- self._file_lock1.acquire_lock()
- self._file_lock1.release_lock()
diff --git a/Tools/Scripts/webkitpy/common/system/file_lock_mock.py b/Tools/Scripts/webkitpy/common/system/file_lock_mock.py
deleted file mode 100644
index f53081d..0000000
--- a/Tools/Scripts/webkitpy/common/system/file_lock_mock.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Copyright (c) 2012 Google Inc. All rights reserved.
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-class MockFileLock(object):
- def __init__(self, lock_file_path, max_wait_time_sec=20):
- pass
-
- def acquire_lock(self):
- return True
-
- def release_lock(self):
- return True
diff --git a/Tools/Scripts/webkitpy/common/system/systemhost.py b/Tools/Scripts/webkitpy/common/system/systemhost.py
index dfec68b..3b4439ee 100644
--- a/Tools/Scripts/webkitpy/common/system/systemhost.py
+++ b/Tools/Scripts/webkitpy/common/system/systemhost.py
@@ -30,7 +30,7 @@
import platform
import sys
-from webkitpy.common.system import environment, executive, file_lock, filesystem, platforminfo, user, workspace
+from webkitpy.common.system import environment, executive, filesystem, platforminfo, user, workspace
class SystemHost(object):
@@ -43,6 +43,3 @@
def copy_current_environment(self):
return environment.Environment(os.environ.copy())
-
- def make_file_lock(self, path):
- return file_lock.FileLock(path)
diff --git a/Tools/Scripts/webkitpy/common/system/systemhost_mock.py b/Tools/Scripts/webkitpy/common/system/systemhost_mock.py
index a529f34..4667b08 100644
--- a/Tools/Scripts/webkitpy/common/system/systemhost_mock.py
+++ b/Tools/Scripts/webkitpy/common/system/systemhost_mock.py
@@ -29,7 +29,6 @@
from webkitpy.common.system.environment import Environment
from webkitpy.common.system.executive_mock import MockExecutive
from webkitpy.common.system.filesystem_mock import MockFileSystem
-from webkitpy.common.system.file_lock_mock import MockFileLock
from webkitpy.common.system.platforminfo_mock import MockPlatformInfo
from webkitpy.common.system.user_mock import MockUser
from webkitpy.common.system.workspace_mock import MockWorkspace
@@ -51,6 +50,3 @@
def copy_current_environment(self):
return Environment({"MOCK_ENVIRON_COPY": '1'})
-
- def make_file_lock(self, path):
- return MockFileLock(path)
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
index ef51c49..515de4d 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
@@ -72,26 +72,18 @@
self._expectations = None
self._test_inputs = []
- self._needs_http = None
- self._needs_websockets = None
self._retrying = False
self._current_run_results = None
- self._remaining_locked_shards = []
- self._has_http_lock = False
- def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying):
+ def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, retrying):
self._expectations = expectations
self._test_inputs = test_inputs
- self._needs_http = needs_http
- self._needs_websockets = needs_websockets
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
- self._remaining_locked_shards = []
- self._has_http_lock = False
self._printer.num_tests = len(test_inputs)
self._printer.num_completed = 0
@@ -106,19 +98,10 @@
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel)
- # FIXME: We don't have a good way to coordinate the workers so that
- # they don't try to run the shards that need a lock if we don't actually
- # have the lock. The easiest solution at the moment is to grab the
- # lock at the beginning of the run, and then run all of the locked
- # shards first. This minimizes the time spent holding the lock, but
- # means that we won't be running tests while we're waiting for the lock.
- # If this becomes a problem in practice we'll need to change this.
-
+ # We don't have a good way to coordinate the workers so that they don't
+ # try to run the shards that need a lock. The easiest solution is to
+ # run all of the locked shards first.
all_shards = locked_shards + unlocked_shards
- self._remaining_locked_shards = locked_shards
- if self._port.requires_http_server() or locked_shards:
- self.start_servers_with_lock(2 * min(num_workers, len(locked_shards)))
-
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
@@ -140,8 +123,6 @@
except Exception, e:
_log.debug('%s("%s") raised, exiting' % (e.__class__.__name__, str(e)))
raise
- finally:
- self.stop_servers_with_lock()
return run_results
@@ -193,29 +174,6 @@
self._interrupt_if_at_failure_limits(run_results)
- def start_servers_with_lock(self, number_of_servers):
- self._printer.write_update('Acquiring http lock ...')
- self._port.acquire_http_lock()
- if self._needs_http:
- self._printer.write_update('Starting HTTP server ...')
- self._port.start_http_server(number_of_servers=number_of_servers)
- if self._needs_websockets:
- self._printer.write_update('Starting WebSocket server ...')
- self._port.start_websocket_server()
- self._has_http_lock = True
-
- def stop_servers_with_lock(self):
- if self._has_http_lock:
- if self._needs_http:
- self._printer.write_update('Stopping HTTP server ...')
- self._port.stop_http_server()
- if self._needs_websockets:
- self._printer.write_update('Stopping WebSocket server ...')
- self._port.stop_websocket_server()
- self._printer.write_update('Releasing server lock ...')
- self._port.release_http_lock()
- self._has_http_lock = False
-
def handle(self, name, source, *args):
method = getattr(self, '_handle_' + name)
if method:
@@ -226,17 +184,7 @@
self._printer.print_started_test(test_input.test_name)
def _handle_finished_test_list(self, worker_name, list_name):
- def find(name, test_lists):
- for i in range(len(test_lists)):
- if test_lists[i].name == name:
- return i
- return -1
-
- index = find(list_name, self._remaining_locked_shards)
- if index >= 0:
- self._remaining_locked_shards.pop(index)
- if not self._remaining_locked_shards and not self._port.requires_http_server():
- self.stop_servers_with_lock()
+ pass
def _handle_finished_test(self, worker_name, result, log_messages=[]):
self._update_summary_with_result(self._current_run_results, result)
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py
index 240c3a6..e2258ac 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py
@@ -104,16 +104,7 @@
def _run_tests(self, runner, tests):
test_inputs = [TestInput(test, 6000) for test in tests]
expectations = TestExpectations(runner._port, tests)
- runner.run_tests(expectations, test_inputs, set(),
- num_workers=1, needs_http=any('http' in test for test in tests), needs_websockets=any(['websocket' in test for test in tests]), retrying=False)
-
- def test_http_locking(self):
- runner = self._runner()
- self._run_tests(runner, ['http/tests/passes/text.html', 'passes/text.html'])
-
- def test_perf_locking(self):
- runner = self._runner()
- self._run_tests(runner, ['http/tests/passes/text.html', 'perf/foo/test.html'])
+ runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False)
def test_interrupt_if_at_failure_limits(self):
runner = self._runner()
@@ -164,58 +155,6 @@
self.assertEqual(0, run_results.expected)
self.assertEqual(1, run_results.unexpected)
- def test_servers_started(self):
-
- def start_http_server(number_of_servers=None):
- self.http_started = True
-
- def start_websocket_server():
- self.websocket_started = True
-
- def stop_http_server():
- self.http_stopped = True
-
- def stop_websocket_server():
- self.websocket_stopped = True
-
- host = MockHost()
- port = host.port_factory.get('test-mac-leopard')
- port.start_http_server = start_http_server
- port.start_websocket_server = start_websocket_server
- port.stop_http_server = stop_http_server
- port.stop_websocket_server = stop_websocket_server
-
- self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
- runner = self._runner(port=port)
- runner._needs_http = True
- runner._needs_websockets = False
- runner.start_servers_with_lock(number_of_servers=4)
- self.assertEqual(self.http_started, True)
- self.assertEqual(self.websocket_started, False)
- runner.stop_servers_with_lock()
- self.assertEqual(self.http_stopped, True)
- self.assertEqual(self.websocket_stopped, False)
-
- self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
- runner._needs_http = True
- runner._needs_websockets = True
- runner.start_servers_with_lock(number_of_servers=4)
- self.assertEqual(self.http_started, True)
- self.assertEqual(self.websocket_started, True)
- runner.stop_servers_with_lock()
- self.assertEqual(self.http_stopped, True)
- self.assertEqual(self.websocket_stopped, True)
-
- self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
- runner._needs_http = False
- runner._needs_websockets = False
- runner.start_servers_with_lock(number_of_servers=4)
- self.assertEqual(self.http_started, False)
- self.assertEqual(self.websocket_started, False)
- runner.stop_servers_with_lock()
- self.assertEqual(self.http_stopped, False)
- self.assertEqual(self.websocket_stopped, False)
-
class SharderTests(unittest.TestCase):
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
index 17b303b..211a851 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
@@ -81,10 +81,8 @@
self.PERF_SUBDIR = 'perf'
self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
-
- # disable wss server. need to install pyOpenSSL on buildbots.
- # self._websocket_secure_server = websocket_server.PyWebSocket(
- # options.results_directory, use_tls=True, port=9323)
+ self._http_server_started = False
+ self._websockets_server_started = False
self._results_directory = self._port.results_directory()
self._finder = LayoutTestFinder(self._port, self._options)
@@ -197,9 +195,12 @@
else:
should_retry_failures = self._options.retry_failures
+
start_time = time.time()
enabled_pixel_tests_in_retry = False
try:
+ self._start_servers(tests_to_run)
+
initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
int(self._options.child_processes), retrying=False)
@@ -218,6 +219,7 @@
else:
retry_results = None
finally:
+ self._stop_servers()
self._clean_up_run()
end_time = time.time()
@@ -247,15 +249,34 @@
return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retrying):
- needs_http = self._port.requires_http_server() or any(self._is_http_test(test) for test in tests_to_run)
- needs_websockets = any(self._is_websocket_test(test) for test in tests_to_run)
test_inputs = []
for _ in xrange(iterations):
for test in tests_to_run:
for _ in xrange(repeat_each):
test_inputs.append(self._test_input_for_file(test))
- return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying)
+ return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, retrying)
+
+ def _start_servers(self, tests_to_run):
+ if self._port.requires_http_server() or any(self._is_http_test(test) for test in tests_to_run):
+ self._printer.write_update('Starting HTTP server ...')
+ self._port.start_http_server(number_of_servers=(2 * self._options.max_locked_shards))
+ self._http_server_started = True
+
+ if any(self._is_websocket_test(test) for test in tests_to_run):
+ self._printer.write_update('Starting WebSocket server ...')
+ self._port.start_websocket_server()
+ self._websockets_server_started = True
+
+ def _stop_servers(self):
+ if self._http_server_started:
+ self._printer.write_update('Stopping HTTP server ...')
+ self._http_server_started = False
+ self._port.stop_http_server()
+ if self._websockets_server_started:
+ self._printer.write_update('Stopping WebSocket server ...')
+ self._websockets_server_started = False
+ self._port.stop_websocket_server()
def _clean_up_run(self):
_log.debug("Flushing stdout")
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
index 4a81548..ccdd4fe 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
@@ -75,6 +75,56 @@
manager = get_manager()
self.assertTrue(manager.needs_servers(['http\\tests\\mime']))
+ def test_servers_started(self):
+ def get_manager(port):
+ manager = Manager(port, options=MockOptions(http=True, max_locked_shards=1), printer=Mock())
+ return manager
+
+ def start_http_server(number_of_servers=None):
+ self.http_started = True
+
+ def start_websocket_server():
+ self.websocket_started = True
+
+ def stop_http_server():
+ self.http_stopped = True
+
+ def stop_websocket_server():
+ self.websocket_stopped = True
+
+ host = MockHost()
+ port = host.port_factory.get('test-mac-leopard')
+ port.start_http_server = start_http_server
+ port.start_websocket_server = start_websocket_server
+ port.stop_http_server = stop_http_server
+ port.stop_websocket_server = stop_websocket_server
+
+ self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
+ manager = get_manager(port)
+ manager._start_servers(['http/tests/foo.html'])
+ self.assertEqual(self.http_started, True)
+ self.assertEqual(self.websocket_started, False)
+ manager._stop_servers()
+ self.assertEqual(self.http_stopped, True)
+ self.assertEqual(self.websocket_stopped, False)
+
+ self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
+ manager._start_servers(['http/tests/websocket/foo.html'])
+ self.assertEqual(self.http_started, True)
+ self.assertEqual(self.websocket_started, True)
+ manager._stop_servers()
+ self.assertEqual(self.http_stopped, True)
+ self.assertEqual(self.websocket_stopped, True)
+
+ self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
+ manager._start_servers(['fast/html/foo.html'])
+ self.assertEqual(self.http_started, False)
+ self.assertEqual(self.websocket_started, False)
+ manager._stop_servers()
+ self.assertEqual(self.http_stopped, False)
+ self.assertEqual(self.websocket_stopped, False)
+
+
def test_look_for_new_crash_logs(self):
def get_manager():
host = MockHost()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py
index 8a24708..3e9e925 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py
@@ -59,7 +59,6 @@
from webkitpy.layout_tests.models.test_configuration import TestConfiguration
from webkitpy.layout_tests.port import config as port_config
from webkitpy.layout_tests.port import driver
-from webkitpy.layout_tests.port import http_lock
from webkitpy.layout_tests.port import server_process
from webkitpy.layout_tests.port.factory import PortFactory
from webkitpy.layout_tests.servers import apache_http_server
@@ -954,10 +953,6 @@
return False
return True
- def acquire_http_lock(self):
- self._http_lock = http_lock.HttpLock(None, filesystem=self._filesystem, executive=self._executive)
- self._http_lock.wait_for_httpd_lock()
-
def stop_helper(self):
"""Shut down the test helper if it is running. Do nothing if
it isn't, or it isn't available. If a port overrides start_helper()
@@ -976,10 +971,6 @@
self._websocket_server.stop()
self._websocket_server = None
- def release_http_lock(self):
- if self._http_lock:
- self._http_lock.cleanup_http_lock()
-
def exit_code_from_summarized_results(self, unexpected_results):
"""Given summarized results, compute the exit code to be returned by new-run-webkit-tests.
Bots turn red when this function returns a non-zero value. By default, return the number of regressions
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
index b4807df..ea226f7 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
@@ -40,7 +40,6 @@
from webkitpy.common.system import executive
from webkitpy.layout_tests.models.test_configuration import TestConfiguration
from webkitpy.layout_tests.port.base import Port, VirtualTestSuite
-from webkitpy.layout_tests.port.http_lock import HttpLock
_log = logging.getLogger(__name__)
@@ -324,17 +323,12 @@
if stderr and 'AddressSanitizer' in stderr:
# Running the AddressSanitizer take a lot of memory, so we need to
# serialize access to it across all the concurrently running drivers.
- lock = HttpLock(lock_path=None, lock_file_prefix='WebKitASAN.lock.',
- filesystem=self._filesystem, executive=self._executive,
- name='ASAN')
- try:
- lock.wait_for_httpd_lock()
- asan_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
- if self._filesystem.exists(asan_filter_path):
- output = self._executive.run_command([asan_filter_path], input=stderr, decode_output=False)
- stderr = self._executive.run_command(['c++filt'], input=output, decode_output=False)
- finally:
- lock.cleanup_http_lock()
+
+ # FIXME: investigate using LLVM_SYMBOLIZER_PATH here to reduce the overhead.
+ asan_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
+ if self._filesystem.exists(asan_filter_path):
+ output = self._executive.run_command(['flock', asan_filter_path], input=stderr, decode_output=False)
+ stderr = self._executive.run_command(['c++filt'], input=output, decode_output=False)
return super(ChromiumPort, self)._get_crash_log(name, pid, stdout, stderr, newer_than)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py b/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py
deleted file mode 100644
index bdde37e..0000000
--- a/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
-# Copyright (C) 2010 Andras Becsi (abecsi@inf.u-szeged.hu), University of Szeged
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# FIXME: rename this file, and add more text about how this is
-# different from the base file_lock class.
-
-"""This class helps to block NRWT threads when more NRWTs run
-perf, http and websocket tests in a same time."""
-
-import logging
-import os
-import sys
-import tempfile
-import time
-
-from webkitpy.common.system.executive import Executive
-from webkitpy.common.system.file_lock import FileLock
-from webkitpy.common.system.filesystem import FileSystem
-
-
-_log = logging.getLogger(__name__)
-
-
-class HttpLock(object):
- def __init__(self, lock_path, lock_file_prefix="WebKitHttpd.lock.", guard_lock="WebKit.lock", filesystem=None, executive=None, name='HTTP'):
- self._executive = executive or Executive()
- self._filesystem = filesystem or FileSystem()
- self._lock_path = lock_path
- if not self._lock_path:
- # FIXME: FileSystem should have an accessor for tempdir()
- self._lock_path = tempfile.gettempdir()
- self._lock_file_prefix = lock_file_prefix
- self._lock_file_path_prefix = self._filesystem.join(self._lock_path, self._lock_file_prefix)
- self._guard_lock_file = self._filesystem.join(self._lock_path, guard_lock)
- self._guard_lock = FileLock(self._guard_lock_file)
- self._process_lock_file_name = ""
- self._name = name
-
- def cleanup_http_lock(self):
- """Delete the lock file if exists."""
- if self._filesystem.exists(self._process_lock_file_name):
- _log.debug("Removing lock file: %s" % self._process_lock_file_name)
- self._filesystem.remove(self._process_lock_file_name)
-
- def _extract_lock_number(self, lock_file_name):
- """Return the lock number from lock file."""
- prefix_length = len(self._lock_file_path_prefix)
- return int(lock_file_name[prefix_length:])
-
- def _lock_file_list(self):
- """Return the list of lock files sequentially."""
- lock_list = self._filesystem.glob(self._lock_file_path_prefix + '*')
- lock_list.sort(key=self._extract_lock_number)
- return lock_list
-
- def _next_lock_number(self):
- """Return the next available lock number."""
- lock_list = self._lock_file_list()
- if not lock_list:
- return 0
- return self._extract_lock_number(lock_list[-1]) + 1
-
- def _current_lock_pid(self):
- """Return with the current lock pid. If the lock is not valid
- it deletes the lock file."""
- lock_list = self._lock_file_list()
- if not lock_list:
- _log.debug("No lock file list")
- return
- try:
- current_pid = self._filesystem.read_text_file(lock_list[0])
- if not (current_pid and self._executive.check_running_pid(int(current_pid))):
- _log.debug("Removing stuck lock file: %s" % lock_list[0])
- self._filesystem.remove(lock_list[0])
- return
- except IOError, e:
- _log.debug("IOError: %s" % e)
- return
- except OSError, e:
- _log.debug("OSError: %s" % e)
- return
- return int(current_pid)
-
- def _create_lock_file(self):
- """The lock files are used to schedule the running test sessions in first
- come first served order. The guard lock ensures that the lock numbers are
- sequential."""
- if not self._filesystem.exists(self._lock_path):
- _log.debug("Lock directory does not exist: %s" % self._lock_path)
- return False
-
- if not self._guard_lock.acquire_lock():
- _log.debug("Guard lock timed out!")
- return False
-
- self._process_lock_file_name = (self._lock_file_path_prefix + str(self._next_lock_number()))
- _log.debug("Creating lock file: %s" % self._process_lock_file_name)
- # FIXME: Executive.py should have an accessor for getpid()
- self._filesystem.write_text_file(self._process_lock_file_name, str(os.getpid()))
- self._guard_lock.release_lock()
- return True
-
- def wait_for_httpd_lock(self):
- """Create a lock file and wait until it's turn comes. If something goes wrong
- it wont do any locking."""
- if not self._create_lock_file():
- _log.debug("Warning, %s locking failed!" % self._name)
- return
-
- # FIXME: This can hang forever!
- while self._current_lock_pid() != os.getpid():
- time.sleep(1)
-
- _log.debug("%s lock acquired" % self._name)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py
deleted file mode 100644
index 25af12f..0000000
--- a/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from http_lock import HttpLock
-import os # Used for os.getpid()
-import unittest2 as unittest
-
-from webkitpy.common.system.filesystem_mock import MockFileSystem
-from webkitpy.common.system.executive_mock import MockExecutive
-
-
-# FIXME: These tests all touch the real disk, but could be written to a MockFileSystem instead.
-class HttpLockTestWithRealFileSystem(unittest.TestCase):
- # FIXME: Unit tests do not use an __init__ method, but rather setUp and tearDown methods.
- def __init__(self, testFunc):
- self.http_lock = HttpLock(None, "WebKitTestHttpd.lock.", "WebKitTest.lock")
- self.filesystem = self.http_lock._filesystem # FIXME: We should be passing in a MockFileSystem instead.
- self.lock_file_path_prefix = self.filesystem.join(self.http_lock._lock_path, self.http_lock._lock_file_prefix)
- self.lock_file_name = self.lock_file_path_prefix + "0"
- self.guard_lock_file = self.http_lock._guard_lock_file
- self.clean_all_lockfile()
- unittest.TestCase.__init__(self, testFunc)
-
- def clean_all_lockfile(self):
- if self.filesystem.exists(self.guard_lock_file):
- self.filesystem.remove(self.guard_lock_file)
- lock_list = self.filesystem.glob(self.lock_file_path_prefix + '*')
- for file_name in lock_list:
- self.filesystem.remove(file_name)
-
- def assertEqual(self, first, second):
- if first != second:
- self.clean_all_lockfile()
- unittest.TestCase.assertEqual(self, first, second)
-
- def _check_lock_file(self):
- if self.filesystem.exists(self.lock_file_name):
- pid = os.getpid()
- lock_file_pid = self.filesystem.read_text_file(self.lock_file_name)
- self.assertEqual(pid, int(lock_file_pid))
- return True
- return False
-
- def test_lock_lifecycle(self):
- self.http_lock._create_lock_file()
-
- self.assertEqual(True, self._check_lock_file())
- self.assertEqual(1, self.http_lock._next_lock_number())
-
- self.http_lock.cleanup_http_lock()
-
- self.assertEqual(False, self._check_lock_file())
- self.assertEqual(0, self.http_lock._next_lock_number())
-
-
-class HttpLockTest(unittest.TestCase):
- def setUp(self):
- self.filesystem = MockFileSystem()
- self.http_lock = HttpLock(None, "WebKitTestHttpd.lock.", "WebKitTest.lock", filesystem=self.filesystem, executive=MockExecutive())
- # FIXME: Shouldn't we be able to get these values from the http_lock object directly?
- self.lock_file_path_prefix = self.filesystem.join(self.http_lock._lock_path, self.http_lock._lock_file_prefix)
- self.lock_file_name = self.lock_file_path_prefix + "0"
-
- def test_current_lock_pid(self):
- # FIXME: Once Executive wraps getpid, we can mock this and not use a real pid.
- current_pid = os.getpid()
- self.http_lock._filesystem.write_text_file(self.lock_file_name, str(current_pid))
- self.assertEqual(self.http_lock._current_lock_pid(), current_pid)
-
- def test_extract_lock_number(self):
- lock_file_list = (
- self.lock_file_path_prefix + "00",
- self.lock_file_path_prefix + "9",
- self.lock_file_path_prefix + "001",
- self.lock_file_path_prefix + "021",
- )
-
- expected_number_list = (0, 9, 1, 21)
-
- for lock_file, expected in zip(lock_file_list, expected_number_list):
- self.assertEqual(self.http_lock._extract_lock_number(lock_file), expected)
-
- def test_lock_file_list(self):
- self.http_lock._filesystem = MockFileSystem({
- self.lock_file_path_prefix + "6": "",
- self.lock_file_path_prefix + "1": "",
- self.lock_file_path_prefix + "4": "",
- self.lock_file_path_prefix + "3": "",
- })
-
- expected_file_list = [
- self.lock_file_path_prefix + "1",
- self.lock_file_path_prefix + "3",
- self.lock_file_path_prefix + "4",
- self.lock_file_path_prefix + "6",
- ]
-
- self.assertEqual(self.http_lock._lock_file_list(), expected_file_list)
