blob: d81211abd318d66372cee072fe59cca1c50f52db [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import os
import shutil
import tempfile
import unittest
import mock
from telemetry.testing import browser_test_runner
from telemetry.internal.platform import system_info
from gpu_tests import path_util
from gpu_tests import gpu_integration_test
path_util.AddDirToPathIfNeeded(path_util.GetChromiumSrcDir(), 'tools', 'perf')
from chrome_telemetry_build import chromium_config
VENDOR_NVIDIA = 0x10DE
VENDOR_AMD = 0x1002
VENDOR_INTEL = 0x8086
VENDOR_STRING_IMAGINATION = 'Imagination Technologies'
DEVICE_STRING_SGX = 'PowerVR SGX 554'
class MockPlatform(object):
def __init__(self, os_name, os_version_name=None):
self.os_name = os_name
self.os_version_name = os_version_name
def GetOSName(self):
return self.os_name
def GetOSVersionName(self):
return self.os_version_name
class MockBrowser(object):
def __init__(self, platform, gpu='', device='', vendor_string='',
device_string='', browser_type=None, gl_renderer=None,
passthrough=False):
self.platform = platform
self.browser_type = browser_type
sys_info = {
'model_name': '',
'gpu': {
'devices': [
{'vendor_id': gpu, 'device_id': device,
'vendor_string': vendor_string, 'device_string': device_string},
],
'aux_attributes': {'passthrough_cmd_decoder': passthrough}
}
}
if gl_renderer:
sys_info['gpu']['aux_attributes']['gl_renderer'] = gl_renderer
self.system_info = system_info.SystemInfo.FromDict(sys_info)
def GetSystemInfo(self):
return self.system_info
def __enter__(self):
return self
def __exit__(self, *args):
pass
class MockArgs(object):
pass
class MockAbstractGpuTestClass(gpu_integration_test.GpuIntegrationTest):
@classmethod
def GenerateGpuTests(cls, options):
pass
def RunActualGpuTest(self, test_path, *args):
pass
@classmethod
def _CreateExpectations(cls):
pass
class MockTestCaseWithoutExpectationsFile(MockAbstractGpuTestClass):
pass
class MockTestCaseWithExpectationsFile(MockAbstractGpuTestClass):
@classmethod
def ExpectationsFiles(cls):
return ['example_test_expectations.txt']
class MockPossibleBrowser(object):
def __init__(self, browser=None):
self._returned_browser = browser
def BrowserSession(self, options):
del options
return self._returned_browser
class GpuIntegrationTestUnittest(unittest.TestCase):
def setUp(self):
self._test_state = {}
self._test_result = {}
def testWithoutExpectationsFilesGenerateTagsReturnsEmptyList(self):
# we need to make sure that GenerateTags() returns an empty list if
# there are no expectations files returned from ExpectationsFiles() or
# else Typ will raise an exception
args = MockArgs()
possible_browser = MockPossibleBrowser()
self.assertFalse(MockTestCaseWithoutExpectationsFile.GenerateTags(
args, possible_browser))
def testGenerateNvidiaExampleTags(self):
args = MockArgs()
platform = MockPlatform('mac', 'mojave')
browser = MockBrowser(
platform, VENDOR_NVIDIA, 0x1cb3, browser_type='release',
gl_renderer='ANGLE Direct3D9')
possible_browser = MockPossibleBrowser(browser)
self.assertEqual(
set(MockTestCaseWithExpectationsFile.GenerateTags(
args, possible_browser)),
set(['mac', 'mojave', 'release', 'nvidia', 'nvidia-0x1cb3',
'd3d9', 'no_passthrough']))
def testGenerateVendorTagUsingVendorString(self):
args = MockArgs()
platform = MockPlatform('mac', 'mojave')
browser = MockBrowser(
platform, browser_type='release',
gl_renderer='ANGLE OpenGL ES', passthrough=True,
vendor_string=VENDOR_STRING_IMAGINATION,
device_string=DEVICE_STRING_SGX)
possible_browser = MockPossibleBrowser(browser)
self.assertEqual(
set(MockTestCaseWithExpectationsFile.GenerateTags(
args, possible_browser)),
set(['mac', 'mojave', 'release', 'imagination',
'imagination-powervr-sgx-554',
'opengles', 'passthrough']))
def testGenerateVendorTagUsingDeviceString(self):
args = MockArgs()
platform = MockPlatform('mac', 'mojave')
browser = MockBrowser(
platform, browser_type='release',
vendor_string='illegal vendor string',
device_string='ANGLE (Imagination, Triangle Monster 3000, 1.0)')
possible_browser = MockPossibleBrowser(browser)
self.assertEqual(
set(MockTestCaseWithExpectationsFile.GenerateTags(
args, possible_browser)),
set(['mac', 'mojave', 'release', 'imagination',
'imagination-triangle-monster-3000',
'no_angle', 'no_passthrough']))
def testSimpleIntegrationTest(self):
self._RunIntegrationTest(
'simple_integration_unittest',
['unittest_data.integration_tests.SimpleTest.unexpected_error',
'unittest_data.integration_tests.SimpleTest.unexpected_failure'],
['unittest_data.integration_tests.SimpleTest.expected_flaky',
'unittest_data.integration_tests.SimpleTest.expected_failure'],
['unittest_data.integration_tests.SimpleTest.expected_skip'],
[])
# It might be nice to be more precise about the order of operations
# with these browser restarts, but this is at least a start.
self.assertEquals(self._test_state['num_browser_starts'], 6)
def testIntegrationTesttWithBrowserFailure(self):
self._RunIntegrationTest(
'browser_start_failure_integration_unittest', [],
['unittest_data.integration_tests.BrowserStartFailureTest.restart'],
[], [])
self.assertEquals(self._test_state['num_browser_crashes'], 2)
self.assertEquals(self._test_state['num_browser_starts'], 3)
def testIntegrationTestWithBrowserCrashUponStart(self):
self._RunIntegrationTest(
'browser_crash_after_start_integration_unittest', [],
[('unittest_data.integration_tests.BrowserCrashAfterStartTest.restart')],
[], [])
self.assertEquals(self._test_state['num_browser_crashes'], 2)
self.assertEquals(self._test_state['num_browser_starts'], 3)
def testRetryLimit(self):
self._RunIntegrationTest(
'test_retry_limit',
['unittest_data.integration_tests.TestRetryLimit.unexpected_failure'],
[],
[],
['--retry-limit=2'])
# The number of attempted runs is 1 + the retry limit.
self.assertEquals(self._test_state['num_test_runs'], 3)
def _RunTestsWithExpectationsFiles(self):
self._RunIntegrationTest(
'run_tests_with_expectations_files',
[('unittest_data.integration_tests'
'.RunTestsWithExpectationsFiles.unexpected_test_failure')],
[('unittest_data.integration_tests'
'.RunTestsWithExpectationsFiles.expected_failure'),
('unittest_data.integration_tests'
'.RunTestsWithExpectationsFiles.expected_flaky')],
[('unittest_data.integration_tests'
'.RunTestsWithExpectationsFiles.expected_skip')],
['--retry-limit=3', '--retry-only-retry-on-failure-tests'])
def testUseTestExpectationsFileToHandleExpectedSkip(self):
self._RunTestsWithExpectationsFiles()
results = (self._test_result['tests']['unittest_data']['integration_tests']
['RunTestsWithExpectationsFiles']['expected_skip'])
self.assertEqual(results['expected'], 'SKIP')
self.assertEqual(results['actual'], 'SKIP')
self.assertNotIn('is_regression', results)
def testUseTestExpectationsFileToHandleUnexpectedTestFailure(self):
self._RunTestsWithExpectationsFiles()
results = (self._test_result['tests']['unittest_data']['integration_tests']
['RunTestsWithExpectationsFiles']['unexpected_test_failure'])
self.assertEqual(results['expected'], 'PASS')
self.assertEqual(results['actual'], 'FAIL')
self.assertIn('is_regression', results)
def testUseTestExpectationsFileToHandleExpectedFailure(self):
self._RunTestsWithExpectationsFiles()
results = (self._test_result['tests']['unittest_data']['integration_tests']
['RunTestsWithExpectationsFiles']['expected_failure'])
self.assertEqual(results['expected'], 'FAIL')
self.assertEqual(results['actual'], 'FAIL')
self.assertNotIn('is_regression', results)
def testUseTestExpectationsFileToHandleExpectedFlakyTest(self):
self._RunTestsWithExpectationsFiles()
results = (self._test_result['tests']['unittest_data']['integration_tests']
['RunTestsWithExpectationsFiles']['expected_flaky'])
self.assertEqual(results['expected'], 'PASS')
self.assertEqual(results['actual'], 'FAIL FAIL FAIL PASS')
self.assertNotIn('is_regression', results)
def testRepeat(self):
self._RunIntegrationTest(
'test_repeat',
[],
['unittest_data.integration_tests.TestRepeat.success'],
[],
['--repeat=3'])
self.assertEquals(self._test_state['num_test_runs'], 3)
def testAlsoRunDisabledTests(self):
self._RunIntegrationTest(
'test_also_run_disabled_tests',
['unittest_data.integration_tests.TestAlsoRunDisabledTests.skip',
'unittest_data.integration_tests.TestAlsoRunDisabledTests.flaky'],
# Tests that are expected to fail and do fail are treated as test passes
[('unittest_data.integration_tests.'
'TestAlsoRunDisabledTests.expected_failure')],
[],
['--also-run-disabled-tests'])
self.assertEquals(self._test_state['num_flaky_test_runs'], 4)
self.assertEquals(self._test_state['num_test_runs'], 6)
def testStartBrowser_Retries(self):
class TestException(Exception):
pass
def SetBrowserAndRaiseTestException():
gpu_integration_test.GpuIntegrationTest.browser = (
mock.MagicMock())
raise TestException
gpu_integration_test.GpuIntegrationTest.browser = None
gpu_integration_test.GpuIntegrationTest.platform = None
with mock.patch.object(
gpu_integration_test.serially_executed_browser_test_case.\
SeriallyExecutedBrowserTestCase,
'StartBrowser',
side_effect=SetBrowserAndRaiseTestException) as mock_start_browser:
with mock.patch.object(
gpu_integration_test.GpuIntegrationTest,
'StopBrowser') as mock_stop_browser:
with self.assertRaises(TestException):
gpu_integration_test.GpuIntegrationTest.StartBrowser()
self.assertEqual(mock_start_browser.call_count,
gpu_integration_test._START_BROWSER_RETRIES)
self.assertEqual(mock_stop_browser.call_count,
gpu_integration_test._START_BROWSER_RETRIES)
def _RunIntegrationTest(self, test_name, failures, successes, skips,
additional_args):
config = chromium_config.ChromiumConfig(
top_level_dir=path_util.GetGpuTestDir(),
benchmark_dirs=[
os.path.join(path_util.GetGpuTestDir(), 'unittest_data')])
temp_dir = tempfile.mkdtemp()
test_results_path = os.path.join(temp_dir, 'test_results.json')
test_state_path = os.path.join(temp_dir, 'test_state.json')
try:
browser_test_runner.Run(
config,
[test_name,
'--write-full-results-to=%s' % test_results_path,
'--test-state-json-path=%s' % test_state_path] + additional_args)
with open(test_results_path) as f:
self._test_result = json.load(f)
with open(test_state_path) as f:
self._test_state = json.load(f)
actual_successes, actual_failures, actual_skips = (
self._ExtractTestResults(self._test_result))
self.assertEquals(set(actual_failures), set(failures))
self.assertEquals(set(actual_successes), set(successes))
self.assertEquals(set(actual_skips), set(skips))
finally:
shutil.rmtree(temp_dir)
def _ExtractTestResults(self, test_result):
delimiter = test_result['path_delimiter']
failures = []
successes = []
skips = []
def _IsLeafNode(node):
test_dict = node[1]
return ('expected' in test_dict and
isinstance(test_dict['expected'], basestring))
node_queues = []
for t in test_result['tests']:
node_queues.append((t, test_result['tests'][t]))
while node_queues:
node = node_queues.pop()
full_test_name, test_dict = node
if _IsLeafNode(node):
if all(res not in test_dict['expected'].split() for res in
test_dict['actual'].split()):
failures.append(full_test_name)
elif test_dict['expected'] == test_dict['actual'] == 'SKIP':
skips.append(full_test_name)
else:
successes.append(full_test_name)
else:
for k in test_dict:
node_queues.append(
('%s%s%s' % (full_test_name, delimiter, k),
test_dict[k]))
return successes, failures, skips