| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import json |
| import os |
| import shutil |
| import tempfile |
| import unittest |
| |
| from telemetry.testing import browser_test_runner |
| |
| from gpu_tests import path_util |
| |
| path_util.AddDirToPathIfNeeded(path_util.GetChromiumSrcDir(), 'tools', 'perf') |
| from chrome_telemetry_build import chromium_config |
| |
| |
| class GpuIntegrationTestUnittest(unittest.TestCase): |
| def setUp(self): |
| self._test_state = {} |
| |
| def testSimpleIntegrationTest(self): |
| self._RunIntegrationTest( |
| 'simple_integration_unittest', |
| ['unittest_data.integration_tests.SimpleTest.unexpected_error', |
| 'unittest_data.integration_tests.SimpleTest.unexpected_failure'], |
| ['unittest_data.integration_tests.SimpleTest.expected_flaky', |
| 'unittest_data.integration_tests.SimpleTest.expected_failure'], |
| ['unittest_data.integration_tests.SimpleTest.expected_skip']) |
| # It might be nice to be more precise about the order of operations |
| # with these browser restarts, but this is at least a start. |
| self.assertEquals(self._test_state['num_browser_starts'], 6) |
| |
| def testIntegrationTesttWithBrowserFailure(self): |
| self._RunIntegrationTest( |
| 'browser_start_failure_integration_unittest', [], |
| ['unittest_data.integration_tests.BrowserStartFailureTest.restart'], |
| []) |
| self.assertEquals(self._test_state['num_browser_crashes'], 2) |
| self.assertEquals(self._test_state['num_browser_starts'], 3) |
| |
| def testIntegrationTestWithBrowserCrashUponStart(self): |
| self._RunIntegrationTest( |
| 'browser_crash_after_start_integration_unittest', [], |
| [('unittest_data.integration_tests.BrowserCrashAfterStartTest.restart')], |
| []) |
| self.assertEquals(self._test_state['num_browser_crashes'], 2) |
| self.assertEquals(self._test_state['num_browser_starts'], 3) |
| |
| def _RunIntegrationTest(self, test_name, failures, successes, skips): |
| config = chromium_config.ChromiumConfig( |
| top_level_dir=path_util.GetGpuTestDir(), |
| benchmark_dirs=[ |
| os.path.join(path_util.GetGpuTestDir(), 'unittest_data')]) |
| temp_dir = tempfile.mkdtemp() |
| test_results_path = os.path.join(temp_dir, 'test_results.json') |
| test_state_path = os.path.join(temp_dir, 'test_state.json') |
| try: |
| browser_test_runner.Run( |
| config, |
| [test_name, |
| '--write-full-results-to=%s' % test_results_path, |
| '--test-state-json-path=%s' % test_state_path]) |
| with open(test_results_path) as f: |
| test_result = json.load(f) |
| with open(test_state_path) as f: |
| self._test_state = json.load(f) |
| actual_successes, actual_failures, actual_skips = ( |
| self._ExtracTestResults(test_result)) |
| self.assertEquals(actual_failures, failures) |
| self.assertEquals(actual_successes, successes) |
| self.assertEquals(actual_skips, skips) |
| finally: |
| shutil.rmtree(temp_dir) |
| |
| def _ExtracTestResults(self, test_result): |
| delimiter = test_result['path_delimiter'] |
| failures = [] |
| successes = [] |
| skips = [] |
| def _IsLeafNode(node): |
| test_dict = node[1] |
| return ('expected' in test_dict and |
| isinstance(test_dict['expected'], basestring)) |
| node_queues = [] |
| for t in test_result['tests']: |
| node_queues.append((t, test_result['tests'][t])) |
| while node_queues: |
| node = node_queues.pop() |
| full_test_name, test_dict = node |
| if _IsLeafNode(node): |
| if all(res not in test_dict['expected'].split() for res in |
| test_dict['actual'].split()): |
| failures.append(full_test_name) |
| elif test_dict['expected'] == test_dict['actual'] == 'SKIP': |
| skips.append(full_test_name) |
| else: |
| successes.append(full_test_name) |
| else: |
| for k in test_dict: |
| node_queues.append( |
| ('%s%s%s' % (full_test_name, delimiter, k), |
| test_dict[k])) |
| return successes, failures, skips |
| |
