| # Copyright 2019 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import copy |
| import unittest |
| |
| from core.results_processor.formatters import json3_output |
| from core.results_processor import testing |
| |
| |
| class Json3OutputTest(unittest.TestCase): |
| def setUp(self): |
| self.base_dir = 'base_dir' |
| |
| def Convert(self, test_results): |
| test_results_copy = copy.deepcopy(test_results) |
| results = json3_output.Convert(test_results_copy, self.base_dir) |
| # Convert should not modify the original intermediate results. |
| self.assertEqual(test_results_copy, test_results) |
| return results |
| |
| def FindTestResult(self, results, benchmark, story): |
| node = results['tests'] |
| for key in (benchmark, story): |
| self.assertIn(key, node) |
| node = node[key] |
| return node |
| |
| def testStartTime(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story', |
| start_time='2009-02-13T23:31:30.987000Z') |
| ]) |
| |
| self.assertFalse(results['interrupted']) |
| self.assertEqual(results['path_delimiter'], '/') |
| self.assertEqual(results['seconds_since_epoch'], 1234567890.987) |
| self.assertEqual(results['version'], 3) |
| |
| def testSingleTestCase(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story', run_duration='1.2s') |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['times'], [1.2]) |
| self.assertEqual(test_result['time'], 1.2) |
| self.assertNotIn('shard', test_result) |
| self.assertEqual(results['num_failures_by_type'], {'PASS': 1}) |
| |
| def testUrlAsStoryName(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/http%3A%2F%2Fexample.com') |
| ]) |
| |
| test_result = self.FindTestResult( |
| results, 'benchmark', 'http://example.com') |
| self.assertEqual(test_result['actual'], 'PASS') |
| |
| def testTwoTestCases(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', tags=['shard:7']), |
| testing.TestResult('benchmark/story2', tags=['shard:3']) |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['shard'], 7) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story2') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['shard'], 3) |
| |
| self.assertEqual(results['num_failures_by_type'], {'PASS': 2}) |
| |
| def testRepeatedTestCases(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', status='PASS', |
| run_duration='1.2s'), |
| testing.TestResult('benchmark/story2', status='SKIP'), |
| testing.TestResult('benchmark/story1', status='PASS', |
| run_duration='3.4s'), |
| testing.TestResult('benchmark/story2', status='SKIP'), |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['times'], [1.2, 3.4]) |
| self.assertEqual(test_result['time'], 1.2) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story2') |
| self.assertEqual(test_result['actual'], 'SKIP') |
| self.assertEqual(test_result['expected'], 'SKIP') |
| |
| self.assertEqual(results['num_failures_by_type'], {'PASS': 2, 'SKIP': 2}) |
| |
| def testFaliedAndSippedTestCases(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', status='PASS'), |
| testing.TestResult('benchmark/story2', status='PASS'), |
| testing.TestResult('benchmark/story1', status='FAIL'), |
| testing.TestResult('benchmark/story2', status='SKIP', |
| is_expected=False), |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS FAIL') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertTrue(test_result['is_unexpected']) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story2') |
| self.assertEqual(test_result['actual'], 'PASS SKIP') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertTrue(test_result['is_unexpected']) |
| |
| self.assertEqual(results['num_failures_by_type'], |
| {'PASS': 2, 'SKIP': 1, 'FAIL': 1}) |
| |
| def testDedupedStatus(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', status='PASS'), |
| testing.TestResult('benchmark/story2', status='SKIP'), |
| testing.TestResult('benchmark/story3', status='FAIL'), |
| testing.TestResult('benchmark/story1', status='PASS'), |
| testing.TestResult('benchmark/story2', status='SKIP'), |
| testing.TestResult('benchmark/story3', status='FAIL'), |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertFalse(test_result['is_unexpected']) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story2') |
| self.assertEqual(test_result['actual'], 'SKIP') |
| self.assertEqual(test_result['expected'], 'SKIP') |
| self.assertFalse(test_result['is_unexpected']) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story3') |
| self.assertEqual(test_result['actual'], 'FAIL FAIL') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertTrue(test_result['is_unexpected']) |
| |
| self.assertEqual(results['num_failures_by_type'], |
| {'PASS': 2, 'SKIP': 2, 'FAIL': 2}) |
| |
| def testRepeatedTestCaseWithArtifacts(self): |
| self.base_dir = 'base' |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', output_artifacts={ |
| 'logs.txt': testing.Artifact('base/artifacts/logs1.txt') |
| }), |
| testing.TestResult('benchmark/story1', output_artifacts={ |
| 'logs.txt': testing.Artifact('base/artifacts/logs2.txt'), |
| 'trace.json': testing.Artifact('base/artifacts/trace2.json') |
| }), |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['artifacts'], { |
| 'logs.txt': ['artifacts/logs1.txt', 'artifacts/logs2.txt'], |
| 'trace.json': ['artifacts/trace2.json'] |
| }) |
| |
| def testRemoteArtifacts(self): |
| results = self.Convert([ |
| testing.TestResult('benchmark/story1', output_artifacts={ |
| 'logs.txt': testing.Artifact( |
| 'base/artifacts/logs1.txt', |
| 'https://example.org/artifacts/logs1.txt') |
| }), |
| testing.TestResult('benchmark/story1', output_artifacts={ |
| 'logs.txt': testing.Artifact( |
| 'base/artifacts/logs2.txt', |
| 'https://example.org/artifacts/logs2.txt'), |
| 'trace.json': testing.Artifact( |
| 'base/artifacts/trace2.json', |
| 'https://example.org/artifacts/trace2.json') |
| }), |
| ]) |
| |
| test_result = self.FindTestResult(results, 'benchmark', 'story1') |
| self.assertEqual(test_result['actual'], 'PASS') |
| self.assertEqual(test_result['expected'], 'PASS') |
| self.assertEqual(test_result['artifacts'], { |
| 'logs.txt': [ |
| 'https://example.org/artifacts/logs1.txt', |
| 'https://example.org/artifacts/logs2.txt' |
| ], |
| 'trace.json': [ |
| 'https://example.org/artifacts/trace2.json' |
| ] |
| }) |
