| #!/usr/bin/env vpython |
| # Copyright (c) 2020 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unit tests for common functionality of wpt testing scripts.""" |
| |
| import base64 |
| import json |
| import os |
| import re |
| import unittest |
| |
| from wpt_common import ( |
| BaseWptScriptAdapter, EXTERNAL_WPT_TESTS_DIR, WEB_TESTS_DIR, |
| LAYOUT_TEST_RESULTS_SUBDIR |
| ) |
| |
| from blinkpy.common.host_mock import MockHost |
| from blinkpy.web_tests.port.factory_mock import MockPortFactory |
| from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME |
| |
| # The path where the output of a wpt run was written. This is the file that |
| # gets processed by BaseWptScriptAdapter. |
| OUTPUT_JSON_FILENAME = "out.json" |
| |
| |
| class BaseWptScriptAdapterTest(unittest.TestCase): |
| def setUp(self): |
| self.host = MockHost() |
| self.host.port_factory = MockPortFactory(self.host) |
| self.port = self.host.port_factory.get() |
| |
| # Create a testing manifest containing any test files that we |
| # might interact with. |
| self.host.filesystem.write_text_file( |
| self.port.web_tests_dir() + '/external/' + BASE_MANIFEST_NAME, |
| json.dumps({ |
| 'items': { |
| 'reftest': { |
| 'reftest.html': [ |
| 'c3f2fb6f436da59d43aeda0a7e8a018084557033', |
| [None, [['reftest-ref.html', '==']], {}], |
| ] |
| }, |
| 'testharness': { |
| 'test.html': [ |
| 'd933fd981d4a33ba82fb2b000234859bdda1494e', |
| [None, {}] |
| ], |
| 'crash.html': [ |
| 'd933fd981d4a33ba82fb2b000234859bdda1494e', |
| [None, {}] |
| ], |
| 'variant.html': [ |
| 'b8db5972284d1ac6bbda0da81621d9bca5d04ee7', |
| ['variant.html?foo=bar/abc', {}], |
| ['variant.html?foo=baz', {}], |
| ], |
| 'dir': { |
| 'multiglob.https.any.js': [ |
| 'd6498c3e388e0c637830fa080cca78b0ab0e5305', |
| ['dir/multiglob.https.any.window.html', {}], |
| ['dir/multiglob.https.any.worker.html', {}], |
| ], |
| }, |
| }, |
| }, |
| })) |
| self.host.filesystem.write_text_file( |
| os.path.join(WEB_TESTS_DIR, "fast", "harness", "results.html"), |
| "results-viewer-body") |
| self.wpt_adapter = BaseWptScriptAdapter(self.host) |
| self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME |
| |
| def _create_json_output(self, json_dict): |
| """Writing some json output for processing.""" |
| self.host.filesystem.write_text_file(OUTPUT_JSON_FILENAME, |
| json.dumps(json_dict)) |
| |
| def _load_json_output(self, filename=OUTPUT_JSON_FILENAME): |
| """Loads the json output after post-processing.""" |
| return json.loads(self.host.filesystem.read_text_file(filename)) |
| |
| def test_write_jsons(self): |
| # Ensure that various JSONs are written to the correct location. |
| |
| json_dict = { |
| 'tests': { |
| 'pass': { |
| 'test.html': { |
| 'expected': 'PASS', |
| 'actual': 'PASS', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| }, |
| }, |
| }, |
| 'unexpected_pass.html': { |
| 'expected': 'FAIL', |
| 'actual': 'PASS', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| }, |
| 'is_unexpected': True, |
| }, |
| 'fail.html': { |
| 'expected': 'PASS', |
| 'actual': 'FAIL', |
| 'artifacts': { |
| 'wpt_actual_status': ['ERROR'], |
| }, |
| 'is_unexpected': True, |
| 'is_regression': True, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| # The correctness of the output JSON is verified by other tests. |
| written_files = self.wpt_adapter.fs.written_files |
| self.assertEqual(written_files[OUTPUT_JSON_FILENAME], |
| written_files[os.path.join( |
| LAYOUT_TEST_RESULTS_SUBDIR, 'full_results.json')]) |
| # Verify JSONP |
| full_results_jsonp = written_files[os.path.join( |
| LAYOUT_TEST_RESULTS_SUBDIR, 'full_results_jsonp.js')] |
| match = re.match(r'ADD_FULL_RESULTS\((.*)\);$', full_results_jsonp) |
| self.assertIsNotNone(match) |
| self.assertEqual(match.group(1), written_files[OUTPUT_JSON_FILENAME]) |
| failing_results_jsonp = written_files[os.path.join( |
| LAYOUT_TEST_RESULTS_SUBDIR, 'failing_results.json')] |
| match = re.match(r'ADD_RESULTS\((.*)\);$', failing_results_jsonp) |
| self.assertIsNotNone(match) |
| failing_results = json.loads(match.group(1)) |
| # Verify filtering of failing_results.json |
| self.assertIn('fail.html', failing_results['tests']['external']['wpt']) |
| # We shouldn't have unexpected passes or empty dirs after filtering. |
| self.assertNotIn('unexpected_pass.html', |
| failing_results['tests']['external']['wpt']) |
| self.assertNotIn('pass', failing_results['tests']['external']['wpt']) |
| |
| def test_write_text_outputs(self): |
| # Ensure that text outputs are written to the correct location. |
| |
| # We only generate an actual.txt if our actual wasn't PASS, so in this |
| # case we shouldn't write anything. |
| json_dict = { |
| 'tests': { |
| 'test.html': { |
| 'expected': 'PASS', |
| 'actual': 'PASS', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| 'wpt_actual_metadata': ['test.html actual text'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| actual_path = os.path.join(artifact_subdir, |
| "test-actual.txt") |
| diff_path = os.path.join(artifact_subdir, "test-diff.txt") |
| pretty_diff_path = os.path.join(artifact_subdir, |
| "test-pretty-diff.html") |
| self.assertNotIn(actual_path, written_files) |
| self.assertNotIn(diff_path, written_files) |
| self.assertNotIn(pretty_diff_path, written_files) |
| |
| # Now we change the outcome to be a FAIL, which should generate an |
| # actual.txt |
| json_dict['tests']['test.html']['actual'] = 'FAIL' |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| actual_path = os.path.join(artifact_subdir, "test-actual.txt") |
| self.assertEqual("test.html actual text", written_files[actual_path]) |
| # Ensure the artifact in the json was replaced with the location of |
| # the newly-created file. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"] |
| ["test.html"]["artifacts"]) |
| self.assertEqual( |
| [actual_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["actual_text"]) |
| |
| # Ensure that a diff was also generated. Since there's no expected |
| # output, the actual text is all new. We don't validate the entire diff |
| # files to avoid checking line numbers/markup. |
| diff_path = os.path.join(artifact_subdir, "test-diff.txt") |
| self.assertIn("+test.html actual text", written_files[diff_path]) |
| self.assertEqual( |
| [diff_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["text_diff"]) |
| pretty_diff_path = os.path.join(artifact_subdir, |
| "test-pretty-diff.html") |
| self.assertIn("test.html actual text", written_files[pretty_diff_path]) |
| self.assertEqual( |
| [pretty_diff_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["pretty_text_diff"]) |
| |
| def test_write_log_artifact(self): |
| # Ensure that crash log artifacts are written to the correct location. |
| json_dict = { |
| 'tests': { |
| 'test.html': { |
| 'expected': 'PASS', |
| 'actual': 'FAIL', |
| 'artifacts': { |
| 'wpt_actual_status': ['ERROR'], |
| 'wpt_log': ['test.html exceptions'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| stderr_path = os.path.join(artifact_subdir, |
| "test-stderr.txt") |
| self.assertEqual("test.html exceptions", written_files[stderr_path]) |
| # Ensure the artifact in the json was replaced with the location of |
| # the newly-created file. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_log" in updated_json["tests"]["external"]["wpt"]["test.html"] |
| ["artifacts"]) |
| self.assertEqual( |
| [stderr_path], updated_json["tests"]["external"]["wpt"]["test.html"] |
| ["artifacts"]["stderr"]) |
| self.assertTrue( |
| updated_json["tests"]["external"]["wpt"]["test.html"]["has_stderr"]) |
| |
| def test_write_crashlog_artifact(self): |
| # Ensure that crash log artifacts are written to the correct location. |
| json_dict = { |
| 'tests': { |
| 'test.html': { |
| 'expected': 'PASS', |
| 'actual': 'CRASH', |
| 'artifacts': { |
| 'wpt_actual_status': ['CRASH'], |
| 'wpt_crash_log': ['test.html crashed!'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| crash_log_path = os.path.join(artifact_subdir, |
| "test-crash-log.txt") |
| self.assertEqual("test.html crashed!", written_files[crash_log_path]) |
| # Ensure the artifact in the json was replaced with the location of |
| # the newly-created file. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_crash_log" in updated_json["tests"]["external"]["wpt"] |
| ["test.html"]["artifacts"]) |
| self.assertEqual( |
| [crash_log_path], updated_json["tests"]["external"]["wpt"] |
| ["test.html"]["artifacts"]["crash_log"]) |
| |
| def test_write_screenshot_artifacts(self): |
| # Ensure that screenshots are written to the correct filenames and |
| # their bytes are base64 decoded. The image diff should also be created. |
| json_dict = { |
| 'tests': { |
| 'reftest.html': { |
| 'expected': 'PASS', |
| 'actual': 'PASS', |
| 'artifacts': { |
| 'wpt_actual_status': ['PASS'], |
| 'screenshots': [ |
| 'reftest.html:abcd', |
| 'reftest-ref.html:bcde', |
| ], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| actual_image_path = os.path.join(artifact_subdir, |
| "reftest-actual.png") |
| self.assertEqual(base64.b64decode('abcd'), |
| written_files[actual_image_path]) |
| expected_image_path = os.path.join(artifact_subdir, |
| "reftest-expected.png") |
| self.assertEqual(base64.b64decode('bcde'), |
| written_files[expected_image_path]) |
| diff_image_path = os.path.join(artifact_subdir, |
| "reftest-diff.png") |
| # Make sure the diff image was written but don't check the contents, |
| # assume that diff_image does the right thing. |
| self.assertIsNotNone(written_files[diff_image_path]) |
| # Ensure the artifacts in the json were replaced with the location of |
| # the newly-created files. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "screenshots" in updated_json["tests"]["external"]["wpt"] |
| ["reftest.html"]["artifacts"]) |
| self.assertEqual( |
| [actual_image_path], |
| updated_json["tests"]["external"]["wpt"]["reftest.html"] |
| ["artifacts"]["actual_image"]) |
| self.assertEqual( |
| [expected_image_path], |
| updated_json["tests"]["external"]["wpt"]["reftest.html"] |
| ["artifacts"]["expected_image"]) |
| self.assertEqual( |
| [diff_image_path], |
| updated_json["tests"]["external"]["wpt"]["reftest.html"] |
| ["artifacts"]["image_diff"]) |
| |
| def test_copy_expected_output(self): |
| # Check that an -expected.txt file is created from a checked-in metadata |
| # ini file if it exists for a test |
| json_dict = { |
| 'tests': { |
| 'test.html': { |
| 'expected': 'PASS', |
| 'actual': 'FAIL', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| 'wpt_actual_metadata': ['test.html actual text'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| # Also create a checked-in metadata file for this test |
| self.host.filesystem.write_text_file( |
| os.path.join(EXTERNAL_WPT_TESTS_DIR, "test.html.ini"), |
| "test.html checked-in metadata") |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| actual_path = os.path.join(artifact_subdir, |
| "test-actual.txt") |
| self.assertEqual("test.html actual text", written_files[actual_path]) |
| # The checked-in metadata file gets renamed from .ini to -expected.txt |
| expected_path = os.path.join(artifact_subdir, |
| "test-expected.txt") |
| self.assertEqual("test.html checked-in metadata", |
| written_files[expected_path]) |
| # Ensure the artifacts in the json were replaced with the locations of |
| # the newly-created files. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"] |
| ["test.html"]["artifacts"]) |
| self.assertEqual( |
| [actual_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["actual_text"]) |
| self.assertEqual( |
| [expected_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["expected_text"]) |
| |
| # Ensure that a diff was also generated. There should be both additions |
| # and deletions for this test since we have expected output. We don't |
| # validate the entire diff files to avoid checking line numbers/markup. |
| diff_path = os.path.join(artifact_subdir, "test-diff.txt") |
| self.assertIn("-test.html checked-in metadata", |
| written_files[diff_path]) |
| self.assertIn("+test.html actual text", written_files[diff_path]) |
| self.assertEqual( |
| [diff_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["text_diff"]) |
| pretty_diff_path = os.path.join(artifact_subdir, |
| "test-pretty-diff.html") |
| self.assertIn("test.html checked-in metadata", |
| written_files[pretty_diff_path]) |
| self.assertIn("test.html actual text", written_files[pretty_diff_path]) |
| self.assertEqual( |
| [pretty_diff_path], |
| updated_json["tests"]["external"]["wpt"]["test.html"]["artifacts"] |
| ["pretty_text_diff"]) |
| |
| def test_expected_output_for_variant(self): |
| # Check that an -expected.txt file is created from a checked-in metadata |
| # ini file for a variant test. Variants are a little different because |
| # we have to use the manifest to map a test name to the test file, and |
| # determine the associated metadata from the test file. |
| # Check that an -expected.txt file is created from a checked-in metadata |
| # ini file if it exists for a test |
| json_dict = { |
| 'tests': { |
| 'variant.html?foo=bar/abc': { |
| 'expected': 'PASS', |
| 'actual': 'FAIL', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| 'wpt_actual_metadata': ['variant bar/abc actual text'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| # Also create a checked-in metadata file for this test. This filename |
| # matches the test *file* name, not the test name (which includes the |
| # variant). |
| self.host.filesystem.write_text_file( |
| os.path.join(EXTERNAL_WPT_TESTS_DIR, "variant.html.ini"), |
| "variant.html checked-in metadata") |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| actual_path = os.path.join(artifact_subdir, |
| "variant_foo=bar_abc-actual.txt") |
| self.assertEqual("variant bar/abc actual text", |
| written_files[actual_path]) |
| # The checked-in metadata file gets renamed from .ini to -expected.txt |
| expected_path = os.path.join(artifact_subdir, |
| "variant_foo=bar_abc-expected.txt") |
| self.assertEqual("variant.html checked-in metadata", |
| written_files[expected_path]) |
| # Ensure the artifacts in the json were replaced with the locations of |
| # the newly-created files. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"] |
| ["variant.html?foo=bar/abc"]["artifacts"]) |
| self.assertEqual( |
| [actual_path], |
| updated_json["tests"]["external"]["wpt"]["variant.html?foo=bar/abc"] |
| ["artifacts"]["actual_text"]) |
| self.assertEqual( |
| [expected_path], |
| updated_json["tests"]["external"]["wpt"]["variant.html?foo=bar/abc"] |
| ["artifacts"]["expected_text"]) |
| |
| def test_expected_output_for_multiglob(self): |
| # Check that an -expected.txt file is created from a checked-in metadata |
| # ini file for a multiglobal test. Multi-globals are a little different |
| # because we have to use the manifest to map a test name to the test |
| # file, and determine the associated metadata from the test file. |
| # |
| # Also note that the "dir" is both a directory and a part of the test |
| # name, so the path delimiter remains a / (ie: dir/multiglob) even on |
| # Windows. |
| json_dict = { |
| 'tests': { |
| 'dir/multiglob.https.any.worker.html': { |
| 'expected': 'PASS', |
| 'actual': 'FAIL', |
| 'artifacts': { |
| 'wpt_actual_status': ['OK'], |
| 'wpt_actual_metadata': |
| ['dir/multiglob worker actual text'], |
| }, |
| }, |
| }, |
| 'path_delimiter': '/', |
| } |
| self._create_json_output(json_dict) |
| # Also create a checked-in metadata file for this test. This filename |
| # matches the test *file* name, not the test name (which includes test |
| # scope). |
| self.host.filesystem.write_text_file( |
| os.path.join(EXTERNAL_WPT_TESTS_DIR, |
| "dir/multiglob.https.any.js.ini"), |
| "dir/multiglob checked-in metadata") |
| self.wpt_adapter.do_post_test_run_tasks() |
| written_files = self.wpt_adapter.fs.written_files |
| artifact_subdir = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, |
| "external", "wpt") |
| actual_path = os.path.join(artifact_subdir, |
| "dir/multiglob.https.any.worker-actual.txt") |
| self.assertEqual("dir/multiglob worker actual text", |
| written_files[actual_path]) |
| # The checked-in metadata file gets renamed from .ini to -expected.txt |
| expected_path = os.path.join( |
| artifact_subdir, |
| "dir/multiglob.https.any.worker-expected.txt") |
| self.assertEqual("dir/multiglob checked-in metadata", |
| written_files[expected_path]) |
| # Ensure the artifacts in the json were replaced with the locations of |
| # the newly-created files. |
| updated_json = self._load_json_output() |
| self.assertFalse( |
| "wpt_actual_metadata" in updated_json["tests"]["external"]["wpt"] |
| ["dir/multiglob.https.any.worker.html"]["artifacts"]) |
| self.assertEqual( |
| [actual_path], |
| updated_json["tests"]["external"]["wpt"] |
| ["dir/multiglob.https.any.worker.html"]["artifacts"] |
| ["actual_text"]) |
| self.assertEqual( |
| [expected_path], |
| updated_json["tests"]["external"]["wpt"] |
| ["dir/multiglob.https.any.worker.html"]["artifacts"] |
| ["expected_text"]) |
| |
| if __name__ == '__main__': |
| unittest.main() |
