third_party/blink/tools/blinkpy/web_tests/controllers/test_result_writer.py

# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import logging

from blinkpy.common.html_diff import html_diff
from blinkpy.common.unified_diff import unified_diff
from blinkpy.web_tests.controllers import repaint_overlay
from blinkpy.web_tests.models import test_failures


_log = logging.getLogger(__name__)


def write_test_result(filesystem, port, results_directory, test_name, driver_output,
                      expected_driver_output, failures):
    """Write the test result to the result output directory."""
    root_output_dir = results_directory
    writer = TestResultWriter(filesystem, port, root_output_dir, test_name)

    # TODO(rmhasan): If the reference file driver output has stderr, we should
    # write out that standard error
    if driver_output.error:
        writer.write_stderr(driver_output.error)

    for failure in failures:
        # FIXME: Instead of this long 'if' block, each failure class might
        # have a responsibility for writing a test result.
        if isinstance(failure, (test_failures.FailureMissingResult,
                                test_failures.FailureTextMismatch,
                                test_failures.FailureTestHarnessAssertion)):
            writer.write_text_files(driver_output.text, expected_driver_output.text)
            writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
        elif isinstance(failure, test_failures.FailureMissingImage):
            writer.write_image_files(driver_output.image, expected_image=None)
        elif isinstance(failure, test_failures.FailureMissingImageHash):
            writer.write_image_files(driver_output.image, expected_driver_output.image)
        elif isinstance(failure, test_failures.FailureImageHashMismatch):
            writer.write_image_files(driver_output.image, expected_driver_output.image)
            writer.write_image_diff_files(driver_output.image_diff)
        elif isinstance(failure, (test_failures.FailureAudioMismatch,
                                  test_failures.FailureMissingAudio)):
            writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
        elif isinstance(failure, test_failures.FailureCrash):
            crashed_driver_output = expected_driver_output if failure.is_reftest else driver_output
            writer.write_crash_log(crashed_driver_output.crash_log)
        elif isinstance(failure, test_failures.FailureLeak):
            writer.write_leak_log(driver_output.leak_log)
        elif isinstance(failure, (
                test_failures.FailureReftestMismatch,
                test_failures.FailureReftestNoImageGenerated,
                test_failures.FailureReftestNoReferenceImageGenerated)):
            writer.write_image_files(driver_output.image, expected_driver_output.image)
            if driver_output.image and expected_driver_output.image:
                diff_image, _ = port.diff_image(expected_driver_output.image, driver_output.image)
                if diff_image:
                    writer.write_image_diff_files(diff_image)
                else:
                    _log.warning('ref test mismatch did not produce an image diff.')
            if filesystem.exists(failure.reference_filename):
                writer.write_reftest(failure.reference_filename)
            else:
                _log.warning("reference %s was not found", failure.reference_filename)
        elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
            writer.write_image_files(driver_output.image, expected_image=None)
            if filesystem.exists(failure.reference_filename):
                writer.write_reftest(failure.reference_filename)
            else:
                _log.warning("reference %s was not found", failure.reference_filename)
        else:
            assert isinstance(failure, test_failures.FailureTimeout)

        if expected_driver_output is not None:
            writer.create_repaint_overlay_result(driver_output.text, expected_driver_output.text)


class TestResultWriter(object):
    """A class which handles all writing operations to the result directory."""

    # Filename pieces when writing failures to the test results directory.
    FILENAME_SUFFIX_ACTUAL = "-actual"
    FILENAME_SUFFIX_EXPECTED = "-expected"
    FILENAME_SUFFIX_DIFF = "-diff"
    FILENAME_SUFFIX_DIFFS = "-diffs"
    FILENAME_SUFFIX_STDERR = "-stderr"
    FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
    FILENAME_SUFFIX_SAMPLE = "-sample"
    FILENAME_SUFFIX_LEAK_LOG = "-leak-log"
    FILENAME_SUFFIX_HTML_DIFF = "-pretty-diff"
    FILENAME_SUFFIX_OVERLAY = "-overlay"

    def __init__(self, filesystem, port, root_output_dir, test_name):
        self._filesystem = filesystem
        self._port = port
        self._root_output_dir = root_output_dir
        self._test_name = test_name

    def _write_file(self, path, contents):
        if contents is not None:
            fs = self._filesystem
            fs.maybe_make_directory(fs.dirname(path))
            fs.write_binary_file(path, contents)

    def _output_filename(self, suffix, extension):
        """Returns a filename based on the given suffix and extension.

        For example, if test name is "fast/dom/foo.html", the suffix is
        "-expected", and the extension is "-expected.txt", the return value is
        "fast/dom/foo-expected.txt".
        """
        return self._port.output_filename(self._test_name, suffix, extension)

    def _output_abspath(self, suffix, extension):
        """Similar to _output_filename, but returns an absolute path in output dir."""
        filename = self._output_filename(suffix, extension)
        return self._filesystem.join(self._root_output_dir, filename)

    def write_output_files(self, file_type, output, expected):
        """Writes the test output, the expected output in the results directory.

        The full output filename of the actual, for example, will be
          <filename>-actual<file_type>
        For instance,
          my_test-actual.txt

        Args:
          file_type: A string describing the test output file type, e.g. ".txt"
          output: A string containing the test output
          expected: A string containing the expected test output
        """
        actual_filename = self._output_abspath(self.FILENAME_SUFFIX_ACTUAL, file_type)
        expected_filename = self._output_abspath(self.FILENAME_SUFFIX_EXPECTED, file_type)

        self._write_file(actual_filename, output)
        self._write_file(expected_filename, expected)

    def write_stderr(self, error):
        filename = self._output_abspath(self.FILENAME_SUFFIX_STDERR, ".txt")
        self._write_file(filename, error)

    def write_crash_log(self, crash_log):
        filename = self._output_abspath(self.FILENAME_SUFFIX_CRASH_LOG, ".txt")
        self._write_file(filename, crash_log.encode('utf8', 'replace'))

    def write_leak_log(self, leak_log):
        filename = self._output_abspath(self.FILENAME_SUFFIX_LEAK_LOG, ".txt")
        self._write_file(filename, leak_log)

    def copy_sample_file(self, sample_file):
        filename = self._output_abspath(self.FILENAME_SUFFIX_SAMPLE, ".txt")
        self._filesystem.copyfile(sample_file, filename)

    def write_text_files(self, actual_text, expected_text):
        self.write_output_files(".txt", actual_text, expected_text)

    def create_text_diff_and_write_result(self, actual_text, expected_text):
        # FIXME: This function is actually doing the diffs as well as writing results.
        # It might be better to extract code which does 'diff' and make it a separate function.
        actual_text = actual_text or ''
        expected_text = expected_text or ''

        # Output a plain-text diff file.
        file_type = '.txt'
        actual_filename = self._output_abspath(self.FILENAME_SUFFIX_ACTUAL, file_type)
        expected_filename = self._output_abspath(self.FILENAME_SUFFIX_EXPECTED, file_type)
        diff = unified_diff(expected_text, actual_text, expected_filename, actual_filename)
        diff_filename = self._output_abspath(self.FILENAME_SUFFIX_DIFF, file_type)
        self._write_file(diff_filename, diff)

        # Output a HTML diff file.
        html_diff_filename = self._output_abspath(self.FILENAME_SUFFIX_HTML_DIFF, '.html')
        html_diff_contents = html_diff(expected_text, actual_text)
        self._write_file(html_diff_filename, html_diff_contents)

    def create_repaint_overlay_result(self, actual_text, expected_text):
        html = repaint_overlay.generate_repaint_overlay_html(self._test_name, actual_text, expected_text)
        if html:
            overlay_filename = self._output_abspath(self.FILENAME_SUFFIX_OVERLAY, '.html')
            self._write_file(overlay_filename, html)

    def write_audio_files(self, actual_audio, expected_audio):
        self.write_output_files('.wav', actual_audio, expected_audio)

    def write_image_files(self, actual_image, expected_image):
        self.write_output_files('.png', actual_image, expected_image)

    def write_image_diff_files(self, image_diff):
        diff_filename = self._output_abspath(self.FILENAME_SUFFIX_DIFF, '.png')
        self._write_file(diff_filename, image_diff)

        diffs_html_filename = self._output_abspath(self.FILENAME_SUFFIX_DIFFS, '.html')
        # FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
        # FIXME: old-run-webkit-tests include a link to the test file.
        html = ("""<!DOCTYPE HTML>
<html>
<head>
<title>%(title)s</title>
<style>.label{font-weight:bold}</style>
</head>
<body>
Difference between images: <a href="%(diff_filename)s">diff</a><br>
<div class=imageText></div>
<div class=imageContainer data-prefix="%(prefix)s">Loading...</div>
<script>
(function() {
    var preloadedImageCount = 0;
    function preloadComplete() {
        ++preloadedImageCount;
        if (preloadedImageCount < 2)
            return;
        toggleImages();
        setInterval(toggleImages, 2000)
    }

    function preloadImage(url) {
        image = new Image();
        image.addEventListener('load', preloadComplete);
        image.src = url;
        return image;
    }

    function toggleImages() {
        if (text.textContent == 'Expected Image') {
            text.textContent = 'Actual Image';
            container.replaceChild(actualImage, container.firstChild);
        } else {
            text.textContent = 'Expected Image';
            container.replaceChild(expectedImage, container.firstChild);
        }
    }

    var text = document.querySelector('.imageText');
    var container = document.querySelector('.imageContainer');
    var actualImage = preloadImage(container.getAttribute('data-prefix') + '-actual.png');
    var expectedImage = preloadImage(container.getAttribute('data-prefix') + '-expected.png');
})();
</script>
</body>
</html>"""
                % {
                    'title': self._test_name,
                    'diff_filename': self._output_filename(self.FILENAME_SUFFIX_DIFF, '.png'),
                    'prefix': self._output_filename('', ''),
                })
        self._write_file(diffs_html_filename, html)

    def write_reftest(self, src_filepath):
        fs = self._filesystem
        dst_dir = fs.dirname(fs.join(self._root_output_dir, self._test_name))
        dst_filepath = fs.join(dst_dir, fs.basename(src_filepath))
        self._write_file(dst_filepath, fs.read_binary_file(src_filepath))