| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| from __future__ import division |
| |
| import os |
| import unittest |
| |
| from telemetry import decorators |
| from telemetry import story |
| from telemetry.testing import legacy_page_test_case |
| from telemetry.timeline import async_slice |
| from telemetry.timeline import model as model_module |
| |
| from benchmarks import blink_perf |
| |
| |
| _BLINK_PERF_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), |
| '..', '..', '..', 'third_party', 'blink', 'perf_tests', 'test_data') |
| _BLINK_PERF_RESOURCES_DIR = os.path.join(os.path.dirname(__file__), |
| '..', '..', '..', 'third_party', 'blink', 'perf_tests', 'resources') |
| |
| |
| def _Mean(values): |
| return sum(values) / len(values) |
| |
| |
| class BlinkPerfTest(legacy_page_test_case.LegacyPageTestCase): |
| def setUp(self): |
| super(BlinkPerfTest, self).setUp() |
| # pylint: disable=protected-access |
| self.blink_page_test = blink_perf._BlinkPerfMeasurement() |
| # pylint: enable=protected-access |
| |
| def HasChromeTraces(self): |
| return any(name.startswith('trace/traceEvents/') |
| for name in self.test_result['outputArtifacts']) |
| |
| @staticmethod |
| def CreateStorySetForTest(url): |
| story_set = story.StorySet( |
| base_dir=_BLINK_PERF_TEST_DATA_DIR, |
| serving_dirs=[_BLINK_PERF_TEST_DATA_DIR, _BLINK_PERF_RESOURCES_DIR]) |
| assert url.startswith('file://'), 'Expected local URI, got %s' % url |
| blink_page = blink_perf._BlinkPerfPage( # pylint: disable=protected-access |
| url, story_set, base_dir=story_set.base_dir, name=url[len('file://'):]) |
| story_set.AddStory(blink_page) |
| return story_set |
| |
| @decorators.Disabled('chromeos') # Flaky: https://crbug.com/1271916 |
| def testBlinkPerfTracingMetricsForMeasureTime(self): |
| measurements = self.RunPageTest( |
| self.blink_page_test, 'file://append-child-measure-time.html') |
| self.assertTrue(self.HasChromeTraces()) |
| |
| frame_view_layouts = measurements['LocalFrameView::layout']['samples'] |
| # append-child-measure-time.html specifies 5 iterationCount. |
| self.assertEqual(len(frame_view_layouts), 5) |
| self.assertGreater(_Mean(frame_view_layouts), 0.001) |
| |
| update_layout_trees = measurements['UpdateLayoutTree']['samples'] |
| # append-child-measure-time.html specifies 5 iterationCount. |
| self.assertEqual(len(update_layout_trees), 5) |
| self.assertGreater(_Mean(update_layout_trees), 0.001) |
| |
| @decorators.Disabled( |
| 'chromeos', # Flaky: https://crbug.com/1284873 |
| 'android-nougat') # Flaky: https://crbug.com/1342706 |
| def testBlinkPerfTracingMetricsForMeasureFrameTime(self): |
| measurements = self.RunPageTest( |
| self.blink_page_test, 'file://color-changes-measure-frame-time.html') |
| self.assertTrue(self.HasChromeTraces()) |
| |
| frame_view_prepaints = measurements[ |
| 'LocalFrameView::RunPrePaintLifecyclePhase']['samples'] |
| |
| # color-changes-measure-frame-time.html specifies 10 iterationCount. |
| self.assertEqual(len(frame_view_prepaints), 10) |
| self.assertGreater(_Mean(frame_view_prepaints), 0.001) |
| |
| frame_view_painttrees = measurements[ |
| 'LocalFrameView::RunPaintLifecyclePhase']['samples'] |
| # color-changes-measure-frame-time.html specifies 10 iterationCount. |
| self.assertEqual(len(frame_view_painttrees), 10) |
| self.assertGreater(_Mean(frame_view_painttrees), 0.001) |
| |
| @decorators.Disabled('linux', |
| 'chromeos') # Disable due to flaky: crbug.com/1240931 |
| def testBlinkPerfTracingMetricsForMeasurePageLoadTime(self): |
| measurements = self.RunPageTest( |
| self.blink_page_test, 'file://simple-html-measure-page-load-time.html') |
| self.assertTrue(self.HasChromeTraces()) |
| |
| create_child_frame = measurements[ |
| 'WebLocalFrameImpl::createChildframe']['samples'] |
| # color-changes-measure-frame-time.html specifies 7 iterationCount. |
| self.assertEqual(len(create_child_frame), 7) |
| self.assertGreater(_Mean(create_child_frame), 0.001) |
| |
| post_layout_task = measurements[ |
| 'LocalFrameView::performPostLayoutTasks']['samples'] |
| # color-changes-measure-frame-time.html specifies 7 iterationCount. |
| self.assertEqual(len(post_layout_task), 7) |
| self.assertGreater(_Mean(post_layout_task), 0.001) |
| |
| @decorators.Disabled('mac', # Flaky on mac: crbug.com/960554 |
| 'chromeos') # Flaky on CrOS: crbug.com/1275110 |
| def testBlinkPerfTracingMetricsForMeasureAsync(self): |
| measurements = self.RunPageTest( |
| self.blink_page_test, 'file://simple-blob-measure-async.html') |
| self.assertTrue(self.HasChromeTraces()) |
| |
| blob_requests = measurements['BlobRequest']['samples'] |
| blob_readers = measurements['BlobReader']['samples'] |
| # simple-blob-measure-async.html specifies 6 iterationCount. |
| self.assertEqual(len(blob_requests), 6) |
| self.assertEqual(len(blob_readers), 6) |
| |
| # TODO(mek): Delete non-mojo code paths when blobs are always using mojo. |
| using_mojo = _Mean(blob_readers) > 0.001 |
| if using_mojo: |
| self.assertEqual(_Mean(blob_requests), 0) |
| self.assertGreater(_Mean(blob_readers), 0.001) |
| else: |
| self.assertGreater(_Mean(blob_requests), 0.001) |
| self.assertEqual(_Mean(blob_readers), 0) |
| |
| if using_mojo: |
| read_data = measurements['BlobReader::ReadMore']['samples'] |
| else: |
| read_data = measurements['BlobRequest::ReadRawData']['samples'] |
| # simple-blob-measure-async.html specifies 6 iterationCount. |
| self.assertEqual(len(read_data), 6) |
| self.assertGreater(_Mean(read_data), 0.001) |
| |
| def testBlinkPerfLifecycleMethods(self): |
| self.RunPageTest(self.blink_page_test, 'file://lifecycle-methods.html') |
| self.assertFalse(self.HasChromeTraces()) |
| |
| @decorators.Disabled('linux', |
| 'chromeos') # Disable due to flaky: crbug.com/1163628 |
| def testExtraChromeCategories(self): |
| self.options.extra_chrome_categories = 'cc,blink' |
| self.RunPageTest(self.blink_page_test, 'file://lifecycle-methods.html') |
| self.assertTrue(self.HasChromeTraces()) |
| |
| |
| # pylint: disable=protected-access |
| # This is needed for testing _ComputeTraceEventsThreadTimeForBlinkPerf method. |
| class ComputeTraceEventsMetricsForBlinkPerfTest(unittest.TestCase): |
| |
| def _AddAsyncSlice(self, renderer_thread, category, name, start, end): |
| s = async_slice.AsyncSlice( |
| category, name, |
| timestamp=start, duration=end - start, start_thread=renderer_thread, |
| end_thread=renderer_thread) |
| renderer_thread.AddAsyncSlice(s) |
| |
| def _AddBlinkTestSlice(self, renderer_thread, start, end): |
| self._AddAsyncSlice( |
| renderer_thread, 'blink', 'blink_perf.runTest', start, end) |
| |
| def testTraceEventMetricsSingleBlinkTest(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] |
| # | [ foo ] [ bar ] [ baz ] |
| # | | | | | | | | |
| # 100 120 140 400 420 500 550 600 |
| # | | | |
| # CPU dur: 15 18 70 |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 550) |
| |
| renderer_main.BeginSlice('blink', 'foo', 120, 122) |
| renderer_main.EndSlice(140, 137) |
| |
| renderer_main.BeginSlice('blink', 'bar', 400, 402) |
| renderer_main.EndSlice(420, 420) |
| |
| # Since this "baz" slice has CPU duration = 70ms, wall-time duration = 100ms |
| # & its overalapped wall-time with "blink_perf.run_test" is 50 ms, its |
| # overlapped CPU time with "blink_perf.run_test" is |
| # 50 * 70 / 100 = 35ms. |
| renderer_main.BeginSlice('blink', 'baz', 500, 520) |
| renderer_main.EndSlice(600, 590) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar', 'baz']), { |
| 'foo': [15], |
| 'bar': [18], |
| 'baz': [35] |
| }) |
| |
| def testTraceEventMetricsMultiBlinkTest(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] [ blink_perf.run_test ] |
| # | [ foo ] [ bar ] | [ | foo ] | |
| # | | | | | | | | | | | |
| # 100 120 140 400 420 440 500 520 600 640 |
| # | | | |
| # CPU dur: 15 18 40 |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 440) |
| self._AddBlinkTestSlice(renderer_main, 520, 640) |
| |
| renderer_main.BeginSlice('blink', 'foo', 120, 122) |
| renderer_main.EndSlice(140, 137) |
| |
| renderer_main.BeginSlice('blink', 'bar', 400, 402) |
| renderer_main.EndSlice(420, 420) |
| |
| # Since this "foo" slice has CPU duration = 40ms, wall-time duration = 100ms |
| # & its overalapped wall-time with "blink_perf.run_test" is 80 ms, its |
| # overlapped CPU time with "blink_perf.run_test" is |
| # 80 * 40 / 100 = 32ms. |
| renderer_main.BeginSlice('blink', 'foo', 500, 520) |
| renderer_main.EndSlice(600, 560) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar', 'baz']), { |
| 'foo': [15, 32], |
| 'bar': [18, 0], |
| 'baz': [0, 0] |
| }) |
| |
| def testTraceEventMetricsNoThreadTimeAvailable(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] |
| # | [ foo ] [ bar ] | |
| # | | | | | | |
| # 100 120 140 400 420 550 |
| # | | |
| # CPU dur: None None |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 550) |
| |
| renderer_main.BeginSlice('blink', 'foo', 120) |
| renderer_main.EndSlice(140) |
| |
| renderer_main.BeginSlice('blink', 'bar', 400) |
| renderer_main.EndSlice(420) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar']), { |
| 'foo': [20], |
| 'bar': [20] |
| }) |
| |
| def testTraceEventMetricsMultiBlinkTestCrossProcesses(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| foo_thread = model.GetOrCreateProcess(2).GetOrCreateThread(4) |
| bar_thread = model.GetOrCreateProcess(2).GetOrCreateThread(5) |
| |
| # Set up a main model that looks like (P1 & P2 are different processes): |
| # P1 [ blink_perf.run_test ] [ blink_perf.run_test ] |
| # | | | | |
| # P2 | [ foo ] | [ | foo ] | |
| # | | | | [ bar ] | | | | | |
| # | | | | | | | | | | | | | |
| # 100 120 | 140 400 | 420 440 500 520 | 600 640 |
| # | | | |
| # CPU dur: 15 N/A 40 |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 440) |
| self._AddBlinkTestSlice(renderer_main, 520, 640) |
| |
| foo_thread.BeginSlice('blink', 'foo', 120, 122) |
| foo_thread.EndSlice(140, 137) |
| |
| bar_thread.BeginSlice('blink', 'bar', 400) |
| bar_thread.EndSlice(420) |
| |
| # Since this "foo" slice has CPU duration = 40ms, wall-time duration = 100ms |
| # & its overalapped wall-time with "blink_perf.run_test" is 80 ms, its |
| # overlapped CPU time with "blink_perf.run_test" is |
| # 80 * 40 / 100 = 32ms. |
| foo_thread.BeginSlice('blink', 'foo', 500, 520) |
| foo_thread.EndSlice(600, 560) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar', 'baz']), { |
| 'foo': [15, 32], |
| 'bar': [20, 0], |
| 'baz': [0, 0] |
| }) |
| |
| def testTraceEventMetricsNoDoubleCountingBasic(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] |
| # | [ foo ] [ foo ] | |
| # | [ foo ] | | | |
| # | | [ foo ] | | | | |
| # | | | | | | | | |
| # 100 120 140 400 420 440 510 550 |
| # | | |
| # CPU dur of | | |
| # of top most event: 280 50 |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 550) |
| |
| renderer_main.BeginSlice('blink', 'foo', 120, 130) |
| renderer_main.BeginSlice('blink', 'foo', 120, 130) |
| renderer_main.BeginSlice('blink', 'foo', 140, 150) |
| renderer_main.EndSlice(400, 390) |
| renderer_main.EndSlice(420, 410) |
| renderer_main.EndSlice(420, 410) |
| |
| renderer_main.BeginSlice('blink', 'foo', 440, 455) |
| renderer_main.EndSlice(510, 505) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo']), {'foo': [330]}) |
| |
| |
| def testTraceEventMetricsNoDoubleCountingWithOtherSlidesMixedIn(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] |
| # | [ foo ] [ bar ] | |
| # | | [ bar ] | | [ foo ] | | |
| # | | | [ foo ] | | | | | | | |
| # | | | | | | | | | | | | |
| # 100 120 130 140 | 400 405 420 440 480 | 510 520 550 |
| # | | |
| # CPU dur of | | |
| # of top most event: 280 (foo) & 270 (bar) 50 (bar) & 20 (foo) |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 550) |
| |
| renderer_main.BeginSlice('blink', 'foo', 120, 130) |
| renderer_main.BeginSlice('blink', 'bar', 130, 135) |
| renderer_main.BeginSlice('blink', 'foo', 140, 150) |
| renderer_main.EndSlice(400, 390) |
| renderer_main.EndSlice(405, 405) |
| renderer_main.EndSlice(420, 410) |
| |
| renderer_main.BeginSlice('blink', 'bar', 440, 455) |
| renderer_main.BeginSlice('blink', 'foo', 480, 490) |
| renderer_main.EndSlice(510, 510) |
| renderer_main.EndSlice(510, 505) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar']), { |
| 'foo': [300], |
| 'bar': [320] |
| }) |
| |
| def testAsyncTraceEventMetricsOverlapping(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main thread model that looks like: |
| # [ blink_perf.run_test ] |
| # | [ foo ] [ bar ] | |
| # | [ foo ] | | | | |
| # | | | | | | | | |
| # 100 110 120 130 140 400 420 550 |
| # CPU dur: None for all. |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 550) |
| |
| self._AddAsyncSlice(renderer_main, 'blink', 'foo', 110, 130) |
| self._AddAsyncSlice(renderer_main, 'blink', 'foo', 120, 140) |
| self._AddAsyncSlice(renderer_main, 'blink', 'bar', 400, 420) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar']), { |
| 'foo': [30], |
| 'bar': [20] |
| }) |
| |
| def testAsyncTraceEventMetricsMultipleTests(self): |
| model = model_module.TimelineModel() |
| renderer_main = model.GetOrCreateProcess(1).GetOrCreateThread(2) |
| renderer_main.name = 'CrRendererMain' |
| |
| # Set up a main model that looks like: |
| # [ blink_perf.run_test ] [ blink_perf.run_test ] |
| # | | | | |
| # [ foo ] |
| # | [ foo ] |
| # | | | | | | | | |
| # 80 90 100 200 300 400 500 510 |
| # CPU dur: None for all |
| # |
| self._AddBlinkTestSlice(renderer_main, 100, 200) |
| self._AddBlinkTestSlice(renderer_main, 300, 400) |
| |
| # Both events totally intersect both tests. |
| self._AddAsyncSlice(renderer_main, 'blink', 'foo', 80, 500) |
| self._AddAsyncSlice(renderer_main, 'blink', 'bar', 90, 510) |
| |
| self.assertEqual( |
| blink_perf._ComputeTraceEventsThreadTimeForBlinkPerf( |
| model, renderer_main, ['foo', 'bar']), { |
| 'foo': [100, 100], |
| 'bar': [100, 100] |
| }) |
