| // Copyright 2016 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <string> |
| |
| #include "base/command_line.h" |
| #include "base/memory/raw_ptr.h" |
| #include "base/test/test_timeouts.h" |
| #include "base/threading/thread_restrictions.h" |
| #include "base/time/time.h" |
| #include "build/build_config.h" |
| #include "chrome/browser/media/webrtc/test_stats_dictionary.h" |
| #include "chrome/browser/media/webrtc/webrtc_browsertest_base.h" |
| #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h" |
| #include "content/public/common/content_switches.h" |
| #include "content/public/test/browser_test.h" |
| #include "media/base/media_switches.h" |
| #include "testing/perf/perf_test.h" |
| #include "third_party/blink/public/common/features.h" |
| |
| namespace content { |
| |
| namespace { |
| |
| const char kMainWebrtcTestHtmlPage[] = "/webrtc/webrtc_jsep01_test.html"; |
| |
| const char kInboundRtp[] = "inbound-rtp"; |
| const char kOutboundRtp[] = "outbound-rtp"; |
| |
| enum class GetStatsVariation { |
| PROMISE_BASED, |
| CALLBACK_BASED |
| }; |
| |
| // Sums up "RTC[In/Out]boundRTPStreamStats.bytes_[received/sent]" values. |
| double GetTotalRTPStreamBytes( |
| TestStatsReportDictionary* report, const char* type, |
| const char* media_type) { |
| DCHECK(type == kInboundRtp || type == kOutboundRtp); |
| const char* bytes_name = |
| (type == kInboundRtp) ? "bytesReceived" : "bytesSent"; |
| double total_bytes = 0.0; |
| report->ForEach([&type, &bytes_name, &media_type, &total_bytes]( |
| const TestStatsDictionary& stats) { |
| if (stats.GetString("type") == type && |
| stats.GetString("mediaType") == media_type) { |
| total_bytes += stats.GetNumber(bytes_name); |
| } |
| }); |
| return total_bytes; |
| } |
| |
| double GetAudioBytesSent(TestStatsReportDictionary* report) { |
| return GetTotalRTPStreamBytes(report, kOutboundRtp, "audio"); |
| } |
| |
| double GetAudioBytesReceived(TestStatsReportDictionary* report) { |
| return GetTotalRTPStreamBytes(report, kInboundRtp, "audio"); |
| } |
| |
| double GetVideoBytesSent(TestStatsReportDictionary* report) { |
| return GetTotalRTPStreamBytes(report, kOutboundRtp, "video"); |
| } |
| |
| double GetVideoBytesReceived(TestStatsReportDictionary* report) { |
| return GetTotalRTPStreamBytes(report, kInboundRtp, "video"); |
| } |
| |
| // Performance browsertest for WebRTC. This test is manual since it takes long |
| // to execute and requires the reference files provided by the webrtc.DEPS |
| // solution (which is only available on WebRTC internal bots). |
| // Gets its metrics from the standards conformant "RTCPeerConnection.getStats". |
| class WebRtcStatsPerfBrowserTest : public WebRtcTestBase { |
| public: |
| void SetUpInProcessBrowserTestFixture() override { |
| DetectErrorsInJavaScript(); |
| } |
| |
| void SetUpCommandLine(base::CommandLine* command_line) override { |
| // Ensure the infobar is enabled, since we expect that in this test. |
| EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream)); |
| |
| // Play a suitable, somewhat realistic video file. |
| base::FilePath input_video = test::GetReferenceFilesDir() |
| .Append(test::kReferenceFileName360p) |
| .AddExtension(test::kY4mFileExtension); |
| command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture, |
| input_video); |
| } |
| |
| void StartCall(const std::string& audio_codec, |
| const std::string& video_codec, |
| bool prefer_hw_video_codec, |
| const std::string& video_codec_profile) { |
| ASSERT_TRUE(test::HasReferenceFilesInCheckout()); |
| ASSERT_TRUE(embedded_test_server()->Start()); |
| |
| ASSERT_GE(TestTimeouts::test_launcher_timeout().InSeconds(), 100) |
| << "This is a long-running test; you must specify " |
| "--test-launcher-timeout to have a value of at least 100000."; |
| |
| ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100) |
| << "This is a long-running test; you must specify " |
| "--ui-test-action-max-timeout to have a value of at least 100000."; |
| |
| ASSERT_LT(TestTimeouts::action_max_timeout(), |
| TestTimeouts::test_launcher_timeout()) |
| << "action_max_timeout needs to be strictly-less-than " |
| "test_launcher_timeout"; |
| |
| left_tab_ = OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage); |
| right_tab_ = OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage); |
| |
| SetupPeerconnectionWithLocalStream(left_tab_); |
| SetupPeerconnectionWithLocalStream(right_tab_); |
| SetDefaultAudioCodec(left_tab_, audio_codec); |
| SetDefaultAudioCodec(right_tab_, audio_codec); |
| SetDefaultVideoCodec(left_tab_, video_codec, prefer_hw_video_codec, |
| video_codec_profile); |
| SetDefaultVideoCodec(right_tab_, video_codec, prefer_hw_video_codec, |
| video_codec_profile); |
| CreateDataChannel(left_tab_, "data"); |
| CreateDataChannel(right_tab_, "data"); |
| NegotiateCall(left_tab_, right_tab_); |
| StartDetectingVideo(left_tab_, "remote-view"); |
| StartDetectingVideo(right_tab_, "remote-view"); |
| WaitForVideoToPlay(left_tab_); |
| WaitForVideoToPlay(right_tab_); |
| } |
| |
| void EndCall() { |
| if (left_tab_) |
| HangUp(left_tab_); |
| if (right_tab_) |
| HangUp(right_tab_); |
| } |
| |
| void RunsAudioAndVideoCallCollectingMetricsWithAudioCodec( |
| const std::string& audio_codec) { |
| RunsAudioAndVideoCallCollectingMetrics( |
| audio_codec, kUseDefaultVideoCodec, false /* prefer_hw_video_codec */, |
| "" /* video_codec_profile */, "" /* video_codec_print_modifier */); |
| } |
| |
| void RunsAudioAndVideoCallCollectingMetricsWithVideoCodec( |
| const std::string& video_codec, |
| bool prefer_hw_video_codec = false, |
| const std::string& video_codec_profile = std::string(), |
| const std::string& video_codec_print_modifier = std::string()) { |
| RunsAudioAndVideoCallCollectingMetrics( |
| kUseDefaultAudioCodec, video_codec, prefer_hw_video_codec, |
| video_codec_profile, video_codec_print_modifier); |
| } |
| |
| void RunsAudioAndVideoCallCollectingMetrics( |
| const std::string& audio_codec, |
| const std::string& video_codec, |
| bool prefer_hw_video_codec, |
| const std::string& video_codec_profile, |
| const std::string& video_codec_print_modifier) { |
| StartCall(audio_codec, video_codec, prefer_hw_video_codec, |
| video_codec_profile); |
| |
| // Call for 60 seconds so that values may stabilize, bandwidth ramp up, etc. |
| test::SleepInJavascript(left_tab_, 60000); |
| |
| // The ramp-up may vary greatly and impact the resulting total bytes, to get |
| // reliable measurements we do two measurements, at 60 and 70 seconds and |
| // look at the average bytes/second in that window. |
| double audio_bytes_sent_before = 0.0; |
| double audio_bytes_received_before = 0.0; |
| double video_bytes_sent_before = 0.0; |
| double video_bytes_received_before = 0.0; |
| |
| scoped_refptr<TestStatsReportDictionary> report = |
| GetStatsReportDictionary(left_tab_); |
| if (audio_codec != kUseDefaultAudioCodec) { |
| audio_bytes_sent_before = GetAudioBytesSent(report.get()); |
| audio_bytes_received_before = GetAudioBytesReceived(report.get()); |
| |
| } |
| if (video_codec != kUseDefaultVideoCodec) { |
| video_bytes_sent_before = GetVideoBytesSent(report.get()); |
| video_bytes_received_before = GetVideoBytesReceived(report.get()); |
| } |
| |
| double measure_duration_seconds = 10.0; |
| test::SleepInJavascript(left_tab_, static_cast<int>( |
| measure_duration_seconds * base::Time::kMillisecondsPerSecond)); |
| |
| report = GetStatsReportDictionary(left_tab_); |
| if (audio_codec != kUseDefaultAudioCodec) { |
| double audio_bytes_sent_after = GetAudioBytesSent(report.get()); |
| double audio_bytes_received_after = GetAudioBytesReceived(report.get()); |
| |
| double audio_send_rate = |
| (audio_bytes_sent_after - audio_bytes_sent_before) / |
| measure_duration_seconds; |
| double audio_receive_rate = |
| (audio_bytes_received_after - audio_bytes_received_before) / |
| measure_duration_seconds; |
| |
| std::string audio_codec_modifier = "_" + audio_codec; |
| perf_test::PrintResult( |
| "audio", audio_codec_modifier, "send_rate", audio_send_rate, |
| "bytes/second", false); |
| perf_test::PrintResult( |
| "audio", audio_codec_modifier, "receive_rate", audio_receive_rate, |
| "bytes/second", false); |
| } |
| if (video_codec != kUseDefaultVideoCodec) { |
| double video_bytes_sent_after = GetVideoBytesSent(report.get()); |
| double video_bytes_received_after = GetVideoBytesReceived(report.get()); |
| |
| double video_send_rate = |
| (video_bytes_sent_after - video_bytes_sent_before) / |
| measure_duration_seconds; |
| double video_receive_rate = |
| (video_bytes_received_after - video_bytes_received_before) / |
| measure_duration_seconds; |
| |
| std::string video_codec_modifier = |
| "_" + (video_codec_print_modifier.empty() |
| ? video_codec |
| : video_codec_print_modifier); |
| perf_test::PrintResult("video", video_codec_modifier, "send_rate", |
| video_send_rate, "bytes/second", false); |
| perf_test::PrintResult( |
| "video", video_codec_modifier, "receive_rate", video_receive_rate, |
| "bytes/second", false); |
| } |
| |
| EndCall(); |
| } |
| |
| void RunsAudioAndVideoCallMeasuringGetStatsPerformance( |
| GetStatsVariation variation) { |
| EXPECT_TRUE(base::TimeTicks::IsHighResolution()); |
| |
| StartCall(kUseDefaultAudioCodec, kUseDefaultVideoCodec, |
| false /* prefer_hw_video_codec */, ""); |
| |
| double invocation_time = 0.0; |
| switch (variation) { |
| case GetStatsVariation::PROMISE_BASED: |
| invocation_time = (MeasureGetStatsPerformance(left_tab_) + |
| MeasureGetStatsPerformance(right_tab_)) / 2.0; |
| break; |
| case GetStatsVariation::CALLBACK_BASED: |
| invocation_time = |
| (MeasureGetStatsCallbackPerformance(left_tab_) + |
| MeasureGetStatsCallbackPerformance(right_tab_)) / 2.0; |
| break; |
| } |
| perf_test::PrintResult( |
| "getStats", |
| (variation == GetStatsVariation::PROMISE_BASED) ? |
| "_promise" : "_callback", |
| "invocation_time", |
| invocation_time, |
| "milliseconds", |
| false); |
| |
| EndCall(); |
| } |
| |
| private: |
| raw_ptr<content::WebContents> left_tab_ = nullptr; |
| raw_ptr<content::WebContents> right_tab_ = nullptr; |
| }; |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_opus) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("opus"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_ISAC) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("ISAC"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_G722) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("G722"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMU) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("PCMU"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMA) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("PCMA"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP8) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithVideoCodec("VP8"); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithVideoCodec("VP9"); |
| } |
| |
| // TODO(crbug.com/1241344): test fails on some mac bots. |
| #if defined(OS_MAC) |
| #define MAYBE_MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2 \ |
| DISABLED_MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2 |
| #else |
| #define MAYBE_MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2 \ |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2 |
| #endif |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MAYBE_MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallCollectingMetricsWithVideoCodec( |
| "VP9", true /* prefer_hw_video_codec */, |
| WebRtcTestBase::kVP9Profile2Specifier, "VP9p2"); |
| } |
| |
| #if BUILDFLAG(RTC_USE_H264) |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_H264) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| // Only run test if run-time feature corresponding to |rtc_use_h264| is on. |
| if (!base::FeatureList::IsEnabled( |
| blink::features::kWebRtcH264WithOpenH264FFmpeg)) { |
| LOG(WARNING) << "Run-time feature WebRTC-H264WithOpenH264FFmpeg disabled. " |
| "Skipping WebRtcPerfBrowserTest." |
| "MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_" |
| "H264 (test " |
| "\"OK\")"; |
| return; |
| } |
| RunsAudioAndVideoCallCollectingMetricsWithVideoCodec( |
| "H264", true /* prefer_hw_video_codec */); |
| } |
| |
| #endif // BUILDFLAG(RTC_USE_H264) |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Promise) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallMeasuringGetStatsPerformance( |
| GetStatsVariation::PROMISE_BASED); |
| } |
| |
| IN_PROC_BROWSER_TEST_F( |
| WebRtcStatsPerfBrowserTest, |
| MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Callback) { |
| base::ScopedAllowBlockingForTesting allow_blocking; |
| RunsAudioAndVideoCallMeasuringGetStatsPerformance( |
| GetStatsVariation::CALLBACK_BASED); |
| } |
| |
| } // namespace |
| |
| } // namespace content |