Handle URLLoaderFactory error in StoragePartitionImpl and SystemNetworkContextManager

This CL:
 1.Adds reconnect logic to |StoragePartitionImpl::GetURLLoaderFactoryForBrowserProcess()|
   and |SystemNetworkContextManager::GetURLLoaderFactory()| to makes sure they return
   valid interface after crash.
 2.Moved two common test APIs into "browser_test_utils.h".
 3.Updated browser tests to run against restarted NetworkContext/URLLoaderFactory.

Note:
 * More tests will be added for other NetworkContext providers (e.g. IOThread),
   |URLLoaderFactoryGetter|, as well as Renderer related stuff.

Bug: 780956
Change-Id: I78cd400263e79f2ba2b6088e108a499ee87ae1c3
Reviewed-on: https://chromium-review.googlesource.com/769855
Commit-Queue: Chong Zhang <chongz@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#523017}
diff --git a/chrome/browser/chrome_network_service_restart_browsertest.cc b/chrome/browser/chrome_network_service_restart_browsertest.cc
index eee1cc5..e89d21bb 100644
--- a/chrome/browser/chrome_network_service_restart_browsertest.cc
+++ b/chrome/browser/chrome_network_service_restart_browsertest.cc
@@ -10,17 +10,10 @@
 #include "chrome/browser/ui/browser.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "content/public/browser/browser_context.h"
-#include "content/public/browser/network_service_instance.h"
 #include "content/public/browser/storage_partition.h"
 #include "content/public/common/content_features.h"
-#include "content/public/common/network_service_test.mojom.h"
-#include "content/public/common/service_manager_connection.h"
-#include "content/public/common/service_names.mojom.h"
 #include "content/public/test/browser_test.h"
 #include "content/public/test/browser_test_utils.h"
-#include "content/public/test/simple_url_loader_test_helper.h"
-#include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
-#include "services/service_manager/public/cpp/connector.h"
 
 namespace content {
 
@@ -34,41 +27,10 @@
     EXPECT_TRUE(embedded_test_server()->Start());
   }
 
-  void SimulateNetworkServiceCrash() {
-    mojom::NetworkServiceTestPtr network_service_test;
-    ServiceManagerConnection::GetForProcess()->GetConnector()->BindInterface(
-        mojom::kNetworkServiceName, &network_service_test);
-
-    base::RunLoop run_loop;
-    network_service_test.set_connection_error_handler(run_loop.QuitClosure());
-
-    network_service_test->SimulateCrash();
-    run_loop.Run();
-
-    // Make sure the cached NetworkServicePtr receives error notification.
-    FlushNetworkServiceInstanceForTesting();
-  }
-
-  int LoadBasicRequest(mojom::NetworkContext* network_context) {
-    mojom::URLLoaderFactoryPtr url_loader_factory;
-    network_context->CreateURLLoaderFactory(MakeRequest(&url_loader_factory),
-                                            0);
-
-    auto request = std::make_unique<ResourceRequest>();
+  GURL GetTestURL() const {
     // Use '/echoheader' instead of '/echo' to avoid a disk_cache bug.
     // See https://crbug.com/792255.
-    request->url = embedded_test_server()->GetURL("/echoheader");
-
-    content::SimpleURLLoaderTestHelper simple_loader_helper;
-    std::unique_ptr<content::SimpleURLLoader> simple_loader =
-        content::SimpleURLLoader::Create(std::move(request),
-                                         TRAFFIC_ANNOTATION_FOR_TESTS);
-
-    simple_loader->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
-        url_loader_factory.get(), simple_loader_helper.GetCallback());
-    simple_loader_helper.WaitForCallback();
-
-    return simple_loader->NetError();
+    return embedded_test_server()->GetURL("/echoheader");
   }
 
  private:
@@ -89,7 +51,7 @@
       BrowserContext::GetDefaultStoragePartition(browser()->profile());
 
   mojom::NetworkContext* old_network_context = partition->GetNetworkContext();
-  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context));
+  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context, GetTestURL()));
 
   // Crash the NetworkService process. Existing interfaces should receive error
   // notifications at some point.
@@ -100,7 +62,8 @@
   // |partition->GetNetworkContext()| should return a valid new pointer after
   // crash.
   EXPECT_NE(old_network_context, partition->GetNetworkContext());
-  EXPECT_EQ(net::OK, LoadBasicRequest(partition->GetNetworkContext()));
+  EXPECT_EQ(net::OK,
+            LoadBasicRequest(partition->GetNetworkContext(), GetTestURL()));
 }
 
 // Make sure |SystemNetworkContextManager::GetContext()| returns valid interface
@@ -116,7 +79,7 @@
 
   mojom::NetworkContext* old_network_context =
       system_network_context_manager->GetContext();
-  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context));
+  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context, GetTestURL()));
 
   // Crash the NetworkService process. Existing interfaces should receive error
   // notifications at some point.
@@ -128,7 +91,8 @@
   // pointer after crash.
   EXPECT_NE(old_network_context, system_network_context_manager->GetContext());
   EXPECT_EQ(net::OK,
-            LoadBasicRequest(system_network_context_manager->GetContext()));
+            LoadBasicRequest(system_network_context_manager->GetContext(),
+                             GetTestURL()));
 }
 
 }  // namespace content
diff --git a/chrome/browser/net/network_context_configuration_browsertest.cc b/chrome/browser/net/network_context_configuration_browsertest.cc
index 02fbd773..bfcd47e 100644
--- a/chrome/browser/net/network_context_configuration_browsertest.cc
+++ b/chrome/browser/net/network_context_configuration_browsertest.cc
@@ -36,6 +36,7 @@
 #include "content/public/common/url_constants.h"
 #include "content/public/common/url_loader.mojom.h"
 #include "content/public/common/url_loader_factory.mojom.h"
+#include "content/public/test/browser_test_utils.h"
 #include "content/public/test/simple_url_loader_test_helper.h"
 #include "content/public/test/test_url_loader_client.h"
 #include "mojo/common/data_pipe_utils.h"
@@ -55,6 +56,9 @@
 enum class NetworkServiceState {
   kDisabled,
   kEnabled,
+  // Similar to |kEnabled|, but will simulate a crash and run tests again the
+  // restarted Network Service process.
+  kRestarted,
 };
 
 enum class NetworkContextType {
@@ -88,39 +92,16 @@
   ~NetworkContextConfigurationBrowserTest() override {}
 
   void SetUpInProcessBrowserTestFixture() override {
-    if (GetParam().network_service_state == NetworkServiceState::kEnabled)
+    if (GetParam().network_service_state != NetworkServiceState::kDisabled)
       feature_list_.InitAndEnableFeature(features::kNetworkService);
   }
 
   void SetUpOnMainThread() override {
-    switch (GetParam().network_context_type) {
-      case NetworkContextType::kSystem: {
-        SystemNetworkContextManager* system_network_context_manager =
-            g_browser_process->system_network_context_manager();
-        network_context_ = system_network_context_manager->GetContext();
-        loader_factory_ = system_network_context_manager->GetURLLoaderFactory();
-        break;
-      }
-      case NetworkContextType::kProfile: {
-        content::StoragePartition* storage_partition =
-            content::BrowserContext::GetDefaultStoragePartition(
-                browser()->profile());
-        network_context_ = storage_partition->GetNetworkContext();
-        loader_factory_ =
-            storage_partition->GetURLLoaderFactoryForBrowserProcess();
-        break;
-      }
-      case NetworkContextType::kIncognitoProfile: {
-        Browser* incognito = CreateIncognitoBrowser();
-        content::StoragePartition* storage_partition =
-            content::BrowserContext::GetDefaultStoragePartition(
-                incognito->profile());
-        network_context_ = storage_partition->GetNetworkContext();
-        loader_factory_ =
-            storage_partition->GetURLLoaderFactoryForBrowserProcess();
-        break;
-      }
+    if (GetParam().network_context_type ==
+        NetworkContextType::kIncognitoProfile) {
+      incognito_ = CreateIncognitoBrowser();
     }
+    SimulateNetworkServiceCrashIfNecessary();
   }
 
   // Returns, as a string, a PAC script that will use the EmbeddedTestServer as
@@ -134,11 +115,41 @@
   }
 
   content::mojom::URLLoaderFactory* loader_factory() const {
-    return loader_factory_;
+    switch (GetParam().network_context_type) {
+      case NetworkContextType::kSystem:
+        return g_browser_process->system_network_context_manager()
+            ->GetURLLoaderFactory();
+      case NetworkContextType::kProfile:
+        return content::BrowserContext::GetDefaultStoragePartition(
+                   browser()->profile())
+            ->GetURLLoaderFactoryForBrowserProcess();
+      case NetworkContextType::kIncognitoProfile:
+        DCHECK(incognito_);
+        return content::BrowserContext::GetDefaultStoragePartition(
+                   incognito_->profile())
+            ->GetURLLoaderFactoryForBrowserProcess();
+    }
+    NOTREACHED();
+    return nullptr;
   }
 
   content::mojom::NetworkContext* network_context() const {
-    return network_context_;
+    switch (GetParam().network_context_type) {
+      case NetworkContextType::kSystem:
+        return g_browser_process->system_network_context_manager()
+            ->GetContext();
+      case NetworkContextType::kProfile:
+        return content::BrowserContext::GetDefaultStoragePartition(
+                   browser()->profile())
+            ->GetNetworkContext();
+      case NetworkContextType::kIncognitoProfile:
+        DCHECK(incognito_);
+        return content::BrowserContext::GetDefaultStoragePartition(
+                   incognito_->profile())
+            ->GetNetworkContext();
+    }
+    NOTREACHED();
+    return nullptr;
   }
 
   StorageType GetHttpCacheType() const {
@@ -220,8 +231,48 @@
   }
 
  private:
-  content::mojom::NetworkContext* network_context_ = nullptr;
-  content::mojom::URLLoaderFactory* loader_factory_ = nullptr;
+  void SimulateNetworkServiceCrashIfNecessary() {
+    if (GetParam().network_service_state != NetworkServiceState::kRestarted)
+      return;
+
+    // Make sure |network_context()| is working as expected. Use '/echoheader'
+    // instead of '/echo' to avoid a disk_cache bug.
+    // See https://crbug.com/792255.
+    int net_error = content::LoadBasicRequest(
+        network_context(), embedded_test_server()->GetURL("/echoheader"));
+    // The error code could be |net::ERR_PROXY_CONNECTION_FAILED| if the test is
+    // using 'bad_server.pac'.
+    EXPECT_TRUE(net_error == net::OK ||
+                net_error == net::ERR_PROXY_CONNECTION_FAILED);
+
+    // Crash the NetworkService process. Existing interfaces should receive
+    // error notifications at some point.
+    content::SimulateNetworkServiceCrash();
+    // Flush the interface to make sure the error notification was received.
+    FlushNetworkInterface();
+  }
+
+  void FlushNetworkInterface() {
+    switch (GetParam().network_context_type) {
+      case NetworkContextType::kSystem:
+        g_browser_process->system_network_context_manager()
+            ->FlushNetworkInterfaceForTesting();
+        break;
+      case NetworkContextType::kProfile:
+        content::BrowserContext::GetDefaultStoragePartition(
+            browser()->profile())
+            ->FlushNetworkInterfaceForTesting();
+        break;
+      case NetworkContextType::kIncognitoProfile:
+        DCHECK(incognito_);
+        content::BrowserContext::GetDefaultStoragePartition(
+            incognito_->profile())
+            ->FlushNetworkInterfaceForTesting();
+        break;
+    }
+  }
+
+  Browser* incognito_ = nullptr;
   base::test::ScopedFeatureList feature_list_;
 
   DISALLOW_COPY_AND_ASSIGN(NetworkContextConfigurationBrowserTest);
@@ -625,6 +676,8 @@
   EXPECT_EQ(net::ERR_PROXY_CONNECTION_FAILED, simple_loader->NetError());
 }
 
+// |NetworkServiceTestHelper| doesn't work on browser_tests on OSX.
+#if defined(OS_MACOSX)
 // Instiates tests with a prefix indicating which NetworkContext is being
 // tested, and a suffix of "/0" if the network service is disabled and "/1" if
 // it's enabled.
@@ -650,6 +703,39 @@
                         TestCase({NetworkServiceState::kEnabled,           \
                                   NetworkContextType::kIncognitoProfile})))
 
+#else  // !defined(OS_MACOSX)
+// Instiates tests with a prefix indicating which NetworkContext is being
+// tested, and a suffix of "/0" if the network service is disabled, "/1" if it's
+// enabled, and "/2" if it's enabled and restarted.
+#define INSTANTIATE_TEST_CASES_FOR_TEST_FIXTURE(TestFixture)               \
+  INSTANTIATE_TEST_CASE_P(                                                 \
+      SystemNetworkContext, TestFixture,                                   \
+      ::testing::Values(TestCase({NetworkServiceState::kDisabled,          \
+                                  NetworkContextType::kSystem}),           \
+                        TestCase({NetworkServiceState::kEnabled,           \
+                                  NetworkContextType::kSystem}),           \
+                        TestCase({NetworkServiceState::kRestarted,         \
+                                  NetworkContextType::kSystem})));         \
+                                                                           \
+  INSTANTIATE_TEST_CASE_P(                                                 \
+      ProfileMainNetworkContext, TestFixture,                              \
+      ::testing::Values(TestCase({NetworkServiceState::kDisabled,          \
+                                  NetworkContextType::kProfile}),          \
+                        TestCase({NetworkServiceState::kEnabled,           \
+                                  NetworkContextType::kProfile}),          \
+                        TestCase({NetworkServiceState::kRestarted,         \
+                                  NetworkContextType::kProfile})));        \
+                                                                           \
+  INSTANTIATE_TEST_CASE_P(                                                 \
+      IncognitoProfileMainNetworkContext, TestFixture,                     \
+      ::testing::Values(TestCase({NetworkServiceState::kDisabled,          \
+                                  NetworkContextType::kIncognitoProfile}), \
+                        TestCase({NetworkServiceState::kEnabled,           \
+                                  NetworkContextType::kIncognitoProfile}), \
+                        TestCase({NetworkServiceState::kRestarted,         \
+                                  NetworkContextType::kIncognitoProfile})))
+#endif  // !defined(OS_MACOSX)
+
 INSTANTIATE_TEST_CASES_FOR_TEST_FIXTURE(NetworkContextConfigurationBrowserTest);
 INSTANTIATE_TEST_CASES_FOR_TEST_FIXTURE(
     NetworkContextConfigurationFixedPortBrowserTest);
diff --git a/chrome/browser/net/system_network_context_manager.cc b/chrome/browser/net/system_network_context_manager.cc
index 78d6d141..c919339 100644
--- a/chrome/browser/net/system_network_context_manager.cc
+++ b/chrome/browser/net/system_network_context_manager.cc
@@ -71,7 +71,7 @@
 
 content::mojom::URLLoaderFactory*
 SystemNetworkContextManager::GetURLLoaderFactory() {
-  if (!url_loader_factory_) {
+  if (!url_loader_factory_ || url_loader_factory_.encountered_error()) {
     GetContext()->CreateURLLoaderFactory(
         mojo::MakeRequest(&url_loader_factory_), 0);
   }
@@ -134,7 +134,10 @@
 }
 
 void SystemNetworkContextManager::FlushNetworkInterfaceForTesting() {
+  DCHECK(network_service_network_context_);
   network_service_network_context_.FlushForTesting();
+  if (url_loader_factory_)
+    url_loader_factory_.FlushForTesting();
 }
 
 content::mojom::NetworkContextParamsPtr
diff --git a/content/browser/network_service_restart_browsertest.cc b/content/browser/network_service_restart_browsertest.cc
index 829ba96e..44e74b42 100644
--- a/content/browser/network_service_restart_browsertest.cc
+++ b/content/browser/network_service_restart_browsertest.cc
@@ -10,15 +10,10 @@
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/content_features.h"
 #include "content/public/common/network_service.mojom.h"
-#include "content/public/common/network_service_test.mojom.h"
-#include "content/public/common/service_manager_connection.h"
-#include "content/public/common/service_names.mojom.h"
+#include "content/public/test/browser_test_utils.h"
 #include "content/public/test/content_browser_test.h"
 #include "content/public/test/content_browser_test_utils.h"
-#include "content/public/test/simple_url_loader_test_helper.h"
 #include "content/shell/browser/shell.h"
-#include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
-#include "services/service_manager/public/cpp/connector.h"
 
 namespace content {
 
@@ -44,45 +39,10 @@
     EXPECT_TRUE(embedded_test_server()->Start());
   }
 
-  void SimulateNetworkServiceCrash() {
-    mojom::NetworkServiceTestPtr network_service_test;
-    ServiceManagerConnection::GetForProcess()->GetConnector()->BindInterface(
-        mojom::kNetworkServiceName, &network_service_test);
-
-    base::RunLoop run_loop;
-    network_service_test.set_connection_error_handler(run_loop.QuitClosure());
-
-    network_service_test->SimulateCrash();
-    run_loop.Run();
-
-    // Make sure the cached NetworkServicePtr receives error notification.
-    FlushNetworkServiceInstanceForTesting();
-  }
-
-  int LoadBasicRequest(mojom::NetworkContext* network_context) {
-    mojom::URLLoaderFactoryPtr url_loader_factory;
-    network_context->CreateURLLoaderFactory(MakeRequest(&url_loader_factory),
-                                            0);
-    // |url_loader_factory| will receive error notification asynchronously if
-    // |network_context| has already encountered error. However it's still false
-    // at this point.
-    EXPECT_FALSE(url_loader_factory.encountered_error());
-
-    std::unique_ptr<ResourceRequest> request =
-        std::make_unique<ResourceRequest>();
+  GURL GetTestURL() const {
     // Use '/echoheader' instead of '/echo' to avoid a disk_cache bug.
     // See https://crbug.com/792255.
-    request->url = embedded_test_server()->GetURL("/echoheader");
-
-    content::SimpleURLLoaderTestHelper simple_loader_helper;
-    std::unique_ptr<content::SimpleURLLoader> simple_loader =
-        content::SimpleURLLoader::Create(std::move(request),
-                                         TRAFFIC_ANNOTATION_FOR_TESTS);
-    simple_loader->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
-        url_loader_factory.get(), simple_loader_helper.GetCallback());
-    simple_loader_helper.WaitForCallback();
-
-    return simple_loader->NetError();
+    return embedded_test_server()->GetURL("/echoheader");
   }
 
  private:
@@ -94,7 +54,7 @@
 IN_PROC_BROWSER_TEST_F(NetworkServiceRestartBrowserTest,
                        NetworkServiceProcessRecovery) {
   mojom::NetworkContextPtr network_context = CreateNetworkContext();
-  EXPECT_EQ(net::OK, LoadBasicRequest(network_context.get()));
+  EXPECT_EQ(net::OK, LoadBasicRequest(network_context.get(), GetTestURL()));
   EXPECT_TRUE(network_context.is_bound());
   EXPECT_FALSE(network_context.encountered_error());
 
@@ -108,11 +68,12 @@
   EXPECT_TRUE(network_context.is_bound());
   EXPECT_TRUE(network_context.encountered_error());
   // Make sure we could get |net::ERR_FAILED| with an invalid |network_context|.
-  EXPECT_EQ(net::ERR_FAILED, LoadBasicRequest(network_context.get()));
+  EXPECT_EQ(net::ERR_FAILED,
+            LoadBasicRequest(network_context.get(), GetTestURL()));
 
   // NetworkService should restart automatically and return valid interface.
   mojom::NetworkContextPtr network_context2 = CreateNetworkContext();
-  EXPECT_EQ(net::OK, LoadBasicRequest(network_context2.get()));
+  EXPECT_EQ(net::OK, LoadBasicRequest(network_context2.get(), GetTestURL()));
   EXPECT_TRUE(network_context2.is_bound());
   EXPECT_FALSE(network_context2.encountered_error());
 }
@@ -126,7 +87,7 @@
           shell()->web_contents()->GetBrowserContext()));
 
   mojom::NetworkContext* old_network_context = partition->GetNetworkContext();
-  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context));
+  EXPECT_EQ(net::OK, LoadBasicRequest(old_network_context, GetTestURL()));
 
   // Crash the NetworkService process. Existing interfaces should receive error
   // notifications at some point.
@@ -137,7 +98,8 @@
   // |partition->GetNetworkContext()| should return a valid new pointer after
   // crash.
   EXPECT_NE(old_network_context, partition->GetNetworkContext());
-  EXPECT_EQ(net::OK, LoadBasicRequest(partition->GetNetworkContext()));
+  EXPECT_EQ(net::OK,
+            LoadBasicRequest(partition->GetNetworkContext(), GetTestURL()));
 }
 
 }  // namespace content
diff --git a/content/browser/storage_partition_impl.cc b/content/browser/storage_partition_impl.cc
index 4c1dc5b..03249343 100644
--- a/content/browser/storage_partition_impl.cc
+++ b/content/browser/storage_partition_impl.cc
@@ -613,7 +613,8 @@
 mojom::URLLoaderFactory*
 StoragePartitionImpl::GetURLLoaderFactoryForBrowserProcess() {
   // Create the URLLoaderFactory as needed.
-  if (!url_loader_factory_for_browser_process_) {
+  if (!url_loader_factory_for_browser_process_ ||
+      url_loader_factory_for_browser_process_.encountered_error()) {
     GetNetworkContext()->CreateURLLoaderFactory(
         mojo::MakeRequest(&url_loader_factory_for_browser_process_), 0);
   }
@@ -1030,7 +1031,10 @@
 }
 
 void StoragePartitionImpl::FlushNetworkInterfaceForTesting() {
+  DCHECK(network_context_);
   network_context_.FlushForTesting();
+  if (url_loader_factory_for_browser_process_)
+    url_loader_factory_for_browser_process_.FlushForTesting();
 }
 
 BrowserContext* StoragePartitionImpl::browser_context() const {
diff --git a/content/public/test/browser_test_utils.cc b/content/public/test/browser_test_utils.cc
index d7c3a2ae..678af54 100644
--- a/content/public/test/browser_test_utils.cc
+++ b/content/public/test/browser_test_utils.cc
@@ -60,6 +60,7 @@
 #include "content/public/browser/navigation_entry.h"
 #include "content/public/browser/navigation_handle.h"
 #include "content/public/browser/navigation_throttle.h"
+#include "content/public/browser/network_service_instance.h"
 #include "content/public/browser/notification_service.h"
 #include "content/public/browser/notification_types.h"
 #include "content/public/browser/render_frame_host.h"
@@ -67,7 +68,12 @@
 #include "content/public/browser/render_view_host.h"
 #include "content/public/browser/storage_partition.h"
 #include "content/public/browser/web_contents.h"
+#include "content/public/common/content_features.h"
 #include "content/public/common/network_service.mojom.h"
+#include "content/public/common/network_service_test.mojom.h"
+#include "content/public/common/service_names.mojom.h"
+#include "content/public/common/simple_url_loader.h"
+#include "content/public/test/simple_url_loader_test_helper.h"
 #include "content/public/test/test_fileapi_operation_waiter.h"
 #include "content/public/test/test_navigation_observer.h"
 #include "content/public/test/test_utils.h"
@@ -83,7 +89,11 @@
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
 #include "net/test/python_utils.h"
+#include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
+#include "net/url_request/url_request_context.h"
+#include "net/url_request/url_request_context_getter.h"
 #include "services/network/public/interfaces/cookie_manager.mojom.h"
+#include "services/service_manager/public/cpp/connector.h"
 #include "storage/browser/fileapi/file_system_context.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "ui/base/clipboard/clipboard.h"
@@ -2351,4 +2361,43 @@
   return static_cast<content::WebContentsImpl*>(guest)->GetOuterWebContents();
 }
 
+void SimulateNetworkServiceCrash() {
+  CHECK(base::FeatureList::IsEnabled(features::kNetworkService));
+  mojom::NetworkServiceTestPtr network_service_test;
+  ServiceManagerConnection::GetForProcess()->GetConnector()->BindInterface(
+      mojom::kNetworkServiceName, &network_service_test);
+
+  base::RunLoop run_loop;
+  network_service_test.set_connection_error_handler(run_loop.QuitClosure());
+
+  network_service_test->SimulateCrash();
+  run_loop.Run();
+
+  // Make sure the cached NetworkServicePtr receives error notification.
+  FlushNetworkServiceInstanceForTesting();
+}
+
+int LoadBasicRequest(mojom::NetworkContext* network_context, const GURL& url) {
+  mojom::URLLoaderFactoryPtr url_loader_factory;
+  network_context->CreateURLLoaderFactory(MakeRequest(&url_loader_factory), 0);
+  // |url_loader_factory| will receive error notification asynchronously if
+  // |network_context| has already encountered error. However it's still false
+  // at this point.
+  EXPECT_FALSE(url_loader_factory.encountered_error());
+
+  auto request = std::make_unique<ResourceRequest>();
+  request->url = url;
+
+  content::SimpleURLLoaderTestHelper simple_loader_helper;
+  std::unique_ptr<content::SimpleURLLoader> simple_loader =
+      content::SimpleURLLoader::Create(std::move(request),
+                                       TRAFFIC_ANNOTATION_FOR_TESTS);
+
+  simple_loader->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
+      url_loader_factory.get(), simple_loader_helper.GetCallback());
+  simple_loader_helper.WaitForCallback();
+
+  return simple_loader->NetError();
+}
+
 }  // namespace content
diff --git a/content/public/test/browser_test_utils.h b/content/public/test/browser_test_utils.h
index 1180be5..9b63012 100644
--- a/content/public/test/browser_test_utils.h
+++ b/content/public/test/browser_test_utils.h
@@ -28,6 +28,7 @@
 #include "content/public/browser/web_contents_delegate.h"
 #include "content/public/browser/web_contents_observer.h"
 #include "content/public/common/context_menu_params.h"
+#include "content/public/common/network_service.mojom.h"
 #include "content/public/common/page_type.h"
 #include "ipc/message_filter.h"
 #include "storage/common/fileapi/file_system_types.h"
@@ -1066,6 +1067,13 @@
 
 WebContents* GetEmbedderForGuest(content::WebContents* guest);
 
+// Crash the Network Service process. Should only be called when out-of-process
+// Network Service is enabled.
+void SimulateNetworkServiceCrash();
+
+// Load the given |url| with |network_context| and return the |net::Error| code.
+int LoadBasicRequest(mojom::NetworkContext* network_context, const GURL& url);
+
 }  // namespace content
 
 #endif  // CONTENT_PUBLIC_TEST_BROWSER_TEST_UTILS_H_