ipcz: Implement multiprocess tests

Introduces real multiprocess support for ipcz multinode tests when the
multiprocess Linux driver is in use. Each test node runs in its own
isolated child process.

Bug: 1299283
Change-Id: I41f3794b6222dbe096c66a755d15ecb2d431341f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3696704
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Commit-Queue: Ken Rockot <rockot@google.com>
Cr-Commit-Position: refs/heads/main@{#1016975}
NOKEYCHECK=True
GitOrigin-RevId: 8d84786eed20832265c253500e1359033198b7aa
diff --git a/src/BUILD.gn b/src/BUILD.gn
index f679514..d73bb4e 100644
--- a/src/BUILD.gn
+++ b/src/BUILD.gn
@@ -2,9 +2,17 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//build/buildflag_header.gni")
 import("//build_overrides/ipcz.gni")
 import("//testing/test.gni")
 
+enable_multiprocess_tests = is_linux
+
+buildflag_header("test_buildflags") {
+  header = "test_buildflags.h"
+  flags = [ "ENABLE_IPCZ_MULTIPROCESS_TESTS=$enable_multiprocess_tests" ]
+}
+
 shared_library("ipcz_shared") {
   output_name = "ipcz"
   sources = [
@@ -142,9 +150,10 @@
     "reference_drivers/single_process_reference_driver.cc",
   ]
 
-  if (is_linux) {
+  if (enable_multiprocess_tests) {
     public += [
       "reference_drivers/file_descriptor.h",
+      "reference_drivers/handle_eintr.h",
       "reference_drivers/memfd_memory.h",
       "reference_drivers/multiprocess_reference_driver.h",
       "reference_drivers/socket_transport.h",
@@ -164,7 +173,7 @@
     ":util",
   ]
   public_deps = [ ":ipcz_header" ]
-  configs = [ ":ipcz_include_src_dir" ]
+  public_configs = [ ":ipcz_include_src_dir" ]
 
   deps = []
   if (is_fuchsia) {
@@ -334,11 +343,13 @@
     "util/stack_trace_test.cc",
   ]
 
-  if (is_linux) {
+  if (enable_multiprocess_tests) {
+    public = [ "test/test_child_launcher.h" ]
     sources += [
       "reference_drivers/memfd_memory_test.cc",
       "reference_drivers/multiprocess_reference_driver_test.cc",
       "reference_drivers/socket_transport_test.cc",
+      "test/test_child_launcher.cc",
     ]
   }
 
@@ -347,12 +358,13 @@
     "//testing/gtest",
     "//third_party/abseil-cpp:absl",
   ]
+  public_deps = [ ":test_buildflags" ]
   ipcz_deps = [
     ":impl",
     ":ipcz",
-    ":reference_drivers",
     ":util",
   ]
+  ipcz_public_deps = [ ":reference_drivers" ]
 
   configs = [ ":ipcz_include_src_dir" ]
 }
@@ -365,6 +377,7 @@
   sources = [ "test/run_all_tests.cc" ]
   deps = [
     ":ipcz_tests_sources_standalone",
+    ":test_buildflags",
     "${ipcz_src_root}/standalone",
     "//testing/gtest",
   ]
diff --git a/src/reference_drivers/handle_eintr.h b/src/reference_drivers/handle_eintr.h
new file mode 100644
index 0000000..927e9bd
--- /dev/null
+++ b/src/reference_drivers/handle_eintr.h
@@ -0,0 +1,20 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IPCZ_INCLUDE_SRC_REFERENCE_DRIVERS_HANDLE_EINTR_H_
+#define IPCZ_INCLUDE_SRC_REFERENCE_DRIVERS_HANDLE_EINTR_H_
+
+// Helper to ignore EINTR errors when making interruptible system calls. The
+// expression `x` is retried until it produces a non-error result or a non-EINTR
+// error.
+#define HANDLE_EINTR(x)                                     \
+  ({                                                        \
+    decltype(x) eintr_wrapper_result;                       \
+    do {                                                    \
+      eintr_wrapper_result = (x);                           \
+    } while (eintr_wrapper_result == -1 && errno == EINTR); \
+    eintr_wrapper_result;                                   \
+  })
+
+#endif  // IPCZ_INCLUDE_SRC_REFERENCE_DRIVERS_HANDLE_EINTR_H_
diff --git a/src/reference_drivers/socket_transport.cc b/src/reference_drivers/socket_transport.cc
index b14175a..74b1912 100644
--- a/src/reference_drivers/socket_transport.cc
+++ b/src/reference_drivers/socket_transport.cc
@@ -19,21 +19,13 @@
 #include <vector>
 
 #include "reference_drivers/file_descriptor.h"
+#include "reference_drivers/handle_eintr.h"
 #include "third_party/abseil-cpp/absl/synchronization/mutex.h"
 #include "third_party/abseil-cpp/absl/types/optional.h"
 #include "third_party/abseil-cpp/absl/types/span.h"
 #include "util/log.h"
 #include "util/safe_math.h"
 
-#define HANDLE_EINTR(x)                                     \
-  ({                                                        \
-    decltype(x) eintr_wrapper_result;                       \
-    do {                                                    \
-      eintr_wrapper_result = (x);                           \
-    } while (eintr_wrapper_result == -1 && errno == EINTR); \
-    eintr_wrapper_result;                                   \
-  })
-
 namespace ipcz::reference_drivers {
 
 namespace {
diff --git a/src/test/multinode_test.cc b/src/test/multinode_test.cc
index 83eb887..ed24c24 100644
--- a/src/test/multinode_test.cc
+++ b/src/test/multinode_test.cc
@@ -8,18 +8,19 @@
 #include <string>
 #include <thread>
 
-#include "build/build_config.h"
 #include "ipcz/ipcz.h"
-#include "reference_drivers/file_descriptor.h"
 #include "reference_drivers/single_process_reference_driver.h"
-#include "reference_drivers/socket_transport.h"
 #include "third_party/abseil-cpp/absl/base/macros.h"
 #include "third_party/abseil-cpp/absl/types/optional.h"
 #include "third_party/abseil-cpp/absl/types/variant.h"
+#include "third_party/ipcz/src/test_buildflags.h"
 #include "util/log.h"
 
-#if BUILDFLAG(IS_LINUX)
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+#include "reference_drivers/file_descriptor.h"
 #include "reference_drivers/multiprocess_reference_driver.h"
+#include "reference_drivers/socket_transport.h"
+#include "test/test_child_launcher.h"
 #endif
 
 namespace ipcz::test {
@@ -68,6 +69,30 @@
   absl::optional<std::thread> client_thread_;
 };
 
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+// Controls a node running within an isolated child process.
+class ChildProcessTestNodeController : public TestNode::TestNodeController {
+ public:
+  explicit ChildProcessTestNodeController(pid_t pid) : pid_(pid) {}
+  ~ChildProcessTestNodeController() override {
+    ABSL_ASSERT(result_.has_value());
+  }
+
+  // TestNode::TestNodeController:
+  bool WaitForShutdown() override {
+    if (result_.has_value()) {
+      return *result_;
+    }
+
+    result_ = TestChildLauncher::WaitForSuccessfulProcessTermination(pid_);
+    return *result_;
+  }
+
+  const pid_t pid_;
+  absl::optional<bool> result_;
+};
+#endif
+
 }  // namespace
 
 TestNode::~TestNode() {
@@ -89,7 +114,7 @@
     case DriverMode::kSync:
       return reference_drivers::kSingleProcessReferenceDriver;
 
-#if BUILDFLAG(IS_LINUX)
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
     case DriverMode::kMultiprocess:
       return reference_drivers::kMultiprocessReferenceDriver;
 #endif
@@ -165,13 +190,23 @@
   Connect connect(*this);
   IpczDriverHandle their_transport = absl::visit(connect, portals_or_transport);
 
-  // TODO: Support a multiprocess mode which launches the new node in a child
-  // child process, passing the transport there.
-  std::unique_ptr<TestNode> test_node = details.factory();
-  test_node->SetTransport(their_transport);
-  Ref<TestNodeController> controller =
-      MakeRefCounted<InProcessTestNodeController>(driver_mode_,
-                                                  std::move(test_node));
+  Ref<TestNodeController> controller;
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+  if (driver_mode_ == DriverMode::kMultiprocess) {
+    reference_drivers::FileDescriptor socket =
+        reference_drivers::TakeMultiprocessTransportDescriptor(their_transport);
+    controller = MakeRefCounted<ChildProcessTestNodeController>(
+        child_launcher_.Launch(details.name, std::move(socket)));
+  }
+#endif
+
+  if (!controller) {
+    std::unique_ptr<TestNode> test_node = details.factory();
+    test_node->SetTransport(their_transport);
+    controller = MakeRefCounted<InProcessTestNodeController>(
+        driver_mode_, std::move(test_node));
+  }
+
   spawned_nodes_.push_back(controller);
   return controller;
 }
@@ -190,4 +225,22 @@
   transport_ = transport;
 }
 
+int TestNode::RunAsChild() {
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+  auto transport = std::make_unique<reference_drivers::SocketTransport>(
+      TestChildLauncher::TakeChildSocketDescriptor());
+  SetTransport(
+      reference_drivers::CreateMultiprocessTransport(std::move(transport)));
+  Initialize(DriverMode::kMultiprocess, IPCZ_NO_FLAGS);
+  NodeBody();
+
+  const int exit_code = ::testing::Test::HasFailure() ? 1 : 0;
+  return exit_code;
+#else
+  // Not supported outside of Linux.
+  ABSL_ASSERT(false);
+  return 0;
+#endif
+}
+
 }  // namespace ipcz::test
diff --git a/src/test/multinode_test.h b/src/test/multinode_test.h
index 037153f..5ae41e8 100644
--- a/src/test/multinode_test.h
+++ b/src/test/multinode_test.h
@@ -11,15 +11,20 @@
 #include <type_traits>
 #include <vector>
 
-#include "build/build_config.h"
 #include "ipcz/ipcz.h"
 #include "test/test_base.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "testing/multiprocess_func_list.h"
 #include "third_party/abseil-cpp/absl/base/macros.h"
 #include "third_party/abseil-cpp/absl/types/span.h"
 #include "third_party/abseil-cpp/absl/types/variant.h"
+#include "third_party/ipcz/src/test_buildflags.h"
 #include "util/ref_counted.h"
 
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+#include "test/test_child_launcher.h"
+#endif
+
 namespace ipcz::test {
 
 class TestNode;
@@ -73,11 +78,9 @@
   // kAsyncDelegatedAlloc and kAsyncObjectBrokering as described above.
   kAsyncObjectBrokeringAndDelegatedAlloc,
 
-#if BUILDFLAG(IS_LINUX)
-  // Use a multiprocess-capable driver (Linux only), with each test node running
-  // in its own isolated child process.
-  //
-  // TODO: Actually run nodes in child processes.
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+  // Use a multiprocess-capable driver (Linux only for now), with each test node
+  // running in its own isolated child process.
   kMultiprocess,
 #endif
 };
@@ -207,6 +210,9 @@
   };
   TransportPair CreateTransports();
 
+  // Helper used to support multiprocess TestNode invocation.
+  int RunAsChild();
+
  private:
   // Sets the transport to use when connecting to a broker via ConnectBroker.
   // Must only be called once.
@@ -238,6 +244,10 @@
   IpczHandle node_ = IPCZ_INVALID_HANDLE;
   IpczDriverHandle transport_ = IPCZ_INVALID_DRIVER_HANDLE;
   std::vector<Ref<TestNodeController>> spawned_nodes_;
+
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+  TestChildLauncher child_launcher_;
+#endif
 };
 
 // Actual parameterized GTest Test fixture for multinode tests. This or a
@@ -257,6 +267,15 @@
 
 }  // namespace ipcz::test
 
+#define MULTINODE_TEST_CHILD_MAIN_HELPER(func, node_name) \
+  MULTIPROCESS_TEST_MAIN(func) {                          \
+    node_name node;                                       \
+    return node.RunAsChild();                             \
+  }
+
+#define MULTINODE_TEST_CHILD_MAIN(fixture, node_name) \
+  MULTINODE_TEST_CHILD_MAIN_HELPER(fixture##_##node_name##_Node, node_name)
+
 // Defines the main body of a non-broker test node for a multinode test. The
 // named node can be spawned by another node using SpawnTestNode<T> where T is
 // the unique name given by `node_name` here. `fixture` must be
@@ -274,9 +293,10 @@
     };                                                                     \
     void NodeBody() override;                                              \
   };                                                                       \
+  MULTINODE_TEST_CHILD_MAIN(fixture, node_name);                           \
   void node_name::NodeBody()
 
-#if BUILDFLAG(IS_LINUX)
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
 #define IPCZ_EXTRA_DRIVER_MODES , ipcz::test::DriverMode::kMultiprocess
 #else
 #define IPCZ_EXTRA_DRIVER_MODES
diff --git a/src/test/run_all_tests.cc b/src/test/run_all_tests.cc
index 4455934..9d8cfda 100644
--- a/src/test/run_all_tests.cc
+++ b/src/test/run_all_tests.cc
@@ -5,10 +5,26 @@
 #include "standalone/base/logging.h"
 #include "standalone/base/stack_trace.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "testing/multiprocess_func_list.h"
+#include "third_party/ipcz/src/test_buildflags.h"
+
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+#include "third_party/ipcz/src/test/test_child_launcher.h"
+#endif
 
 int main(int argc, char** argv) {
   // TODO(rockot): Implement basic command line parsing for standalone builds.
   ipcz::standalone::StackTrace::EnableStackTraceSymbolization(argv[0]);
   testing::InitGoogleTest(&argc, argv);
+
+#if BUILDFLAG(ENABLE_IPCZ_MULTIPROCESS_TESTS)
+  ipcz::test::TestChildLauncher::Initialize(argc, argv);
+
+  int exit_code;
+  if (ipcz::test::TestChildLauncher::RunTestChild(exit_code)) {
+    return exit_code;
+  }
+#endif
+
   return RUN_ALL_TESTS();
 }
diff --git a/src/test/test_child_launcher.cc b/src/test/test_child_launcher.cc
new file mode 100644
index 0000000..b6d9b0e
--- /dev/null
+++ b/src/test/test_child_launcher.cc
@@ -0,0 +1,176 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "test/test_child_launcher.h"
+
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "reference_drivers/handle_eintr.h"
+#include "test/multinode_test.h"
+#include "testing/multiprocess_func_list.h"
+#include "third_party/abseil-cpp/absl/base/macros.h"
+#include "third_party/abseil-cpp/absl/strings/str_cat.h"
+#include "util/safe_math.h"
+
+namespace ipcz::test {
+
+namespace {
+
+// NOTE: This switch name must be identical to Chromium's kTestChildProcess
+// switch in //base/base_switches.h in order for these tests to work properly as
+// part of any base::TestSuite based unit tests.
+constexpr std::string_view kTestChildProcess = "test-child-process";
+
+// Used to tell a forked child process which open file descriptor corresponds to
+// its ipcz driver's SocketTransport.
+constexpr std::string_view kSocketFd = "test-child-socket-fd";
+
+using ArgList = std::vector<std::string>;
+ArgList& GetArgList() {
+  static ArgList* list = new ArgList();
+  return *list;
+}
+
+std::string& GetTestNodeName() {
+  static std::string* name = new std::string();
+  return *name;
+}
+
+reference_drivers::FileDescriptor& GetSocketFd() {
+  static reference_drivers::FileDescriptor* descriptor =
+      new reference_drivers::FileDescriptor();
+  return *descriptor;
+}
+
+template <typename ValueType>
+std::string MakeSwitch(std::string_view name, ValueType value) {
+  return absl::StrCat("--", name.data(), "=", value);
+}
+
+// Produces an argv-style data representation of a vector of strings.
+std::vector<char*> MakeExecArgv(ArgList& args) {
+  // +1 to NULL-terminate.
+  std::vector<char*> argv(args.size() + 1);
+  for (size_t i = 0; i < args.size(); ++i) {
+    argv[i] = args[i].data();
+  }
+  return argv;
+}
+
+int GetMaxFds() {
+  struct rlimit num_files;
+  int result = getrlimit(RLIMIT_NOFILE, &num_files);
+  ABSL_ASSERT(result == 0);
+  return checked_cast<int>(num_files.rlim_cur);
+}
+
+}  // namespace
+
+TestChildLauncher::TestChildLauncher() = default;
+
+TestChildLauncher::~TestChildLauncher() = default;
+
+// static
+void TestChildLauncher::Initialize(int argc, char** argv) {
+  // This implements extremely cheesy command-line "parsing" by scanning for the
+  // only two switches we'll ever care about.
+
+  const std::string kTestChildSwitchPrefix =
+      absl::StrCat("--", kTestChildProcess.data(), "=");
+  const std::string kSocketFdSwitchPrefix =
+      absl::StrCat("--", kSocketFd.data(), "=");
+
+  // We stash a complete copy of the command line in GetArgList(), modulo the
+  // switches above which are specific to a single child process. This copy is
+  // used to stamp out new command lines for future fork/exec'd processes.
+  ArgList& args = GetArgList();
+  args.resize(argc);
+  for (int i = 0; i < argc; ++i) {
+    std::string_view value(argv[i]);
+    if (value.rfind(kTestChildSwitchPrefix) != std::string::npos) {
+      GetTestNodeName() = value.substr(kTestChildSwitchPrefix.size());
+    } else if (value.rfind(kSocketFdSwitchPrefix) != std::string::npos) {
+      int fd;
+      const bool ok = absl::SimpleAtoi(
+          value.substr(kSocketFdSwitchPrefix.size()).data(), &fd);
+      ABSL_HARDENING_ASSERT(ok);
+      GetSocketFd() = reference_drivers::FileDescriptor(fd);
+    } else {
+      args[i] = value;
+    }
+  }
+}
+
+// static
+bool TestChildLauncher::RunTestChild(int& exit_code) {
+  if (GetTestNodeName().empty()) {
+    return false;
+  }
+
+  // Run the function emitted by named test node's MUTLTIPROCESS_TEST_MAIN()
+  // invocation. Note that this only occurs in upstream ipcz_tests. If these
+  // tests are run as part of a base::TestSuite in the Chromium repository,
+  // the TestSuite itself is responsible for invoking this function in child
+  // processes. See base::TestSuite::Run() for that.
+  exit_code =
+      multi_process_function_list::InvokeChildProcessTest(GetTestNodeName());
+  return true;
+}
+
+// static
+reference_drivers::FileDescriptor
+TestChildLauncher::TakeChildSocketDescriptor() {
+  reference_drivers::FileDescriptor& fd = GetSocketFd();
+  ABSL_HARDENING_ASSERT(fd.is_valid());
+  return std::move(fd);
+}
+
+// static
+bool TestChildLauncher::WaitForSuccessfulProcessTermination(pid_t pid) {
+  int status;
+  pid_t result = HANDLE_EINTR(waitpid(pid, &status, 0));
+  return result == pid && WIFEXITED(status) && WEXITSTATUS(status) == 0;
+}
+
+pid_t TestChildLauncher::Launch(std::string_view node_name,
+                                reference_drivers::FileDescriptor socket) {
+  pid_t child_pid = fork();
+  ABSL_HARDENING_ASSERT(child_pid >= 0);
+
+  if (child_pid > 0) {
+    // In the parent.
+    socket.reset();
+    return child_pid;
+  }
+
+  // In the child. First clean up all file descriptors other than the ones we
+  // want to keep open.
+  for (int i = STDERR_FILENO + 1; i < GetMaxFds(); ++i) {
+    if (i != socket.get()) {
+      close(i);
+    }
+  }
+
+  // Execute the test binary with an extra command-line switch that circumvents
+  // the normal test runner path and instead runs the named TestNode's body.
+  ArgList child_args = GetArgList();
+  child_args.push_back(MakeSwitch(kTestChildProcess, node_name.data()));
+  child_args.push_back(MakeSwitch(kSocketFd, socket.release()));
+
+  std::vector<char*> child_argv = MakeExecArgv(child_args);
+  execv(child_argv[0], child_argv.data());
+
+  // Should never be reached.
+  ABSL_HARDENING_ASSERT(false);
+  return 0;
+}
+
+}  // namespace ipcz::test
diff --git a/src/test/test_child_launcher.h b/src/test/test_child_launcher.h
new file mode 100644
index 0000000..0651141
--- /dev/null
+++ b/src/test/test_child_launcher.h
@@ -0,0 +1,50 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IPCZ_SRC_TEST_TEST_CHILD_LAUNCHER_H_
+#define IPCZ_SRC_TEST_TEST_CHILD_LAUNCHER_H_
+
+#include <sys/types.h>
+
+#include "reference_drivers/file_descriptor.h"
+
+#include <string_view>
+
+namespace ipcz::test {
+
+// This class helps multinode tests launch child processes to run test nodes in
+// isolation. This is only supported on Linux.
+class TestChildLauncher {
+ public:
+  TestChildLauncher();
+  ~TestChildLauncher();
+
+  // Must be called early in process startup with inputs from main().
+  static void Initialize(int argc, char** argv);
+
+  // Called by standalone ipcz_tests main() to run a TestNode body within a
+  // child process. Must be called after Initialize(). Returns true if and only
+  // if the calling process was launched as a test child process and has
+  // appropriate command line arguments to initialize a TestNode.
+  static bool RunTestChild(int& exit_code);
+
+  // Extracts a FileDescriptor passed on the command line to this (child)
+  // process.
+  static reference_drivers::FileDescriptor TakeChildSocketDescriptor();
+
+  // Waits for a child process to terminate, and returns true if and only if
+  // the process terminated normally with an exit code of 0.
+  static bool WaitForSuccessfulProcessTermination(pid_t pid);
+
+  // Launches a new child process to run the TestNode identified by `node_name`,
+  // using `socket` as the basis for a SocketTransport which will connect to the
+  // test's main broker node. Returns the PID of the new child process. This
+  // call either succeeds or crashes, so the return PID will always be valid.
+  pid_t Launch(std::string_view node_name,
+               reference_drivers::FileDescriptor socket);
+};
+
+}  // namespace ipcz::test
+
+#endif  // IPCZ_SRC_TEST_TEST_CHILD_LAUNCHER_H_