chrome/browser/metrics/thread_watcher.h

// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This file defines a WatchDog thread that monitors the responsiveness of other
// browser threads like UI, IO, DB, FILE and CACHED threads. It also defines
// ThreadWatcher class which performs health check on threads that would like to
// be watched. This file also defines ThreadWatcherList class that has list of
// all active ThreadWatcher objects.
//
// ThreadWatcher class sends ping message to the watched thread and the watched
// thread responds back with a pong message. It uploads response time
// (difference between ping and pong times) as a histogram.
//
// TODO(raman): ThreadWatcher can detect hung threads. If a hung thread is
// detected, we should probably just crash, and allow the crash system to gather
// then stack trace.
//
// Example Usage:
//
//   The following is an example for watching responsiveness of watched (IO)
//   thread. |sleep_time| specifies how often ping messages have to be sent to
//   watched (IO) thread. |unresponsive_time| is the wait time after ping
//   message is sent, to check if we have received pong message or not.
//   |unresponsive_threshold| specifies the number of unanswered ping messages
//   after which watched (IO) thread is considered as not responsive.
//   |crash_on_hang| specifies if we want to crash the browser when the watched
//   (IO) thread has become sufficiently unresponsive, while other threads are
//   sufficiently responsive.
//
//   base::TimeDelta sleep_time = base::TimeDelta::FromSeconds(5);
//   base::TimeDelta unresponsive_time = base::TimeDelta::FromSeconds(10);
//   uint32_t unresponsive_threshold = ThreadWatcherList::kUnresponsiveCount;
//   bool crash_on_hang = false;
//   ThreadWatcher::StartWatching(
//       BrowserThread::IO, "IO", sleep_time, unresponsive_time,
//       unresponsive_threshold, crash_on_hang);

#ifndef CHROME_BROWSER_METRICS_THREAD_WATCHER_H_
#define CHROME_BROWSER_METRICS_THREAD_WATCHER_H_

#include <stdint.h>

#include <map>
#include <string>
#include <vector>

#include "base/command_line.h"
#include "base/gtest_prod_util.h"
#include "base/macros.h"
#include "base/memory/memory_pressure_listener.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/synchronization/lock.h"
#include "base/threading/platform_thread.h"
#include "base/threading/thread.h"
#include "base/threading/watchdog.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "components/metrics/call_stack_profile_params.h"
#include "content/public/browser/browser_thread.h"

class CustomThreadWatcher;
class ThreadWatcherList;

namespace base {
class HistogramBase;
}

// This class performs health check on threads that would like to be watched.
class ThreadWatcher {
 public:
  // base::Bind supports methods with up to 6 parameters. WatchingParams is used
  // as a workaround that limitation for invoking ThreadWatcher::StartWatching.
  struct WatchingParams {
    content::BrowserThread::ID thread_id;
    std::string thread_name;
    base::TimeDelta sleep_time;
    base::TimeDelta unresponsive_time;
    uint32_t unresponsive_threshold;
    bool crash_on_hang;

    WatchingParams(const content::BrowserThread::ID& thread_id_in,
                   const std::string& thread_name_in,
                   const base::TimeDelta& sleep_time_in,
                   const base::TimeDelta& unresponsive_time_in,
                   uint32_t unresponsive_threshold_in,
                   bool crash_on_hang_in)
        : thread_id(thread_id_in),
          thread_name(thread_name_in),
          sleep_time(sleep_time_in),
          unresponsive_time(unresponsive_time_in),
          unresponsive_threshold(unresponsive_threshold_in),
          crash_on_hang(crash_on_hang_in) {}
  };

  virtual ~ThreadWatcher();

  // This method starts performing health check on the given |thread_id|. It
  // will create ThreadWatcher object for the given |thread_id|, |thread_name|.
  // |sleep_time| is the wait time between ping messages. |unresponsive_time| is
  // the wait time after ping message is sent, to check if we have received pong
  // message or not. |unresponsive_threshold| is used to determine if the thread
  // is responsive or not. The watched thread is considered unresponsive if it
  // hasn't responded with a pong message for |unresponsive_threshold| number of
  // ping messages. |crash_on_hang| specifies if browser should be crashed when
  // the watched thread is unresponsive. It will register that ThreadWatcher
  // object and activate the thread watching of the given thread_id.
  static void StartWatching(const WatchingParams& params);

  // Return the |thread_id_| of the thread being watched.
  content::BrowserThread::ID thread_id() const { return thread_id_; }

  // Return the name of the thread being watched.
  std::string thread_name() const { return thread_name_; }

  // Return the sleep time between ping messages to be sent to the thread.
  base::TimeDelta sleep_time() const { return sleep_time_; }

  // Return the the wait time to check the responsiveness of the thread.
  base::TimeDelta unresponsive_time() const { return unresponsive_time_; }

  // Returns true if we are montioring the thread.
  bool active() const { return active_; }

  // Returns |ping_time_| (used by unit tests).
  base::TimeTicks ping_time() const { return ping_time_; }

  // Returns |ping_sequence_number_| (used by unit tests).
  uint64_t ping_sequence_number() const { return ping_sequence_number_; }

 protected:
  // Construct a ThreadWatcher for the given |thread_id|. |sleep_time| is the
  // wait time between ping messages. |unresponsive_time| is the wait time after
  // ping message is sent, to check if we have received pong message or not.
  explicit ThreadWatcher(const WatchingParams& params);

  // This method activates the thread watching which starts ping/pong messaging.
  virtual void ActivateThreadWatching();

  // This method de-activates the thread watching and revokes all tasks.
  virtual void DeActivateThreadWatching();

  // This will ensure that the watching is actively taking place, and awaken
  // (i.e., post a PostPingMessage()) if the watcher has stopped pinging due to
  // lack of user activity. It will also reset |ping_count_| to
  // |unresponsive_threshold_|.
  virtual void WakeUp();

  // This method records when ping message was sent and it will Post a task
  // (OnPingMessage()) to the watched thread that does nothing but respond with
  // OnPongMessage(). It also posts a task (OnCheckResponsiveness()) to check
  // responsiveness of monitored thread that would be called after waiting
  // |unresponsive_time_|.
  // This method is accessible on WatchDogThread.
  virtual void PostPingMessage();

  // This method handles a Pong Message from watched thread. It will track the
  // response time (pong time minus ping time) via histograms. It posts a
  // PostPingMessage() task that would be called after waiting |sleep_time_|. It
  // increments |ping_sequence_number_| by 1.
  // This method is accessible on WatchDogThread.
  virtual void OnPongMessage(uint64_t ping_sequence_number);

  // This method will determine if the watched thread is responsive or not. If
  // the latest |ping_sequence_number_| is not same as the
  // |ping_sequence_number| that is passed in, then we can assume that watched
  // thread has responded with a pong message.
  // This method is accessible on WatchDogThread.
  virtual void OnCheckResponsiveness(uint64_t ping_sequence_number);

  // Set by OnCheckResponsiveness when it determines if the watched thread is
  // responsive or not.
  bool responsive_;

 private:
  friend class ThreadWatcherList;
  friend class CustomThreadWatcher;

  // Allow tests to access our innards for testing purposes.
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, Registration);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, ThreadResponding);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, ThreadNotResponding);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, MultipleThreadsResponding);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, MultipleThreadsNotResponding);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTestWithMockTime,
                           MemoryPressureCrashKey);

  // Post constructor initialization.
  void Initialize();

  // Watched thread does nothing except post callback_task to the WATCHDOG
  // Thread. This method is called on watched thread.
  static void OnPingMessage(const content::BrowserThread::ID& thread_id,
                            base::OnceClosure callback_task);

  // This method resets |unresponsive_count_| to zero because watched thread is
  // responding to the ping message with a pong message.
  void ResetHangCounters();

  // This method records watched thread is not responding to the ping message.
  // It increments |unresponsive_count_| by 1.
  void GotNoResponse();

  // Sets a crash key with the time since last critical memory pressure signal.
  void SetTimeSinceLastCriticalMemoryPressureCrashKey();

  // This method returns true if the watched thread has not responded with a
  // pong message for |unresponsive_threshold_| number of ping messages.
  bool IsVeryUnresponsive();

  // The |thread_id_| of the thread being watched. Only one instance can exist
  // for the given |thread_id_| of the thread being watched.
  const content::BrowserThread::ID thread_id_;

  // The name of the thread being watched.
  const std::string thread_name_;

  // Used to post messages to watched thread.
  const scoped_refptr<base::SingleThreadTaskRunner> watched_runner_;

  // It is the sleep time between the receipt of a pong message back, and the
  // sending of another ping message.
  const base::TimeDelta sleep_time_;

  // It is the duration from sending a ping message, until we check status to be
  // sure a pong message has been returned.
  const base::TimeDelta unresponsive_time_;

  // This is the last time when ping message was sent.
  base::TimeTicks ping_time_;

  // This is the last time when we got pong message.
  base::TimeTicks pong_time_;

  // This is the sequence number of the next ping for which there is no pong. If
  // the instance is sleeping, then it will be the sequence number for the next
  // ping.
  uint64_t ping_sequence_number_;

  // This is set to true if thread watcher is watching.
  bool active_;

  // The counter tracks least number of ping messages that will be sent to
  // watched thread before the ping-pong mechanism will go into an extended
  // sleep. If this value is zero, then the mechanism is in an extended sleep,
  // and awaiting some observed user action before continuing.
  int ping_count_;

  // Histogram that keeps track of response times for the watched thread.
  base::HistogramBase* response_time_histogram_;

  // Histogram that keeps track of unresponsive time since the last pong message
  // when we got no response (GotNoResponse()) from the watched thread.
  base::HistogramBase* unresponsive_time_histogram_;

  // Histogram that keeps track of how many threads are responding when we got
  // no response (GotNoResponse()) from the watched thread.
  base::HistogramBase* responsive_count_histogram_;

  // Histogram that keeps track of how many threads are not responding when we
  // got no response (GotNoResponse()) from the watched thread. Count includes
  // the thread that got no response.
  base::HistogramBase* unresponsive_count_histogram_;

  // This counter tracks the unresponsiveness of watched thread. If this value
  // is zero then watched thread has responded with a pong message. This is
  // incremented by 1 when we got no response (GotNoResponse()) from the watched
  // thread.
  uint32_t unresponsive_count_;

  // This is set to true when we would have crashed the browser because the
  // watched thread hasn't responded at least |unresponsive_threshold_| times.
  // It is reset to false when watched thread responds with a pong message.
  bool hung_processing_complete_;

  // This is used to determine if the watched thread is responsive or not. If
  // watched thread's |unresponsive_count_| is greater than or equal to
  // |unresponsive_threshold_| then we would consider it as unresponsive.
  uint32_t unresponsive_threshold_;

  // This is set to true if we want to crash the browser when the watched thread
  // has become sufficiently unresponsive, while other threads are sufficiently
  // responsive.
  bool crash_on_hang_;

  // The last time at which a critical memory pressure signal was received, or
  // null if no signal was ever received. Maintained by ThreadWatcherList.
  base::TimeTicks last_critical_memory_pressure_;

  // We use this factory to create callback tasks for ThreadWatcher object. We
  // use this during ping-pong messaging between WatchDog thread and watched
  // thread.
  base::WeakPtrFactory<ThreadWatcher> weak_ptr_factory_{this};

  DISALLOW_COPY_AND_ASSIGN(ThreadWatcher);
};

// Class with a list of all active thread watchers.  A thread watcher is active
// if it has been registered, which includes determing the histogram name. This
// class provides utility functions to start and stop watching all browser
// threads. Only one instance of this class exists.
class ThreadWatcherList {
 public:
  // A map from BrowserThread to the actual instances.
  typedef std::map<content::BrowserThread::ID, ThreadWatcher*> RegistrationList;

  // A map from thread names (UI/IO) to |UnresponsiveCountThreshold|.
  //
  // UnresponsiveCountThreshold specifies the number of unanswered ping messages
  // after which watched (UI/IO) thread is considered as not responsive. We
  // translate "time" (given in seconds) into a number of pings. As a result, we
  // only declare a thread unresponsive when a lot of "time" has passed (many
  // pings), and yet our pinging thread has continued to process messages (so we
  // know the entire PC is not hung). Set this number higher to crash less
  // often, and lower to crash more often.
  //
  // The map lists all threads (by name) that can induce a crash by hanging. It
  // is populated from the command line, or given a default list.  See
  // InitializeAndStartWatching() for the separate list of all threads that are
  // watched, as they provide the system context of how hung *other* threads
  // are.
  //
  // ThreadWatcher monitors UI and IO browser threads.
  //
  // The example command line argument consists of "UI:18,IO:18". In that
  // string, the first parameter specifies the thread_id: UI or IO. The second
  // parameter specifies the unresponsive threshold seconds. This number is used
  // to calculate the UnresponsiveCountThreshold. In this example for UI and IO
  // threads, we would crash if those threads don't respond for 18 seconds (i.e.
  // 9 unanswered ping messages with a 2 seconds timeout).
  using UnresponsiveCountThreshold = uint32_t;
  typedef std::map<std::string, UnresponsiveCountThreshold>
      CrashOnHangThreadMap;

  // This method posts a task on WatchDogThread to start watching all browser
  // threads.
  // This method is accessible on UI thread.
  static void StartWatchingAll(const base::CommandLine& command_line);

  // This method posts a task on WatchDogThread to RevokeAll tasks and to
  // deactive thread watching of other threads and tell NotificationService to
  // stop calling Observe.
  // This method is accessible on UI thread.
  static void StopWatchingAll();

  // Register() stores a pointer to the given ThreadWatcher in a global map.
  // Returns the pointer if it was successfully registered, null otherwise.
  static ThreadWatcher* Register(std::unique_ptr<ThreadWatcher> watcher);

  // This method returns number of responsive and unresponsive watched threads.
  static void GetStatusOfThreads(uint32_t* responding_thread_count,
                                 uint32_t* unresponding_thread_count);

  // This will ensure that the watching is actively taking place, and awaken
  // all thread watchers that are registered.
  static void WakeUpAll();

 private:
  // Allow tests to access our innards for testing purposes.
  friend class CustomThreadWatcher;
  friend class ThreadWatcherListTest;
  friend class ThreadWatcherTest;
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherAndroidTest,
                           ApplicationStatusNotification);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherListTest, Restart);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, ThreadNamesOnlyArgs);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherTest, CrashOnHangThreadsAllArgs);
  FRIEND_TEST_ALL_PREFIXES(ThreadWatcherCrashKeyTest, MemoryPressureCrashKey);

  // This singleton holds the global list of registered ThreadWatchers.
  ThreadWatcherList();

  // Destructor deletes all registered ThreadWatcher instances.
  virtual ~ThreadWatcherList();

  // Parses the command line to get |crash_on_hang_threads| map from
  // switches::kCrashOnHangThreads. |crash_on_hang_threads| is a map of
  // |crash_on_hang| thread's names to |CrashDataThresholds|.
  static void ParseCommandLine(const base::CommandLine& command_line,
                               uint32_t* unresponsive_threshold,
                               CrashOnHangThreadMap* crash_on_hang_threads);

  // Parses the argument |crash_on_hang_thread_names| and creates
  // |crash_on_hang_threads| map of |crash_on_hang| thread's names to
  // |CrashDataThresholds|. If |crash_on_hang_thread_names| doesn't specify
  // |crash_seconds|, then it uses |default_crash_seconds| as the value.
  static void ParseCommandLineCrashOnHangThreads(
      const std::string& crash_on_hang_thread_names,
      uint32_t default_crash_seconds,
      CrashOnHangThreadMap* crash_on_hang_threads);

  // This constructs the |ThreadWatcherList| singleton and starts watching
  // browser threads by calling StartWatching() on each browser thread that is
  // watched.
  static void InitializeAndStartWatching(
      uint32_t unresponsive_threshold,
      const CrashOnHangThreadMap& crash_on_hang_threads);

  // This method calls ThreadWatcher::StartWatching() to perform health check on
  // the given |thread_id|.
  static void StartWatching(const content::BrowserThread::ID& thread_id,
                            const std::string& thread_name,
                            const base::TimeDelta& sleep_time,
                            const base::TimeDelta& unresponsive_time,
                            uint32_t unresponsive_threshold,
                            const CrashOnHangThreadMap& crash_on_hang_threads);

  // Delete all thread watcher objects and remove them from global map. It also
  // deletes |g_thread_watcher_list_|.
  static void DeleteAll();

  // The Find() method can be used to test to see if a given ThreadWatcher was
  // already registered, or to retrieve a pointer to it from the global map.
  static ThreadWatcher* Find(const content::BrowserThread::ID& thread_id);

  // Sets |g_stopped_| on the WatchDogThread. This is necessary to reflect the
  // state between the delayed |StartWatchingAll| and the immediate
  // |StopWatchingAll|.
  static void SetStopped(bool stopped);

  // Invoked on memory pressure signal.
  void OnMemoryPressure(
      base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level);

  // The singleton of this class and is used to keep track of information about
  // threads that are being watched.
  static ThreadWatcherList* g_thread_watcher_list_;

  // StartWatchingAll() is delayed in relation to StopWatchingAll(), so if
  // a Stop comes first, prevent further initialization.
  static bool g_stopped_;

  // This is the wait time between ping messages.
  static const int kSleepSeconds;

  // This is the wait time after ping message is sent, to check if we have
  // received pong message or not.
  static const int kUnresponsiveSeconds;

  // Default values for |unresponsive_threshold|.
  static const int kUnresponsiveCount;

  // Default value for the delay until |InitializeAndStartWatching| is called.
  // Non-const for tests.
  static int g_initialize_delay_seconds;

  // Registration to receive memory pressure signals.
  base::MemoryPressureListener memory_pressure_listener_;

  // Map of all registered watched threads, from thread_id to ThreadWatcher.
  RegistrationList registered_;

  DISALLOW_COPY_AND_ASSIGN(ThreadWatcherList);
};

// Class for WatchDogThread and in its Init method, we start watching UI, IO,
// DB, FILE, CACHED threads.
class WatchDogThread : public base::Thread {
 public:
  // Constructor.
  WatchDogThread();

  // Destroys the thread and stops the thread.
  ~WatchDogThread() override;

  // Callable on any thread.  Returns whether you're currently on a
  // WatchDogThread.
  static bool CurrentlyOnWatchDogThread();

  // These are the same methods in message_loop.h, but are guaranteed to either
  // get posted to the MessageLoop if it's still alive, or be deleted otherwise.
  // They return true iff the watchdog thread existed and the task was posted.
  // Note that even if the task is posted, there's no guarantee that it will
  // run, since the target thread may already have a Quit message in its queue.
  static bool PostTask(const base::Location& from_here, base::OnceClosure task);
  static bool PostDelayedTask(const base::Location& from_here,
                              base::OnceClosure task,
                              base::TimeDelta delay);

 protected:
  void Init() override;
  void CleanUp() override;

 private:
  // This method returns true if Init() is called.
  bool Started() const;

  static bool PostTaskHelper(const base::Location& from_here,
                             base::OnceClosure task,
                             base::TimeDelta delay);

  DISALLOW_COPY_AND_ASSIGN(WatchDogThread);
};

// ShutdownWatcherHelper is useless on Android because there is no shutdown,
// Chrome is always killed one way or another (swiped away in the task
// switcher, OOM-killed, etc.).
#if !defined(OS_ANDROID)
// This is a wrapper class for detecting hangs during shutdown.
class ShutdownWatcherHelper {
 public:
  // Create an empty holder for |shutdown_watchdog_|.
  ShutdownWatcherHelper();

  // Destructor disarm's shutdown_watchdog_ so that alarm doesn't go off.
  ~ShutdownWatcherHelper();

  // Constructs ShutdownWatchDogThread which spawns a thread and starts timer.
  // |duration| specifies how long it will wait before it calls alarm.
  void Arm(const base::TimeDelta& duration);

 private:
  // shutdown_watchdog_ watches for hangs during shutdown.
  base::Watchdog* shutdown_watchdog_;

  // The |thread_id_| on which this object is constructed.
  const base::PlatformThreadId thread_id_;

  DISALLOW_COPY_AND_ASSIGN(ShutdownWatcherHelper);
};

#endif  // !defined(OS_ANDROID)

#endif  // CHROME_BROWSER_METRICS_THREAD_WATCHER_H_