15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2012 The Chromium Authors. All rights reserved.
25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be
35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// found in the LICENSE file.
45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/browser/metrics/thread_watcher.h"
65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <math.h>  // ceil
85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/bind.h"
105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/compiler_specific.h"
115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/debug/alias.h"
126e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)#include "base/debug/debugger.h"
135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/debug/dump_without_crashing.h"
145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/lazy_instance.h"
15cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)#include "base/metrics/field_trial.h"
162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_number_conversions.h"
172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_split.h"
182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_tokenizer.h"
19868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)#include "base/strings/stringprintf.h"
205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/threading/thread_restrictions.h"
215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "build/build_config.h"
22cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)#include "chrome/browser/chrome_notification_types.h"
235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/common/chrome_switches.h"
245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/common/chrome_version_info.h"
255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/common/logging_chrome.h"
26cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)#include "content/public/browser/notification_service.h"
275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(OS_WIN)
295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/win/windows_version.h"
305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif
315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using content::BrowserThread;
335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace {
355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The following are unique function names for forcing the crash when a thread
375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// is unresponsive. This makes it possible to tell from the callstack alone what
385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// thread was unresponsive.
395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We disable optimizations for this block of functions so the compiler doesn't
415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// merge them all together.
425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)MSVC_DISABLE_OPTIMIZE()
435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)MSVC_PUSH_DISABLE_WARNING(4748)
445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
456e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)void ReportThreadHang() {
466e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)#if defined(NDEBUG)
475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)  base::debug::DumpWithoutCrashing();
486e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)#else
496e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  base::debug::BreakDebugger();
502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#endif
512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)}
522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
53010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#if !defined(OS_ANDROID) || !defined(NDEBUG)
54010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)// TODO(rtenneti): Enabled crashing, after getting data.
556e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)NOINLINE void StartupHang() {
566e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
57010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)}
58010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#endif  // OS_ANDROID
59010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)
606e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)NOINLINE void ShutdownHang() {
616e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
62c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)}
63c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_UI() {
656e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_DB() {
696e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_FILE() {
736e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_FILE_USER_BLOCKING() {
776e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_PROCESS_LAUNCHER() {
816e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_CACHE() {
856e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NOINLINE void ThreadUnresponsive_IO() {
896e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)  ReportThreadHang();
905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)MSVC_POP_WARNING()
935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)MSVC_ENABLE_OPTIMIZE();
945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void CrashBecauseThreadWasUnresponsive(BrowserThread::ID thread_id) {
965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::debug::Alias(&thread_id);
975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  switch (thread_id) {
995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::UI:
1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_UI();
1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::DB:
1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_DB();
1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::FILE:
1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_FILE();
1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::FILE_USER_BLOCKING:
1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_FILE_USER_BLOCKING();
1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::PROCESS_LAUNCHER:
1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_PROCESS_LAUNCHER();
1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::CACHE:
1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_CACHE();
1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::IO:
1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return ThreadUnresponsive_IO();
1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    case BrowserThread::ID_COUNT:
1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      CHECK(false);  // This shouldn't actually be reached!
1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      break;
1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    // Omission of the default hander is intentional -- that way the compiler
1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    // should warn if our switch becomes outdated.
1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  CHECK(false) << "Unknown thread was unresponsive.";  // Shouldn't be reached.
1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}  // namespace
1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ThreadWatcher methods and members.
1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcher::ThreadWatcher(const WatchingParams& params)
1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    : thread_id_(params.thread_id),
1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      thread_name_(params.thread_name),
1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      watched_loop_(
1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      sleep_time_(params.sleep_time),
1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_time_(params.unresponsive_time),
1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ping_time_(base::TimeTicks::Now()),
1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      pong_time_(ping_time_),
1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ping_sequence_number_(0),
1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      active_(false),
1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ping_count_(params.unresponsive_threshold),
1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      response_time_histogram_(NULL),
1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_time_histogram_(NULL),
1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_count_(0),
1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      hung_processing_complete_(false),
1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_threshold_(params.unresponsive_threshold),
1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      crash_on_hang_(params.crash_on_hang),
1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      live_threads_threshold_(params.live_threads_threshold),
146c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      weak_ptr_factory_(this) {
1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Initialize();
1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcher::~ThreadWatcher() {}
1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::StartWatching(const WatchingParams& params) {
1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK_GE(params.unresponsive_time.InMilliseconds(),
1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)            params.sleep_time.InMilliseconds());
1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If we are not on WatchDogThread, then post a task to call StartWatching on
1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // WatchDogThread.
1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    WatchDogThread::PostTask(
1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)        FROM_HERE,
1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)        base::Bind(&ThreadWatcher::StartWatching, params));
1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Create a new thread watcher object for the given thread and activate it.
1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcher* watcher = new ThreadWatcher(params);
1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(watcher);
1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If we couldn't register the thread watcher object, we are shutting down,
1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // then don't activate thread watching.
1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!ThreadWatcherList::IsRegistered(params.thread_id))
1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  watcher->ActivateThreadWatching();
1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::ActivateThreadWatching() {
1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (active_) return;
1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  active_ = true;
1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ping_count_ = unresponsive_threshold_;
1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ResetHangCounters();
18790dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  base::MessageLoop::current()->PostTask(
1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      FROM_HERE,
1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcher::PostPingMessage,
1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                 weak_ptr_factory_.GetWeakPtr()));
1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::DeActivateThreadWatching() {
1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  active_ = false;
1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ping_count_ = 0;
1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  weak_ptr_factory_.InvalidateWeakPtrs();
1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::WakeUp() {
2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // There is some user activity, PostPingMessage task of thread watcher if
2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // needed.
2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!active_) return;
2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Throw away the previous |unresponsive_count_| and start over again. Just
2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // before going to sleep, |unresponsive_count_| could be very close to
2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // |unresponsive_threshold_| and when user becomes active,
2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // response for ping messages. Reset |unresponsive_count_| to start measuring
2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // the unresponsiveness of the threads when system becomes active.
2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_count_ = 0;
2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (ping_count_ <= 0) {
2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    ping_count_ = unresponsive_threshold_;
2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    ResetHangCounters();
2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    PostPingMessage();
2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  } else {
2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    ping_count_ = unresponsive_threshold_;
2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::PostPingMessage() {
2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If we have stopped watching or if the user is idle, then stop sending
2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // ping messages.
2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!active_ || ping_count_ <= 0)
2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Save the current time when we have sent ping message.
2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ping_time_ = base::TimeTicks::Now();
2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Send a ping message to the watched thread. Callback will be called on
2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // the WatchDogThread.
2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::Closure callback(
2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                 ping_sequence_number_));
2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (watched_loop_->PostTask(
2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          FROM_HERE,
2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                     callback))) {
2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      // Post a task to check the responsiveness of watched thread.
24390dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)      base::MessageLoop::current()->PostDelayedTask(
2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          FROM_HERE,
2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          base::Bind(&ThreadWatcher::OnCheckResponsiveness,
2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                     weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)          unresponsive_time_);
2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  } else {
2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    // Watched thread might have gone away, stop watching it.
2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    DeActivateThreadWatching();
2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record watched thread's response time.
2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::TimeTicks now = base::TimeTicks::Now();
2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::TimeDelta response_time = now - ping_time_;
2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  response_time_histogram_->AddTime(response_time);
2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Save the current time when we have got pong message.
2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  pong_time_ = now;
2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Check if there are any extra pings in flight.
2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (ping_sequence_number_ != ping_sequence_number)
2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Increment sequence number for the next ping message to indicate watched
2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // thread is responsive.
2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ++ping_sequence_number_;
2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If we have stopped watching or if the user is idle, then stop sending
2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // ping messages.
2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!active_ || --ping_count_ <= 0)
2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
27990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  base::MessageLoop::current()->PostDelayedTask(
2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      FROM_HERE,
2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcher::PostPingMessage,
2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                 weak_ptr_factory_.GetWeakPtr()),
2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      sleep_time_);
2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If we have stopped watching then consider thread as responding.
2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!active_) {
2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    responsive_ = true;
2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // If the latest ping_sequence_number_ is not same as the ping_sequence_number
2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // that is passed in, then we can assume OnPongMessage was called.
2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // OnPongMessage increments ping_sequence_number_.
2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (ping_sequence_number_ != ping_sequence_number) {
2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    // Reset unresponsive_count_ to zero because we got a response from the
2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    // watched thread.
2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    ResetHangCounters();
3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    responsive_ = true;
3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record that we got no response from watched thread.
3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  GotNoResponse();
3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Post a task to check the responsiveness of watched thread.
30890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  base::MessageLoop::current()->PostDelayedTask(
3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      FROM_HERE,
3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcher::OnCheckResponsiveness,
3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                 weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_time_);
3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  responsive_ = false;
3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::Initialize() {
3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherList::Register(this);
3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const std::string response_time_histogram_name =
3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      "ThreadWatcher.ResponseTime." + thread_name_;
3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  response_time_histogram_ = base::Histogram::FactoryTimeGet(
3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      response_time_histogram_name,
3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromMilliseconds(1),
3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromSeconds(100), 50,
3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Histogram::kUmaTargetedHistogramFlag);
3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const std::string unresponsive_time_histogram_name =
3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      "ThreadWatcher.Unresponsive." + thread_name_;
3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_time_histogram_name,
3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromMilliseconds(1),
3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromSeconds(100), 50,
3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Histogram::kUmaTargetedHistogramFlag);
3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const std::string responsive_count_histogram_name =
3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      "ThreadWatcher.ResponsiveThreads." + thread_name_;
3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      responsive_count_histogram_name, 1, 10, 11,
3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Histogram::kUmaTargetedHistogramFlag);
3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const std::string unresponsive_count_histogram_name =
3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      "ThreadWatcher.UnresponsiveThreads." + thread_name_;
3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      unresponsive_count_histogram_name, 1, 10, 11,
3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Histogram::kUmaTargetedHistogramFlag);
3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                  const base::Closure& callback_task) {
3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // This method is called on watched thread.
3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(BrowserThread::CurrentlyOn(thread_id));
3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WatchDogThread::PostTask(FROM_HERE, callback_task);
3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::ResetHangCounters() {
3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_count_ = 0;
3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  hung_processing_complete_ = false;
3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcher::GotNoResponse() {
3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ++unresponsive_count_;
3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!IsVeryUnresponsive())
3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record total unresponsive_time since last pong message.
3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_time_histogram_->AddTime(unresponse_time);
3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // We have already collected stats for the non-responding watched thread.
3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (hung_processing_complete_)
3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record how other threads are responding.
3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  uint32 responding_thread_count = 0;
3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  uint32 unresponding_thread_count = 0;
3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                        &unresponding_thread_count);
3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record how many watched threads are responding.
3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  responsive_count_histogram_->Add(responding_thread_count);
3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Record how many watched threads are not responding.
3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  unresponsive_count_histogram_->Add(unresponding_thread_count);
3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Crash the browser if the watched thread is to be crashed on hang and if the
3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // number of other threads responding is less than or equal to
3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // live_threads_threshold_ and at least one other thread is responding.
3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (crash_on_hang_ &&
3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      responding_thread_count > 0 &&
3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      responding_thread_count <= live_threads_threshold_) {
3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    static bool crashed_once = false;
3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    if (!crashed_once) {
3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      crashed_once = true;
3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      CrashBecauseThreadWasUnresponsive(thread_id_);
4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    }
4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  hung_processing_complete_ = true;
4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool ThreadWatcher::IsVeryUnresponsive() {
4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return unresponsive_count_ >= unresponsive_threshold_;
4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ThreadWatcherList methods and members.
4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
416a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)bool ThreadWatcherList::g_stopped_ = false;
417a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// static
4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const int ThreadWatcherList::kSleepSeconds = 1;
4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const int ThreadWatcherList::kUnresponsiveSeconds = 2;
4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
42258e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdochconst int ThreadWatcherList::kUnresponsiveCount = 9;
4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
424c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)const int ThreadWatcherList::kLiveThreadsThreshold = 2;
425a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// static, non-const for tests.
426a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)int ThreadWatcherList::g_initialize_delay_seconds = 120;
4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
4292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 live_threads_threshold,
4302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 unresponsive_threshold)
4312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    : live_threads_threshold(live_threads_threshold),
4322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      unresponsive_threshold(unresponsive_threshold) {
4332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)}
4342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
4352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
4362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    : live_threads_threshold(kLiveThreadsThreshold),
4372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      unresponsive_threshold(kUnresponsiveCount) {
4382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)}
4392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
4424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  // TODO(rtenneti): Enable ThreadWatcher.
4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  uint32 unresponsive_threshold;
4442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  CrashOnHangThreadMap crash_on_hang_threads;
4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ParseCommandLine(command_line,
4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                   &unresponsive_threshold,
4472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                   &crash_on_hang_threads);
4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherObserver::SetupNotifications(
4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
452a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  WatchDogThread::PostTask(
453a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      FROM_HERE,
454a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      base::Bind(&ThreadWatcherList::SetStopped, false));
455a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WatchDogThread::PostDelayedTask(
4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      FROM_HERE,
4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                 unresponsive_threshold,
4602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                 crash_on_hang_threads),
461a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      base::TimeDelta::FromSeconds(g_initialize_delay_seconds));
4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::StopWatchingAll() {
4664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  // TODO(rtenneti): Enable ThreadWatcher.
4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherObserver::RemoveNotifications();
4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DeleteAll();
4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::Register(ThreadWatcher* watcher) {
4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_list_)
4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return NULL != ThreadWatcherList::Find(thread_id);
4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                           uint32* unresponding_thread_count) {
4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  *responding_thread_count = 0;
4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  *unresponding_thread_count = 0;
4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_list_)
4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  for (RegistrationList::iterator it =
4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)           g_thread_watcher_list_->registered_.begin();
4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)       g_thread_watcher_list_->registered_.end() != it;
4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)       ++it) {
4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    if (it->second->IsVeryUnresponsive())
5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ++(*unresponding_thread_count);
5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    else
5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ++(*responding_thread_count);
5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::WakeUpAll() {
5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_list_)
5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  for (RegistrationList::iterator it =
5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)           g_thread_watcher_list_->registered_.begin();
5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)       g_thread_watcher_list_->registered_.end() != it;
5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)       ++it)
5165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    it->second->WakeUp();
5175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherList::ThreadWatcherList() {
5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  CHECK(!g_thread_watcher_list_);
5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_list_ = this;
5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherList::~ThreadWatcherList() {
5265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(this == g_thread_watcher_list_);
5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_list_ = NULL;
5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::ParseCommandLine(
5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const CommandLine& command_line,
5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    uint32* unresponsive_threshold,
5352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    CrashOnHangThreadMap* crash_on_hang_threads) {
5362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  // Initialize |unresponsive_threshold| to a default value.
5371320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  // TODO(rtenneti): Changed the default value to 4 times, until we can triage
5381320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  // hangs automatically (and to reduce the crash dumps).
5391320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  *unresponsive_threshold = kUnresponsiveCount * 4;
5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Increase the unresponsive_threshold on the Stable and Beta channels to
5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // reduce the number of crashes due to ThreadWatcher.
5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    *unresponsive_threshold *= 4;
5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    *unresponsive_threshold *= 2;
5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(OS_WIN)
5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // For Windows XP (old systems), double the unresponsive_threshold to give
5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // the OS a chance to schedule UI/IO threads a time slice to respond with a
5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // pong message (to get around limitations with the OS).
5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (base::win::GetVersion() <= base::win::VERSION_XP)
5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    *unresponsive_threshold *= 2;
5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif
5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
5592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  std::string crash_on_hang_thread_names;
5602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  bool has_command_line_overwrite = false;
5612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
5622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    crash_on_hang_thread_names =
5632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
5642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    has_command_line_overwrite = true;
5652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
5662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    // Default to crashing the browser if UI or IO or FILE threads are not
5672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    // responsive except in stable channel.
5682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    crash_on_hang_thread_names = base::StringPrintf(
5692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
5702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        kLiveThreadsThreshold, crash_seconds,
5712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        kLiveThreadsThreshold, crash_seconds,
5722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        kLiveThreadsThreshold, crash_seconds * 5);
5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
5762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                                     kLiveThreadsThreshold,
5772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                                     crash_seconds,
5782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                                     crash_on_hang_threads);
5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  if (channel != chrome::VersionInfo::CHANNEL_CANARY ||
5812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      has_command_line_overwrite) {
5822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    return;
5832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  }
5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
5850529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  const char* kFieldTrialName = "ThreadWatcher";
5860529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch
5870529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  // Nothing else to be done if the trial has already been set (i.e., when
5880529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  // StartWatchingAll() has been already called once).
5890529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  if (base::FieldTrialList::TrialExists(kFieldTrialName))
5900529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    return;
5910529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch
5922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  // Set up a field trial for 100% of the users to crash if either UI or IO
5932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  // thread is not responsive for 30 seconds (or 15 pings).
5942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  scoped_refptr<base::FieldTrial> field_trial(
5952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      base::FieldTrialList::FactoryGetFieldTrial(
5960529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch          kFieldTrialName, 100, "default_hung_threads",
5974e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)          2014, 10, 30, base::FieldTrial::SESSION_RANDOMIZED, NULL));
5982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  int hung_thread_group = field_trial->AppendGroup("hung_thread", 100);
5992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  if (field_trial->group() == hung_thread_group) {
6002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    for (CrashOnHangThreadMap::iterator it = crash_on_hang_threads->begin();
6012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)         crash_on_hang_threads->end() != it;
6022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)         ++it) {
6034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      if (it->first == "FILE")
6042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        continue;
6052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      it->second.live_threads_threshold = INT_MAX;
6064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      if (it->first == "UI") {
6074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        // TODO(rtenneti): set unresponsive threshold to 120 seconds to catch
6084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        // the worst UI hangs and for fewer crashes due to ThreadWatcher. Reduce
6094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        // it to a more reasonable time ala IO thread.
6104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        it->second.unresponsive_threshold = 60;
6114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      } else {
6124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)        it->second.unresponsive_threshold = 15;
6134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      }
6142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    }
6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
6162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)}
6172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
6182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)// static
6192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
6202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    const std::string& crash_on_hang_thread_names,
6212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 default_live_threads_threshold,
6222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 default_crash_seconds,
6232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    CrashOnHangThreadMap* crash_on_hang_threads) {
6242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
6252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  std::vector<std::string> values;
6262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  while (tokens.GetNext()) {
6272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    const std::string& token = tokens.token();
6282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    base::SplitString(token, ':', &values);
6292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    std::string thread_name = values[0];
6302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
6312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 live_threads_threshold = default_live_threads_threshold;
6322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 crash_seconds = default_crash_seconds;
6332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    if (values.size() >= 2 &&
6342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        (!base::StringToUint(values[1], &live_threads_threshold))) {
6352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      continue;
6362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    }
6372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    if (values.size() >= 3 &&
6382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        (!base::StringToUint(values[2], &crash_seconds))) {
6392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      continue;
6402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    }
6412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    uint32 unresponsive_threshold = static_cast<uint32>(
6422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
6432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
6442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    CrashDataThresholds crash_data(live_threads_threshold,
6452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                                   unresponsive_threshold);
6462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    // Use the last specifier.
6472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    (*crash_on_hang_threads)[thread_name] = crash_data;
6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
6495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
6505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
6515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::InitializeAndStartWatching(
6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    uint32 unresponsive_threshold,
6542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    const CrashOnHangThreadMap& crash_on_hang_threads) {
6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
657f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  // Disarm the startup timebomb, even if stop has been called.
658f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  BrowserThread::PostTask(
659f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      BrowserThread::UI,
660f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      FROM_HERE,
661f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
662f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
663a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  // This method is deferred in relationship to its StopWatchingAll()
664a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  // counterpart. If a previous initialization has already happened, or if
665a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  // stop has been called, there's nothing left to do here.
666a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  if (g_thread_watcher_list_ || g_stopped_)
667a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    return;
668a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  CHECK(thread_watcher_list);
6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const base::TimeDelta kSleepTime =
6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromSeconds(kSleepSeconds);
6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  const base::TimeDelta kUnresponsiveTime =
6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
6782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                unresponsive_threshold, crash_on_hang_threads);
6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
6802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                unresponsive_threshold, crash_on_hang_threads);
6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
6822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                unresponsive_threshold, crash_on_hang_threads);
6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
6842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                unresponsive_threshold, crash_on_hang_threads);
6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
6862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)                unresponsive_threshold, crash_on_hang_threads);
6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::StartWatching(
6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const BrowserThread::ID& thread_id,
6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const std::string& thread_name,
6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const base::TimeDelta& sleep_time,
6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const base::TimeDelta& unresponsive_time,
6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    uint32 unresponsive_threshold,
6962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    const CrashOnHangThreadMap& crash_on_hang_threads) {
6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
6992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  CrashOnHangThreadMap::const_iterator it =
7002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      crash_on_hang_threads.find(thread_name);
7012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  bool crash_on_hang = false;
7022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  uint32 live_threads_threshold = 0;
7032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  if (it != crash_on_hang_threads.end()) {
7042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    crash_on_hang = true;
7052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    live_threads_threshold = it->second.live_threads_threshold;
7062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    unresponsive_threshold = it->second.unresponsive_threshold;
7072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  }
7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcher::StartWatching(
7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      ThreadWatcher::WatchingParams(thread_id,
7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    thread_name,
7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    sleep_time,
7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    unresponsive_time,
7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    unresponsive_threshold,
7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    crash_on_hang,
7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                    live_threads_threshold));
7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherList::DeleteAll() {
7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    WatchDogThread::PostTask(
7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)        FROM_HERE,
7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)        base::Bind(&ThreadWatcherList::DeleteAll));
7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
7265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
7275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
729a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
730a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  SetStopped(true);
731a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
7325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_list_)
7335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
7345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Delete all thread watcher objects.
7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  while (!g_thread_watcher_list_->registered_.empty()) {
7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    delete it->second;
7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    g_thread_watcher_list_->registered_.erase(it);
7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  delete g_thread_watcher_list_;
7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_list_)
7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return NULL;
7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  RegistrationList::iterator it =
7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      g_thread_watcher_list_->registered_.find(thread_id);
7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (g_thread_watcher_list_->registered_.end() == it)
7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return NULL;
7545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return it->second;
7555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
7565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
757a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// static
758a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void ThreadWatcherList::SetStopped(bool stopped) {
759a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
760a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  g_stopped_ = stopped;
761a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)}
762a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ThreadWatcherObserver methods and members.
7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherObserver::ThreadWatcherObserver(
7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const base::TimeDelta& wakeup_interval)
7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    : last_wakeup_time_(base::TimeTicks::Now()),
7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      wakeup_interval_(wakeup_interval) {
7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  CHECK(!g_thread_watcher_observer_);
7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_observer_ = this;
7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ThreadWatcherObserver::~ThreadWatcherObserver() {
7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(this == g_thread_watcher_observer_);
7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_observer_ = NULL;
7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherObserver::SetupNotifications(
7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const base::TimeDelta& wakeup_interval) {
7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
786cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(
787cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      observer,
788cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      chrome::NOTIFICATION_BROWSER_OPENED,
789cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      content::NotificationService::AllBrowserContextsAndSources());
790cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
791cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           chrome::NOTIFICATION_BROWSER_CLOSED,
792cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
793cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
794cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           chrome::NOTIFICATION_TAB_PARENTED,
795cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
796cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
797cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           chrome::NOTIFICATION_TAB_CLOSING,
798cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
799cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
800cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NOTIFICATION_LOAD_START,
801cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
802cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
803cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NOTIFICATION_LOAD_STOP,
804cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
805cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
806cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
807cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
808cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
809cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
810cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
811cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  observer->registrar_.Add(observer,
812cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
813cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           content::NotificationService::AllSources());
8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherObserver::RemoveNotifications() {
8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!g_thread_watcher_observer_)
8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_thread_watcher_observer_->registrar_.RemoveAll();
8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  delete g_thread_watcher_observer_;
8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void ThreadWatcherObserver::Observe(
8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    int type,
8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const content::NotificationSource& source,
8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const content::NotificationDetails& details) {
8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // There is some user activity, see if thread watchers are to be awakened.
8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::TimeTicks now = base::TimeTicks::Now();
8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if ((now - last_wakeup_time_) < wakeup_interval_)
8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return;
8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  last_wakeup_time_ = now;
8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WatchDogThread::PostTask(
8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      FROM_HERE,
8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      base::Bind(&ThreadWatcherList::WakeUpAll));
8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// WatchDogThread methods and members.
8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This lock protects g_watchdog_thread.
8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static base::LazyInstance<base::Lock>::Leaky
8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The singleton of this class.
8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static WatchDogThread* g_watchdog_thread = NULL;
8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)WatchDogThread::~WatchDogThread() {
8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Stop();
8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool WatchDogThread::CurrentlyOnWatchDogThread() {
8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::AutoLock lock(g_watchdog_lock.Get());
8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return g_watchdog_thread &&
85990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)      g_watchdog_thread->message_loop() == base::MessageLoop::current();
8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                              const base::Closure& task) {
8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return PostTaskHelper(from_here, task, base::TimeDelta());
8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                     const base::Closure& task,
8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)                                     base::TimeDelta delay) {
8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return PostTaskHelper(from_here, task, delay);
8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool WatchDogThread::PostTaskHelper(
8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const tracked_objects::Location& from_here,
8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    const base::Closure& task,
8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    base::TimeDelta delay) {
8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  {
8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    base::AutoLock lock(g_watchdog_lock.Get());
8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
88390dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)    base::MessageLoop* message_loop = g_watchdog_thread ?
8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)        g_watchdog_thread->message_loop() : NULL;
8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    if (message_loop) {
8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      message_loop->PostDelayedTask(from_here, task, delay);
8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      return true;
8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    }
8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return false;
8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void WatchDogThread::Init() {
8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // This thread shouldn't be allowed to perform any blocking disk I/O.
8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::ThreadRestrictions::SetIOAllowed(false);
8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::AutoLock lock(g_watchdog_lock.Get());
8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  CHECK(!g_watchdog_thread);
9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_watchdog_thread = this;
9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void WatchDogThread::CleanUp() {
9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  base::AutoLock lock(g_watchdog_lock.Get());
9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  g_watchdog_thread = NULL;
9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace {
9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// StartupWatchDogThread methods and members.
9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Class for detecting hangs during startup.
9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class StartupWatchDogThread : public base::Watchdog {
9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public:
9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Constructor specifies how long the StartupWatchDogThread will wait before
9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // alarming.
9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  explicit StartupWatchDogThread(const base::TimeDelta& duration)
9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      : base::Watchdog(duration, "Startup watchdog thread", true) {
919010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#if defined(OS_ANDROID)
920010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    // TODO(rtenneti): Delete this code, after getting data.
921010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    start_time_clock_= base::Time::Now();
922010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    start_time_monotonic_ = base::TimeTicks::Now();
923010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    start_time_thread_now_ = base::TimeTicks::IsThreadNowSupported()
924010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)        ? base::TimeTicks::ThreadNow() : base::TimeTicks::Now();
925010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#endif  // OS_ANDROID
9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Alarm is called if the time expires after an Arm() without someone calling
9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Disarm(). When Alarm goes off, in release mode we get the crash dump
9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // without crashing and in debug mode we break into the debugger.
9312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  virtual void Alarm() OVERRIDE {
932010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#if !defined(NDEBUG)
9336e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)    StartupHang();
934010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    return;
935010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#elif !defined(OS_ANDROID)
9366e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)    WatchDogThread::PostTask(FROM_HERE, base::Bind(&StartupHang));
937010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    return;
938010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#else  // Android release: gather stats to figure out when to crash.
939010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    // TODO(rtenneti): Delete this code, after getting data.
940cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    UMA_HISTOGRAM_TIMES("StartupTimeBomb.Alarm.TimeDuration",
941010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)                        base::Time::Now() - start_time_clock_);
942cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    UMA_HISTOGRAM_TIMES("StartupTimeBomb.Alarm.TimeTicksDuration",
943010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)                        base::TimeTicks::Now() - start_time_monotonic_);
944010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    if (base::TimeTicks::IsThreadNowSupported()) {
945010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)      UMA_HISTOGRAM_TIMES(
946cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          "StartupTimeBomb.Alarm.ThreadNowDuration",
947010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)          base::TimeTicks::ThreadNow() - start_time_thread_now_);
948010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    }
949010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)    return;
950010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#endif  // OS_ANDROID
9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
953010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) private:
954010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#if defined(OS_ANDROID)
955010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  // TODO(rtenneti): Delete this code, after getting data.
956010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  base::Time start_time_clock_;
957010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  base::TimeTicks start_time_monotonic_;
958010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  base::TimeTicks start_time_thread_now_;
959010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)#endif  // OS_ANDROID
960010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)
9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)};
9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ShutdownWatchDogThread methods and members.
9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Class for detecting hangs during shutdown.
9675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ShutdownWatchDogThread : public base::Watchdog {
9685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public:
9695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Constructor specifies how long the ShutdownWatchDogThread will wait before
9705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // alarming.
9715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
9725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      : base::Watchdog(duration, "Shutdown watchdog thread", true) {
9735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
9745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Alarm is called if the time expires after an Arm() without someone calling
9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  // Disarm(). We crash the browser if this method is called.
9772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  virtual void Alarm() OVERRIDE {
9786e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)    ShutdownHang();
9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
981010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) private:
9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)};
9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}  // namespace
9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// StartupTimeBomb methods and members.
9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static
989