1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <math.h>
6
7#include "base/basictypes.h"
8#include "base/bind.h"
9#include "base/logging.h"
10#include "base/memory/scoped_ptr.h"
11#include "base/message_loop/message_loop.h"
12#include "base/message_loop/message_loop_proxy.h"
13#include "base/strings/string_number_conversions.h"
14#include "base/strings/string_split.h"
15#include "base/strings/string_tokenizer.h"
16#include "base/synchronization/condition_variable.h"
17#include "base/synchronization/lock.h"
18#include "base/threading/platform_thread.h"
19#include "base/time/time.h"
20#include "build/build_config.h"
21#include "chrome/browser/metrics/thread_watcher.h"
22#include "chrome/common/chrome_switches.h"
23#include "content/public/test/test_browser_thread.h"
24#include "testing/gtest/include/gtest/gtest.h"
25#include "testing/platform_test.h"
26
27using base::TimeDelta;
28using base::TimeTicks;
29using content::BrowserThread;
30
31enum State {
32  INITIALIZED,        // Created ThreadWatch object.
33  ACTIVATED,          // Thread watching activated.
34  SENT_PING,          // Sent ping message to watched thread.
35  RECEIVED_PONG,      // Received Pong message.
36  DEACTIVATED,        // Thread watching de-activated.
37};
38
39enum WaitState {
40  UNINITIALIZED,
41  STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
42  STOPPED_WAITING,    // Done with the waiting.
43  ALL_DONE,           // Done with waiting for STOPPED_WAITING.
44};
45
46enum CheckResponseState {
47  UNKNOWN,
48  SUCCESSFUL,         // CheckResponse was successful.
49  FAILED,             // CheckResponse has failed.
50};
51
52// This class helps to track and manipulate thread state during tests. This
53// class also has utility method to simulate hanging of watched thread by making
54// the watched thread wait for a very long time by posting a task on watched
55// thread that keeps it busy. It also has an utility method to block running of
56// tests until ThreadWatcher object's post-condition state changes to an
57// expected state.
58class CustomThreadWatcher : public ThreadWatcher {
59 public:
60  base::Lock custom_lock_;
61  base::ConditionVariable state_changed_;
62  State thread_watcher_state_;
63  WaitState wait_state_;
64  CheckResponseState check_response_state_;
65  uint64 ping_sent_;
66  uint64 pong_received_;
67  uint64 success_response_;
68  uint64 failed_response_;
69  base::TimeTicks saved_ping_time_;
70  uint64 saved_ping_sequence_number_;
71
72  CustomThreadWatcher(const BrowserThread::ID thread_id,
73                      const std::string thread_name,
74                      const TimeDelta& sleep_time,
75                      const TimeDelta& unresponsive_time)
76      : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
77                      unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
78                      true, ThreadWatcherList::kLiveThreadsThreshold)),
79        state_changed_(&custom_lock_),
80        thread_watcher_state_(INITIALIZED),
81        wait_state_(UNINITIALIZED),
82        check_response_state_(UNKNOWN),
83        ping_sent_(0),
84        pong_received_(0),
85        success_response_(0),
86        failed_response_(0),
87        saved_ping_time_(base::TimeTicks::Now()),
88        saved_ping_sequence_number_(0) {
89  }
90
91  State UpdateState(State new_state) {
92    State old_state;
93    {
94      base::AutoLock auto_lock(custom_lock_);
95      old_state = thread_watcher_state_;
96      if (old_state != DEACTIVATED)
97        thread_watcher_state_ = new_state;
98      if (new_state == SENT_PING)
99        ++ping_sent_;
100      if (new_state == RECEIVED_PONG)
101        ++pong_received_;
102      saved_ping_time_ = ping_time();
103      saved_ping_sequence_number_ = ping_sequence_number();
104    }
105    state_changed_.Broadcast();
106    return old_state;
107  }
108
109  WaitState UpdateWaitState(WaitState new_state) {
110    WaitState old_state;
111    {
112      base::AutoLock auto_lock(custom_lock_);
113      old_state = wait_state_;
114      wait_state_ = new_state;
115    }
116    state_changed_.Broadcast();
117    return old_state;
118  }
119
120  virtual void ActivateThreadWatching() OVERRIDE {
121    State old_state = UpdateState(ACTIVATED);
122    EXPECT_EQ(old_state, INITIALIZED);
123    ThreadWatcher::ActivateThreadWatching();
124  }
125
126  virtual void DeActivateThreadWatching() OVERRIDE {
127    State old_state = UpdateState(DEACTIVATED);
128    EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
129                old_state == RECEIVED_PONG);
130    ThreadWatcher::DeActivateThreadWatching();
131  }
132
133  virtual void PostPingMessage() OVERRIDE {
134    State old_state = UpdateState(SENT_PING);
135    EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
136    ThreadWatcher::PostPingMessage();
137  }
138
139  virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE {
140    State old_state = UpdateState(RECEIVED_PONG);
141    EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
142    ThreadWatcher::OnPongMessage(ping_sequence_number);
143  }
144
145  virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE {
146    ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
147    {
148      base::AutoLock auto_lock(custom_lock_);
149      if (responsive_) {
150        ++success_response_;
151        check_response_state_ = SUCCESSFUL;
152      } else {
153        ++failed_response_;
154        check_response_state_ = FAILED;
155      }
156    }
157    // Broadcast to indicate we have checked responsiveness of the thread that
158    // is watched.
159    state_changed_.Broadcast();
160  }
161
162  void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
163    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
164    TimeTicks end_time = TimeTicks::Now() + wait_time;
165    {
166      base::AutoLock auto_lock(custom_lock_);
167      while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
168        state_changed_.TimedWait(end_time - TimeTicks::Now());
169    }
170  }
171
172  void VeryLongMethod(TimeDelta wait_time) {
173    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
174    WaitForWaitStateChange(wait_time, STOPPED_WAITING);
175    UpdateWaitState(ALL_DONE);
176  }
177
178  State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
179    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
180    UpdateWaitState(STARTED_WAITING);
181
182    State exit_state = INITIALIZED;
183    // Keep the thread that is running the tests waiting until ThreadWatcher
184    // object's state changes to the expected_state or until wait_time elapses.
185    for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
186        TimeTicks end_time = TimeTicks::Now() + wait_time;
187        {
188          base::AutoLock auto_lock(custom_lock_);
189          while (thread_watcher_state_ != expected_state &&
190                 TimeTicks::Now() < end_time) {
191            TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
192            state_changed_.TimedWait(state_change_wait_time);
193          }
194          // Capture the thread_watcher_state_ before it changes and return it
195          // to the caller.
196          exit_state = thread_watcher_state_;
197          if (exit_state == expected_state)
198            break;
199        }
200    }
201    UpdateWaitState(STOPPED_WAITING);
202    return exit_state;
203  }
204
205  CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
206                                          CheckResponseState expected_state) {
207    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
208    UpdateWaitState(STARTED_WAITING);
209
210    CheckResponseState exit_state = UNKNOWN;
211    // Keep the thread that is running the tests waiting until ThreadWatcher
212    // object's check_response_state_ changes to the expected_state or until
213    // wait_time elapses.
214    for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
215        TimeTicks end_time = TimeTicks::Now() + wait_time;
216        {
217          base::AutoLock auto_lock(custom_lock_);
218          while (check_response_state_ != expected_state &&
219                 TimeTicks::Now() < end_time) {
220            TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
221            state_changed_.TimedWait(state_change_wait_time);
222          }
223          // Capture the check_response_state_ before it changes and return it
224          // to the caller.
225          exit_state = check_response_state_;
226          if (exit_state == expected_state)
227            break;
228        }
229    }
230    UpdateWaitState(STOPPED_WAITING);
231    return exit_state;
232  }
233};
234
235class ThreadWatcherTest : public ::testing::Test {
236 public:
237  static const TimeDelta kSleepTime;
238  static const TimeDelta kUnresponsiveTime;
239  static const BrowserThread::ID io_thread_id;
240  static const std::string io_thread_name;
241  static const BrowserThread::ID db_thread_id;
242  static const std::string db_thread_name;
243  static const std::string crash_on_hang_seconds;
244  static const std::string crash_on_hang_thread_names;
245  static const std::string thread_names_and_live_threshold;
246  static const std::string crash_on_hang_thread_data;
247  CustomThreadWatcher* io_watcher_;
248  CustomThreadWatcher* db_watcher_;
249  ThreadWatcherList* thread_watcher_list_;
250
251  ThreadWatcherTest()
252      : setup_complete_(&lock_),
253        initialized_(false) {
254    db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
255    io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
256    watchdog_thread_.reset(new WatchDogThread());
257    db_thread_->Start();
258    io_thread_->Start();
259    watchdog_thread_->Start();
260
261    WatchDogThread::PostTask(
262        FROM_HERE,
263        base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
264
265    WaitForSetUp(TimeDelta::FromMinutes(1));
266  }
267
268  void SetUpObjects() {
269    DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
270
271    // Setup the registry for thread watchers.
272    thread_watcher_list_ = new ThreadWatcherList();
273
274    // Create thread watcher object for the IO thread.
275    io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
276                                          kSleepTime, kUnresponsiveTime);
277    EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
278
279    // Create thread watcher object for the DB thread.
280    db_watcher_ = new CustomThreadWatcher(
281        db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
282    EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
283
284    {
285      base::AutoLock lock(lock_);
286      initialized_ = true;
287    }
288    setup_complete_.Signal();
289  }
290
291  void WaitForSetUp(TimeDelta wait_time) {
292    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
293    TimeTicks end_time = TimeTicks::Now() + wait_time;
294    {
295      base::AutoLock auto_lock(lock_);
296      while (!initialized_ && TimeTicks::Now() < end_time)
297        setup_complete_.TimedWait(end_time - TimeTicks::Now());
298    }
299  }
300
301  virtual ~ThreadWatcherTest() {
302    ThreadWatcherList::DeleteAll();
303    io_watcher_ = NULL;
304    db_watcher_ = NULL;
305    io_thread_.reset();
306    db_thread_.reset();
307    watchdog_thread_.reset();
308    thread_watcher_list_ = NULL;
309  }
310
311 private:
312  base::Lock lock_;
313  base::ConditionVariable setup_complete_;
314  bool initialized_;
315  scoped_ptr<content::TestBrowserThread> db_thread_;
316  scoped_ptr<content::TestBrowserThread> io_thread_;
317  scoped_ptr<WatchDogThread> watchdog_thread_;
318};
319
320// Define static constants.
321const TimeDelta ThreadWatcherTest::kSleepTime =
322    TimeDelta::FromMilliseconds(50);
323const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
324    TimeDelta::FromMilliseconds(500);
325const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
326const std::string ThreadWatcherTest::io_thread_name = "IO";
327const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
328const std::string ThreadWatcherTest::db_thread_name = "DB";
329const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
330const std::string ThreadWatcherTest::thread_names_and_live_threshold =
331    "UI:4,IO:4";
332const std::string ThreadWatcherTest::crash_on_hang_thread_data =
333    "UI:5:12,IO:5:12,FILE:5:12";
334
335TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
336  // Setup command_line arguments.
337  CommandLine command_line(CommandLine::NO_PROGRAM);
338  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
339                                 crash_on_hang_thread_names);
340
341  // Parse command_line arguments.
342  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
343  uint32 unresponsive_threshold;
344  ThreadWatcherList::ParseCommandLine(command_line,
345                                      &unresponsive_threshold,
346                                      &crash_on_hang_threads);
347
348  // Verify the data.
349  base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
350  std::vector<std::string> values;
351  while (tokens.GetNext()) {
352    const std::string& token = tokens.token();
353    base::SplitString(token, ':', &values);
354    std::string thread_name = values[0];
355
356    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
357        crash_on_hang_threads.find(thread_name);
358    bool crash_on_hang = (it != crash_on_hang_threads.end());
359    EXPECT_TRUE(crash_on_hang);
360    EXPECT_LT(0u, it->second.live_threads_threshold);
361    EXPECT_LT(0u, it->second.unresponsive_threshold);
362  }
363}
364
365TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
366  // Setup command_line arguments.
367  CommandLine command_line(CommandLine::NO_PROGRAM);
368  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
369                                 thread_names_and_live_threshold);
370
371  // Parse command_line arguments.
372  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
373  uint32 unresponsive_threshold;
374  ThreadWatcherList::ParseCommandLine(command_line,
375                                      &unresponsive_threshold,
376                                      &crash_on_hang_threads);
377
378  // Verify the data.
379  base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
380  std::vector<std::string> values;
381  while (tokens.GetNext()) {
382    const std::string& token = tokens.token();
383    base::SplitString(token, ':', &values);
384    std::string thread_name = values[0];
385
386    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
387        crash_on_hang_threads.find(thread_name);
388    bool crash_on_hang = (it != crash_on_hang_threads.end());
389    EXPECT_TRUE(crash_on_hang);
390    EXPECT_EQ(4u, it->second.live_threads_threshold);
391    EXPECT_LT(0u, it->second.unresponsive_threshold);
392  }
393}
394
395TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
396  // Setup command_line arguments.
397  CommandLine command_line(CommandLine::NO_PROGRAM);
398  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
399                                 crash_on_hang_thread_data);
400
401  // Parse command_line arguments.
402  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
403  uint32 unresponsive_threshold;
404  ThreadWatcherList::ParseCommandLine(command_line,
405                                      &unresponsive_threshold,
406                                      &crash_on_hang_threads);
407
408  // Verify the data.
409  base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
410  std::vector<std::string> values;
411  while (tokens.GetNext()) {
412    const std::string& token = tokens.token();
413    base::SplitString(token, ':', &values);
414    std::string thread_name = values[0];
415
416    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
417        crash_on_hang_threads.find(thread_name);
418
419    bool crash_on_hang = (it != crash_on_hang_threads.end());
420    EXPECT_TRUE(crash_on_hang);
421
422    uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
423    EXPECT_EQ(5u, crash_live_threads_threshold);
424
425    uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
426    uint32 crash_on_unresponsive_seconds =
427        ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
428    EXPECT_EQ(12u, crash_on_unresponsive_seconds);
429  }
430}
431
432// Test registration. When thread_watcher_list_ goes out of scope after
433// TearDown, all thread watcher objects will be deleted.
434TEST_F(ThreadWatcherTest, Registration) {
435  // Check ThreadWatcher object has all correct parameters.
436  EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
437  EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
438  EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
439  EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
440  EXPECT_FALSE(io_watcher_->active());
441
442  // Check ThreadWatcher object of watched DB thread has correct data.
443  EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
444  EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
445  EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
446  EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
447  EXPECT_FALSE(db_watcher_->active());
448}
449
450// Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
451// method also checks that pong message was sent by the watched thread and pong
452// message was received by the WatchDogThread. It also checks that
453// OnCheckResponsiveness has verified the ping-pong mechanism and the watched
454// thread is not hung.
455TEST_F(ThreadWatcherTest, ThreadResponding) {
456  TimeTicks time_before_ping = TimeTicks::Now();
457  // Activate watching IO thread.
458  WatchDogThread::PostTask(
459      FROM_HERE,
460      base::Bind(&ThreadWatcher::ActivateThreadWatching,
461                 base::Unretained(io_watcher_)));
462
463  // Activate would have started ping/pong messaging. Expect atleast one
464  // ping/pong messaging sequence to happen.
465  io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
466                                  RECEIVED_PONG);
467  EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
468  EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
469  EXPECT_TRUE(io_watcher_->active());
470  EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
471  EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
472
473  // Verify watched thread is responding with ping/pong messaging.
474  io_watcher_->WaitForCheckResponse(
475      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
476  EXPECT_GT(io_watcher_->success_response_, static_cast<uint64>(0));
477  EXPECT_EQ(io_watcher_->failed_response_, static_cast<uint64>(0));
478
479  // DeActivate thread watching for shutdown.
480  WatchDogThread::PostTask(
481      FROM_HERE,
482      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
483      base::Unretained(io_watcher_)));
484}
485
486// This test posts a task on watched thread that takes very long time (this is
487// to simulate hanging of watched thread). It then checks for
488// OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
489// if the watched thread is not responding).
490TEST_F(ThreadWatcherTest, ThreadNotResponding) {
491  // Simulate hanging of watched thread by making the watched thread wait for a
492  // very long time by posting a task on watched thread that keeps it busy.
493  // It is safe to use base::Unretained because test is waiting for the method
494  // to finish.
495  BrowserThread::PostTask(
496      io_thread_id,
497      FROM_HERE,
498      base::Bind(&CustomThreadWatcher::VeryLongMethod,
499                 base::Unretained(io_watcher_),
500                 kUnresponsiveTime * 10));
501
502  // Activate thread watching.
503  WatchDogThread::PostTask(
504      FROM_HERE,
505      base::Bind(&ThreadWatcher::ActivateThreadWatching,
506                 base::Unretained(io_watcher_)));
507
508  // Verify watched thread is not responding for ping messages.
509  io_watcher_->WaitForCheckResponse(
510      kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
511  EXPECT_EQ(io_watcher_->success_response_, static_cast<uint64>(0));
512  EXPECT_GT(io_watcher_->failed_response_, static_cast<uint64>(0));
513
514  // DeActivate thread watching for shutdown.
515  WatchDogThread::PostTask(
516      FROM_HERE,
517      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
518                 base::Unretained(io_watcher_)));
519
520  // Wait for the io_watcher_'s VeryLongMethod to finish.
521  io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
522}
523
524// Test watching of multiple threads with all threads not responding.
525TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
526  // Check for DB thread to perform ping/pong messaging.
527  WatchDogThread::PostTask(
528      FROM_HERE,
529      base::Bind(&ThreadWatcher::ActivateThreadWatching,
530                 base::Unretained(db_watcher_)));
531
532  // Check for IO thread to perform ping/pong messaging.
533  WatchDogThread::PostTask(
534      FROM_HERE,
535      base::Bind(&ThreadWatcher::ActivateThreadWatching,
536                 base::Unretained(io_watcher_)));
537
538  // Verify DB thread is responding with ping/pong messaging.
539  db_watcher_->WaitForCheckResponse(
540      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
541  EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
542  EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
543  EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
544  EXPECT_GT(db_watcher_->success_response_, static_cast<uint64>(0));
545  EXPECT_EQ(db_watcher_->failed_response_, static_cast<uint64>(0));
546
547  // Verify IO thread is responding with ping/pong messaging.
548  io_watcher_->WaitForCheckResponse(
549      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
550  EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
551  EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
552  EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
553  EXPECT_GT(io_watcher_->success_response_, static_cast<uint64>(0));
554  EXPECT_EQ(io_watcher_->failed_response_, static_cast<uint64>(0));
555
556  // DeActivate thread watching for shutdown.
557  WatchDogThread::PostTask(
558      FROM_HERE,
559      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
560                 base::Unretained(io_watcher_)));
561
562  WatchDogThread::PostTask(
563      FROM_HERE,
564      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
565                 base::Unretained(db_watcher_)));
566}
567
568// Test watching of multiple threads with one of the threads not responding.
569TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
570  // Simulate hanging of watched thread by making the watched thread wait for a
571  // very long time by posting a task on watched thread that keeps it busy.
572  // It is safe ot use base::Unretained because test is waiting for the method
573  // to finish.
574  BrowserThread::PostTask(
575      io_thread_id,
576      FROM_HERE,
577      base::Bind(&CustomThreadWatcher::VeryLongMethod,
578                 base::Unretained(io_watcher_),
579                 kUnresponsiveTime * 10));
580
581  // Activate watching of DB thread.
582  WatchDogThread::PostTask(
583      FROM_HERE,
584      base::Bind(&ThreadWatcher::ActivateThreadWatching,
585                 base::Unretained(db_watcher_)));
586
587  // Activate watching of IO thread.
588  WatchDogThread::PostTask(
589      FROM_HERE,
590      base::Bind(&ThreadWatcher::ActivateThreadWatching,
591                 base::Unretained(io_watcher_)));
592
593  // Verify DB thread is responding with ping/pong messaging.
594  db_watcher_->WaitForCheckResponse(
595      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
596  EXPECT_GT(db_watcher_->success_response_, static_cast<uint64>(0));
597  EXPECT_EQ(db_watcher_->failed_response_, static_cast<uint64>(0));
598
599  // Verify IO thread is not responding for ping messages.
600  io_watcher_->WaitForCheckResponse(
601      kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
602  EXPECT_EQ(io_watcher_->success_response_, static_cast<uint64>(0));
603  EXPECT_GT(io_watcher_->failed_response_, static_cast<uint64>(0));
604
605  // DeActivate thread watching for shutdown.
606  WatchDogThread::PostTask(
607      FROM_HERE,
608      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
609                 base::Unretained(io_watcher_)));
610  WatchDogThread::PostTask(
611      FROM_HERE,
612      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
613                 base::Unretained(db_watcher_)));
614
615  // Wait for the io_watcher_'s VeryLongMethod to finish.
616  io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
617}
618