1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <math.h>
6
7#include "base/basictypes.h"
8#include "base/bind.h"
9#include "base/logging.h"
10#include "base/memory/scoped_ptr.h"
11#include "base/message_loop/message_loop.h"
12#include "base/message_loop/message_loop_proxy.h"
13#include "base/run_loop.h"
14#include "base/strings/string_number_conversions.h"
15#include "base/strings/string_split.h"
16#include "base/strings/string_tokenizer.h"
17#include "base/synchronization/condition_variable.h"
18#include "base/synchronization/lock.h"
19#include "base/threading/platform_thread.h"
20#include "base/time/time.h"
21#include "build/build_config.h"
22#include "chrome/browser/metrics/thread_watcher.h"
23#include "chrome/common/chrome_switches.h"
24#include "content/public/test/test_browser_thread.h"
25#include "testing/gtest/include/gtest/gtest.h"
26#include "testing/platform_test.h"
27
28using base::TimeDelta;
29using base::TimeTicks;
30using content::BrowserThread;
31
32enum State {
33  INITIALIZED,        // Created ThreadWatch object.
34  ACTIVATED,          // Thread watching activated.
35  SENT_PING,          // Sent ping message to watched thread.
36  RECEIVED_PONG,      // Received Pong message.
37  DEACTIVATED,        // Thread watching de-activated.
38};
39
40enum WaitState {
41  UNINITIALIZED,
42  STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
43  STOPPED_WAITING,    // Done with the waiting.
44  ALL_DONE,           // Done with waiting for STOPPED_WAITING.
45};
46
47enum CheckResponseState {
48  UNKNOWN,
49  SUCCESSFUL,         // CheckResponse was successful.
50  FAILED,             // CheckResponse has failed.
51};
52
53// This class helps to track and manipulate thread state during tests. This
54// class also has utility method to simulate hanging of watched thread by making
55// the watched thread wait for a very long time by posting a task on watched
56// thread that keeps it busy. It also has an utility method to block running of
57// tests until ThreadWatcher object's post-condition state changes to an
58// expected state.
59class CustomThreadWatcher : public ThreadWatcher {
60 public:
61  base::Lock custom_lock_;
62  base::ConditionVariable state_changed_;
63  State thread_watcher_state_;
64  WaitState wait_state_;
65  CheckResponseState check_response_state_;
66  uint64 ping_sent_;
67  uint64 pong_received_;
68  base::subtle::Atomic32 success_response_;
69  base::subtle::Atomic32 failed_response_;
70  base::TimeTicks saved_ping_time_;
71  uint64 saved_ping_sequence_number_;
72
73  CustomThreadWatcher(const BrowserThread::ID thread_id,
74                      const std::string thread_name,
75                      const TimeDelta& sleep_time,
76                      const TimeDelta& unresponsive_time)
77      : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
78                      unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
79                      true, ThreadWatcherList::kLiveThreadsThreshold)),
80        state_changed_(&custom_lock_),
81        thread_watcher_state_(INITIALIZED),
82        wait_state_(UNINITIALIZED),
83        check_response_state_(UNKNOWN),
84        ping_sent_(0),
85        pong_received_(0),
86        success_response_(0),
87        failed_response_(0),
88        saved_ping_time_(base::TimeTicks::Now()),
89        saved_ping_sequence_number_(0) {
90  }
91
92  State UpdateState(State new_state) {
93    State old_state;
94    {
95      base::AutoLock auto_lock(custom_lock_);
96      old_state = thread_watcher_state_;
97      if (old_state != DEACTIVATED)
98        thread_watcher_state_ = new_state;
99      if (new_state == SENT_PING)
100        ++ping_sent_;
101      if (new_state == RECEIVED_PONG)
102        ++pong_received_;
103      saved_ping_time_ = ping_time();
104      saved_ping_sequence_number_ = ping_sequence_number();
105    }
106    state_changed_.Broadcast();
107    return old_state;
108  }
109
110  WaitState UpdateWaitState(WaitState new_state) {
111    WaitState old_state;
112    {
113      base::AutoLock auto_lock(custom_lock_);
114      old_state = wait_state_;
115      wait_state_ = new_state;
116    }
117    state_changed_.Broadcast();
118    return old_state;
119  }
120
121  virtual void ActivateThreadWatching() OVERRIDE {
122    State old_state = UpdateState(ACTIVATED);
123    EXPECT_EQ(old_state, INITIALIZED);
124    ThreadWatcher::ActivateThreadWatching();
125  }
126
127  virtual void DeActivateThreadWatching() OVERRIDE {
128    State old_state = UpdateState(DEACTIVATED);
129    EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
130                old_state == RECEIVED_PONG);
131    ThreadWatcher::DeActivateThreadWatching();
132  }
133
134  virtual void PostPingMessage() OVERRIDE {
135    State old_state = UpdateState(SENT_PING);
136    EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
137    ThreadWatcher::PostPingMessage();
138  }
139
140  virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE {
141    State old_state = UpdateState(RECEIVED_PONG);
142    EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
143    ThreadWatcher::OnPongMessage(ping_sequence_number);
144  }
145
146  virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE {
147    ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
148    {
149      base::AutoLock auto_lock(custom_lock_);
150      if (responsive_) {
151        base::subtle::Release_Store(&success_response_,
152            base::subtle::Acquire_Load(&success_response_) + 1);
153        check_response_state_ = SUCCESSFUL;
154      } else {
155        base::subtle::Release_Store(&failed_response_,
156            base::subtle::Acquire_Load(&failed_response_) + 1);
157        check_response_state_ = FAILED;
158      }
159    }
160    // Broadcast to indicate we have checked responsiveness of the thread that
161    // is watched.
162    state_changed_.Broadcast();
163  }
164
165  void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
166    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
167    TimeTicks end_time = TimeTicks::Now() + wait_time;
168    {
169      base::AutoLock auto_lock(custom_lock_);
170      while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
171        state_changed_.TimedWait(end_time - TimeTicks::Now());
172    }
173  }
174
175  void VeryLongMethod(TimeDelta wait_time) {
176    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
177    WaitForWaitStateChange(wait_time, STOPPED_WAITING);
178    UpdateWaitState(ALL_DONE);
179  }
180
181  State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
182    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
183    UpdateWaitState(STARTED_WAITING);
184
185    State exit_state = INITIALIZED;
186    // Keep the thread that is running the tests waiting until ThreadWatcher
187    // object's state changes to the expected_state or until wait_time elapses.
188    for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
189        TimeTicks end_time = TimeTicks::Now() + wait_time;
190        {
191          base::AutoLock auto_lock(custom_lock_);
192          while (thread_watcher_state_ != expected_state &&
193                 TimeTicks::Now() < end_time) {
194            TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
195            state_changed_.TimedWait(state_change_wait_time);
196          }
197          // Capture the thread_watcher_state_ before it changes and return it
198          // to the caller.
199          exit_state = thread_watcher_state_;
200          if (exit_state == expected_state)
201            break;
202        }
203    }
204    UpdateWaitState(STOPPED_WAITING);
205    return exit_state;
206  }
207
208  CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
209                                          CheckResponseState expected_state) {
210    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
211    UpdateWaitState(STARTED_WAITING);
212
213    CheckResponseState exit_state = UNKNOWN;
214    // Keep the thread that is running the tests waiting until ThreadWatcher
215    // object's check_response_state_ changes to the expected_state or until
216    // wait_time elapses.
217    for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
218        TimeTicks end_time = TimeTicks::Now() + wait_time;
219        {
220          base::AutoLock auto_lock(custom_lock_);
221          while (check_response_state_ != expected_state &&
222                 TimeTicks::Now() < end_time) {
223            TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
224            state_changed_.TimedWait(state_change_wait_time);
225          }
226          // Capture the check_response_state_ before it changes and return it
227          // to the caller.
228          exit_state = check_response_state_;
229          if (exit_state == expected_state)
230            break;
231        }
232    }
233    UpdateWaitState(STOPPED_WAITING);
234    return exit_state;
235  }
236};
237
238class ThreadWatcherTest : public ::testing::Test {
239 public:
240  static const TimeDelta kSleepTime;
241  static const TimeDelta kUnresponsiveTime;
242  static const BrowserThread::ID io_thread_id;
243  static const std::string io_thread_name;
244  static const BrowserThread::ID db_thread_id;
245  static const std::string db_thread_name;
246  static const std::string crash_on_hang_seconds;
247  static const std::string crash_on_hang_thread_names;
248  static const std::string thread_names_and_live_threshold;
249  static const std::string crash_on_hang_thread_data;
250  CustomThreadWatcher* io_watcher_;
251  CustomThreadWatcher* db_watcher_;
252  ThreadWatcherList* thread_watcher_list_;
253
254  ThreadWatcherTest()
255      : setup_complete_(&lock_),
256        initialized_(false) {
257    db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
258    io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
259    watchdog_thread_.reset(new WatchDogThread());
260    db_thread_->Start();
261    io_thread_->Start();
262    watchdog_thread_->Start();
263
264    WatchDogThread::PostTask(
265        FROM_HERE,
266        base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
267
268    WaitForSetUp(TimeDelta::FromMinutes(1));
269  }
270
271  void SetUpObjects() {
272    DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
273
274    // Setup the registry for thread watchers.
275    thread_watcher_list_ = new ThreadWatcherList();
276
277    // Create thread watcher object for the IO thread.
278    io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
279                                          kSleepTime, kUnresponsiveTime);
280    EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
281
282    // Create thread watcher object for the DB thread.
283    db_watcher_ = new CustomThreadWatcher(
284        db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
285    EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
286
287    {
288      base::AutoLock lock(lock_);
289      initialized_ = true;
290    }
291    setup_complete_.Signal();
292  }
293
294  void WaitForSetUp(TimeDelta wait_time) {
295    DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
296    TimeTicks end_time = TimeTicks::Now() + wait_time;
297    {
298      base::AutoLock auto_lock(lock_);
299      while (!initialized_ && TimeTicks::Now() < end_time)
300        setup_complete_.TimedWait(end_time - TimeTicks::Now());
301    }
302  }
303
304  virtual ~ThreadWatcherTest() {
305    ThreadWatcherList::DeleteAll();
306    io_watcher_ = NULL;
307    db_watcher_ = NULL;
308    io_thread_.reset();
309    db_thread_.reset();
310    watchdog_thread_.reset();
311    thread_watcher_list_ = NULL;
312  }
313
314 private:
315  base::Lock lock_;
316  base::ConditionVariable setup_complete_;
317  bool initialized_;
318  scoped_ptr<content::TestBrowserThread> db_thread_;
319  scoped_ptr<content::TestBrowserThread> io_thread_;
320  scoped_ptr<WatchDogThread> watchdog_thread_;
321};
322
323// Define static constants.
324const TimeDelta ThreadWatcherTest::kSleepTime =
325    TimeDelta::FromMilliseconds(50);
326const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
327    TimeDelta::FromMilliseconds(500);
328const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
329const std::string ThreadWatcherTest::io_thread_name = "IO";
330const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
331const std::string ThreadWatcherTest::db_thread_name = "DB";
332const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
333const std::string ThreadWatcherTest::thread_names_and_live_threshold =
334    "UI:4,IO:4";
335const std::string ThreadWatcherTest::crash_on_hang_thread_data =
336    "UI:5:12,IO:5:12,FILE:5:12";
337
338TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
339  // Setup command_line arguments.
340  CommandLine command_line(CommandLine::NO_PROGRAM);
341  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
342                                 crash_on_hang_thread_names);
343
344  // Parse command_line arguments.
345  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
346  uint32 unresponsive_threshold;
347  ThreadWatcherList::ParseCommandLine(command_line,
348                                      &unresponsive_threshold,
349                                      &crash_on_hang_threads);
350
351  // Verify the data.
352  base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
353  std::vector<std::string> values;
354  while (tokens.GetNext()) {
355    const std::string& token = tokens.token();
356    base::SplitString(token, ':', &values);
357    std::string thread_name = values[0];
358
359    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
360        crash_on_hang_threads.find(thread_name);
361    bool crash_on_hang = (it != crash_on_hang_threads.end());
362    EXPECT_TRUE(crash_on_hang);
363    EXPECT_LT(0u, it->second.live_threads_threshold);
364    EXPECT_LT(0u, it->second.unresponsive_threshold);
365  }
366}
367
368TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
369  // Setup command_line arguments.
370  CommandLine command_line(CommandLine::NO_PROGRAM);
371  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
372                                 thread_names_and_live_threshold);
373
374  // Parse command_line arguments.
375  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
376  uint32 unresponsive_threshold;
377  ThreadWatcherList::ParseCommandLine(command_line,
378                                      &unresponsive_threshold,
379                                      &crash_on_hang_threads);
380
381  // Verify the data.
382  base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
383  std::vector<std::string> values;
384  while (tokens.GetNext()) {
385    const std::string& token = tokens.token();
386    base::SplitString(token, ':', &values);
387    std::string thread_name = values[0];
388
389    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
390        crash_on_hang_threads.find(thread_name);
391    bool crash_on_hang = (it != crash_on_hang_threads.end());
392    EXPECT_TRUE(crash_on_hang);
393    EXPECT_EQ(4u, it->second.live_threads_threshold);
394    EXPECT_LT(0u, it->second.unresponsive_threshold);
395  }
396}
397
398TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
399  // Setup command_line arguments.
400  CommandLine command_line(CommandLine::NO_PROGRAM);
401  command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
402                                 crash_on_hang_thread_data);
403
404  // Parse command_line arguments.
405  ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
406  uint32 unresponsive_threshold;
407  ThreadWatcherList::ParseCommandLine(command_line,
408                                      &unresponsive_threshold,
409                                      &crash_on_hang_threads);
410
411  // Verify the data.
412  base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
413  std::vector<std::string> values;
414  while (tokens.GetNext()) {
415    const std::string& token = tokens.token();
416    base::SplitString(token, ':', &values);
417    std::string thread_name = values[0];
418
419    ThreadWatcherList::CrashOnHangThreadMap::iterator it =
420        crash_on_hang_threads.find(thread_name);
421
422    bool crash_on_hang = (it != crash_on_hang_threads.end());
423    EXPECT_TRUE(crash_on_hang);
424
425    uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
426    EXPECT_EQ(5u, crash_live_threads_threshold);
427
428    uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
429    uint32 crash_on_unresponsive_seconds =
430        ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
431    EXPECT_EQ(12u, crash_on_unresponsive_seconds);
432  }
433}
434
435// Test registration. When thread_watcher_list_ goes out of scope after
436// TearDown, all thread watcher objects will be deleted.
437TEST_F(ThreadWatcherTest, Registration) {
438  // Check ThreadWatcher object has all correct parameters.
439  EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
440  EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
441  EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
442  EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
443  EXPECT_FALSE(io_watcher_->active());
444
445  // Check ThreadWatcher object of watched DB thread has correct data.
446  EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
447  EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
448  EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
449  EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
450  EXPECT_FALSE(db_watcher_->active());
451}
452
453// Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
454// method also checks that pong message was sent by the watched thread and pong
455// message was received by the WatchDogThread. It also checks that
456// OnCheckResponsiveness has verified the ping-pong mechanism and the watched
457// thread is not hung.
458TEST_F(ThreadWatcherTest, ThreadResponding) {
459  TimeTicks time_before_ping = TimeTicks::Now();
460  // Activate watching IO thread.
461  WatchDogThread::PostTask(
462      FROM_HERE,
463      base::Bind(&ThreadWatcher::ActivateThreadWatching,
464                 base::Unretained(io_watcher_)));
465
466  // Activate would have started ping/pong messaging. Expect atleast one
467  // ping/pong messaging sequence to happen.
468  io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
469                                  RECEIVED_PONG);
470  EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
471  EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
472  EXPECT_TRUE(io_watcher_->active());
473  EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
474  EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
475
476  // Verify watched thread is responding with ping/pong messaging.
477  io_watcher_->WaitForCheckResponse(
478      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
479  EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
480      static_cast<base::subtle::Atomic32>(0));
481  EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
482      static_cast<base::subtle::Atomic32>(0));
483
484  // DeActivate thread watching for shutdown.
485  WatchDogThread::PostTask(
486      FROM_HERE,
487      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
488      base::Unretained(io_watcher_)));
489}
490
491// This test posts a task on watched thread that takes very long time (this is
492// to simulate hanging of watched thread). It then checks for
493// OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
494// if the watched thread is not responding).
495TEST_F(ThreadWatcherTest, ThreadNotResponding) {
496  // Simulate hanging of watched thread by making the watched thread wait for a
497  // very long time by posting a task on watched thread that keeps it busy.
498  // It is safe to use base::Unretained because test is waiting for the method
499  // to finish.
500  BrowserThread::PostTask(
501      io_thread_id,
502      FROM_HERE,
503      base::Bind(&CustomThreadWatcher::VeryLongMethod,
504                 base::Unretained(io_watcher_),
505                 kUnresponsiveTime * 10));
506
507  // Activate thread watching.
508  WatchDogThread::PostTask(
509      FROM_HERE,
510      base::Bind(&ThreadWatcher::ActivateThreadWatching,
511                 base::Unretained(io_watcher_)));
512
513  // Verify watched thread is not responding for ping messages.
514  io_watcher_->WaitForCheckResponse(
515      kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
516  EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
517      static_cast<base::subtle::Atomic32>(0));
518  EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
519      static_cast<base::subtle::Atomic32>(0));
520
521  // DeActivate thread watching for shutdown.
522  WatchDogThread::PostTask(
523      FROM_HERE,
524      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
525                 base::Unretained(io_watcher_)));
526
527  // Wait for the io_watcher_'s VeryLongMethod to finish.
528  io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
529}
530
531// Test watching of multiple threads with all threads not responding.
532TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
533  // Check for DB thread to perform ping/pong messaging.
534  WatchDogThread::PostTask(
535      FROM_HERE,
536      base::Bind(&ThreadWatcher::ActivateThreadWatching,
537                 base::Unretained(db_watcher_)));
538
539  // Check for IO thread to perform ping/pong messaging.
540  WatchDogThread::PostTask(
541      FROM_HERE,
542      base::Bind(&ThreadWatcher::ActivateThreadWatching,
543                 base::Unretained(io_watcher_)));
544
545  // Verify DB thread is responding with ping/pong messaging.
546  db_watcher_->WaitForCheckResponse(
547      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
548  EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
549  EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
550  EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
551  EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
552      static_cast<base::subtle::Atomic32>(0));
553  EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
554      static_cast<base::subtle::Atomic32>(0));
555
556  // Verify IO thread is responding with ping/pong messaging.
557  io_watcher_->WaitForCheckResponse(
558      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
559  EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
560  EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
561  EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
562  EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
563      static_cast<base::subtle::Atomic32>(0));
564  EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
565      static_cast<base::subtle::Atomic32>(0));
566
567  // DeActivate thread watching for shutdown.
568  WatchDogThread::PostTask(
569      FROM_HERE,
570      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
571                 base::Unretained(io_watcher_)));
572
573  WatchDogThread::PostTask(
574      FROM_HERE,
575      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
576                 base::Unretained(db_watcher_)));
577}
578
579// Test watching of multiple threads with one of the threads not responding.
580TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
581  // Simulate hanging of watched thread by making the watched thread wait for a
582  // very long time by posting a task on watched thread that keeps it busy.
583  // It is safe ot use base::Unretained because test is waiting for the method
584  // to finish.
585  BrowserThread::PostTask(
586      io_thread_id,
587      FROM_HERE,
588      base::Bind(&CustomThreadWatcher::VeryLongMethod,
589                 base::Unretained(io_watcher_),
590                 kUnresponsiveTime * 10));
591
592  // Activate watching of DB thread.
593  WatchDogThread::PostTask(
594      FROM_HERE,
595      base::Bind(&ThreadWatcher::ActivateThreadWatching,
596                 base::Unretained(db_watcher_)));
597
598  // Activate watching of IO thread.
599  WatchDogThread::PostTask(
600      FROM_HERE,
601      base::Bind(&ThreadWatcher::ActivateThreadWatching,
602                 base::Unretained(io_watcher_)));
603
604  // Verify DB thread is responding with ping/pong messaging.
605  db_watcher_->WaitForCheckResponse(
606      kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
607  EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
608      static_cast<base::subtle::Atomic32>(0));
609  EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
610      static_cast<base::subtle::Atomic32>(0));
611
612  // Verify IO thread is not responding for ping messages.
613  io_watcher_->WaitForCheckResponse(
614      kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
615  EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
616      static_cast<base::subtle::Atomic32>(0));
617  EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
618      static_cast<base::subtle::Atomic32>(0));
619
620  // DeActivate thread watching for shutdown.
621  WatchDogThread::PostTask(
622      FROM_HERE,
623      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
624                 base::Unretained(io_watcher_)));
625  WatchDogThread::PostTask(
626      FROM_HERE,
627      base::Bind(&ThreadWatcher::DeActivateThreadWatching,
628                 base::Unretained(db_watcher_)));
629
630  // Wait for the io_watcher_'s VeryLongMethod to finish.
631  io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
632}
633
634class ThreadWatcherListTest : public ::testing::Test {
635 protected:
636  ThreadWatcherListTest()
637      : done_(&lock_),
638        state_available_(false),
639        has_thread_watcher_list_(false),
640        stopped_(false) {
641  }
642
643  void ReadStateOnWatchDogThread() {
644    CHECK(WatchDogThread::CurrentlyOnWatchDogThread());
645    {
646      base::AutoLock auto_lock(lock_);
647      has_thread_watcher_list_ =
648          ThreadWatcherList::g_thread_watcher_list_ != NULL;
649      stopped_ = ThreadWatcherList::g_stopped_;
650      state_available_ = true;
651    }
652    done_.Signal();
653  }
654
655  void CheckState(bool has_thread_watcher_list,
656                  bool stopped,
657                  const char* const msg) {
658    CHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
659    {
660      base::AutoLock auto_lock(lock_);
661      state_available_ = false;
662    }
663
664    WatchDogThread::PostTask(
665        FROM_HERE,
666        base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread,
667                   base::Unretained(this)));
668    {
669      base::AutoLock auto_lock(lock_);
670      while (!state_available_)
671        done_.Wait();
672
673      EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg;
674      EXPECT_EQ(stopped, stopped_) << msg;
675    }
676  }
677
678  base::Lock lock_;
679  base::ConditionVariable done_;
680
681  bool state_available_;
682  bool has_thread_watcher_list_;
683  bool stopped_;
684};
685
686TEST_F(ThreadWatcherListTest, Restart) {
687  ThreadWatcherList::g_initialize_delay_seconds = 1;
688
689  base::MessageLoopForUI message_loop_for_ui;
690  content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui);
691
692  scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread());
693  watchdog_thread_->Start();
694
695  // See http://crbug.com/347887.
696  // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_,
697  // whilst StopWatchingAll() will just PostTask to destroy it.
698  // Ensure that when Stop is called, Start will NOT create
699  // g_thread_watcher_list_ later on.
700  ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess());
701  ThreadWatcherList::StopWatchingAll();
702  message_loop_for_ui.PostDelayedTask(
703      FROM_HERE,
704      message_loop_for_ui.QuitClosure(),
705      base::TimeDelta::FromSeconds(
706          ThreadWatcherList::g_initialize_delay_seconds));
707  message_loop_for_ui.Run();
708
709  CheckState(false /* has_thread_watcher_list */,
710             true /* stopped */,
711             "Start / Stopped");
712
713  // Proceed with just |StartWatchingAll| and ensure it'll be started.
714  ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess());
715  message_loop_for_ui.PostDelayedTask(
716      FROM_HERE,
717      message_loop_for_ui.QuitClosure(),
718      base::TimeDelta::FromSeconds(
719          ThreadWatcherList::g_initialize_delay_seconds + 1));
720  message_loop_for_ui.Run();
721
722  CheckState(true /* has_thread_watcher_list */,
723             false /* stopped */,
724             "Started");
725
726  // Finally, StopWatchingAll() must stop.
727  ThreadWatcherList::StopWatchingAll();
728  message_loop_for_ui.PostDelayedTask(
729      FROM_HERE,
730      message_loop_for_ui.QuitClosure(),
731      base::TimeDelta::FromSeconds(
732          ThreadWatcherList::g_initialize_delay_seconds));
733  message_loop_for_ui.Run();
734
735  CheckState(false /* has_thread_watcher_list */,
736             true /* stopped */,
737             "Stopped");
738}
739