15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2012 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <math.h> 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/bind.h" 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/logging.h" 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/memory/scoped_ptr.h" 11ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch#include "base/message_loop/message_loop.h" 12b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)#include "base/message_loop/message_loop_proxy.h" 13a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)#include "base/run_loop.h" 142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_number_conversions.h" 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_split.h" 162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "base/strings/string_tokenizer.h" 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/synchronization/condition_variable.h" 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/synchronization/lock.h" 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/threading/platform_thread.h" 20eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/time/time.h" 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "build/build_config.h" 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/browser/metrics/thread_watcher.h" 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "chrome/common/chrome_switches.h" 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "content/public/test/test_browser_thread.h" 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "testing/gtest/include/gtest/gtest.h" 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "testing/platform_test.h" 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using base::TimeDelta; 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using base::TimeTicks; 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using content::BrowserThread; 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)enum State { 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) INITIALIZED, // Created ThreadWatch object. 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ACTIVATED, // Thread watching activated. 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SENT_PING, // Sent ping message to watched thread. 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) RECEIVED_PONG, // Received Pong message. 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DEACTIVATED, // Thread watching de-activated. 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)enum WaitState { 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNINITIALIZED, 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) STARTED_WAITING, // Start waiting for state_ to change to expected_state. 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) STOPPED_WAITING, // Done with the waiting. 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ALL_DONE, // Done with waiting for STOPPED_WAITING. 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)enum CheckResponseState { 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNKNOWN, 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SUCCESSFUL, // CheckResponse was successful. 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FAILED, // CheckResponse has failed. 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This class helps to track and manipulate thread state during tests. This 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// class also has utility method to simulate hanging of watched thread by making 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the watched thread wait for a very long time by posting a task on watched 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// thread that keeps it busy. It also has an utility method to block running of 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// tests until ThreadWatcher object's post-condition state changes to an 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// expected state. 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class CustomThreadWatcher : public ThreadWatcher { 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Lock custom_lock_; 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::ConditionVariable state_changed_; 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State thread_watcher_state_; 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WaitState wait_state_; 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CheckResponseState check_response_state_; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64 ping_sent_; 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64 pong_received_; 688bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Atomic32 success_response_; 698bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Atomic32 failed_response_; 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::TimeTicks saved_ping_time_; 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64 saved_ping_sequence_number_; 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CustomThreadWatcher(const BrowserThread::ID thread_id, 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const std::string thread_name, 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const TimeDelta& sleep_time, 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const TimeDelta& unresponsive_time) 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time, 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unresponsive_time, ThreadWatcherList::kUnresponsiveCount, 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) true, ThreadWatcherList::kLiveThreadsThreshold)), 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_(&custom_lock_), 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_watcher_state_(INITIALIZED), 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) wait_state_(UNINITIALIZED), 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) check_response_state_(UNKNOWN), 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ping_sent_(0), 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pong_received_(0), 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) success_response_(0), 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) failed_response_(0), 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) saved_ping_time_(base::TimeTicks::Now()), 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) saved_ping_sequence_number_(0) { 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State UpdateState(State new_state) { 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State old_state; 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) old_state = thread_watcher_state_; 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (old_state != DEACTIVATED) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_watcher_state_ = new_state; 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_state == SENT_PING) 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ++ping_sent_; 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_state == RECEIVED_PONG) 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ++pong_received_; 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) saved_ping_time_ = ping_time(); 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) saved_ping_sequence_number_ = ping_sequence_number(); 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.Broadcast(); 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old_state; 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WaitState UpdateWaitState(WaitState new_state) { 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WaitState old_state; 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) old_state = wait_state_; 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) wait_state_ = new_state; 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.Broadcast(); 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old_state; 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual void ActivateThreadWatching() OVERRIDE { 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State old_state = UpdateState(ACTIVATED); 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(old_state, INITIALIZED); 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcher::ActivateThreadWatching(); 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual void DeActivateThreadWatching() OVERRIDE { 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State old_state = UpdateState(DEACTIVATED); 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING || 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) old_state == RECEIVED_PONG); 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcher::DeActivateThreadWatching(); 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual void PostPingMessage() OVERRIDE { 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State old_state = UpdateState(SENT_PING); 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG); 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcher::PostPingMessage(); 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE { 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State old_state = UpdateState(RECEIVED_PONG); 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED); 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcher::OnPongMessage(ping_sequence_number); 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE { 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcher::OnCheckResponsiveness(ping_sequence_number); 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (responsive_) { 1518bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Release_Store(&success_response_, 1528bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Acquire_Load(&success_response_) + 1); 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) check_response_state_ = SUCCESSFUL; 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 1558bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Release_Store(&failed_response_, 1568bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) base::subtle::Acquire_Load(&failed_response_) + 1); 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) check_response_state_ = FAILED; 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Broadcast to indicate we have checked responsiveness of the thread that 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // is watched. 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.Broadcast(); 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) { 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks end_time = TimeTicks::Now() + wait_time; 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (wait_state_ != expected_state && TimeTicks::Now() < end_time) 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.TimedWait(end_time - TimeTicks::Now()); 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void VeryLongMethod(TimeDelta wait_time) { 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WaitForWaitStateChange(wait_time, STOPPED_WAITING); 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UpdateWaitState(ALL_DONE); 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State WaitForStateChange(const TimeDelta& wait_time, State expected_state) { 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UpdateWaitState(STARTED_WAITING); 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) State exit_state = INITIALIZED; 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Keep the thread that is running the tests waiting until ThreadWatcher 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // object's state changes to the expected_state or until wait_time elapses. 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks end_time = TimeTicks::Now() + wait_time; 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (thread_watcher_state_ != expected_state && 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks::Now() < end_time) { 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.TimedWait(state_change_wait_time); 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Capture the thread_watcher_state_ before it changes and return it 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to the caller. 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) exit_state = thread_watcher_state_; 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (exit_state == expected_state) 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UpdateWaitState(STOPPED_WAITING); 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return exit_state; 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time, 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CheckResponseState expected_state) { 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UpdateWaitState(STARTED_WAITING); 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CheckResponseState exit_state = UNKNOWN; 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Keep the thread that is running the tests waiting until ThreadWatcher 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // object's check_response_state_ changes to the expected_state or until 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // wait_time elapses. 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks end_time = TimeTicks::Now() + wait_time; 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(custom_lock_); 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (check_response_state_ != expected_state && 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks::Now() < end_time) { 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state_changed_.TimedWait(state_change_wait_time); 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Capture the check_response_state_ before it changes and return it 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to the caller. 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) exit_state = check_response_state_; 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (exit_state == expected_state) 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UpdateWaitState(STOPPED_WAITING); 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return exit_state; 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ThreadWatcherTest : public ::testing::Test { 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const TimeDelta kSleepTime; 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const TimeDelta kUnresponsiveTime; 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const BrowserThread::ID io_thread_id; 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const std::string io_thread_name; 244ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch static const BrowserThread::ID db_thread_id; 245ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch static const std::string db_thread_name; 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const std::string crash_on_hang_seconds; 2472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) static const std::string crash_on_hang_thread_names; 2482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) static const std::string thread_names_and_live_threshold; 2492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) static const std::string crash_on_hang_thread_data; 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CustomThreadWatcher* io_watcher_; 251ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch CustomThreadWatcher* db_watcher_; 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcherList* thread_watcher_list_; 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcherTest() 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : setup_complete_(&lock_), 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) initialized_(false) { 257ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB)); 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO)); 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) watchdog_thread_.reset(new WatchDogThread()); 260ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_thread_->Start(); 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_thread_->Start(); 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) watchdog_thread_->Start(); 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this))); 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WaitForSetUp(TimeDelta::FromMinutes(1)); 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void SetUpObjects() { 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(WatchDogThread::CurrentlyOnWatchDogThread()); 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Setup the registry for thread watchers. 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_watcher_list_ = new ThreadWatcherList(); 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Create thread watcher object for the IO thread. 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name, 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kSleepTime, kUnresponsiveTime); 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id)); 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 282ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Create thread watcher object for the DB thread. 283ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_watcher_ = new CustomThreadWatcher( 284ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime); 285ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id)); 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock lock(lock_); 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) initialized_ = true; 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) setup_complete_.Signal(); 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void WaitForSetUp(TimeDelta wait_time) { 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks end_time = TimeTicks::Now() + wait_time; 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::AutoLock auto_lock(lock_); 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (!initialized_ && TimeTicks::Now() < end_time) 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) setup_complete_.TimedWait(end_time - TimeTicks::Now()); 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) virtual ~ThreadWatcherTest() { 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcherList::DeleteAll(); 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_ = NULL; 307ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_watcher_ = NULL; 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_thread_.reset(); 309ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_thread_.reset(); 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) watchdog_thread_.reset(); 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_watcher_list_ = NULL; 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) private: 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Lock lock_; 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::ConditionVariable setup_complete_; 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool initialized_; 318ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch scoped_ptr<content::TestBrowserThread> db_thread_; 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scoped_ptr<content::TestBrowserThread> io_thread_; 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scoped_ptr<WatchDogThread> watchdog_thread_; 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Define static constants. 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const TimeDelta ThreadWatcherTest::kSleepTime = 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeDelta::FromMilliseconds(50); 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const TimeDelta ThreadWatcherTest::kUnresponsiveTime = 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeDelta::FromMilliseconds(500); 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO; 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const std::string ThreadWatcherTest::io_thread_name = "IO"; 330ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochconst BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB; 331ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochconst std::string ThreadWatcherTest::db_thread_name = "DB"; 3322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO"; 3332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)const std::string ThreadWatcherTest::thread_names_and_live_threshold = 3342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) "UI:4,IO:4"; 3352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)const std::string ThreadWatcherTest::crash_on_hang_thread_data = 3362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) "UI:5:12,IO:5:12,FILE:5:12"; 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) { 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Setup command_line arguments. 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CommandLine command_line(CommandLine::NO_PROGRAM); 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 3422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) crash_on_hang_thread_names); 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Parse command_line arguments. 3452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32 unresponsive_threshold; 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadWatcherList::ParseCommandLine(command_line, 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) &unresponsive_threshold, 3492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) &crash_on_hang_threads); 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Verify the data. 3522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::StringTokenizer tokens(crash_on_hang_thread_names, ","); 3532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::vector<std::string> values; 3542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) while (tokens.GetNext()) { 3552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const std::string& token = tokens.token(); 3562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::SplitString(token, ':', &values); 3572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::string thread_name = values[0]; 3582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap::iterator it = 3602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) crash_on_hang_threads.find(thread_name); 3612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) bool crash_on_hang = (it != crash_on_hang_threads.end()); 3622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_TRUE(crash_on_hang); 3632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_LT(0u, it->second.live_threads_threshold); 3642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_LT(0u, it->second.unresponsive_threshold); 3652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 3662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 3672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) { 3692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Setup command_line arguments. 3702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) CommandLine command_line(CommandLine::NO_PROGRAM); 3712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 3722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) thread_names_and_live_threshold); 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Parse command_line arguments. 3752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 3762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uint32 unresponsive_threshold; 3772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::ParseCommandLine(command_line, 3782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) &unresponsive_threshold, 3792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) &crash_on_hang_threads); 3802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Verify the data. 3822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::StringTokenizer tokens(thread_names_and_live_threshold, ","); 3832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::vector<std::string> values; 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (tokens.GetNext()) { 3852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const std::string& token = tokens.token(); 3862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::SplitString(token, ':', &values); 3872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::string thread_name = values[0]; 3882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap::iterator it = 3902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) crash_on_hang_threads.find(thread_name); 3912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) bool crash_on_hang = (it != crash_on_hang_threads.end()); 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_TRUE(crash_on_hang); 3932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_EQ(4u, it->second.live_threads_threshold); 3942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_LT(0u, it->second.unresponsive_threshold); 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) { 3992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Setup command_line arguments. 4002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) CommandLine command_line(CommandLine::NO_PROGRAM); 4012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 4022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) crash_on_hang_thread_data); 4032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Parse command_line arguments. 4052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 4062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uint32 unresponsive_threshold; 4072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::ParseCommandLine(command_line, 4082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) &unresponsive_threshold, 4092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) &crash_on_hang_threads); 4102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Verify the data. 4122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::StringTokenizer tokens(crash_on_hang_thread_data, ","); 4132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::vector<std::string> values; 4142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) while (tokens.GetNext()) { 4152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const std::string& token = tokens.token(); 4162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) base::SplitString(token, ':', &values); 4172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) std::string thread_name = values[0]; 4182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::CrashOnHangThreadMap::iterator it = 4202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) crash_on_hang_threads.find(thread_name); 4212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) bool crash_on_hang = (it != crash_on_hang_threads.end()); 4232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_TRUE(crash_on_hang); 4242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uint32 crash_live_threads_threshold = it->second.live_threads_threshold; 4262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_EQ(5u, crash_live_threads_threshold); 4272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold; 4292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uint32 crash_on_unresponsive_seconds = 4302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold; 4312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPECT_EQ(12u, crash_on_unresponsive_seconds); 4322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Test registration. When thread_watcher_list_ goes out of scope after 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TearDown, all thread watcher objects will be deleted. 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TEST_F(ThreadWatcherTest, Registration) { 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Check ThreadWatcher object has all correct parameters. 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(io_thread_id, io_watcher_->thread_id()); 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(io_thread_name, io_watcher_->thread_name()); 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(kSleepTime, io_watcher_->sleep_time()); 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time()); 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_FALSE(io_watcher_->active()); 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 445ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Check ThreadWatcher object of watched DB thread has correct data. 446ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_EQ(db_thread_id, db_watcher_->thread_id()); 447ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_EQ(db_thread_name, db_watcher_->thread_name()); 448ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_EQ(kSleepTime, db_watcher_->sleep_time()); 449ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time()); 450ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_FALSE(db_watcher_->active()); 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// method also checks that pong message was sent by the watched thread and pong 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// message was received by the WatchDogThread. It also checks that 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OnCheckResponsiveness has verified the ping-pong mechanism and the watched 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// thread is not hung. 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TEST_F(ThreadWatcherTest, ThreadResponding) { 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TimeTicks time_before_ping = TimeTicks::Now(); 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Activate watching IO thread. 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Activate would have started ping/pong messaging. Expect atleast one 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // ping/pong messaging sequence to happen. 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1), 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) RECEIVED_PONG); 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_TRUE(io_watcher_->active()); 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping); 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0)); 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Verify watched thread is responding with ping/pong messaging. 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForCheckResponse( 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 4798bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 4808bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 4818bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 4828bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // DeActivate thread watching for shutdown. 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This test posts a task on watched thread that takes very long time (this is 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// to simulate hanging of watched thread). It then checks for 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// if the watched thread is not responding). 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TEST_F(ThreadWatcherTest, ThreadNotResponding) { 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Simulate hanging of watched thread by making the watched thread wait for a 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // very long time by posting a task on watched thread that keeps it busy. 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // It is safe to use base::Unretained because test is waiting for the method 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to finish. 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BrowserThread::PostTask( 5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_thread_id, 5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&CustomThreadWatcher::VeryLongMethod, 5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_), 5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime * 10)); 5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Activate thread watching. 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Verify watched thread is not responding for ping messages. 5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForCheckResponse( 5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 5168bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 5178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5188bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 5198bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // DeActivate thread watching for shutdown. 5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 5265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Wait for the io_watcher_'s VeryLongMethod to finish. 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Test watching of multiple threads with all threads not responding. 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TEST_F(ThreadWatcherTest, MultipleThreadsResponding) { 533ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Check for DB thread to perform ping/pong messaging. 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 537ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch base::Unretained(db_watcher_))); 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Check for IO thread to perform ping/pong messaging. 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 545ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Verify DB thread is responding with ping/pong messaging. 546ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_watcher_->WaitForCheckResponse( 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 548ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0)); 549ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0)); 550ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 5518bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)), 5528bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5538bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)), 5548bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Verify IO thread is responding with ping/pong messaging. 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForCheckResponse( 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 5628bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 5638bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5648bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 5658bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // DeActivate thread watching for shutdown. 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 576ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch base::Unretained(db_watcher_))); 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Test watching of multiple threads with one of the threads not responding. 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) { 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Simulate hanging of watched thread by making the watched thread wait for a 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // very long time by posting a task on watched thread that keeps it busy. 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // It is safe ot use base::Unretained because test is waiting for the method 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to finish. 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BrowserThread::PostTask( 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_thread_id, 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&CustomThreadWatcher::VeryLongMethod, 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_), 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime * 10)); 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 592ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Activate watching of DB thread. 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 596ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch base::Unretained(db_watcher_))); 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Activate watching of IO thread. 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::ActivateThreadWatching, 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 604ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch // Verify DB thread is responding with ping/pong messaging. 605ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch db_watcher_->WaitForCheckResponse( 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 6078bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)), 6088bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 6098bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)), 6108bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Verify IO thread is not responding for ping messages. 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForCheckResponse( 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 6158bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 6168bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 6178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 6188bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) static_cast<base::subtle::Atomic32>(0)); 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // DeActivate thread watching for shutdown. 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Unretained(io_watcher_))); 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WatchDogThread::PostTask( 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FROM_HERE, 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::Bind(&ThreadWatcher::DeActivateThreadWatching, 628ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch base::Unretained(db_watcher_))); 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Wait for the io_watcher_'s VeryLongMethod to finish. 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 633a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 634a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)class ThreadWatcherListTest : public ::testing::Test { 635a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) protected: 636a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherListTest() 637a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) : done_(&lock_), 638a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) state_available_(false), 639a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) has_thread_watcher_list_(false), 640a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) stopped_(false) { 641a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 642a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 643a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) void ReadStateOnWatchDogThread() { 644a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) CHECK(WatchDogThread::CurrentlyOnWatchDogThread()); 645a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) { 646a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::AutoLock auto_lock(lock_); 647a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) has_thread_watcher_list_ = 648a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::g_thread_watcher_list_ != NULL; 649a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) stopped_ = ThreadWatcherList::g_stopped_; 650a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) state_available_ = true; 651a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 652a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) done_.Signal(); 653a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 654a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 655a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) void CheckState(bool has_thread_watcher_list, 656a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool stopped, 657a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) const char* const msg) { 658a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) CHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 659a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) { 660a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::AutoLock auto_lock(lock_); 661a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) state_available_ = false; 662a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 663a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 664a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) WatchDogThread::PostTask( 665a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) FROM_HERE, 666a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread, 667a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::Unretained(this))); 668a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) { 669a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::AutoLock auto_lock(lock_); 670a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) while (!state_available_) 671a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) done_.Wait(); 672a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 673a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg; 674a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) EXPECT_EQ(stopped, stopped_) << msg; 675a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 676a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 677a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 678a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::Lock lock_; 679a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::ConditionVariable done_; 680a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 681a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool state_available_; 682a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool has_thread_watcher_list_; 683a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool stopped_; 684a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)}; 685a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 686a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)TEST_F(ThreadWatcherListTest, Restart) { 687a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::g_initialize_delay_seconds = 1; 688a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 689a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::MessageLoopForUI message_loop_for_ui; 690a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui); 691a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 692a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread()); 693a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) watchdog_thread_->Start(); 694a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 695a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // See http://crbug.com/347887. 696a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_, 697a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // whilst StopWatchingAll() will just PostTask to destroy it. 698a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // Ensure that when Stop is called, Start will NOT create 699a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // g_thread_watcher_list_ later on. 700a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess()); 701a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::StopWatchingAll(); 702a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.PostDelayedTask( 703a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) FROM_HERE, 704a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.QuitClosure(), 705a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::TimeDelta::FromSeconds( 706a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::g_initialize_delay_seconds)); 707a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.Run(); 708a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 709a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) CheckState(false /* has_thread_watcher_list */, 710a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) true /* stopped */, 711a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) "Start / Stopped"); 712a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 713a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // Proceed with just |StartWatchingAll| and ensure it'll be started. 714a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess()); 715a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.PostDelayedTask( 716a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) FROM_HERE, 717a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.QuitClosure(), 718a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::TimeDelta::FromSeconds( 719a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::g_initialize_delay_seconds + 1)); 720a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.Run(); 721a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 722a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) CheckState(true /* has_thread_watcher_list */, 723a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) false /* stopped */, 724a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) "Started"); 725a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 726a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // Finally, StopWatchingAll() must stop. 727a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::StopWatchingAll(); 728a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.PostDelayedTask( 729a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) FROM_HERE, 730a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.QuitClosure(), 731a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::TimeDelta::FromSeconds( 732a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ThreadWatcherList::g_initialize_delay_seconds)); 733a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) message_loop_for_ui.Run(); 734a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 735a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) CheckState(false /* has_thread_watcher_list */, 736a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) true /* stopped */, 737a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) "Stopped"); 738a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 739