mutex.cc revision 59d9d668d4f4286813afe2b4e7c6db839222ce96
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "mutex.h" 18 19#include <errno.h> 20#include <sys/time.h> 21 22#include "atomic.h" 23#include "base/logging.h" 24#include "mutex-inl.h" 25#include "runtime.h" 26#include "scoped_thread_state_change.h" 27#include "thread-inl.h" 28#include "utils.h" 29 30namespace art { 31 32Mutex* Locks::abort_lock_ = nullptr; 33Mutex* Locks::allocated_monitor_ids_lock_ = nullptr; 34Mutex* Locks::allocated_thread_ids_lock_ = nullptr; 35ReaderWriterMutex* Locks::breakpoint_lock_ = nullptr; 36ReaderWriterMutex* Locks::classlinker_classes_lock_ = nullptr; 37ReaderWriterMutex* Locks::heap_bitmap_lock_ = nullptr; 38Mutex* Locks::logging_lock_ = nullptr; 39Mutex* Locks::mem_maps_lock_ = nullptr; 40Mutex* Locks::modify_ldt_lock_ = nullptr; 41ReaderWriterMutex* Locks::mutator_lock_ = nullptr; 42Mutex* Locks::runtime_shutdown_lock_ = nullptr; 43Mutex* Locks::thread_list_lock_ = nullptr; 44Mutex* Locks::thread_list_suspend_thread_lock_ = nullptr; 45Mutex* Locks::thread_suspend_count_lock_ = nullptr; 46Mutex* Locks::trace_lock_ = nullptr; 47Mutex* Locks::profiler_lock_ = nullptr; 48Mutex* Locks::unexpected_signal_lock_ = nullptr; 49Mutex* Locks::intern_table_lock_ = nullptr; 50 51struct AllMutexData { 52 // A guard for all_mutexes_ that's not a mutex (Mutexes must CAS to acquire and busy wait). 53 Atomic<const BaseMutex*> all_mutexes_guard; 54 // All created mutexes guarded by all_mutexes_guard_. 55 std::set<BaseMutex*>* all_mutexes; 56 AllMutexData() : all_mutexes(NULL) {} 57}; 58static struct AllMutexData gAllMutexData[kAllMutexDataSize]; 59 60#if ART_USE_FUTEXES 61static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) { 62 const int32_t one_sec = 1000 * 1000 * 1000; // one second in nanoseconds. 63 result_ts->tv_sec = lhs.tv_sec - rhs.tv_sec; 64 result_ts->tv_nsec = lhs.tv_nsec - rhs.tv_nsec; 65 if (result_ts->tv_nsec < 0) { 66 result_ts->tv_sec--; 67 result_ts->tv_nsec += one_sec; 68 } else if (result_ts->tv_nsec > one_sec) { 69 result_ts->tv_sec++; 70 result_ts->tv_nsec -= one_sec; 71 } 72 return result_ts->tv_sec < 0; 73} 74#endif 75 76class ScopedAllMutexesLock { 77 public: 78 explicit ScopedAllMutexesLock(const BaseMutex* mutex) : mutex_(mutex) { 79 while (!gAllMutexData->all_mutexes_guard.CompareExchangeWeakAcquire(0, mutex)) { 80 NanoSleep(100); 81 } 82 } 83 ~ScopedAllMutexesLock() { 84 while (!gAllMutexData->all_mutexes_guard.CompareExchangeWeakRelease(mutex_, 0)) { 85 NanoSleep(100); 86 } 87 } 88 private: 89 const BaseMutex* const mutex_; 90}; 91 92BaseMutex::BaseMutex(const char* name, LockLevel level) : level_(level), name_(name) { 93 if (kLogLockContentions) { 94 ScopedAllMutexesLock mu(this); 95 std::set<BaseMutex*>** all_mutexes_ptr = &gAllMutexData->all_mutexes; 96 if (*all_mutexes_ptr == NULL) { 97 // We leak the global set of all mutexes to avoid ordering issues in global variable 98 // construction/destruction. 99 *all_mutexes_ptr = new std::set<BaseMutex*>(); 100 } 101 (*all_mutexes_ptr)->insert(this); 102 } 103} 104 105BaseMutex::~BaseMutex() { 106 if (kLogLockContentions) { 107 ScopedAllMutexesLock mu(this); 108 gAllMutexData->all_mutexes->erase(this); 109 } 110} 111 112void BaseMutex::DumpAll(std::ostream& os) { 113 if (kLogLockContentions) { 114 os << "Mutex logging:\n"; 115 ScopedAllMutexesLock mu(reinterpret_cast<const BaseMutex*>(-1)); 116 std::set<BaseMutex*>* all_mutexes = gAllMutexData->all_mutexes; 117 if (all_mutexes == NULL) { 118 // No mutexes have been created yet during at startup. 119 return; 120 } 121 typedef std::set<BaseMutex*>::const_iterator It; 122 os << "(Contended)\n"; 123 for (It it = all_mutexes->begin(); it != all_mutexes->end(); ++it) { 124 BaseMutex* mutex = *it; 125 if (mutex->HasEverContended()) { 126 mutex->Dump(os); 127 os << "\n"; 128 } 129 } 130 os << "(Never contented)\n"; 131 for (It it = all_mutexes->begin(); it != all_mutexes->end(); ++it) { 132 BaseMutex* mutex = *it; 133 if (!mutex->HasEverContended()) { 134 mutex->Dump(os); 135 os << "\n"; 136 } 137 } 138 } 139} 140 141void BaseMutex::CheckSafeToWait(Thread* self) { 142 if (self == NULL) { 143 CheckUnattachedThread(level_); 144 return; 145 } 146 if (kDebugLocking) { 147 CHECK(self->GetHeldMutex(level_) == this || level_ == kMonitorLock) 148 << "Waiting on unacquired mutex: " << name_; 149 bool bad_mutexes_held = false; 150 for (int i = kLockLevelCount - 1; i >= 0; --i) { 151 if (i != level_) { 152 BaseMutex* held_mutex = self->GetHeldMutex(static_cast<LockLevel>(i)); 153 // We expect waits to happen while holding the thread list suspend thread lock. 154 if (held_mutex != NULL && i != kThreadListSuspendThreadLock) { 155 LOG(ERROR) << "Holding \"" << held_mutex->name_ << "\" " 156 << "(level " << LockLevel(i) << ") while performing wait on " 157 << "\"" << name_ << "\" (level " << level_ << ")"; 158 bad_mutexes_held = true; 159 } 160 } 161 } 162 CHECK(!bad_mutexes_held); 163 } 164} 165 166void BaseMutex::ContentionLogData::AddToWaitTime(uint64_t value) { 167 if (kLogLockContentions) { 168 // Atomically add value to wait_time. 169 wait_time.FetchAndAddSequentiallyConsistent(value); 170 } 171} 172 173void BaseMutex::RecordContention(uint64_t blocked_tid, 174 uint64_t owner_tid, 175 uint64_t nano_time_blocked) { 176 if (kLogLockContentions) { 177 ContentionLogData* data = contention_log_data_; 178 ++(data->contention_count); 179 data->AddToWaitTime(nano_time_blocked); 180 ContentionLogEntry* log = data->contention_log; 181 // This code is intentionally racy as it is only used for diagnostics. 182 uint32_t slot = data->cur_content_log_entry.LoadRelaxed(); 183 if (log[slot].blocked_tid == blocked_tid && 184 log[slot].owner_tid == blocked_tid) { 185 ++log[slot].count; 186 } else { 187 uint32_t new_slot; 188 do { 189 slot = data->cur_content_log_entry.LoadRelaxed(); 190 new_slot = (slot + 1) % kContentionLogSize; 191 } while (!data->cur_content_log_entry.CompareExchangeWeakRelaxed(slot, new_slot)); 192 log[new_slot].blocked_tid = blocked_tid; 193 log[new_slot].owner_tid = owner_tid; 194 log[new_slot].count.StoreRelaxed(1); 195 } 196 } 197} 198 199void BaseMutex::DumpContention(std::ostream& os) const { 200 if (kLogLockContentions) { 201 const ContentionLogData* data = contention_log_data_; 202 const ContentionLogEntry* log = data->contention_log; 203 uint64_t wait_time = data->wait_time.LoadRelaxed(); 204 uint32_t contention_count = data->contention_count.LoadRelaxed(); 205 if (contention_count == 0) { 206 os << "never contended"; 207 } else { 208 os << "contended " << contention_count 209 << " total wait of contender " << PrettyDuration(wait_time) 210 << " average " << PrettyDuration(wait_time / contention_count); 211 SafeMap<uint64_t, size_t> most_common_blocker; 212 SafeMap<uint64_t, size_t> most_common_blocked; 213 for (size_t i = 0; i < kContentionLogSize; ++i) { 214 uint64_t blocked_tid = log[i].blocked_tid; 215 uint64_t owner_tid = log[i].owner_tid; 216 uint32_t count = log[i].count.LoadRelaxed(); 217 if (count > 0) { 218 auto it = most_common_blocked.find(blocked_tid); 219 if (it != most_common_blocked.end()) { 220 most_common_blocked.Overwrite(blocked_tid, it->second + count); 221 } else { 222 most_common_blocked.Put(blocked_tid, count); 223 } 224 it = most_common_blocker.find(owner_tid); 225 if (it != most_common_blocker.end()) { 226 most_common_blocker.Overwrite(owner_tid, it->second + count); 227 } else { 228 most_common_blocker.Put(owner_tid, count); 229 } 230 } 231 } 232 uint64_t max_tid = 0; 233 size_t max_tid_count = 0; 234 for (const auto& pair : most_common_blocked) { 235 if (pair.second > max_tid_count) { 236 max_tid = pair.first; 237 max_tid_count = pair.second; 238 } 239 } 240 if (max_tid != 0) { 241 os << " sample shows most blocked tid=" << max_tid; 242 } 243 max_tid = 0; 244 max_tid_count = 0; 245 for (const auto& pair : most_common_blocker) { 246 if (pair.second > max_tid_count) { 247 max_tid = pair.first; 248 max_tid_count = pair.second; 249 } 250 } 251 if (max_tid != 0) { 252 os << " sample shows tid=" << max_tid << " owning during this time"; 253 } 254 } 255 } 256} 257 258 259Mutex::Mutex(const char* name, LockLevel level, bool recursive) 260 : BaseMutex(name, level), recursive_(recursive), recursion_count_(0) { 261#if ART_USE_FUTEXES 262 DCHECK_EQ(0, state_.LoadRelaxed()); 263 DCHECK_EQ(0, num_contenders_.LoadRelaxed()); 264#else 265 CHECK_MUTEX_CALL(pthread_mutex_init, (&mutex_, nullptr)); 266#endif 267 exclusive_owner_ = 0; 268} 269 270Mutex::~Mutex() { 271#if ART_USE_FUTEXES 272 if (state_.LoadRelaxed() != 0) { 273 Runtime* runtime = Runtime::Current(); 274 bool shutting_down = runtime == nullptr || runtime->IsShuttingDown(Thread::Current()); 275 LOG(shutting_down ? WARNING : FATAL) << "destroying mutex with owner: " << exclusive_owner_; 276 } else { 277 CHECK_EQ(exclusive_owner_, 0U) << "unexpectedly found an owner on unlocked mutex " << name_; 278 CHECK_EQ(num_contenders_.LoadSequentiallyConsistent(), 0) 279 << "unexpectedly found a contender on mutex " << name_; 280 } 281#else 282 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread 283 // may still be using locks. 284 int rc = pthread_mutex_destroy(&mutex_); 285 if (rc != 0) { 286 errno = rc; 287 // TODO: should we just not log at all if shutting down? this could be the logging mutex! 288 MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_); 289 Runtime* runtime = Runtime::Current(); 290 bool shutting_down = (runtime == NULL) || runtime->IsShuttingDownLocked(); 291 PLOG(shutting_down ? WARNING : FATAL) << "pthread_mutex_destroy failed for " << name_; 292 } 293#endif 294} 295 296void Mutex::ExclusiveLock(Thread* self) { 297 DCHECK(self == NULL || self == Thread::Current()); 298 if (kDebugLocking && !recursive_) { 299 AssertNotHeld(self); 300 } 301 if (!recursive_ || !IsExclusiveHeld(self)) { 302#if ART_USE_FUTEXES 303 bool done = false; 304 do { 305 int32_t cur_state = state_.LoadRelaxed(); 306 if (LIKELY(cur_state == 0)) { 307 // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition. 308 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, 1 /* new state */); 309 } else { 310 // Failed to acquire, hang up. 311 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid()); 312 num_contenders_++; 313 if (futex(state_.Address(), FUTEX_WAIT, 1, NULL, NULL, 0) != 0) { 314 // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning. 315 // We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock. 316 if ((errno != EAGAIN) && (errno != EINTR)) { 317 PLOG(FATAL) << "futex wait failed for " << name_; 318 } 319 } 320 num_contenders_--; 321 } 322 } while (!done); 323 DCHECK_EQ(state_.LoadRelaxed(), 1); 324#else 325 CHECK_MUTEX_CALL(pthread_mutex_lock, (&mutex_)); 326#endif 327 DCHECK_EQ(exclusive_owner_, 0U); 328 exclusive_owner_ = SafeGetTid(self); 329 RegisterAsLocked(self); 330 } 331 recursion_count_++; 332 if (kDebugLocking) { 333 CHECK(recursion_count_ == 1 || recursive_) << "Unexpected recursion count on mutex: " 334 << name_ << " " << recursion_count_; 335 AssertHeld(self); 336 } 337} 338 339bool Mutex::ExclusiveTryLock(Thread* self) { 340 DCHECK(self == NULL || self == Thread::Current()); 341 if (kDebugLocking && !recursive_) { 342 AssertNotHeld(self); 343 } 344 if (!recursive_ || !IsExclusiveHeld(self)) { 345#if ART_USE_FUTEXES 346 bool done = false; 347 do { 348 int32_t cur_state = state_.LoadRelaxed(); 349 if (cur_state == 0) { 350 // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition. 351 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, 1 /* new state */); 352 } else { 353 return false; 354 } 355 } while (!done); 356 DCHECK_EQ(state_.LoadRelaxed(), 1); 357#else 358 int result = pthread_mutex_trylock(&mutex_); 359 if (result == EBUSY) { 360 return false; 361 } 362 if (result != 0) { 363 errno = result; 364 PLOG(FATAL) << "pthread_mutex_trylock failed for " << name_; 365 } 366#endif 367 DCHECK_EQ(exclusive_owner_, 0U); 368 exclusive_owner_ = SafeGetTid(self); 369 RegisterAsLocked(self); 370 } 371 recursion_count_++; 372 if (kDebugLocking) { 373 CHECK(recursion_count_ == 1 || recursive_) << "Unexpected recursion count on mutex: " 374 << name_ << " " << recursion_count_; 375 AssertHeld(self); 376 } 377 return true; 378} 379 380void Mutex::ExclusiveUnlock(Thread* self) { 381 DCHECK(self == NULL || self == Thread::Current()); 382 AssertHeld(self); 383 DCHECK_NE(exclusive_owner_, 0U); 384 recursion_count_--; 385 if (!recursive_ || recursion_count_ == 0) { 386 if (kDebugLocking) { 387 CHECK(recursion_count_ == 0 || recursive_) << "Unexpected recursion count on mutex: " 388 << name_ << " " << recursion_count_; 389 } 390 RegisterAsUnlocked(self); 391#if ART_USE_FUTEXES 392 bool done = false; 393 do { 394 int32_t cur_state = state_.LoadRelaxed(); 395 if (LIKELY(cur_state == 1)) { 396 // We're no longer the owner. 397 exclusive_owner_ = 0; 398 // Change state to 0 and impose load/store ordering appropriate for lock release. 399 // Note, the relaxed loads below musn't reorder before the CompareExchange. 400 // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing 401 // a status bit into the state on contention. 402 done = state_.CompareExchangeWeakSequentiallyConsistent(cur_state, 0 /* new state */); 403 if (LIKELY(done)) { // Spurious fail? 404 // Wake a contender. 405 if (UNLIKELY(num_contenders_.LoadRelaxed() > 0)) { 406 futex(state_.Address(), FUTEX_WAKE, 1, NULL, NULL, 0); 407 } 408 } 409 } else { 410 // Logging acquires the logging lock, avoid infinite recursion in that case. 411 if (this != Locks::logging_lock_) { 412 LOG(FATAL) << "Unexpected state_ in unlock " << cur_state << " for " << name_; 413 } else { 414 LogMessageData data(__FILE__, __LINE__, INTERNAL_FATAL, -1); 415 LogMessage::LogLine(data, StringPrintf("Unexpected state_ %d in unlock for %s", 416 cur_state, name_).c_str()); 417 _exit(1); 418 } 419 } 420 } while (!done); 421#else 422 exclusive_owner_ = 0; 423 CHECK_MUTEX_CALL(pthread_mutex_unlock, (&mutex_)); 424#endif 425 } 426} 427 428void Mutex::Dump(std::ostream& os) const { 429 os << (recursive_ ? "recursive " : "non-recursive ") 430 << name_ 431 << " level=" << static_cast<int>(level_) 432 << " rec=" << recursion_count_ 433 << " owner=" << GetExclusiveOwnerTid() << " "; 434 DumpContention(os); 435} 436 437std::ostream& operator<<(std::ostream& os, const Mutex& mu) { 438 mu.Dump(os); 439 return os; 440} 441 442ReaderWriterMutex::ReaderWriterMutex(const char* name, LockLevel level) 443 : BaseMutex(name, level) 444#if ART_USE_FUTEXES 445 , state_(0), num_pending_readers_(0), num_pending_writers_(0) 446#endif 447{ // NOLINT(whitespace/braces) 448#if !ART_USE_FUTEXES 449 CHECK_MUTEX_CALL(pthread_rwlock_init, (&rwlock_, nullptr)); 450#endif 451 exclusive_owner_ = 0; 452} 453 454ReaderWriterMutex::~ReaderWriterMutex() { 455#if ART_USE_FUTEXES 456 CHECK_EQ(state_.LoadRelaxed(), 0); 457 CHECK_EQ(exclusive_owner_, 0U); 458 CHECK_EQ(num_pending_readers_.LoadRelaxed(), 0); 459 CHECK_EQ(num_pending_writers_.LoadRelaxed(), 0); 460#else 461 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread 462 // may still be using locks. 463 int rc = pthread_rwlock_destroy(&rwlock_); 464 if (rc != 0) { 465 errno = rc; 466 // TODO: should we just not log at all if shutting down? this could be the logging mutex! 467 MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_); 468 Runtime* runtime = Runtime::Current(); 469 bool shutting_down = runtime == NULL || runtime->IsShuttingDownLocked(); 470 PLOG(shutting_down ? WARNING : FATAL) << "pthread_rwlock_destroy failed for " << name_; 471 } 472#endif 473} 474 475void ReaderWriterMutex::ExclusiveLock(Thread* self) { 476 DCHECK(self == NULL || self == Thread::Current()); 477 AssertNotExclusiveHeld(self); 478#if ART_USE_FUTEXES 479 bool done = false; 480 do { 481 int32_t cur_state = state_.LoadRelaxed(); 482 if (LIKELY(cur_state == 0)) { 483 // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition. 484 done = state_.CompareExchangeWeakAcquire(0 /* cur_state*/, -1 /* new state */); 485 } else { 486 // Failed to acquire, hang up. 487 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid()); 488 ++num_pending_writers_; 489 if (futex(state_.Address(), FUTEX_WAIT, cur_state, NULL, NULL, 0) != 0) { 490 // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning. 491 // We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock. 492 if ((errno != EAGAIN) && (errno != EINTR)) { 493 PLOG(FATAL) << "futex wait failed for " << name_; 494 } 495 } 496 --num_pending_writers_; 497 } 498 } while (!done); 499 DCHECK_EQ(state_.LoadRelaxed(), -1); 500#else 501 CHECK_MUTEX_CALL(pthread_rwlock_wrlock, (&rwlock_)); 502#endif 503 DCHECK_EQ(exclusive_owner_, 0U); 504 exclusive_owner_ = SafeGetTid(self); 505 RegisterAsLocked(self); 506 AssertExclusiveHeld(self); 507} 508 509void ReaderWriterMutex::ExclusiveUnlock(Thread* self) { 510 DCHECK(self == NULL || self == Thread::Current()); 511 AssertExclusiveHeld(self); 512 RegisterAsUnlocked(self); 513 DCHECK_NE(exclusive_owner_, 0U); 514#if ART_USE_FUTEXES 515 bool done = false; 516 do { 517 int32_t cur_state = state_.LoadRelaxed(); 518 if (LIKELY(cur_state == -1)) { 519 // We're no longer the owner. 520 exclusive_owner_ = 0; 521 // Change state from -1 to 0 and impose load/store ordering appropriate for lock release. 522 // Note, the relaxed loads below musn't reorder before the CompareExchange. 523 // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing 524 // a status bit into the state on contention. 525 done = state_.CompareExchangeWeakSequentiallyConsistent(-1 /* cur_state*/, 0 /* new state */); 526 if (LIKELY(done)) { // Weak CAS may fail spuriously. 527 // Wake any waiters. 528 if (UNLIKELY(num_pending_readers_.LoadRelaxed() > 0 || 529 num_pending_writers_.LoadRelaxed() > 0)) { 530 futex(state_.Address(), FUTEX_WAKE, -1, NULL, NULL, 0); 531 } 532 } 533 } else { 534 LOG(FATAL) << "Unexpected state_:" << cur_state << " for " << name_; 535 } 536 } while (!done); 537#else 538 exclusive_owner_ = 0; 539 CHECK_MUTEX_CALL(pthread_rwlock_unlock, (&rwlock_)); 540#endif 541} 542 543#if HAVE_TIMED_RWLOCK 544bool ReaderWriterMutex::ExclusiveLockWithTimeout(Thread* self, int64_t ms, int32_t ns) { 545 DCHECK(self == NULL || self == Thread::Current()); 546#if ART_USE_FUTEXES 547 bool done = false; 548 timespec end_abs_ts; 549 InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &end_abs_ts); 550 do { 551 int32_t cur_state = state_.LoadRelaxed(); 552 if (cur_state == 0) { 553 // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition. 554 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, -1 /* new state */); 555 } else { 556 // Failed to acquire, hang up. 557 timespec now_abs_ts; 558 InitTimeSpec(true, CLOCK_REALTIME, 0, 0, &now_abs_ts); 559 timespec rel_ts; 560 if (ComputeRelativeTimeSpec(&rel_ts, end_abs_ts, now_abs_ts)) { 561 return false; // Timed out. 562 } 563 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid()); 564 ++num_pending_writers_; 565 if (futex(state_.Address(), FUTEX_WAIT, cur_state, &rel_ts, NULL, 0) != 0) { 566 if (errno == ETIMEDOUT) { 567 --num_pending_writers_; 568 return false; // Timed out. 569 } else if ((errno != EAGAIN) && (errno != EINTR)) { 570 // EAGAIN and EINTR both indicate a spurious failure, 571 // recompute the relative time out from now and try again. 572 // We don't use TEMP_FAILURE_RETRY so we can recompute rel_ts; 573 PLOG(FATAL) << "timed futex wait failed for " << name_; 574 } 575 } 576 --num_pending_writers_; 577 } 578 } while (!done); 579#else 580 timespec ts; 581 InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts); 582 int result = pthread_rwlock_timedwrlock(&rwlock_, &ts); 583 if (result == ETIMEDOUT) { 584 return false; 585 } 586 if (result != 0) { 587 errno = result; 588 PLOG(FATAL) << "pthread_rwlock_timedwrlock failed for " << name_; 589 } 590#endif 591 exclusive_owner_ = SafeGetTid(self); 592 RegisterAsLocked(self); 593 AssertSharedHeld(self); 594 return true; 595} 596#endif 597 598bool ReaderWriterMutex::SharedTryLock(Thread* self) { 599 DCHECK(self == NULL || self == Thread::Current()); 600#if ART_USE_FUTEXES 601 bool done = false; 602 do { 603 int32_t cur_state = state_.LoadRelaxed(); 604 if (cur_state >= 0) { 605 // Add as an extra reader and impose load/store ordering appropriate for lock acquisition. 606 done = state_.CompareExchangeWeakAcquire(cur_state, cur_state + 1); 607 } else { 608 // Owner holds it exclusively. 609 return false; 610 } 611 } while (!done); 612#else 613 int result = pthread_rwlock_tryrdlock(&rwlock_); 614 if (result == EBUSY) { 615 return false; 616 } 617 if (result != 0) { 618 errno = result; 619 PLOG(FATAL) << "pthread_mutex_trylock failed for " << name_; 620 } 621#endif 622 RegisterAsLocked(self); 623 AssertSharedHeld(self); 624 return true; 625} 626 627bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const { 628 DCHECK(self == NULL || self == Thread::Current()); 629 bool result; 630 if (UNLIKELY(self == NULL)) { // Handle unattached threads. 631 result = IsExclusiveHeld(self); // TODO: a better best effort here. 632 } else { 633 result = (self->GetHeldMutex(level_) == this); 634 } 635 return result; 636} 637 638void ReaderWriterMutex::Dump(std::ostream& os) const { 639 os << name_ 640 << " level=" << static_cast<int>(level_) 641 << " owner=" << GetExclusiveOwnerTid() << " "; 642 DumpContention(os); 643} 644 645std::ostream& operator<<(std::ostream& os, const ReaderWriterMutex& mu) { 646 mu.Dump(os); 647 return os; 648} 649 650ConditionVariable::ConditionVariable(const char* name, Mutex& guard) 651 : name_(name), guard_(guard) { 652#if ART_USE_FUTEXES 653 DCHECK_EQ(0, sequence_.LoadRelaxed()); 654 num_waiters_ = 0; 655#else 656 pthread_condattr_t cond_attrs; 657 CHECK_MUTEX_CALL(pthread_condattr_init, (&cond_attrs)); 658#if !defined(__APPLE__) 659 // Apple doesn't have CLOCK_MONOTONIC or pthread_condattr_setclock. 660 CHECK_MUTEX_CALL(pthread_condattr_setclock(&cond_attrs, CLOCK_MONOTONIC)); 661#endif 662 CHECK_MUTEX_CALL(pthread_cond_init, (&cond_, &cond_attrs)); 663#endif 664} 665 666ConditionVariable::~ConditionVariable() { 667#if ART_USE_FUTEXES 668 if (num_waiters_!= 0) { 669 Runtime* runtime = Runtime::Current(); 670 bool shutting_down = runtime == nullptr || runtime->IsShuttingDown(Thread::Current()); 671 LOG(shutting_down ? WARNING : FATAL) << "ConditionVariable::~ConditionVariable for " << name_ 672 << " called with " << num_waiters_ << " waiters."; 673 } 674#else 675 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread 676 // may still be using condition variables. 677 int rc = pthread_cond_destroy(&cond_); 678 if (rc != 0) { 679 errno = rc; 680 MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_); 681 Runtime* runtime = Runtime::Current(); 682 bool shutting_down = (runtime == NULL) || runtime->IsShuttingDownLocked(); 683 PLOG(shutting_down ? WARNING : FATAL) << "pthread_cond_destroy failed for " << name_; 684 } 685#endif 686} 687 688void ConditionVariable::Broadcast(Thread* self) { 689 DCHECK(self == NULL || self == Thread::Current()); 690 // TODO: enable below, there's a race in thread creation that causes false failures currently. 691 // guard_.AssertExclusiveHeld(self); 692 DCHECK_EQ(guard_.GetExclusiveOwnerTid(), SafeGetTid(self)); 693#if ART_USE_FUTEXES 694 if (num_waiters_ > 0) { 695 sequence_++; // Indicate the broadcast occurred. 696 bool done = false; 697 do { 698 int32_t cur_sequence = sequence_.LoadRelaxed(); 699 // Requeue waiters onto mutex. The waiter holds the contender count on the mutex high ensuring 700 // mutex unlocks will awaken the requeued waiter thread. 701 done = futex(sequence_.Address(), FUTEX_CMP_REQUEUE, 0, 702 reinterpret_cast<const timespec*>(std::numeric_limits<int32_t>::max()), 703 guard_.state_.Address(), cur_sequence) != -1; 704 if (!done) { 705 if (errno != EAGAIN) { 706 PLOG(FATAL) << "futex cmp requeue failed for " << name_; 707 } 708 } 709 } while (!done); 710 } 711#else 712 CHECK_MUTEX_CALL(pthread_cond_broadcast, (&cond_)); 713#endif 714} 715 716void ConditionVariable::Signal(Thread* self) { 717 DCHECK(self == NULL || self == Thread::Current()); 718 guard_.AssertExclusiveHeld(self); 719#if ART_USE_FUTEXES 720 if (num_waiters_ > 0) { 721 sequence_++; // Indicate a signal occurred. 722 // Futex wake 1 waiter who will then come and in contend on mutex. It'd be nice to requeue them 723 // to avoid this, however, requeueing can only move all waiters. 724 int num_woken = futex(sequence_.Address(), FUTEX_WAKE, 1, NULL, NULL, 0); 725 // Check something was woken or else we changed sequence_ before they had chance to wait. 726 CHECK((num_woken == 0) || (num_woken == 1)); 727 } 728#else 729 CHECK_MUTEX_CALL(pthread_cond_signal, (&cond_)); 730#endif 731} 732 733void ConditionVariable::Wait(Thread* self) { 734 guard_.CheckSafeToWait(self); 735 WaitHoldingLocks(self); 736} 737 738void ConditionVariable::WaitHoldingLocks(Thread* self) { 739 DCHECK(self == NULL || self == Thread::Current()); 740 guard_.AssertExclusiveHeld(self); 741 unsigned int old_recursion_count = guard_.recursion_count_; 742#if ART_USE_FUTEXES 743 num_waiters_++; 744 // Ensure the Mutex is contended so that requeued threads are awoken. 745 guard_.num_contenders_++; 746 guard_.recursion_count_ = 1; 747 int32_t cur_sequence = sequence_.LoadRelaxed(); 748 guard_.ExclusiveUnlock(self); 749 if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, NULL, NULL, 0) != 0) { 750 // Futex failed, check it is an expected error. 751 // EAGAIN == EWOULDBLK, so we let the caller try again. 752 // EINTR implies a signal was sent to this thread. 753 if ((errno != EINTR) && (errno != EAGAIN)) { 754 PLOG(FATAL) << "futex wait failed for " << name_; 755 } 756 } 757 guard_.ExclusiveLock(self); 758 CHECK_GE(num_waiters_, 0); 759 num_waiters_--; 760 // We awoke and so no longer require awakes from the guard_'s unlock. 761 CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0); 762 guard_.num_contenders_--; 763#else 764 uint64_t old_owner = guard_.exclusive_owner_; 765 guard_.exclusive_owner_ = 0; 766 guard_.recursion_count_ = 0; 767 CHECK_MUTEX_CALL(pthread_cond_wait, (&cond_, &guard_.mutex_)); 768 guard_.exclusive_owner_ = old_owner; 769#endif 770 guard_.recursion_count_ = old_recursion_count; 771} 772 773void ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) { 774 DCHECK(self == NULL || self == Thread::Current()); 775 guard_.AssertExclusiveHeld(self); 776 guard_.CheckSafeToWait(self); 777 unsigned int old_recursion_count = guard_.recursion_count_; 778#if ART_USE_FUTEXES 779 timespec rel_ts; 780 InitTimeSpec(false, CLOCK_REALTIME, ms, ns, &rel_ts); 781 num_waiters_++; 782 // Ensure the Mutex is contended so that requeued threads are awoken. 783 guard_.num_contenders_++; 784 guard_.recursion_count_ = 1; 785 int32_t cur_sequence = sequence_.LoadRelaxed(); 786 guard_.ExclusiveUnlock(self); 787 if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, &rel_ts, NULL, 0) != 0) { 788 if (errno == ETIMEDOUT) { 789 // Timed out we're done. 790 } else if ((errno == EAGAIN) || (errno == EINTR)) { 791 // A signal or ConditionVariable::Signal/Broadcast has come in. 792 } else { 793 PLOG(FATAL) << "timed futex wait failed for " << name_; 794 } 795 } 796 guard_.ExclusiveLock(self); 797 CHECK_GE(num_waiters_, 0); 798 num_waiters_--; 799 // We awoke and so no longer require awakes from the guard_'s unlock. 800 CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0); 801 guard_.num_contenders_--; 802#else 803#if !defined(__APPLE__) 804 int clock = CLOCK_MONOTONIC; 805#else 806 int clock = CLOCK_REALTIME; 807#endif 808 uint64_t old_owner = guard_.exclusive_owner_; 809 guard_.exclusive_owner_ = 0; 810 guard_.recursion_count_ = 0; 811 timespec ts; 812 InitTimeSpec(true, clock, ms, ns, &ts); 813 int rc = TEMP_FAILURE_RETRY(pthread_cond_timedwait(&cond_, &guard_.mutex_, &ts)); 814 if (rc != 0 && rc != ETIMEDOUT) { 815 errno = rc; 816 PLOG(FATAL) << "TimedWait failed for " << name_; 817 } 818 guard_.exclusive_owner_ = old_owner; 819#endif 820 guard_.recursion_count_ = old_recursion_count; 821} 822 823void Locks::Init() { 824 if (logging_lock_ != nullptr) { 825 // Already initialized. 826 if (kRuntimeISA == kX86 || kRuntimeISA == kX86_64) { 827 DCHECK(modify_ldt_lock_ != nullptr); 828 } else { 829 DCHECK(modify_ldt_lock_ == nullptr); 830 } 831 DCHECK(abort_lock_ != nullptr); 832 DCHECK(allocated_monitor_ids_lock_ != nullptr); 833 DCHECK(allocated_thread_ids_lock_ != nullptr); 834 DCHECK(breakpoint_lock_ != nullptr); 835 DCHECK(classlinker_classes_lock_ != nullptr); 836 DCHECK(heap_bitmap_lock_ != nullptr); 837 DCHECK(logging_lock_ != nullptr); 838 DCHECK(mutator_lock_ != nullptr); 839 DCHECK(thread_list_lock_ != nullptr); 840 DCHECK(thread_list_suspend_thread_lock_ != nullptr); 841 DCHECK(thread_suspend_count_lock_ != nullptr); 842 DCHECK(trace_lock_ != nullptr); 843 DCHECK(profiler_lock_ != nullptr); 844 DCHECK(unexpected_signal_lock_ != nullptr); 845 DCHECK(intern_table_lock_ != nullptr); 846 } else { 847 // Create global locks in level order from highest lock level to lowest. 848 LockLevel current_lock_level = kThreadListSuspendThreadLock; 849 DCHECK(thread_list_suspend_thread_lock_ == nullptr); 850 thread_list_suspend_thread_lock_ = 851 new Mutex("thread list suspend thread by .. lock", current_lock_level); 852 853 #define UPDATE_CURRENT_LOCK_LEVEL(new_level) \ 854 DCHECK_LT(new_level, current_lock_level); \ 855 current_lock_level = new_level; 856 857 UPDATE_CURRENT_LOCK_LEVEL(kMutatorLock); 858 DCHECK(mutator_lock_ == nullptr); 859 mutator_lock_ = new ReaderWriterMutex("mutator lock", current_lock_level); 860 861 UPDATE_CURRENT_LOCK_LEVEL(kHeapBitmapLock); 862 DCHECK(heap_bitmap_lock_ == nullptr); 863 heap_bitmap_lock_ = new ReaderWriterMutex("heap bitmap lock", current_lock_level); 864 865 UPDATE_CURRENT_LOCK_LEVEL(kRuntimeShutdownLock); 866 DCHECK(runtime_shutdown_lock_ == nullptr); 867 runtime_shutdown_lock_ = new Mutex("runtime shutdown lock", current_lock_level); 868 869 UPDATE_CURRENT_LOCK_LEVEL(kProfilerLock); 870 DCHECK(profiler_lock_ == nullptr); 871 profiler_lock_ = new Mutex("profiler lock", current_lock_level); 872 873 UPDATE_CURRENT_LOCK_LEVEL(kTraceLock); 874 DCHECK(trace_lock_ == nullptr); 875 trace_lock_ = new Mutex("trace lock", current_lock_level); 876 877 UPDATE_CURRENT_LOCK_LEVEL(kThreadListLock); 878 DCHECK(thread_list_lock_ == nullptr); 879 thread_list_lock_ = new Mutex("thread list lock", current_lock_level); 880 881 UPDATE_CURRENT_LOCK_LEVEL(kBreakpointLock); 882 DCHECK(breakpoint_lock_ == nullptr); 883 breakpoint_lock_ = new ReaderWriterMutex("breakpoint lock", current_lock_level); 884 885 UPDATE_CURRENT_LOCK_LEVEL(kClassLinkerClassesLock); 886 DCHECK(classlinker_classes_lock_ == nullptr); 887 classlinker_classes_lock_ = new ReaderWriterMutex("ClassLinker classes lock", 888 current_lock_level); 889 890 UPDATE_CURRENT_LOCK_LEVEL(kMonitorPoolLock); 891 DCHECK(allocated_monitor_ids_lock_ == nullptr); 892 allocated_monitor_ids_lock_ = new Mutex("allocated monitor ids lock", current_lock_level); 893 894 UPDATE_CURRENT_LOCK_LEVEL(kAllocatedThreadIdsLock); 895 DCHECK(allocated_thread_ids_lock_ == nullptr); 896 allocated_thread_ids_lock_ = new Mutex("allocated thread ids lock", current_lock_level); 897 898 if (kRuntimeISA == kX86 || kRuntimeISA == kX86_64) { 899 UPDATE_CURRENT_LOCK_LEVEL(kModifyLdtLock); 900 DCHECK(modify_ldt_lock_ == nullptr); 901 modify_ldt_lock_ = new Mutex("modify_ldt lock", current_lock_level); 902 } 903 904 UPDATE_CURRENT_LOCK_LEVEL(kInternTableLock); 905 DCHECK(intern_table_lock_ == nullptr); 906 intern_table_lock_ = new Mutex("InternTable lock", current_lock_level); 907 908 909 UPDATE_CURRENT_LOCK_LEVEL(kAbortLock); 910 DCHECK(abort_lock_ == nullptr); 911 abort_lock_ = new Mutex("abort lock", current_lock_level, true); 912 913 UPDATE_CURRENT_LOCK_LEVEL(kThreadSuspendCountLock); 914 DCHECK(thread_suspend_count_lock_ == nullptr); 915 thread_suspend_count_lock_ = new Mutex("thread suspend count lock", current_lock_level); 916 917 UPDATE_CURRENT_LOCK_LEVEL(kUnexpectedSignalLock); 918 DCHECK(unexpected_signal_lock_ == nullptr); 919 unexpected_signal_lock_ = new Mutex("unexpected signal lock", current_lock_level, true); 920 921 UPDATE_CURRENT_LOCK_LEVEL(kMemMapsLock); 922 DCHECK(mem_maps_lock_ == nullptr); 923 mem_maps_lock_ = new Mutex("mem maps lock", current_lock_level); 924 925 UPDATE_CURRENT_LOCK_LEVEL(kLoggingLock); 926 DCHECK(logging_lock_ == nullptr); 927 logging_lock_ = new Mutex("logging lock", current_lock_level, true); 928 929 #undef UPDATE_CURRENT_LOCK_LEVEL 930 } 931} 932 933 934} // namespace art 935