12d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===-- sanitizer_deadlock_detector1.cc -----------------------------------===//
22d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//
32d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//                     The LLVM Compiler Infrastructure
42d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//
52d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// This file is distributed under the University of Illinois Open Source
62d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// License. See LICENSE.TXT for details.
72d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//
82d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===//
92d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//
102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Deadlock detector implementation based on NxN adjacency bit matrix.
112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//
122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===//
132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_deadlock_detector_interface.h"
152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_deadlock_detector.h"
162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_allocator_internal.h"
172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_placement_new.h"
182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_mutex.h"
192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1
212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesnamespace __sanitizer {
232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef TwoLevelBitVector<> DDBV;  // DeadlockDetector's bit vector.
252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstruct DDPhysicalThread {
272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines};
282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstruct DDLogicalThread {
302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  u64 ctx;
312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DeadlockDetectorTLS<DDBV> dd;
322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDReport rep;
332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  bool report_pending;
342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines};
352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstruct DD : public DDetector {
372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  SpinMutex mtx;
382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DeadlockDetector<DDBV> dd;
392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDFlags flags;
402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  explicit DD(const DDFlags *flags);
422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDPhysicalThread* CreatePhysicalThread();
442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void DestroyPhysicalThread(DDPhysicalThread *pt);
452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDLogicalThread* CreateLogicalThread(u64 ctx);
472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void DestroyLogicalThread(DDLogicalThread *lt);
482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexInit(DDCallback *cb, DDMutex *m);
502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock);
512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock, bool trylock);
522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock);
532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexDestroy(DDCallback *cb, DDMutex *m);
542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDReport *GetReport(DDCallback *cb);
562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void MutexEnsureID(DDLogicalThread *lt, DDMutex *m);
582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void ReportDeadlock(DDCallback *cb, DDMutex *m);
592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines};
602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesDDetector *DDetector::Create(const DDFlags *flags) {
622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  (void)flags;
632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  void *mem = MmapOrDie(sizeof(DD), "deadlock detector");
642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  return new(mem) DD(flags);
652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesDD::DD(const DDFlags *flags)
682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    : flags(*flags) {
692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  dd.clear();
702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesDDPhysicalThread* DD::CreatePhysicalThread() {
732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  return 0;
742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::DestroyPhysicalThread(DDPhysicalThread *pt) {
772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesDDLogicalThread* DD::CreateLogicalThread(u64 ctx) {
802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDLogicalThread *lt = (DDLogicalThread*)InternalAlloc(sizeof(*lt));
812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  lt->ctx = ctx;
822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  lt->dd.clear();
832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  lt->report_pending = false;
842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  return lt;
852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::DestroyLogicalThread(DDLogicalThread *lt) {
882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  lt->~DDLogicalThread();
892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  InternalFree(lt);
902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexInit(DDCallback *cb, DDMutex *m) {
932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  m->id = 0;
942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  m->stk = cb->Unwind();
952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexEnsureID(DDLogicalThread *lt, DDMutex *m) {
982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (!dd.nodeBelongsToCurrentEpoch(m->id))
992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    m->id = dd.newNode(reinterpret_cast<uptr>(m));
1002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  dd.ensureCurrentEpoch(&lt->dd);
1012d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexBeforeLock(DDCallback *cb,
1042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    DDMutex *m, bool wlock) {
1052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDLogicalThread *lt = cb->lt;
1062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (lt->dd.empty()) return;  // This will be the first lock held by lt.
1072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.hasAllEdges(&lt->dd, m->id)) return;  // We already have all edges.
1082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  SpinMutexLock lk(&mtx);
1092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  MutexEnsureID(lt, m);
1102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.isHeld(&lt->dd, m->id))
1112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    return;  // FIXME: allow this only for recursive locks.
1122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.onLockBefore(&lt->dd, m->id)) {
1132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    // Actually add this edge now so that we have all the stack traces.
1142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    dd.addEdges(&lt->dd, m->id, cb->Unwind(), cb->UniqueTid());
1152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    ReportDeadlock(cb, m);
1162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  }
1172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::ReportDeadlock(DDCallback *cb, DDMutex *m) {
1202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDLogicalThread *lt = cb->lt;
1212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  uptr path[10];
1222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  uptr len = dd.findPathToLock(&lt->dd, m->id, path, ARRAY_SIZE(path));
1232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  CHECK_GT(len, 0U);  // Hm.. cycle of 10 locks? I'd like to see that.
1242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  CHECK_EQ(m->id, path[0]);
1252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  lt->report_pending = true;
1262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDReport *rep = &lt->rep;
1272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  rep->n = len;
1282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  for (uptr i = 0; i < len; i++) {
1292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    uptr from = path[i];
1302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    uptr to = path[(i + 1) % len];
1312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    DDMutex *m0 = (DDMutex*)dd.getData(from);
1322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    DDMutex *m1 = (DDMutex*)dd.getData(to);
1332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    u32 stk_from = -1U, stk_to = -1U;
1352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    int unique_tid = 0;
1362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    dd.findEdge(from, to, &stk_from, &stk_to, &unique_tid);
1372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    // Printf("Edge: %zd=>%zd: %u/%u T%d\n", from, to, stk_from, stk_to,
1382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    //    unique_tid);
1392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    rep->loop[i].thr_ctx = unique_tid;
1402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    rep->loop[i].mtx_ctx0 = m0->ctx;
1412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    rep->loop[i].mtx_ctx1 = m1->ctx;
1422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    rep->loop[i].stk[0] = stk_to;
1432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    rep->loop[i].stk[1] = stk_from;
1442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  }
1452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock, bool trylock) {
1482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  DDLogicalThread *lt = cb->lt;
1492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  u32 stk = 0;
1502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (flags.second_deadlock_stack)
1512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    stk = cb->Unwind();
1522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  // Printf("T%p MutexLock:   %zx stk %u\n", lt, m->id, stk);
1532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.onFirstLock(&lt->dd, m->id, stk))
1542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    return;
1552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.onLockFast(&lt->dd, m->id, stk))
1562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    return;
1572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  SpinMutexLock lk(&mtx);
1592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  MutexEnsureID(lt, m);
1602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (wlock)  // Only a recursive rlock may be held.
1612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    CHECK(!dd.isHeld(&lt->dd, m->id));
1622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (!trylock)
1632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    dd.addEdges(&lt->dd, m->id, stk ? stk : cb->Unwind(), cb->UniqueTid());
1642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  dd.onLockAfter(&lt->dd, m->id, stk);
1652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {
1682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  // Printf("T%p MutexUnLock: %zx\n", cb->lt, m->id);
1692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  dd.onUnlock(&cb->lt->dd, m->id);
1702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DD::MutexDestroy(DDCallback *cb,
1732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    DDMutex *m) {
1742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (!m->id) return;
1752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  SpinMutexLock lk(&mtx);
1762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (dd.nodeBelongsToCurrentEpoch(m->id))
1772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    dd.removeNode(m->id);
1782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  m->id = 0;
1792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesDDReport *DD::GetReport(DDCallback *cb) {
1822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  if (!cb->lt->report_pending)
1832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines    return 0;
1842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  cb->lt->report_pending = false;
1852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines  return &cb->lt->rep;
1862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}
1872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines
1882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}  // namespace __sanitizer
1892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif  // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1
190