1//===-- tsan_interceptors_mac.cc ------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer (TSan), a race detector. 11// 12// Mac-specific interceptors. 13//===----------------------------------------------------------------------===// 14 15#include "sanitizer_common/sanitizer_platform.h" 16#if SANITIZER_MAC 17 18#include "interception/interception.h" 19#include "tsan_interceptors.h" 20#include "tsan_interface.h" 21#include "tsan_interface_ann.h" 22 23#include <libkern/OSAtomic.h> 24#include <xpc/xpc.h> 25 26typedef long long_t; // NOLINT 27 28namespace __tsan { 29 30// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 31// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 32// actually aliases of each other, and we cannot have different interceptors for 33// them, because they're actually the same function. Thus, we have to stay 34// conservative and treat the non-barrier versions as mo_acq_rel. 35static const morder kMacOrderBarrier = mo_acq_rel; 36static const morder kMacOrderNonBarrier = mo_acq_rel; 37 38#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 39 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 40 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 41 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 42 } 43 44#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 45 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 46 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 47 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 48 } 49 50#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 51 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 52 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 53 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 54 } 55 56#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 57 mo) \ 58 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 59 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 60 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 61 } 62 63#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 64 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 65 kMacOrderNonBarrier) \ 66 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 67 kMacOrderBarrier) \ 68 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 69 kMacOrderNonBarrier) \ 70 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ 71 kMacOrderBarrier) 72 73#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 74 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 75 kMacOrderNonBarrier) \ 76 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 77 kMacOrderBarrier) \ 78 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ 79 kMacOrderNonBarrier) \ 80 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 81 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 82 83OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 84 OSATOMIC_INTERCEPTOR_PLUS_X) 85OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 86 OSATOMIC_INTERCEPTOR_PLUS_1) 87OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 88 OSATOMIC_INTERCEPTOR_MINUS_1) 89OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 90 OSATOMIC_INTERCEPTOR) 91OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 92 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 93OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 94 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 95 96#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 97 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 98 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 99 return tsan_atomic_f##_compare_exchange_strong( \ 100 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 101 kMacOrderNonBarrier, kMacOrderNonBarrier); \ 102 } \ 103 \ 104 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 105 t volatile *ptr) { \ 106 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 107 return tsan_atomic_f##_compare_exchange_strong( \ 108 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 109 kMacOrderBarrier, kMacOrderNonBarrier); \ 110 } 111 112OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 113OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 114 long_t) 115OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 116 void *) 117OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 118 int32_t) 119OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 120 int64_t) 121 122#define OSATOMIC_INTERCEPTOR_BITOP(f, op, m, mo) \ 123 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 124 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 125 char *byte_ptr = ((char *)ptr) + (n >> 3); \ 126 char bit_index = n & 7; \ 127 char mask = m; \ 128 char orig_byte = op((a8 *)byte_ptr, mask, mo); \ 129 return orig_byte & mask; \ 130 } 131 132#define OSATOMIC_INTERCEPTORS_BITOP(f, op, m) \ 133 OSATOMIC_INTERCEPTOR_BITOP(f, op, m, kMacOrderNonBarrier) \ 134 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, m, kMacOrderBarrier) 135 136OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, 137 0x80u >> bit_index) 138OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 139 ~(0x80u >> bit_index)) 140 141TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 142 size_t offset) { 143 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 144 __tsan_release(item); 145 REAL(OSAtomicEnqueue)(list, item, offset); 146} 147 148TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 149 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 150 void *item = REAL(OSAtomicDequeue)(list, offset); 151 if (item) __tsan_acquire(item); 152 return item; 153} 154 155// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 156#if !SANITIZER_IOS 157 158TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 159 size_t offset) { 160 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 161 __tsan_release(item); 162 REAL(OSAtomicFifoEnqueue)(list, item, offset); 163} 164 165TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 166 size_t offset) { 167 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 168 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 169 if (item) __tsan_acquire(item); 170 return item; 171} 172 173#endif 174 175TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 176 CHECK(!cur_thread()->is_dead); 177 if (!cur_thread()->is_inited) { 178 return REAL(OSSpinLockLock)(lock); 179 } 180 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 181 REAL(OSSpinLockLock)(lock); 182 Acquire(thr, pc, (uptr)lock); 183} 184 185TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 186 CHECK(!cur_thread()->is_dead); 187 if (!cur_thread()->is_inited) { 188 return REAL(OSSpinLockTry)(lock); 189 } 190 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 191 bool result = REAL(OSSpinLockTry)(lock); 192 if (result) 193 Acquire(thr, pc, (uptr)lock); 194 return result; 195} 196 197TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 198 CHECK(!cur_thread()->is_dead); 199 if (!cur_thread()->is_inited) { 200 return REAL(OSSpinLockUnlock)(lock); 201 } 202 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 203 Release(thr, pc, (uptr)lock); 204 REAL(OSSpinLockUnlock)(lock); 205} 206 207TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 208 CHECK(!cur_thread()->is_dead); 209 if (!cur_thread()->is_inited) { 210 return REAL(os_lock_lock)(lock); 211 } 212 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 213 REAL(os_lock_lock)(lock); 214 Acquire(thr, pc, (uptr)lock); 215} 216 217TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 218 CHECK(!cur_thread()->is_dead); 219 if (!cur_thread()->is_inited) { 220 return REAL(os_lock_trylock)(lock); 221 } 222 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 223 bool result = REAL(os_lock_trylock)(lock); 224 if (result) 225 Acquire(thr, pc, (uptr)lock); 226 return result; 227} 228 229TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 230 CHECK(!cur_thread()->is_dead); 231 if (!cur_thread()->is_inited) { 232 return REAL(os_lock_unlock)(lock); 233 } 234 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 235 Release(thr, pc, (uptr)lock); 236 REAL(os_lock_unlock)(lock); 237} 238 239TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 240 xpc_connection_t connection, xpc_handler_t handler) { 241 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 242 handler); 243 Release(thr, pc, (uptr)connection); 244 xpc_handler_t new_handler = ^(xpc_object_t object) { 245 { 246 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 247 Acquire(thr, pc, (uptr)connection); 248 } 249 handler(object); 250 }; 251 REAL(xpc_connection_set_event_handler)(connection, new_handler); 252} 253 254TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 255 dispatch_block_t barrier) { 256 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 257 Release(thr, pc, (uptr)connection); 258 dispatch_block_t new_barrier = ^() { 259 { 260 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 261 Acquire(thr, pc, (uptr)connection); 262 } 263 barrier(); 264 }; 265 REAL(xpc_connection_send_barrier)(connection, new_barrier); 266} 267 268TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 269 xpc_connection_t connection, xpc_object_t message, 270 dispatch_queue_t replyq, xpc_handler_t handler) { 271 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 272 message, replyq, handler); 273 Release(thr, pc, (uptr)connection); 274 xpc_handler_t new_handler = ^(xpc_object_t object) { 275 { 276 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 277 Acquire(thr, pc, (uptr)connection); 278 } 279 handler(object); 280 }; 281 REAL(xpc_connection_send_message_with_reply) 282 (connection, message, replyq, new_handler); 283} 284 285// On macOS, libc++ is always linked dynamically, so intercepting works the 286// usual way. 287#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 288 289namespace { 290struct fake_shared_weak_count { 291 volatile a64 shared_owners; 292 volatile a64 shared_weak_owners; 293 virtual void _unused_0x0() = 0; 294 virtual void _unused_0x8() = 0; 295 virtual void on_zero_shared() = 0; 296 virtual void _unused_0x18() = 0; 297 virtual void on_zero_shared_weak() = 0; 298}; 299} // namespace 300 301// This adds a libc++ interceptor for: 302// void __shared_weak_count::__release_shared() _NOEXCEPT; 303// Shared and weak pointers in C++ maintain reference counts via atomics in 304// libc++.dylib, which are TSan-invisible, and this leads to false positives in 305// destructor code. This interceptor re-implements the whole function so that 306// the mo_acq_rel semantics of the atomic decrement are visible. 307// 308// Unfortunately, this interceptor cannot simply Acquire/Release some sync 309// object and call the original function, because it would have a race between 310// the sync and the destruction of the object. Calling both under a lock will 311// not work because the destructor can invoke this interceptor again (and even 312// in a different thread, so recursive locks don't help). 313STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 314 fake_shared_weak_count *o) { 315 if (!flags()->shared_ptr_interceptor) 316 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 317 318 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 319 o); 320 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 321 Acquire(thr, pc, (uptr)&o->shared_owners); 322 o->on_zero_shared(); 323 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 324 0) { 325 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 326 o->on_zero_shared_weak(); 327 } 328 } 329} 330 331} // namespace __tsan 332 333#endif // SANITIZER_MAC 334