tsan_interceptors.cc revision 9a949a8909f652b28e9084de785c848743139fd5
1//===-- tsan_interceptors.cc ----------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer (TSan), a race detector. 11// 12// FIXME: move as many interceptors as possible into 13// sanitizer_common/sanitizer_common_interceptors.inc 14//===----------------------------------------------------------------------===// 15 16#include "sanitizer_common/sanitizer_atomic.h" 17#include "sanitizer_common/sanitizer_libc.h" 18#include "sanitizer_common/sanitizer_linux.h" 19#include "sanitizer_common/sanitizer_platform_limits_posix.h" 20#include "sanitizer_common/sanitizer_placement_new.h" 21#include "sanitizer_common/sanitizer_stacktrace.h" 22#include "interception/interception.h" 23#include "tsan_interface.h" 24#include "tsan_platform.h" 25#include "tsan_rtl.h" 26#include "tsan_mman.h" 27#include "tsan_fd.h" 28 29using namespace __tsan; // NOLINT 30 31const int kSigCount = 64; 32 33struct my_siginfo_t { 34 // The size is determined by looking at sizeof of real siginfo_t on linux. 35 u64 opaque[128 / sizeof(u64)]; 36}; 37 38struct ucontext_t { 39 // The size is determined by looking at sizeof of real ucontext_t on linux. 40 u64 opaque[936 / sizeof(u64) + 1]; 41}; 42 43extern "C" int pthread_attr_init(void *attr); 44extern "C" int pthread_attr_destroy(void *attr); 45extern "C" int pthread_attr_getdetachstate(void *attr, int *v); 46extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 47extern "C" int pthread_attr_getstacksize(void *attr, uptr *stacksize); 48extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 49extern "C" int pthread_setspecific(unsigned key, const void *v); 50extern "C" int pthread_mutexattr_gettype(void *a, int *type); 51extern "C" int pthread_yield(); 52extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, 53 __sanitizer_sigset_t *oldset); 54// REAL(sigfillset) defined in common interceptors. 55DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) 56extern "C" void *pthread_self(); 57extern "C" void _exit(int status); 58extern "C" int *__errno_location(); 59extern "C" int fileno_unlocked(void *stream); 60extern "C" void *__libc_malloc(uptr size); 61extern "C" void *__libc_calloc(uptr size, uptr n); 62extern "C" void *__libc_realloc(void *ptr, uptr size); 63extern "C" void __libc_free(void *ptr); 64extern "C" int mallopt(int param, int value); 65const int PTHREAD_MUTEX_RECURSIVE = 1; 66const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 67const int EINVAL = 22; 68const int EBUSY = 16; 69const int EPOLL_CTL_ADD = 1; 70const int SIGILL = 4; 71const int SIGABRT = 6; 72const int SIGFPE = 8; 73const int SIGSEGV = 11; 74const int SIGPIPE = 13; 75const int SIGBUS = 7; 76const int SIGSYS = 31; 77void *const MAP_FAILED = (void*)-1; 78const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 79const int MAP_FIXED = 0x10; 80typedef long long_t; // NOLINT 81 82// From /usr/include/unistd.h 83# define F_ULOCK 0 /* Unlock a previously locked region. */ 84# define F_LOCK 1 /* Lock a region for exclusive use. */ 85# define F_TLOCK 2 /* Test and lock a region for exclusive use. */ 86# define F_TEST 3 /* Test a region for other processes locks. */ 87 88typedef void (*sighandler_t)(int sig); 89 90#define errno (*__errno_location()) 91 92struct sigaction_t { 93 union { 94 sighandler_t sa_handler; 95 void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); 96 }; 97 __sanitizer_sigset_t sa_mask; 98 int sa_flags; 99 void (*sa_restorer)(); 100}; 101 102const sighandler_t SIG_DFL = (sighandler_t)0; 103const sighandler_t SIG_IGN = (sighandler_t)1; 104const sighandler_t SIG_ERR = (sighandler_t)-1; 105const int SA_SIGINFO = 4; 106const int SIG_SETMASK = 2; 107 108namespace std { 109struct nothrow_t {}; 110} // namespace std 111 112static sigaction_t sigactions[kSigCount]; 113 114namespace __tsan { 115struct SignalDesc { 116 bool armed; 117 bool sigaction; 118 my_siginfo_t siginfo; 119 ucontext_t ctx; 120}; 121 122struct SignalContext { 123 int in_blocking_func; 124 int int_signal_send; 125 int pending_signal_count; 126 SignalDesc pending_signals[kSigCount]; 127}; 128 129// Used to ignore interceptors coming directly from libjvm.so. 130atomic_uintptr_t libjvm_begin; 131atomic_uintptr_t libjvm_end; 132 133static bool libjvm_check(uptr pc) { 134 uptr begin = atomic_load(&libjvm_begin, memory_order_relaxed); 135 if (begin != 0 && pc >= begin) { 136 uptr end = atomic_load(&libjvm_end, memory_order_relaxed); 137 if (end != 0 && pc < end) 138 return true; 139 } 140 return false; 141} 142 143} // namespace __tsan 144 145static SignalContext *SigCtx(ThreadState *thr) { 146 SignalContext *ctx = (SignalContext*)thr->signal_ctx; 147 if (ctx == 0 && thr->is_alive) { 148 ScopedInRtl in_rtl; 149 ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext"); 150 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); 151 thr->signal_ctx = ctx; 152 } 153 return ctx; 154} 155 156static unsigned g_thread_finalize_key; 157 158class ScopedInterceptor { 159 public: 160 ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); 161 ~ScopedInterceptor(); 162 private: 163 ThreadState *const thr_; 164 const int in_rtl_; 165}; 166 167ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, 168 uptr pc) 169 : thr_(thr) 170 , in_rtl_(thr->in_rtl) { 171 if (thr_->in_rtl == 0) { 172 Initialize(thr); 173 FuncEntry(thr, pc); 174 thr_->in_rtl++; 175 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 176 } else { 177 thr_->in_rtl++; 178 } 179} 180 181ScopedInterceptor::~ScopedInterceptor() { 182 thr_->in_rtl--; 183 if (thr_->in_rtl == 0) { 184 FuncExit(thr_); 185 ProcessPendingSignals(thr_); 186 } 187 CHECK_EQ(in_rtl_, thr_->in_rtl); 188} 189 190#define SCOPED_INTERCEPTOR_RAW(func, ...) \ 191 ThreadState *thr = cur_thread(); \ 192 StatInc(thr, StatInterceptor); \ 193 StatInc(thr, StatInt_##func); \ 194 const uptr caller_pc = GET_CALLER_PC(); \ 195 ScopedInterceptor si(thr, #func, caller_pc); \ 196 const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ 197 (void)pc; \ 198/**/ 199 200#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ 201 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 202 if (REAL(func) == 0) { \ 203 Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ 204 Die(); \ 205 } \ 206 if (thr->in_rtl > 1 || libjvm_check(pc)) \ 207 return REAL(func)(__VA_ARGS__); \ 208/**/ 209 210#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) 211#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 212 213#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) 214 215struct BlockingCall { 216 explicit BlockingCall(ThreadState *thr) 217 : ctx(SigCtx(thr)) { 218 ctx->in_blocking_func++; 219 } 220 221 ~BlockingCall() { 222 ctx->in_blocking_func--; 223 } 224 225 SignalContext *ctx; 226}; 227 228TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 229 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 230 unsigned res = BLOCK_REAL(sleep)(sec); 231 AfterSleep(thr, pc); 232 return res; 233} 234 235TSAN_INTERCEPTOR(int, usleep, long_t usec) { 236 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 237 int res = BLOCK_REAL(usleep)(usec); 238 AfterSleep(thr, pc); 239 return res; 240} 241 242TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 243 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 244 int res = BLOCK_REAL(nanosleep)(req, rem); 245 AfterSleep(thr, pc); 246 return res; 247} 248 249class AtExitContext { 250 public: 251 AtExitContext() 252 : mtx_(MutexTypeAtExit, StatMtxAtExit) 253 , pos_() { 254 } 255 256 typedef void(*atexit_t)(); 257 258 int atexit(ThreadState *thr, uptr pc, bool is_on_exit, 259 atexit_t f, void *arg) { 260 Lock l(&mtx_); 261 if (pos_ == kMaxAtExit) 262 return 1; 263 Release(thr, pc, (uptr)this); 264 stack_[pos_] = f; 265 args_[pos_] = arg; 266 is_on_exits_[pos_] = is_on_exit; 267 pos_++; 268 return 0; 269 } 270 271 void exit(ThreadState *thr, uptr pc) { 272 CHECK_EQ(thr->in_rtl, 0); 273 for (;;) { 274 atexit_t f = 0; 275 void *arg = 0; 276 bool is_on_exit = false; 277 { 278 Lock l(&mtx_); 279 if (pos_) { 280 pos_--; 281 f = stack_[pos_]; 282 arg = args_[pos_]; 283 is_on_exit = is_on_exits_[pos_]; 284 ScopedInRtl in_rtl; 285 Acquire(thr, pc, (uptr)this); 286 } 287 } 288 if (f == 0) 289 break; 290 DPrintf("#%d: executing atexit func %p\n", thr->tid, f); 291 CHECK_EQ(thr->in_rtl, 0); 292 if (is_on_exit) 293 ((void(*)(int status, void *arg))f)(0, arg); 294 else 295 ((void(*)(void *arg, void *dso))f)(arg, 0); 296 } 297 } 298 299 private: 300 static const int kMaxAtExit = 128; 301 Mutex mtx_; 302 atexit_t stack_[kMaxAtExit]; 303 void *args_[kMaxAtExit]; 304 bool is_on_exits_[kMaxAtExit]; 305 int pos_; 306}; 307 308static AtExitContext *atexit_ctx; 309 310TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 311 if (cur_thread()->in_symbolizer) 312 return 0; 313 SCOPED_TSAN_INTERCEPTOR(atexit, f); 314 return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0); 315} 316 317TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { 318 if (cur_thread()->in_symbolizer) 319 return 0; 320 SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); 321 return atexit_ctx->atexit(thr, pc, true, (void(*)())f, arg); 322} 323 324TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { 325 if (cur_thread()->in_symbolizer) 326 return 0; 327 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); 328 if (dso) { 329 // Memory allocation in __cxa_atexit will race with free during exit, 330 // because we do not see synchronization around atexit callback list. 331 ThreadIgnoreBegin(thr); 332 int res = REAL(__cxa_atexit)(f, arg, dso); 333 ThreadIgnoreEnd(thr); 334 return res; 335 } 336 return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg); 337} 338 339// Cleanup old bufs. 340static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { 341 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 342 JmpBuf *buf = &thr->jmp_bufs[i]; 343 if (buf->sp <= sp) { 344 uptr sz = thr->jmp_bufs.Size(); 345 thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; 346 thr->jmp_bufs.PopBack(); 347 i--; 348 } 349 } 350} 351 352static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { 353 if (thr->shadow_stack_pos == 0) // called from libc guts during bootstrap 354 return; 355 // Cleanup old bufs. 356 JmpBufGarbageCollect(thr, sp); 357 // Remember the buf. 358 JmpBuf *buf = thr->jmp_bufs.PushBack(); 359 buf->sp = sp; 360 buf->mangled_sp = mangled_sp; 361 buf->shadow_stack_pos = thr->shadow_stack_pos; 362} 363 364static void LongJmp(ThreadState *thr, uptr *env) { 365 uptr mangled_sp = env[6]; 366 // Find the saved buf by mangled_sp. 367 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 368 JmpBuf *buf = &thr->jmp_bufs[i]; 369 if (buf->mangled_sp == mangled_sp) { 370 CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); 371 // Unwind the stack. 372 while (thr->shadow_stack_pos > buf->shadow_stack_pos) 373 FuncExit(thr); 374 JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp 375 return; 376 } 377 } 378 Printf("ThreadSanitizer: can't find longjmp buf\n"); 379 CHECK(0); 380} 381 382// FIXME: put everything below into a common extern "C" block? 383extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { 384 ScopedInRtl in_rtl; 385 SetJmp(cur_thread(), sp, mangled_sp); 386} 387 388// Not called. Merely to satisfy TSAN_INTERCEPT(). 389extern "C" SANITIZER_INTERFACE_ATTRIBUTE 390int __interceptor_setjmp(void *env); 391extern "C" int __interceptor_setjmp(void *env) { 392 CHECK(0); 393 return 0; 394} 395 396// FIXME: any reason to have a separate declaration? 397extern "C" SANITIZER_INTERFACE_ATTRIBUTE 398int __interceptor__setjmp(void *env); 399extern "C" int __interceptor__setjmp(void *env) { 400 CHECK(0); 401 return 0; 402} 403 404extern "C" SANITIZER_INTERFACE_ATTRIBUTE 405int __interceptor_sigsetjmp(void *env); 406extern "C" int __interceptor_sigsetjmp(void *env) { 407 CHECK(0); 408 return 0; 409} 410 411extern "C" SANITIZER_INTERFACE_ATTRIBUTE 412int __interceptor___sigsetjmp(void *env); 413extern "C" int __interceptor___sigsetjmp(void *env) { 414 CHECK(0); 415 return 0; 416} 417 418extern "C" int setjmp(void *env); 419extern "C" int _setjmp(void *env); 420extern "C" int sigsetjmp(void *env); 421extern "C" int __sigsetjmp(void *env); 422DEFINE_REAL(int, setjmp, void *env) 423DEFINE_REAL(int, _setjmp, void *env) 424DEFINE_REAL(int, sigsetjmp, void *env) 425DEFINE_REAL(int, __sigsetjmp, void *env) 426 427TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { 428 { 429 SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); 430 } 431 LongJmp(cur_thread(), env); 432 REAL(longjmp)(env, val); 433} 434 435TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { 436 { 437 SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); 438 } 439 LongJmp(cur_thread(), env); 440 REAL(siglongjmp)(env, val); 441} 442 443TSAN_INTERCEPTOR(void*, malloc, uptr size) { 444 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) 445 return __libc_malloc(size); 446 void *p = 0; 447 { 448 SCOPED_INTERCEPTOR_RAW(malloc, size); 449 p = user_alloc(thr, pc, size); 450 } 451 invoke_malloc_hook(p, size); 452 return p; 453} 454 455TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { 456 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); 457 return user_alloc(thr, pc, sz, align); 458} 459 460TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 461 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) 462 return __libc_calloc(size, n); 463 if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) 464 return AllocatorReturnNull(); 465 void *p = 0; 466 { 467 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 468 p = user_alloc(thr, pc, n * size); 469 if (p) 470 internal_memset(p, 0, n * size); 471 } 472 invoke_malloc_hook(p, n * size); 473 return p; 474} 475 476TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 477 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) 478 return __libc_realloc(p, size); 479 if (p) 480 invoke_free_hook(p); 481 { 482 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 483 p = user_realloc(thr, pc, p, size); 484 } 485 invoke_malloc_hook(p, size); 486 return p; 487} 488 489TSAN_INTERCEPTOR(void, free, void *p) { 490 if (p == 0) 491 return; 492 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) 493 return __libc_free(p); 494 invoke_free_hook(p); 495 SCOPED_INTERCEPTOR_RAW(free, p); 496 user_free(thr, pc, p); 497} 498 499TSAN_INTERCEPTOR(void, cfree, void *p) { 500 if (p == 0) 501 return; 502 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) 503 return __libc_free(p); 504 invoke_free_hook(p); 505 SCOPED_INTERCEPTOR_RAW(cfree, p); 506 user_free(thr, pc, p); 507} 508 509TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { 510 SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); 511 if (libjvm_check(pc)) 512 return malloc_usable_size(p); 513 return user_alloc_usable_size(thr, pc, p); 514} 515 516#define OPERATOR_NEW_BODY(mangled_name) \ 517 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) \ 518 return __libc_malloc(size); \ 519 void *p = 0; \ 520 { \ 521 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ 522 p = user_alloc(thr, pc, size); \ 523 } \ 524 invoke_malloc_hook(p, size); \ 525 return p; 526 527SANITIZER_INTERFACE_ATTRIBUTE 528void *operator new(__sanitizer::uptr size); 529void *operator new(__sanitizer::uptr size) { 530 OPERATOR_NEW_BODY(_Znwm); 531} 532 533SANITIZER_INTERFACE_ATTRIBUTE 534void *operator new[](__sanitizer::uptr size); 535void *operator new[](__sanitizer::uptr size) { 536 OPERATOR_NEW_BODY(_Znam); 537} 538 539SANITIZER_INTERFACE_ATTRIBUTE 540void *operator new(__sanitizer::uptr size, std::nothrow_t const&); 541void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { 542 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t); 543} 544 545SANITIZER_INTERFACE_ATTRIBUTE 546void *operator new[](__sanitizer::uptr size, std::nothrow_t const&); 547void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { 548 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t); 549} 550 551#define OPERATOR_DELETE_BODY(mangled_name) \ 552 if (ptr == 0) return; \ 553 if (cur_thread()->in_symbolizer || libjvm_check(GET_CALLER_PC())) \ 554 return __libc_free(ptr); \ 555 invoke_free_hook(ptr); \ 556 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ 557 user_free(thr, pc, ptr); 558 559SANITIZER_INTERFACE_ATTRIBUTE 560void operator delete(void *ptr); 561void operator delete(void *ptr) { 562 OPERATOR_DELETE_BODY(_ZdlPv); 563} 564 565SANITIZER_INTERFACE_ATTRIBUTE 566void operator delete[](void *ptr); 567void operator delete[](void *ptr) { 568 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); 569} 570 571SANITIZER_INTERFACE_ATTRIBUTE 572void operator delete(void *ptr, std::nothrow_t const&); 573void operator delete(void *ptr, std::nothrow_t const&) { 574 OPERATOR_DELETE_BODY(_ZdaPv); 575} 576 577SANITIZER_INTERFACE_ATTRIBUTE 578void operator delete[](void *ptr, std::nothrow_t const&); 579void operator delete[](void *ptr, std::nothrow_t const&) { 580 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); 581} 582 583TSAN_INTERCEPTOR(uptr, strlen, const char *s) { 584 SCOPED_TSAN_INTERCEPTOR(strlen, s); 585 uptr len = internal_strlen(s); 586 MemoryAccessRange(thr, pc, (uptr)s, len + 1, false); 587 return len; 588} 589 590TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { 591 SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size); 592 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 593 return internal_memset(dst, v, size); 594} 595 596TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { 597 SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size); 598 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 599 MemoryAccessRange(thr, pc, (uptr)src, size, false); 600 return internal_memcpy(dst, src, size); 601} 602 603TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { 604 SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n); 605 int res = 0; 606 uptr len = 0; 607 for (; len < n; len++) { 608 if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len])) 609 break; 610 } 611 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); 612 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); 613 return res; 614} 615 616TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) { 617 SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n); 618 void *res = REAL(memchr)(s, c, n); 619 uptr len = res ? (char*)res - (char*)s + 1 : n; 620 MemoryAccessRange(thr, pc, (uptr)s, len, false); 621 return res; 622} 623 624TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) { 625 SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n); 626 MemoryAccessRange(thr, pc, (uptr)s, n, false); 627 return REAL(memrchr)(s, c, n); 628} 629 630TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { 631 SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); 632 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 633 MemoryAccessRange(thr, pc, (uptr)src, n, false); 634 return REAL(memmove)(dst, src, n); 635} 636 637TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { 638 SCOPED_TSAN_INTERCEPTOR(strchr, s, c); 639 char *res = REAL(strchr)(s, c); 640 uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1; 641 MemoryAccessRange(thr, pc, (uptr)s, len, false); 642 return res; 643} 644 645TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { 646 SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); 647 char *res = REAL(strchrnul)(s, c); 648 uptr len = (char*)res - (char*)s + 1; 649 MemoryAccessRange(thr, pc, (uptr)s, len, false); 650 return res; 651} 652 653TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { 654 SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); 655 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false); 656 return REAL(strrchr)(s, c); 657} 658 659TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT 660 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT 661 uptr srclen = internal_strlen(src); 662 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 663 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 664 return REAL(strcpy)(dst, src); // NOLINT 665} 666 667TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 668 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 669 uptr srclen = internal_strnlen(src, n); 670 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 671 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 672 return REAL(strncpy)(dst, src, n); 673} 674 675TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) { 676 SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2); 677 const char *res = REAL(strstr)(s1, s2); 678 uptr len1 = internal_strlen(s1); 679 uptr len2 = internal_strlen(s2); 680 MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false); 681 MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false); 682 return res; 683} 684 685TSAN_INTERCEPTOR(char*, strdup, const char *str) { 686 SCOPED_TSAN_INTERCEPTOR(strdup, str); 687 if (libjvm_check(pc)) { 688 // The memory must come from libc malloc, 689 // and we must not instrument accesses in this case. 690 uptr n = internal_strlen(str) + 1; 691 void *p = __libc_malloc(n); 692 if (p == 0) 693 return 0; 694 return (char*)internal_memcpy(p, str, n); 695 } 696 // strdup will call malloc, so no instrumentation is required here. 697 return REAL(strdup)(str); 698} 699 700static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 701 if (*addr) { 702 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 703 if (flags & MAP_FIXED) { 704 errno = EINVAL; 705 return false; 706 } else { 707 *addr = 0; 708 } 709 } 710 } 711 return true; 712} 713 714TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, 715 int flags, int fd, unsigned off) { 716 SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); 717 if (!fix_mmap_addr(&addr, sz, flags)) 718 return MAP_FAILED; 719 void *res = REAL(mmap)(addr, sz, prot, flags, fd, off); 720 if (res != MAP_FAILED) { 721 if (fd > 0) 722 FdAccess(thr, pc, fd); 723 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 724 } 725 return res; 726} 727 728TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, 729 int flags, int fd, u64 off) { 730 SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); 731 if (!fix_mmap_addr(&addr, sz, flags)) 732 return MAP_FAILED; 733 void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off); 734 if (res != MAP_FAILED) { 735 if (fd > 0) 736 FdAccess(thr, pc, fd); 737 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 738 } 739 return res; 740} 741 742TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 743 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 744 DontNeedShadowFor((uptr)addr, sz); 745 int res = REAL(munmap)(addr, sz); 746 return res; 747} 748 749TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 750 SCOPED_TSAN_INTERCEPTOR(memalign, align, sz); 751 return user_alloc(thr, pc, sz, align); 752} 753 754TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 755 SCOPED_TSAN_INTERCEPTOR(valloc, sz); 756 return user_alloc(thr, pc, sz, GetPageSizeCached()); 757} 758 759TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 760 SCOPED_TSAN_INTERCEPTOR(pvalloc, sz); 761 sz = RoundUp(sz, GetPageSizeCached()); 762 return user_alloc(thr, pc, sz, GetPageSizeCached()); 763} 764 765TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 766 SCOPED_TSAN_INTERCEPTOR(posix_memalign, memptr, align, sz); 767 *memptr = user_alloc(thr, pc, sz, align); 768 return 0; 769} 770 771// Used in thread-safe function static initialization. 772extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { 773 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); 774 for (;;) { 775 u32 cmp = atomic_load(g, memory_order_acquire); 776 if (cmp == 0) { 777 if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) 778 return 1; 779 } else if (cmp == 1) { 780 Acquire(thr, pc, (uptr)g); 781 return 0; 782 } else { 783 internal_sched_yield(); 784 } 785 } 786} 787 788extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { 789 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); 790 Release(thr, pc, (uptr)g); 791 atomic_store(g, 1, memory_order_release); 792} 793 794extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { 795 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); 796 atomic_store(g, 0, memory_order_relaxed); 797} 798 799static void thread_finalize(void *v) { 800 uptr iter = (uptr)v; 801 if (iter > 1) { 802 if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { 803 Printf("ThreadSanitizer: failed to set thread key\n"); 804 Die(); 805 } 806 return; 807 } 808 { 809 ScopedInRtl in_rtl; 810 ThreadState *thr = cur_thread(); 811 ThreadFinish(thr); 812 SignalContext *sctx = thr->signal_ctx; 813 if (sctx) { 814 thr->signal_ctx = 0; 815 UnmapOrDie(sctx, sizeof(*sctx)); 816 } 817 } 818} 819 820 821struct ThreadParam { 822 void* (*callback)(void *arg); 823 void *param; 824 atomic_uintptr_t tid; 825}; 826 827extern "C" void *__tsan_thread_start_func(void *arg) { 828 ThreadParam *p = (ThreadParam*)arg; 829 void* (*callback)(void *arg) = p->callback; 830 void *param = p->param; 831 int tid = 0; 832 { 833 ThreadState *thr = cur_thread(); 834 ScopedInRtl in_rtl; 835 if (pthread_setspecific(g_thread_finalize_key, (void*)4)) { 836 Printf("ThreadSanitizer: failed to set thread key\n"); 837 Die(); 838 } 839 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) 840 pthread_yield(); 841 atomic_store(&p->tid, 0, memory_order_release); 842 ThreadStart(thr, tid, GetTid()); 843 CHECK_EQ(thr->in_rtl, 1); 844 } 845 void *res = callback(param); 846 // Prevent the callback from being tail called, 847 // it mixes up stack traces. 848 volatile int foo = 42; 849 foo++; 850 return res; 851} 852 853TSAN_INTERCEPTOR(int, pthread_create, 854 void *th, void *attr, void *(*callback)(void*), void * param) { 855 SCOPED_TSAN_INTERCEPTOR(pthread_create, th, attr, callback, param); 856 __sanitizer_pthread_attr_t myattr; 857 if (attr == 0) { 858 pthread_attr_init(&myattr); 859 attr = &myattr; 860 } 861 int detached = 0; 862 pthread_attr_getdetachstate(attr, &detached); 863 864#if defined(TSAN_DEBUG_OUTPUT) 865 int verbosity = (TSAN_DEBUG_OUTPUT); 866#else 867 int verbosity = 0; 868#endif 869 AdjustStackSizeLinux(attr, verbosity); 870 871 ThreadParam p; 872 p.callback = callback; 873 p.param = param; 874 atomic_store(&p.tid, 0, memory_order_relaxed); 875 int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 876 if (res == 0) { 877 int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); 878 CHECK_NE(tid, 0); 879 atomic_store(&p.tid, tid, memory_order_release); 880 while (atomic_load(&p.tid, memory_order_acquire) != 0) 881 pthread_yield(); 882 } 883 if (attr == &myattr) 884 pthread_attr_destroy(&myattr); 885 return res; 886} 887 888TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 889 SCOPED_TSAN_INTERCEPTOR(pthread_join, th, ret); 890 int tid = ThreadTid(thr, pc, (uptr)th); 891 int res = BLOCK_REAL(pthread_join)(th, ret); 892 if (res == 0) { 893 ThreadJoin(thr, pc, tid); 894 } 895 return res; 896} 897 898TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 899 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); 900 int tid = ThreadTid(thr, pc, (uptr)th); 901 int res = REAL(pthread_detach)(th); 902 if (res == 0) { 903 ThreadDetach(thr, pc, tid); 904 } 905 return res; 906} 907 908TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 909 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 910 int res = REAL(pthread_mutex_init)(m, a); 911 if (res == 0) { 912 bool recursive = false; 913 if (a) { 914 int type = 0; 915 if (pthread_mutexattr_gettype(a, &type) == 0) 916 recursive = (type == PTHREAD_MUTEX_RECURSIVE 917 || type == PTHREAD_MUTEX_RECURSIVE_NP); 918 } 919 MutexCreate(thr, pc, (uptr)m, false, recursive, false); 920 } 921 return res; 922} 923 924TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 925 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 926 int res = REAL(pthread_mutex_destroy)(m); 927 if (res == 0 || res == EBUSY) { 928 MutexDestroy(thr, pc, (uptr)m); 929 } 930 return res; 931} 932 933TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { 934 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); 935 int res = REAL(pthread_mutex_lock)(m); 936 if (res == 0) { 937 MutexLock(thr, pc, (uptr)m); 938 } 939 return res; 940} 941 942TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 943 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 944 int res = REAL(pthread_mutex_trylock)(m); 945 if (res == 0) { 946 MutexLock(thr, pc, (uptr)m); 947 } 948 return res; 949} 950 951TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 952 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 953 int res = REAL(pthread_mutex_timedlock)(m, abstime); 954 if (res == 0) { 955 MutexLock(thr, pc, (uptr)m); 956 } 957 return res; 958} 959 960TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { 961 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); 962 MutexUnlock(thr, pc, (uptr)m); 963 int res = REAL(pthread_mutex_unlock)(m); 964 return res; 965} 966 967TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 968 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 969 int res = REAL(pthread_spin_init)(m, pshared); 970 if (res == 0) { 971 MutexCreate(thr, pc, (uptr)m, false, false, false); 972 } 973 return res; 974} 975 976TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 977 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 978 int res = REAL(pthread_spin_destroy)(m); 979 if (res == 0) { 980 MutexDestroy(thr, pc, (uptr)m); 981 } 982 return res; 983} 984 985TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 986 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 987 int res = REAL(pthread_spin_lock)(m); 988 if (res == 0) { 989 MutexLock(thr, pc, (uptr)m); 990 } 991 return res; 992} 993 994TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 995 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 996 int res = REAL(pthread_spin_trylock)(m); 997 if (res == 0) { 998 MutexLock(thr, pc, (uptr)m); 999 } 1000 return res; 1001} 1002 1003TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 1004 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 1005 MutexUnlock(thr, pc, (uptr)m); 1006 int res = REAL(pthread_spin_unlock)(m); 1007 return res; 1008} 1009 1010TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 1011 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 1012 int res = REAL(pthread_rwlock_init)(m, a); 1013 if (res == 0) { 1014 MutexCreate(thr, pc, (uptr)m, true, false, false); 1015 } 1016 return res; 1017} 1018 1019TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 1020 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 1021 int res = REAL(pthread_rwlock_destroy)(m); 1022 if (res == 0) { 1023 MutexDestroy(thr, pc, (uptr)m); 1024 } 1025 return res; 1026} 1027 1028TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 1029 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 1030 int res = REAL(pthread_rwlock_rdlock)(m); 1031 if (res == 0) { 1032 MutexReadLock(thr, pc, (uptr)m); 1033 } 1034 return res; 1035} 1036 1037TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 1038 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 1039 int res = REAL(pthread_rwlock_tryrdlock)(m); 1040 if (res == 0) { 1041 MutexReadLock(thr, pc, (uptr)m); 1042 } 1043 return res; 1044} 1045 1046TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 1047 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 1048 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 1049 if (res == 0) { 1050 MutexReadLock(thr, pc, (uptr)m); 1051 } 1052 return res; 1053} 1054 1055TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 1056 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 1057 int res = REAL(pthread_rwlock_wrlock)(m); 1058 if (res == 0) { 1059 MutexLock(thr, pc, (uptr)m); 1060 } 1061 return res; 1062} 1063 1064TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 1065 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 1066 int res = REAL(pthread_rwlock_trywrlock)(m); 1067 if (res == 0) { 1068 MutexLock(thr, pc, (uptr)m); 1069 } 1070 return res; 1071} 1072 1073TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 1074 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 1075 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 1076 if (res == 0) { 1077 MutexLock(thr, pc, (uptr)m); 1078 } 1079 return res; 1080} 1081 1082TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 1083 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 1084 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 1085 int res = REAL(pthread_rwlock_unlock)(m); 1086 return res; 1087} 1088 1089TSAN_INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 1090 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, c, a); 1091 MemoryWrite(thr, pc, (uptr)c, kSizeLog1); 1092 int res = REAL(pthread_cond_init)(c, a); 1093 return res; 1094} 1095 1096TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) { 1097 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c); 1098 MemoryWrite(thr, pc, (uptr)c, kSizeLog1); 1099 int res = REAL(pthread_cond_destroy)(c); 1100 return res; 1101} 1102 1103TSAN_INTERCEPTOR(int, pthread_cond_signal, void *c) { 1104 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, c); 1105 MemoryRead(thr, pc, (uptr)c, kSizeLog1); 1106 int res = REAL(pthread_cond_signal)(c); 1107 return res; 1108} 1109 1110TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 1111 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, c); 1112 MemoryRead(thr, pc, (uptr)c, kSizeLog1); 1113 int res = REAL(pthread_cond_broadcast)(c); 1114 return res; 1115} 1116 1117TSAN_INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1118 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, c, m); 1119 MutexUnlock(thr, pc, (uptr)m); 1120 MemoryRead(thr, pc, (uptr)c, kSizeLog1); 1121 int res = REAL(pthread_cond_wait)(c, m); 1122 MutexLock(thr, pc, (uptr)m); 1123 return res; 1124} 1125 1126TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, 1127 void *abstime) { 1128 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime); 1129 MutexUnlock(thr, pc, (uptr)m); 1130 MemoryRead(thr, pc, (uptr)c, kSizeLog1); 1131 int res = REAL(pthread_cond_timedwait)(c, m, abstime); 1132 MutexLock(thr, pc, (uptr)m); 1133 return res; 1134} 1135 1136TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1137 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1138 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1139 int res = REAL(pthread_barrier_init)(b, a, count); 1140 return res; 1141} 1142 1143TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1144 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1145 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1146 int res = REAL(pthread_barrier_destroy)(b); 1147 return res; 1148} 1149 1150TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1151 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1152 Release(thr, pc, (uptr)b); 1153 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1154 int res = REAL(pthread_barrier_wait)(b); 1155 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1156 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1157 Acquire(thr, pc, (uptr)b); 1158 } 1159 return res; 1160} 1161 1162TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1163 SCOPED_TSAN_INTERCEPTOR(pthread_once, o, f); 1164 if (o == 0 || f == 0) 1165 return EINVAL; 1166 atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); 1167 u32 v = atomic_load(a, memory_order_acquire); 1168 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, 1169 memory_order_relaxed)) { 1170 const int old_in_rtl = thr->in_rtl; 1171 thr->in_rtl = 0; 1172 (*f)(); 1173 CHECK_EQ(thr->in_rtl, 0); 1174 thr->in_rtl = old_in_rtl; 1175 Release(thr, pc, (uptr)o); 1176 atomic_store(a, 2, memory_order_release); 1177 } else { 1178 while (v != 2) { 1179 pthread_yield(); 1180 v = atomic_load(a, memory_order_acquire); 1181 } 1182 Acquire(thr, pc, (uptr)o); 1183 } 1184 return 0; 1185} 1186 1187TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) { 1188 SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value); 1189 int res = REAL(sem_init)(s, pshared, value); 1190 return res; 1191} 1192 1193TSAN_INTERCEPTOR(int, sem_destroy, void *s) { 1194 SCOPED_TSAN_INTERCEPTOR(sem_destroy, s); 1195 int res = REAL(sem_destroy)(s); 1196 return res; 1197} 1198 1199TSAN_INTERCEPTOR(int, sem_wait, void *s) { 1200 SCOPED_TSAN_INTERCEPTOR(sem_wait, s); 1201 int res = BLOCK_REAL(sem_wait)(s); 1202 if (res == 0) { 1203 Acquire(thr, pc, (uptr)s); 1204 } 1205 return res; 1206} 1207 1208TSAN_INTERCEPTOR(int, sem_trywait, void *s) { 1209 SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); 1210 int res = BLOCK_REAL(sem_trywait)(s); 1211 if (res == 0) { 1212 Acquire(thr, pc, (uptr)s); 1213 } 1214 return res; 1215} 1216 1217TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { 1218 SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); 1219 int res = BLOCK_REAL(sem_timedwait)(s, abstime); 1220 if (res == 0) { 1221 Acquire(thr, pc, (uptr)s); 1222 } 1223 return res; 1224} 1225 1226TSAN_INTERCEPTOR(int, sem_post, void *s) { 1227 SCOPED_TSAN_INTERCEPTOR(sem_post, s); 1228 Release(thr, pc, (uptr)s); 1229 int res = REAL(sem_post)(s); 1230 return res; 1231} 1232 1233TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { 1234 SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); 1235 int res = REAL(sem_getvalue)(s, sval); 1236 if (res == 0) { 1237 Acquire(thr, pc, (uptr)s); 1238 } 1239 return res; 1240} 1241 1242TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { 1243 SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf); 1244 return REAL(__xstat)(version, path, buf); 1245} 1246 1247TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { 1248 SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf); 1249 return REAL(__xstat)(0, path, buf); 1250} 1251 1252TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { 1253 SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf); 1254 return REAL(__xstat64)(version, path, buf); 1255} 1256 1257TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { 1258 SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf); 1259 return REAL(__xstat64)(0, path, buf); 1260} 1261 1262TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { 1263 SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf); 1264 return REAL(__lxstat)(version, path, buf); 1265} 1266 1267TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { 1268 SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf); 1269 return REAL(__lxstat)(0, path, buf); 1270} 1271 1272TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { 1273 SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf); 1274 return REAL(__lxstat64)(version, path, buf); 1275} 1276 1277TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { 1278 SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf); 1279 return REAL(__lxstat64)(0, path, buf); 1280} 1281 1282TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { 1283 SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); 1284 if (fd > 0) 1285 FdAccess(thr, pc, fd); 1286 return REAL(__fxstat)(version, fd, buf); 1287} 1288 1289TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { 1290 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); 1291 if (fd > 0) 1292 FdAccess(thr, pc, fd); 1293 return REAL(__fxstat)(0, fd, buf); 1294} 1295 1296TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { 1297 SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); 1298 if (fd > 0) 1299 FdAccess(thr, pc, fd); 1300 return REAL(__fxstat64)(version, fd, buf); 1301} 1302 1303TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { 1304 SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); 1305 if (fd > 0) 1306 FdAccess(thr, pc, fd); 1307 return REAL(__fxstat64)(0, fd, buf); 1308} 1309 1310TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { 1311 SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); 1312 int fd = REAL(open)(name, flags, mode); 1313 if (fd >= 0) 1314 FdFileCreate(thr, pc, fd); 1315 return fd; 1316} 1317 1318TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { 1319 SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); 1320 int fd = REAL(open64)(name, flags, mode); 1321 if (fd >= 0) 1322 FdFileCreate(thr, pc, fd); 1323 return fd; 1324} 1325 1326TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { 1327 SCOPED_TSAN_INTERCEPTOR(creat, name, mode); 1328 int fd = REAL(creat)(name, mode); 1329 if (fd >= 0) 1330 FdFileCreate(thr, pc, fd); 1331 return fd; 1332} 1333 1334TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { 1335 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); 1336 int fd = REAL(creat64)(name, mode); 1337 if (fd >= 0) 1338 FdFileCreate(thr, pc, fd); 1339 return fd; 1340} 1341 1342TSAN_INTERCEPTOR(int, dup, int oldfd) { 1343 SCOPED_TSAN_INTERCEPTOR(dup, oldfd); 1344 int newfd = REAL(dup)(oldfd); 1345 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) 1346 FdDup(thr, pc, oldfd, newfd); 1347 return newfd; 1348} 1349 1350TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { 1351 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); 1352 int newfd2 = REAL(dup2)(oldfd, newfd); 1353 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1354 FdDup(thr, pc, oldfd, newfd2); 1355 return newfd2; 1356} 1357 1358TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { 1359 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); 1360 int newfd2 = REAL(dup3)(oldfd, newfd, flags); 1361 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1362 FdDup(thr, pc, oldfd, newfd2); 1363 return newfd2; 1364} 1365 1366TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { 1367 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); 1368 int fd = REAL(eventfd)(initval, flags); 1369 if (fd >= 0) 1370 FdEventCreate(thr, pc, fd); 1371 return fd; 1372} 1373 1374TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { 1375 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); 1376 if (fd >= 0) 1377 FdClose(thr, pc, fd); 1378 fd = REAL(signalfd)(fd, mask, flags); 1379 if (fd >= 0) 1380 FdSignalCreate(thr, pc, fd); 1381 return fd; 1382} 1383 1384TSAN_INTERCEPTOR(int, inotify_init, int fake) { 1385 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); 1386 int fd = REAL(inotify_init)(fake); 1387 if (fd >= 0) 1388 FdInotifyCreate(thr, pc, fd); 1389 return fd; 1390} 1391 1392TSAN_INTERCEPTOR(int, inotify_init1, int flags) { 1393 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); 1394 int fd = REAL(inotify_init1)(flags); 1395 if (fd >= 0) 1396 FdInotifyCreate(thr, pc, fd); 1397 return fd; 1398} 1399 1400TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { 1401 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); 1402 int fd = REAL(socket)(domain, type, protocol); 1403 if (fd >= 0) 1404 FdSocketCreate(thr, pc, fd); 1405 return fd; 1406} 1407 1408TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { 1409 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); 1410 int res = REAL(socketpair)(domain, type, protocol, fd); 1411 if (res == 0 && fd[0] >= 0 && fd[1] >= 0) 1412 FdPipeCreate(thr, pc, fd[0], fd[1]); 1413 return res; 1414} 1415 1416TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { 1417 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); 1418 FdSocketConnecting(thr, pc, fd); 1419 int res = REAL(connect)(fd, addr, addrlen); 1420 if (res == 0 && fd >= 0) 1421 FdSocketConnect(thr, pc, fd); 1422 return res; 1423} 1424 1425TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { 1426 SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); 1427 int res = REAL(bind)(fd, addr, addrlen); 1428 if (fd > 0 && res == 0) 1429 FdAccess(thr, pc, fd); 1430 return res; 1431} 1432 1433TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { 1434 SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); 1435 int res = REAL(listen)(fd, backlog); 1436 if (fd > 0 && res == 0) 1437 FdAccess(thr, pc, fd); 1438 return res; 1439} 1440 1441TSAN_INTERCEPTOR(int, epoll_create, int size) { 1442 SCOPED_TSAN_INTERCEPTOR(epoll_create, size); 1443 int fd = REAL(epoll_create)(size); 1444 if (fd >= 0) 1445 FdPollCreate(thr, pc, fd); 1446 return fd; 1447} 1448 1449TSAN_INTERCEPTOR(int, epoll_create1, int flags) { 1450 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); 1451 int fd = REAL(epoll_create1)(flags); 1452 if (fd >= 0) 1453 FdPollCreate(thr, pc, fd); 1454 return fd; 1455} 1456 1457TSAN_INTERCEPTOR(int, close, int fd) { 1458 SCOPED_TSAN_INTERCEPTOR(close, fd); 1459 if (fd >= 0) 1460 FdClose(thr, pc, fd); 1461 return REAL(close)(fd); 1462} 1463 1464TSAN_INTERCEPTOR(int, __close, int fd) { 1465 SCOPED_TSAN_INTERCEPTOR(__close, fd); 1466 if (fd >= 0) 1467 FdClose(thr, pc, fd); 1468 return REAL(__close)(fd); 1469} 1470 1471// glibc guts 1472TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { 1473 SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); 1474 int fds[64]; 1475 int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); 1476 for (int i = 0; i < cnt; i++) { 1477 if (fds[i] > 0) 1478 FdClose(thr, pc, fds[i]); 1479 } 1480 REAL(__res_iclose)(state, free_addr); 1481} 1482 1483TSAN_INTERCEPTOR(int, pipe, int *pipefd) { 1484 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); 1485 int res = REAL(pipe)(pipefd); 1486 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1487 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1488 return res; 1489} 1490 1491TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { 1492 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); 1493 int res = REAL(pipe2)(pipefd, flags); 1494 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1495 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1496 return res; 1497} 1498 1499TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { 1500 SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); 1501 if (fd >= 0) 1502 FdRelease(thr, pc, fd); 1503 int res = REAL(send)(fd, buf, len, flags); 1504 return res; 1505} 1506 1507TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { 1508 SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); 1509 if (fd >= 0) 1510 FdRelease(thr, pc, fd); 1511 int res = REAL(sendmsg)(fd, msg, flags); 1512 return res; 1513} 1514 1515TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { 1516 SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); 1517 int res = REAL(recv)(fd, buf, len, flags); 1518 if (res >= 0 && fd >= 0) { 1519 FdAcquire(thr, pc, fd); 1520 } 1521 return res; 1522} 1523 1524TSAN_INTERCEPTOR(int, unlink, char *path) { 1525 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1526 Release(thr, pc, File2addr(path)); 1527 int res = REAL(unlink)(path); 1528 return res; 1529} 1530 1531TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) { 1532 SCOPED_TSAN_INTERCEPTOR(fopen, path, mode); 1533 void *res = REAL(fopen)(path, mode); 1534 Acquire(thr, pc, File2addr(path)); 1535 if (res) { 1536 int fd = fileno_unlocked(res); 1537 if (fd >= 0) 1538 FdFileCreate(thr, pc, fd); 1539 } 1540 return res; 1541} 1542 1543TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) { 1544 SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream); 1545 if (stream) { 1546 int fd = fileno_unlocked(stream); 1547 if (fd >= 0) 1548 FdClose(thr, pc, fd); 1549 } 1550 void *res = REAL(freopen)(path, mode, stream); 1551 Acquire(thr, pc, File2addr(path)); 1552 if (res) { 1553 int fd = fileno_unlocked(res); 1554 if (fd >= 0) 1555 FdFileCreate(thr, pc, fd); 1556 } 1557 return res; 1558} 1559 1560TSAN_INTERCEPTOR(int, fclose, void *stream) { 1561 { 1562 SCOPED_TSAN_INTERCEPTOR(fclose, stream); 1563 if (stream) { 1564 int fd = fileno_unlocked(stream); 1565 if (fd >= 0) 1566 FdClose(thr, pc, fd); 1567 } 1568 } 1569 return REAL(fclose)(stream); 1570} 1571 1572TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { 1573 { 1574 SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); 1575 MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); 1576 } 1577 return REAL(fread)(ptr, size, nmemb, f); 1578} 1579 1580TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { 1581 { 1582 SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); 1583 MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); 1584 } 1585 return REAL(fwrite)(p, size, nmemb, f); 1586} 1587 1588TSAN_INTERCEPTOR(int, fflush, void *stream) { 1589 SCOPED_TSAN_INTERCEPTOR(fflush, stream); 1590 return REAL(fflush)(stream); 1591} 1592 1593TSAN_INTERCEPTOR(void, abort, int fake) { 1594 SCOPED_TSAN_INTERCEPTOR(abort, fake); 1595 REAL(fflush)(0); 1596 REAL(abort)(fake); 1597} 1598 1599TSAN_INTERCEPTOR(int, puts, const char *s) { 1600 SCOPED_TSAN_INTERCEPTOR(puts, s); 1601 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); 1602 return REAL(puts)(s); 1603} 1604 1605TSAN_INTERCEPTOR(int, rmdir, char *path) { 1606 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1607 Release(thr, pc, Dir2addr(path)); 1608 int res = REAL(rmdir)(path); 1609 return res; 1610} 1611 1612TSAN_INTERCEPTOR(void*, opendir, char *path) { 1613 SCOPED_TSAN_INTERCEPTOR(opendir, path); 1614 void *res = REAL(opendir)(path); 1615 if (res != 0) 1616 Acquire(thr, pc, Dir2addr(path)); 1617 return res; 1618} 1619 1620TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1621 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1622 if (op == EPOLL_CTL_ADD && epfd >= 0) { 1623 FdRelease(thr, pc, epfd); 1624 } 1625 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1626 if (fd >= 0) 1627 FdAccess(thr, pc, fd); 1628 return res; 1629} 1630 1631TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1632 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1633 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); 1634 if (res > 0 && epfd >= 0) { 1635 FdAcquire(thr, pc, epfd); 1636 } 1637 return res; 1638} 1639 1640void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, 1641 my_siginfo_t *info, void *ctx) { 1642 ThreadState *thr = cur_thread(); 1643 SignalContext *sctx = SigCtx(thr); 1644 // Don't mess with synchronous signals. 1645 if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || 1646 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || 1647 // If we are sending signal to ourselves, we must process it now. 1648 (sctx && sig == sctx->int_signal_send) || 1649 // If we are in blocking function, we can safely process it now 1650 // (but check if we are in a recursive interceptor, 1651 // i.e. pthread_join()->munmap()). 1652 (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) { 1653 int in_rtl = thr->in_rtl; 1654 thr->in_rtl = 0; 1655 CHECK_EQ(thr->in_signal_handler, false); 1656 thr->in_signal_handler = true; 1657 if (sigact) 1658 sigactions[sig].sa_sigaction(sig, info, ctx); 1659 else 1660 sigactions[sig].sa_handler(sig); 1661 CHECK_EQ(thr->in_signal_handler, true); 1662 thr->in_signal_handler = false; 1663 thr->in_rtl = in_rtl; 1664 return; 1665 } 1666 1667 if (sctx == 0) 1668 return; 1669 SignalDesc *signal = &sctx->pending_signals[sig]; 1670 if (signal->armed == false) { 1671 signal->armed = true; 1672 signal->sigaction = sigact; 1673 if (info) 1674 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 1675 if (ctx) 1676 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 1677 sctx->pending_signal_count++; 1678 } 1679} 1680 1681static void rtl_sighandler(int sig) { 1682 rtl_generic_sighandler(false, sig, 0, 0); 1683} 1684 1685static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { 1686 rtl_generic_sighandler(true, sig, info, ctx); 1687} 1688 1689TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { 1690 SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old); 1691 if (old) 1692 internal_memcpy(old, &sigactions[sig], sizeof(*old)); 1693 if (act == 0) 1694 return 0; 1695 internal_memcpy(&sigactions[sig], act, sizeof(*act)); 1696 sigaction_t newact; 1697 internal_memcpy(&newact, act, sizeof(newact)); 1698 REAL(sigfillset)(&newact.sa_mask); 1699 if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { 1700 if (newact.sa_flags & SA_SIGINFO) 1701 newact.sa_sigaction = rtl_sigaction; 1702 else 1703 newact.sa_handler = rtl_sighandler; 1704 } 1705 int res = REAL(sigaction)(sig, &newact, 0); 1706 return res; 1707} 1708 1709TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { 1710 sigaction_t act; 1711 act.sa_handler = h; 1712 REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); 1713 act.sa_flags = 0; 1714 sigaction_t old; 1715 int res = sigaction(sig, &act, &old); 1716 if (res) 1717 return SIG_ERR; 1718 return old.sa_handler; 1719} 1720 1721TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { 1722 SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); 1723 return REAL(sigsuspend)(mask); 1724} 1725 1726TSAN_INTERCEPTOR(int, raise, int sig) { 1727 SCOPED_TSAN_INTERCEPTOR(raise, sig); 1728 SignalContext *sctx = SigCtx(thr); 1729 CHECK_NE(sctx, 0); 1730 int prev = sctx->int_signal_send; 1731 sctx->int_signal_send = sig; 1732 int res = REAL(raise)(sig); 1733 CHECK_EQ(sctx->int_signal_send, sig); 1734 sctx->int_signal_send = prev; 1735 return res; 1736} 1737 1738TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 1739 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 1740 SignalContext *sctx = SigCtx(thr); 1741 CHECK_NE(sctx, 0); 1742 int prev = sctx->int_signal_send; 1743 if (pid == (int)internal_getpid()) { 1744 sctx->int_signal_send = sig; 1745 } 1746 int res = REAL(kill)(pid, sig); 1747 if (pid == (int)internal_getpid()) { 1748 CHECK_EQ(sctx->int_signal_send, sig); 1749 sctx->int_signal_send = prev; 1750 } 1751 return res; 1752} 1753 1754TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 1755 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 1756 SignalContext *sctx = SigCtx(thr); 1757 CHECK_NE(sctx, 0); 1758 int prev = sctx->int_signal_send; 1759 if (tid == pthread_self()) { 1760 sctx->int_signal_send = sig; 1761 } 1762 int res = REAL(pthread_kill)(tid, sig); 1763 if (tid == pthread_self()) { 1764 CHECK_EQ(sctx->int_signal_send, sig); 1765 sctx->int_signal_send = prev; 1766 } 1767 return res; 1768} 1769 1770TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { 1771 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); 1772 // It's intercepted merely to process pending signals. 1773 return REAL(gettimeofday)(tv, tz); 1774} 1775 1776TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, 1777 void *hints, void *rv) { 1778 SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); 1779 // We miss atomic synchronization in getaddrinfo, 1780 // and can report false race between malloc and free 1781 // inside of getaddrinfo. So ignore memory accesses. 1782 ThreadIgnoreBegin(thr); 1783 // getaddrinfo calls fopen, which can be intercepted by user. 1784 thr->in_rtl--; 1785 CHECK_EQ(thr->in_rtl, 0); 1786 int res = REAL(getaddrinfo)(node, service, hints, rv); 1787 thr->in_rtl++; 1788 ThreadIgnoreEnd(thr); 1789 return res; 1790} 1791 1792// Linux kernel has a bug that leads to kernel deadlock if a process 1793// maps TBs of memory and then calls mlock(). 1794static void MlockIsUnsupported() { 1795 static atomic_uint8_t printed; 1796 if (atomic_exchange(&printed, 1, memory_order_relaxed)) 1797 return; 1798 if (flags()->verbosity > 0) 1799 Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n"); 1800} 1801 1802TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) { 1803 MlockIsUnsupported(); 1804 return 0; 1805} 1806 1807TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) { 1808 MlockIsUnsupported(); 1809 return 0; 1810} 1811 1812TSAN_INTERCEPTOR(int, mlockall, int flags) { 1813 MlockIsUnsupported(); 1814 return 0; 1815} 1816 1817TSAN_INTERCEPTOR(int, munlockall, void) { 1818 MlockIsUnsupported(); 1819 return 0; 1820} 1821 1822TSAN_INTERCEPTOR(int, fork, int fake) { 1823 SCOPED_TSAN_INTERCEPTOR(fork, fake); 1824 int pid = REAL(fork)(fake); 1825 if (pid == 0) { 1826 // child 1827 FdOnFork(thr, pc); 1828 } else if (pid > 0) { 1829 // parent 1830 } 1831 return pid; 1832} 1833 1834struct TsanInterceptorContext { 1835 ThreadState *thr; 1836 const uptr caller_pc; 1837 const uptr pc; 1838}; 1839 1840#include "sanitizer_common/sanitizer_platform_interceptors.h" 1841// Causes interceptor recursion (getpwuid_r() calls fopen()) 1842#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS 1843#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS 1844// Causes interceptor recursion (getaddrinfo() and fopen()) 1845#undef SANITIZER_INTERCEPT_GETADDRINFO 1846#undef SANITIZER_INTERCEPT_GETNAMEINFO 1847// Causes interceptor recursion (glob64() calls lstat64()) 1848#undef SANITIZER_INTERCEPT_GLOB 1849 1850#define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count) \ 1851 do { \ 1852 } while (false) 1853#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ 1854 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ 1855 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ 1856 true) 1857#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ 1858 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ 1859 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ 1860 false) 1861#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ 1862 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ 1863 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 1864 ctx = (void *)&_ctx; \ 1865 (void) ctx; 1866#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ 1867 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) 1868#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ 1869 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) 1870#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ 1871 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) 1872#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ 1873 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) 1874#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) 1875#include "sanitizer_common/sanitizer_common_interceptors.inc" 1876 1877#define TSAN_SYSCALL() \ 1878 ThreadState *thr = cur_thread(); \ 1879 ScopedSyscall scoped_syscall(thr) \ 1880/**/ 1881 1882struct ScopedSyscall { 1883 ThreadState *thr; 1884 1885 explicit ScopedSyscall(ThreadState *thr) 1886 : thr(thr) { 1887 if (thr->in_rtl == 0) 1888 Initialize(thr); 1889 thr->in_rtl++; 1890 } 1891 1892 ~ScopedSyscall() { 1893 thr->in_rtl--; 1894 if (thr->in_rtl == 0) 1895 ProcessPendingSignals(thr); 1896 } 1897}; 1898 1899static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { 1900 TSAN_SYSCALL(); 1901 MemoryAccessRange(thr, pc, p, s, write); 1902} 1903 1904static void syscall_fd_close(uptr pc, int fd) { 1905 TSAN_SYSCALL(); 1906 if (fd >= 0) 1907 FdClose(thr, pc, fd); 1908} 1909 1910static void syscall_pre_fork(uptr pc) { 1911 TSAN_SYSCALL(); 1912} 1913 1914static void syscall_post_fork(uptr pc, int res) { 1915 TSAN_SYSCALL(); 1916 if (res == 0) { 1917 // child 1918 FdOnFork(thr, pc); 1919 } else if (res > 0) { 1920 // parent 1921 } 1922} 1923 1924#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ 1925 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) 1926#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ 1927 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) 1928#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ 1929 do { } while (false) 1930#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ 1931 do { } while (false) 1932#define COMMON_SYSCALL_FD_CLOSE(fd) \ 1933 syscall_fd_close(GET_CALLER_PC(), fd) 1934#define COMMON_SYSCALL_PRE_FORK() \ 1935 syscall_pre_fork(GET_CALLER_PC()) 1936#define COMMON_SYSCALL_POST_FORK(res) \ 1937 syscall_post_fork(GET_CALLER_PC(), res) 1938#include "sanitizer_common/sanitizer_common_syscalls.inc" 1939 1940namespace __tsan { 1941 1942void ProcessPendingSignals(ThreadState *thr) { 1943 CHECK_EQ(thr->in_rtl, 0); 1944 SignalContext *sctx = SigCtx(thr); 1945 if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler) 1946 return; 1947 Context *ctx = CTX(); 1948 thr->in_signal_handler = true; 1949 sctx->pending_signal_count = 0; 1950 // These are too big for stack. 1951 static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; 1952 REAL(sigfillset)(&emptyset); 1953 pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); 1954 for (int sig = 0; sig < kSigCount; sig++) { 1955 SignalDesc *signal = &sctx->pending_signals[sig]; 1956 if (signal->armed) { 1957 signal->armed = false; 1958 if (sigactions[sig].sa_handler != SIG_DFL 1959 && sigactions[sig].sa_handler != SIG_IGN) { 1960 // Insure that the handler does not spoil errno. 1961 const int saved_errno = errno; 1962 errno = 0; 1963 if (signal->sigaction) 1964 sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx); 1965 else 1966 sigactions[sig].sa_handler(sig); 1967 if (flags()->report_bugs && errno != 0) { 1968 ScopedInRtl in_rtl; 1969 __tsan::StackTrace stack; 1970 uptr pc = signal->sigaction ? 1971 (uptr)sigactions[sig].sa_sigaction : 1972 (uptr)sigactions[sig].sa_handler; 1973 pc += 1; // return address is expected, OutputReport() will undo this 1974 stack.Init(&pc, 1); 1975 ThreadRegistryLock l(ctx->thread_registry); 1976 ScopedReport rep(ReportTypeErrnoInSignal); 1977 if (!IsFiredSuppression(ctx, rep, stack)) { 1978 rep.AddStack(&stack); 1979 OutputReport(ctx, rep, rep.GetReport()->stacks[0]); 1980 } 1981 } 1982 errno = saved_errno; 1983 } 1984 } 1985 } 1986 pthread_sigmask(SIG_SETMASK, &oldset, 0); 1987 CHECK_EQ(thr->in_signal_handler, true); 1988 thr->in_signal_handler = false; 1989} 1990 1991static void finalize(void *arg) { 1992 ThreadState * thr = cur_thread(); 1993 uptr pc = 0; 1994 atexit_ctx->exit(thr, pc); 1995 int status = Finalize(cur_thread()); 1996 REAL(fflush)(0); 1997 if (status) 1998 _exit(status); 1999} 2000 2001static void unreachable() { 2002 Printf("FATAL: ThreadSanitizer: unreachable called\n"); 2003 Die(); 2004} 2005 2006void InitializeInterceptors() { 2007 CHECK_GT(cur_thread()->in_rtl, 0); 2008 2009 // We need to setup it early, because functions like dlsym() can call it. 2010 REAL(memset) = internal_memset; 2011 REAL(memcpy) = internal_memcpy; 2012 REAL(memcmp) = internal_memcmp; 2013 2014 // Instruct libc malloc to consume less memory. 2015 mallopt(1, 0); // M_MXFAST 2016 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD 2017 2018 SANITIZER_COMMON_INTERCEPTORS_INIT; 2019 2020 TSAN_INTERCEPT(setjmp); 2021 TSAN_INTERCEPT(_setjmp); 2022 TSAN_INTERCEPT(sigsetjmp); 2023 TSAN_INTERCEPT(__sigsetjmp); 2024 TSAN_INTERCEPT(longjmp); 2025 TSAN_INTERCEPT(siglongjmp); 2026 2027 TSAN_INTERCEPT(malloc); 2028 TSAN_INTERCEPT(__libc_memalign); 2029 TSAN_INTERCEPT(calloc); 2030 TSAN_INTERCEPT(realloc); 2031 TSAN_INTERCEPT(free); 2032 TSAN_INTERCEPT(cfree); 2033 TSAN_INTERCEPT(mmap); 2034 TSAN_INTERCEPT(mmap64); 2035 TSAN_INTERCEPT(munmap); 2036 TSAN_INTERCEPT(memalign); 2037 TSAN_INTERCEPT(valloc); 2038 TSAN_INTERCEPT(pvalloc); 2039 TSAN_INTERCEPT(posix_memalign); 2040 2041 TSAN_INTERCEPT(strlen); 2042 TSAN_INTERCEPT(memset); 2043 TSAN_INTERCEPT(memcpy); 2044 TSAN_INTERCEPT(memchr); 2045 TSAN_INTERCEPT(memrchr); 2046 TSAN_INTERCEPT(memmove); 2047 TSAN_INTERCEPT(memcmp); 2048 TSAN_INTERCEPT(strchr); 2049 TSAN_INTERCEPT(strchrnul); 2050 TSAN_INTERCEPT(strrchr); 2051 TSAN_INTERCEPT(strcpy); // NOLINT 2052 TSAN_INTERCEPT(strncpy); 2053 TSAN_INTERCEPT(strstr); 2054 TSAN_INTERCEPT(strdup); 2055 2056 TSAN_INTERCEPT(pthread_create); 2057 TSAN_INTERCEPT(pthread_join); 2058 TSAN_INTERCEPT(pthread_detach); 2059 2060 TSAN_INTERCEPT(pthread_mutex_init); 2061 TSAN_INTERCEPT(pthread_mutex_destroy); 2062 TSAN_INTERCEPT(pthread_mutex_lock); 2063 TSAN_INTERCEPT(pthread_mutex_trylock); 2064 TSAN_INTERCEPT(pthread_mutex_timedlock); 2065 TSAN_INTERCEPT(pthread_mutex_unlock); 2066 2067 TSAN_INTERCEPT(pthread_spin_init); 2068 TSAN_INTERCEPT(pthread_spin_destroy); 2069 TSAN_INTERCEPT(pthread_spin_lock); 2070 TSAN_INTERCEPT(pthread_spin_trylock); 2071 TSAN_INTERCEPT(pthread_spin_unlock); 2072 2073 TSAN_INTERCEPT(pthread_rwlock_init); 2074 TSAN_INTERCEPT(pthread_rwlock_destroy); 2075 TSAN_INTERCEPT(pthread_rwlock_rdlock); 2076 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 2077 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 2078 TSAN_INTERCEPT(pthread_rwlock_wrlock); 2079 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 2080 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 2081 TSAN_INTERCEPT(pthread_rwlock_unlock); 2082 2083 INTERCEPT_FUNCTION_VER(pthread_cond_init, GLIBC_2.3.2); 2084 INTERCEPT_FUNCTION_VER(pthread_cond_destroy, GLIBC_2.3.2); 2085 INTERCEPT_FUNCTION_VER(pthread_cond_signal, GLIBC_2.3.2); 2086 INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, GLIBC_2.3.2); 2087 INTERCEPT_FUNCTION_VER(pthread_cond_wait, GLIBC_2.3.2); 2088 INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, GLIBC_2.3.2); 2089 2090 TSAN_INTERCEPT(pthread_barrier_init); 2091 TSAN_INTERCEPT(pthread_barrier_destroy); 2092 TSAN_INTERCEPT(pthread_barrier_wait); 2093 2094 TSAN_INTERCEPT(pthread_once); 2095 2096 TSAN_INTERCEPT(sem_init); 2097 TSAN_INTERCEPT(sem_destroy); 2098 TSAN_INTERCEPT(sem_wait); 2099 TSAN_INTERCEPT(sem_trywait); 2100 TSAN_INTERCEPT(sem_timedwait); 2101 TSAN_INTERCEPT(sem_post); 2102 TSAN_INTERCEPT(sem_getvalue); 2103 2104 TSAN_INTERCEPT(stat); 2105 TSAN_INTERCEPT(__xstat); 2106 TSAN_INTERCEPT(stat64); 2107 TSAN_INTERCEPT(__xstat64); 2108 TSAN_INTERCEPT(lstat); 2109 TSAN_INTERCEPT(__lxstat); 2110 TSAN_INTERCEPT(lstat64); 2111 TSAN_INTERCEPT(__lxstat64); 2112 TSAN_INTERCEPT(fstat); 2113 TSAN_INTERCEPT(__fxstat); 2114 TSAN_INTERCEPT(fstat64); 2115 TSAN_INTERCEPT(__fxstat64); 2116 TSAN_INTERCEPT(open); 2117 TSAN_INTERCEPT(open64); 2118 TSAN_INTERCEPT(creat); 2119 TSAN_INTERCEPT(creat64); 2120 TSAN_INTERCEPT(dup); 2121 TSAN_INTERCEPT(dup2); 2122 TSAN_INTERCEPT(dup3); 2123 TSAN_INTERCEPT(eventfd); 2124 TSAN_INTERCEPT(signalfd); 2125 TSAN_INTERCEPT(inotify_init); 2126 TSAN_INTERCEPT(inotify_init1); 2127 TSAN_INTERCEPT(socket); 2128 TSAN_INTERCEPT(socketpair); 2129 TSAN_INTERCEPT(connect); 2130 TSAN_INTERCEPT(bind); 2131 TSAN_INTERCEPT(listen); 2132 TSAN_INTERCEPT(epoll_create); 2133 TSAN_INTERCEPT(epoll_create1); 2134 TSAN_INTERCEPT(close); 2135 TSAN_INTERCEPT(__close); 2136 TSAN_INTERCEPT(__res_iclose); 2137 TSAN_INTERCEPT(pipe); 2138 TSAN_INTERCEPT(pipe2); 2139 2140 TSAN_INTERCEPT(send); 2141 TSAN_INTERCEPT(sendmsg); 2142 TSAN_INTERCEPT(recv); 2143 2144 TSAN_INTERCEPT(unlink); 2145 TSAN_INTERCEPT(fopen); 2146 TSAN_INTERCEPT(freopen); 2147 TSAN_INTERCEPT(fclose); 2148 TSAN_INTERCEPT(fread); 2149 TSAN_INTERCEPT(fwrite); 2150 TSAN_INTERCEPT(fflush); 2151 TSAN_INTERCEPT(abort); 2152 TSAN_INTERCEPT(puts); 2153 TSAN_INTERCEPT(rmdir); 2154 TSAN_INTERCEPT(opendir); 2155 2156 TSAN_INTERCEPT(epoll_ctl); 2157 TSAN_INTERCEPT(epoll_wait); 2158 2159 TSAN_INTERCEPT(sigaction); 2160 TSAN_INTERCEPT(signal); 2161 TSAN_INTERCEPT(sigsuspend); 2162 TSAN_INTERCEPT(raise); 2163 TSAN_INTERCEPT(kill); 2164 TSAN_INTERCEPT(pthread_kill); 2165 TSAN_INTERCEPT(sleep); 2166 TSAN_INTERCEPT(usleep); 2167 TSAN_INTERCEPT(nanosleep); 2168 TSAN_INTERCEPT(gettimeofday); 2169 TSAN_INTERCEPT(getaddrinfo); 2170 2171 TSAN_INTERCEPT(mlock); 2172 TSAN_INTERCEPT(munlock); 2173 TSAN_INTERCEPT(mlockall); 2174 TSAN_INTERCEPT(munlockall); 2175 2176 TSAN_INTERCEPT(fork); 2177 TSAN_INTERCEPT(on_exit); 2178 TSAN_INTERCEPT(__cxa_atexit); 2179 2180 // Need to setup it, because interceptors check that the function is resolved. 2181 // But atexit is emitted directly into the module, so can't be resolved. 2182 REAL(atexit) = (int(*)(void(*)()))unreachable; 2183 atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext))) 2184 AtExitContext(); 2185 2186 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { 2187 Printf("ThreadSanitizer: failed to setup atexit callback\n"); 2188 Die(); 2189 } 2190 2191 if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { 2192 Printf("ThreadSanitizer: failed to create thread key\n"); 2193 Die(); 2194 } 2195 2196 FdInit(); 2197} 2198 2199void internal_start_thread(void(*func)(void *arg), void *arg) { 2200 void *th; 2201 REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); 2202 REAL(pthread_detach)(th); 2203} 2204 2205} // namespace __tsan 2206