1//===-- tsan_interceptors.cc ----------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12// FIXME: move as many interceptors as possible into
13// sanitizer_common/sanitizer_common_interceptors.inc
14//===----------------------------------------------------------------------===//
15
16#include "sanitizer_common/sanitizer_atomic.h"
17#include "sanitizer_common/sanitizer_libc.h"
18#include "sanitizer_common/sanitizer_linux.h"
19#include "sanitizer_common/sanitizer_platform_limits_posix.h"
20#include "sanitizer_common/sanitizer_placement_new.h"
21#include "sanitizer_common/sanitizer_stacktrace.h"
22#include "interception/interception.h"
23#include "tsan_interface.h"
24#include "tsan_platform.h"
25#include "tsan_suppressions.h"
26#include "tsan_rtl.h"
27#include "tsan_mman.h"
28#include "tsan_fd.h"
29
30using namespace __tsan;  // NOLINT
31
32const int kSigCount = 65;
33
34struct my_siginfo_t {
35  // The size is determined by looking at sizeof of real siginfo_t on linux.
36  u64 opaque[128 / sizeof(u64)];
37};
38
39struct ucontext_t {
40  // The size is determined by looking at sizeof of real ucontext_t on linux.
41  u64 opaque[936 / sizeof(u64) + 1];
42};
43
44extern "C" int pthread_attr_init(void *attr);
45extern "C" int pthread_attr_destroy(void *attr);
46DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
47extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
48extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
49extern "C" int pthread_setspecific(unsigned key, const void *v);
50DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
51extern "C" int pthread_yield();
52extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set,
53                               __sanitizer_sigset_t *oldset);
54// REAL(sigfillset) defined in common interceptors.
55DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set)
56DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
57extern "C" void *pthread_self();
58extern "C" void _exit(int status);
59extern "C" int *__errno_location();
60extern "C" int fileno_unlocked(void *stream);
61extern "C" void *__libc_malloc(uptr size);
62extern "C" void *__libc_calloc(uptr size, uptr n);
63extern "C" void *__libc_realloc(void *ptr, uptr size);
64extern "C" void __libc_free(void *ptr);
65extern "C" int mallopt(int param, int value);
66extern __sanitizer_FILE *stdout, *stderr;
67const int PTHREAD_MUTEX_RECURSIVE = 1;
68const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
69const int EINVAL = 22;
70const int EBUSY = 16;
71const int EOWNERDEAD = 130;
72const int EPOLL_CTL_ADD = 1;
73const int SIGILL = 4;
74const int SIGABRT = 6;
75const int SIGFPE = 8;
76const int SIGSEGV = 11;
77const int SIGPIPE = 13;
78const int SIGTERM = 15;
79const int SIGBUS = 7;
80const int SIGSYS = 31;
81void *const MAP_FAILED = (void*)-1;
82const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
83const int MAP_FIXED = 0x10;
84typedef long long_t;  // NOLINT
85
86// From /usr/include/unistd.h
87# define F_ULOCK 0      /* Unlock a previously locked region.  */
88# define F_LOCK  1      /* Lock a region for exclusive use.  */
89# define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
90# define F_TEST  3      /* Test a region for other processes locks.  */
91
92typedef void (*sighandler_t)(int sig);
93
94#define errno (*__errno_location())
95
96struct sigaction_t {
97  union {
98    sighandler_t sa_handler;
99    void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
100  };
101  __sanitizer_sigset_t sa_mask;
102  int sa_flags;
103  void (*sa_restorer)();
104};
105
106const sighandler_t SIG_DFL = (sighandler_t)0;
107const sighandler_t SIG_IGN = (sighandler_t)1;
108const sighandler_t SIG_ERR = (sighandler_t)-1;
109const int SA_SIGINFO = 4;
110const int SIG_SETMASK = 2;
111
112namespace std {
113struct nothrow_t {};
114}  // namespace std
115
116static sigaction_t sigactions[kSigCount];
117
118namespace __tsan {
119struct SignalDesc {
120  bool armed;
121  bool sigaction;
122  my_siginfo_t siginfo;
123  ucontext_t ctx;
124};
125
126struct SignalContext {
127  int in_blocking_func;
128  int int_signal_send;
129  int pending_signal_count;
130  SignalDesc pending_signals[kSigCount];
131};
132
133// The object is 64-byte aligned, because we want hot data to be located in
134// a single cache line if possible (it's accessed in every interceptor).
135static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
136static LibIgnore *libignore() {
137  return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
138}
139
140void InitializeLibIgnore() {
141  libignore()->Init(*GetSuppressionContext());
142  libignore()->OnLibraryLoaded(0);
143}
144
145}  // namespace __tsan
146
147static SignalContext *SigCtx(ThreadState *thr) {
148  SignalContext *ctx = (SignalContext*)thr->signal_ctx;
149  if (ctx == 0 && thr->is_alive) {
150    ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext");
151    MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
152    thr->signal_ctx = ctx;
153  }
154  return ctx;
155}
156
157static unsigned g_thread_finalize_key;
158
159class ScopedInterceptor {
160 public:
161  ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
162  ~ScopedInterceptor();
163 private:
164  ThreadState *const thr_;
165  const uptr pc_;
166  bool in_ignored_lib_;
167};
168
169ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
170                                     uptr pc)
171    : thr_(thr)
172    , pc_(pc)
173    , in_ignored_lib_(false) {
174  if (!thr_->ignore_interceptors) {
175    Initialize(thr);
176    FuncEntry(thr, pc);
177  }
178  DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
179  if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
180    in_ignored_lib_ = true;
181    thr_->in_ignored_lib = true;
182    ThreadIgnoreBegin(thr_, pc_);
183  }
184}
185
186ScopedInterceptor::~ScopedInterceptor() {
187  if (in_ignored_lib_) {
188    thr_->in_ignored_lib = false;
189    ThreadIgnoreEnd(thr_, pc_);
190  }
191  if (!thr_->ignore_interceptors) {
192    ProcessPendingSignals(thr_);
193    FuncExit(thr_);
194    CheckNoLocks(thr_);
195  }
196}
197
198#define SCOPED_INTERCEPTOR_RAW(func, ...) \
199    ThreadState *thr = cur_thread(); \
200    const uptr caller_pc = GET_CALLER_PC(); \
201    ScopedInterceptor si(thr, #func, caller_pc); \
202    const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
203    (void)pc; \
204/**/
205
206#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
207    SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
208    if (REAL(func) == 0) { \
209      Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
210      Die(); \
211    }                                                    \
212    if (thr->ignore_interceptors || thr->in_ignored_lib) \
213      return REAL(func)(__VA_ARGS__); \
214/**/
215
216#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
217#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
218#define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
219
220#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
221
222struct BlockingCall {
223  explicit BlockingCall(ThreadState *thr)
224      : ctx(SigCtx(thr)) {
225    ctx->in_blocking_func++;
226  }
227
228  ~BlockingCall() {
229    ctx->in_blocking_func--;
230  }
231
232  SignalContext *ctx;
233
234  // When we are in a "blocking call", we process signals asynchronously
235  // (right when they arrive). In this context we do not expect to be
236  // executing any user/runtime code. The known interceptor sequence when
237  // this is not true is: pthread_join -> munmap(stack). It's fine
238  // to ignore munmap in this case -- we handle stack shadow separately.
239  ScopedIgnoreInterceptors ignore_interceptors;
240};
241
242TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
243  SCOPED_TSAN_INTERCEPTOR(sleep, sec);
244  unsigned res = BLOCK_REAL(sleep)(sec);
245  AfterSleep(thr, pc);
246  return res;
247}
248
249TSAN_INTERCEPTOR(int, usleep, long_t usec) {
250  SCOPED_TSAN_INTERCEPTOR(usleep, usec);
251  int res = BLOCK_REAL(usleep)(usec);
252  AfterSleep(thr, pc);
253  return res;
254}
255
256TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
257  SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
258  int res = BLOCK_REAL(nanosleep)(req, rem);
259  AfterSleep(thr, pc);
260  return res;
261}
262
263class AtExitContext {
264 public:
265  AtExitContext()
266    : mtx_(MutexTypeAtExit, StatMtxAtExit)
267    , pos_() {
268  }
269
270  typedef void(*atexit_t)();
271
272  int atexit(ThreadState *thr, uptr pc, bool is_on_exit,
273             atexit_t f, void *arg) {
274    Lock l(&mtx_);
275    if (pos_ == kMaxAtExit)
276      return 1;
277    Release(thr, pc, (uptr)this);
278    stack_[pos_] = f;
279    args_[pos_] = arg;
280    is_on_exits_[pos_] = is_on_exit;
281    pos_++;
282    return 0;
283  }
284
285  void exit(ThreadState *thr, uptr pc) {
286    for (;;) {
287      atexit_t f = 0;
288      void *arg = 0;
289      bool is_on_exit = false;
290      {
291        Lock l(&mtx_);
292        if (pos_) {
293          pos_--;
294          f = stack_[pos_];
295          arg = args_[pos_];
296          is_on_exit = is_on_exits_[pos_];
297          Acquire(thr, pc, (uptr)this);
298        }
299      }
300      if (f == 0)
301        break;
302      DPrintf("#%d: executing atexit func %p\n", thr->tid, f);
303      if (is_on_exit)
304        ((void(*)(int status, void *arg))f)(0, arg);
305      else
306        ((void(*)(void *arg, void *dso))f)(arg, 0);
307    }
308  }
309
310 private:
311  static const int kMaxAtExit = 128;
312  Mutex mtx_;
313  atexit_t stack_[kMaxAtExit];
314  void *args_[kMaxAtExit];
315  bool is_on_exits_[kMaxAtExit];
316  int pos_;
317};
318
319static AtExitContext *atexit_ctx;
320
321TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
322  if (cur_thread()->in_symbolizer)
323    return 0;
324  // We want to setup the atexit callback even if we are in ignored lib
325  // or after fork.
326  SCOPED_INTERCEPTOR_RAW(atexit, f);
327  return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0);
328}
329
330TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
331  if (cur_thread()->in_symbolizer)
332    return 0;
333  SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
334  return atexit_ctx->atexit(thr, pc, true, (void(*)())f, arg);
335}
336
337TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
338  if (cur_thread()->in_symbolizer)
339    return 0;
340  SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
341  if (dso) {
342    // Memory allocation in __cxa_atexit will race with free during exit,
343    // because we do not see synchronization around atexit callback list.
344    ThreadIgnoreBegin(thr, pc);
345    int res = REAL(__cxa_atexit)(f, arg, dso);
346    ThreadIgnoreEnd(thr, pc);
347    return res;
348  }
349  return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg);
350}
351
352// Cleanup old bufs.
353static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
354  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
355    JmpBuf *buf = &thr->jmp_bufs[i];
356    if (buf->sp <= sp) {
357      uptr sz = thr->jmp_bufs.Size();
358      thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1];
359      thr->jmp_bufs.PopBack();
360      i--;
361    }
362  }
363}
364
365static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
366  if (thr->shadow_stack_pos == 0)  // called from libc guts during bootstrap
367    return;
368  // Cleanup old bufs.
369  JmpBufGarbageCollect(thr, sp);
370  // Remember the buf.
371  JmpBuf *buf = thr->jmp_bufs.PushBack();
372  buf->sp = sp;
373  buf->mangled_sp = mangled_sp;
374  buf->shadow_stack_pos = thr->shadow_stack_pos;
375}
376
377static void LongJmp(ThreadState *thr, uptr *env) {
378  uptr mangled_sp = env[6];
379  // Find the saved buf by mangled_sp.
380  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
381    JmpBuf *buf = &thr->jmp_bufs[i];
382    if (buf->mangled_sp == mangled_sp) {
383      CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
384      // Unwind the stack.
385      while (thr->shadow_stack_pos > buf->shadow_stack_pos)
386        FuncExit(thr);
387      JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
388      return;
389    }
390  }
391  Printf("ThreadSanitizer: can't find longjmp buf\n");
392  CHECK(0);
393}
394
395// FIXME: put everything below into a common extern "C" block?
396extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
397  SetJmp(cur_thread(), sp, mangled_sp);
398}
399
400// Not called.  Merely to satisfy TSAN_INTERCEPT().
401extern "C" SANITIZER_INTERFACE_ATTRIBUTE
402int __interceptor_setjmp(void *env);
403extern "C" int __interceptor_setjmp(void *env) {
404  CHECK(0);
405  return 0;
406}
407
408// FIXME: any reason to have a separate declaration?
409extern "C" SANITIZER_INTERFACE_ATTRIBUTE
410int __interceptor__setjmp(void *env);
411extern "C" int __interceptor__setjmp(void *env) {
412  CHECK(0);
413  return 0;
414}
415
416extern "C" SANITIZER_INTERFACE_ATTRIBUTE
417int __interceptor_sigsetjmp(void *env);
418extern "C" int __interceptor_sigsetjmp(void *env) {
419  CHECK(0);
420  return 0;
421}
422
423extern "C" SANITIZER_INTERFACE_ATTRIBUTE
424int __interceptor___sigsetjmp(void *env);
425extern "C" int __interceptor___sigsetjmp(void *env) {
426  CHECK(0);
427  return 0;
428}
429
430extern "C" int setjmp(void *env);
431extern "C" int _setjmp(void *env);
432extern "C" int sigsetjmp(void *env);
433extern "C" int __sigsetjmp(void *env);
434DEFINE_REAL(int, setjmp, void *env)
435DEFINE_REAL(int, _setjmp, void *env)
436DEFINE_REAL(int, sigsetjmp, void *env)
437DEFINE_REAL(int, __sigsetjmp, void *env)
438
439TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) {
440  {
441    SCOPED_TSAN_INTERCEPTOR(longjmp, env, val);
442  }
443  LongJmp(cur_thread(), env);
444  REAL(longjmp)(env, val);
445}
446
447TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
448  {
449    SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val);
450  }
451  LongJmp(cur_thread(), env);
452  REAL(siglongjmp)(env, val);
453}
454
455TSAN_INTERCEPTOR(void*, malloc, uptr size) {
456  if (cur_thread()->in_symbolizer)
457    return __libc_malloc(size);
458  void *p = 0;
459  {
460    SCOPED_INTERCEPTOR_RAW(malloc, size);
461    p = user_alloc(thr, pc, size);
462  }
463  invoke_malloc_hook(p, size);
464  return p;
465}
466
467TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
468  SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
469  return user_alloc(thr, pc, sz, align);
470}
471
472TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
473  if (cur_thread()->in_symbolizer)
474    return __libc_calloc(size, n);
475  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
476    return AllocatorReturnNull();
477  void *p = 0;
478  {
479    SCOPED_INTERCEPTOR_RAW(calloc, size, n);
480    p = user_alloc(thr, pc, n * size);
481    if (p)
482      internal_memset(p, 0, n * size);
483  }
484  invoke_malloc_hook(p, n * size);
485  return p;
486}
487
488TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
489  if (cur_thread()->in_symbolizer)
490    return __libc_realloc(p, size);
491  if (p)
492    invoke_free_hook(p);
493  {
494    SCOPED_INTERCEPTOR_RAW(realloc, p, size);
495    p = user_realloc(thr, pc, p, size);
496  }
497  invoke_malloc_hook(p, size);
498  return p;
499}
500
501TSAN_INTERCEPTOR(void, free, void *p) {
502  if (p == 0)
503    return;
504  if (cur_thread()->in_symbolizer)
505    return __libc_free(p);
506  invoke_free_hook(p);
507  SCOPED_INTERCEPTOR_RAW(free, p);
508  user_free(thr, pc, p);
509}
510
511TSAN_INTERCEPTOR(void, cfree, void *p) {
512  if (p == 0)
513    return;
514  if (cur_thread()->in_symbolizer)
515    return __libc_free(p);
516  invoke_free_hook(p);
517  SCOPED_INTERCEPTOR_RAW(cfree, p);
518  user_free(thr, pc, p);
519}
520
521TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
522  SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
523  return user_alloc_usable_size(p);
524}
525
526#define OPERATOR_NEW_BODY(mangled_name) \
527  if (cur_thread()->in_symbolizer) \
528    return __libc_malloc(size); \
529  void *p = 0; \
530  {  \
531    SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
532    p = user_alloc(thr, pc, size); \
533  }  \
534  invoke_malloc_hook(p, size);  \
535  return p;
536
537SANITIZER_INTERFACE_ATTRIBUTE
538void *operator new(__sanitizer::uptr size);
539void *operator new(__sanitizer::uptr size) {
540  OPERATOR_NEW_BODY(_Znwm);
541}
542
543SANITIZER_INTERFACE_ATTRIBUTE
544void *operator new[](__sanitizer::uptr size);
545void *operator new[](__sanitizer::uptr size) {
546  OPERATOR_NEW_BODY(_Znam);
547}
548
549SANITIZER_INTERFACE_ATTRIBUTE
550void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
551void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
552  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
553}
554
555SANITIZER_INTERFACE_ATTRIBUTE
556void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
557void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
558  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
559}
560
561#define OPERATOR_DELETE_BODY(mangled_name) \
562  if (ptr == 0) return;  \
563  if (cur_thread()->in_symbolizer) \
564    return __libc_free(ptr); \
565  invoke_free_hook(ptr);  \
566  SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
567  user_free(thr, pc, ptr);
568
569SANITIZER_INTERFACE_ATTRIBUTE
570void operator delete(void *ptr) throw();
571void operator delete(void *ptr) throw() {
572  OPERATOR_DELETE_BODY(_ZdlPv);
573}
574
575SANITIZER_INTERFACE_ATTRIBUTE
576void operator delete[](void *ptr) throw();
577void operator delete[](void *ptr) throw() {
578  OPERATOR_DELETE_BODY(_ZdaPv);
579}
580
581SANITIZER_INTERFACE_ATTRIBUTE
582void operator delete(void *ptr, std::nothrow_t const&);
583void operator delete(void *ptr, std::nothrow_t const&) {
584  OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
585}
586
587SANITIZER_INTERFACE_ATTRIBUTE
588void operator delete[](void *ptr, std::nothrow_t const&);
589void operator delete[](void *ptr, std::nothrow_t const&) {
590  OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
591}
592
593TSAN_INTERCEPTOR(uptr, strlen, const char *s) {
594  SCOPED_TSAN_INTERCEPTOR(strlen, s);
595  uptr len = internal_strlen(s);
596  MemoryAccessRange(thr, pc, (uptr)s, len + 1, false);
597  return len;
598}
599
600TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
601  SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size);
602  MemoryAccessRange(thr, pc, (uptr)dst, size, true);
603  return internal_memset(dst, v, size);
604}
605
606TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
607  SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size);
608  MemoryAccessRange(thr, pc, (uptr)dst, size, true);
609  MemoryAccessRange(thr, pc, (uptr)src, size, false);
610  return internal_memcpy(dst, src, size);
611}
612
613TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) {
614  SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n);
615  int res = 0;
616  uptr len = 0;
617  for (; len < n; len++) {
618    if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len]))
619      break;
620  }
621  MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
622  MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
623  return res;
624}
625
626TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) {
627  SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n);
628  MemoryAccessRange(thr, pc, (uptr)dst, n, true);
629  MemoryAccessRange(thr, pc, (uptr)src, n, false);
630  return REAL(memmove)(dst, src, n);
631}
632
633TSAN_INTERCEPTOR(char*, strchr, char *s, int c) {
634  SCOPED_TSAN_INTERCEPTOR(strchr, s, c);
635  char *res = REAL(strchr)(s, c);
636  uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1;
637  MemoryAccessRange(thr, pc, (uptr)s, len, false);
638  return res;
639}
640
641TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) {
642  SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c);
643  char *res = REAL(strchrnul)(s, c);
644  uptr len = (char*)res - (char*)s + 1;
645  MemoryAccessRange(thr, pc, (uptr)s, len, false);
646  return res;
647}
648
649TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) {
650  SCOPED_TSAN_INTERCEPTOR(strrchr, s, c);
651  MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false);
652  return REAL(strrchr)(s, c);
653}
654
655TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) {  // NOLINT
656  SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);  // NOLINT
657  uptr srclen = internal_strlen(src);
658  MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
659  MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
660  return REAL(strcpy)(dst, src);  // NOLINT
661}
662
663TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
664  SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
665  uptr srclen = internal_strnlen(src, n);
666  MemoryAccessRange(thr, pc, (uptr)dst, n, true);
667  MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
668  return REAL(strncpy)(dst, src, n);
669}
670
671TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) {
672  SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2);
673  const char *res = REAL(strstr)(s1, s2);
674  uptr len1 = internal_strlen(s1);
675  uptr len2 = internal_strlen(s2);
676  MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false);
677  MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false);
678  return res;
679}
680
681TSAN_INTERCEPTOR(char*, strdup, const char *str) {
682  SCOPED_TSAN_INTERCEPTOR(strdup, str);
683  // strdup will call malloc, so no instrumentation is required here.
684  return REAL(strdup)(str);
685}
686
687static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
688  if (*addr) {
689    if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
690      if (flags & MAP_FIXED) {
691        errno = EINVAL;
692        return false;
693      } else {
694        *addr = 0;
695      }
696    }
697  }
698  return true;
699}
700
701TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot,
702                         int flags, int fd, unsigned off) {
703  SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off);
704  if (!fix_mmap_addr(&addr, sz, flags))
705    return MAP_FAILED;
706  void *res = REAL(mmap)(addr, sz, prot, flags, fd, off);
707  if (res != MAP_FAILED) {
708    if (fd > 0)
709      FdAccess(thr, pc, fd);
710    MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
711  }
712  return res;
713}
714
715TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot,
716                           int flags, int fd, u64 off) {
717  SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off);
718  if (!fix_mmap_addr(&addr, sz, flags))
719    return MAP_FAILED;
720  void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off);
721  if (res != MAP_FAILED) {
722    if (fd > 0)
723      FdAccess(thr, pc, fd);
724    MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
725  }
726  return res;
727}
728
729TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
730  SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
731  DontNeedShadowFor((uptr)addr, sz);
732  int res = REAL(munmap)(addr, sz);
733  return res;
734}
735
736TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
737  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
738  return user_alloc(thr, pc, sz, align);
739}
740
741TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
742  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
743  return user_alloc(thr, pc, sz, align);
744}
745
746TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
747  SCOPED_INTERCEPTOR_RAW(valloc, sz);
748  return user_alloc(thr, pc, sz, GetPageSizeCached());
749}
750
751TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
752  SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
753  sz = RoundUp(sz, GetPageSizeCached());
754  return user_alloc(thr, pc, sz, GetPageSizeCached());
755}
756
757TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
758  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
759  *memptr = user_alloc(thr, pc, sz, align);
760  return 0;
761}
762
763// Used in thread-safe function static initialization.
764extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) {
765  SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
766  for (;;) {
767    u32 cmp = atomic_load(g, memory_order_acquire);
768    if (cmp == 0) {
769      if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
770        return 1;
771    } else if (cmp == 1) {
772      Acquire(thr, pc, (uptr)g);
773      return 0;
774    } else {
775      internal_sched_yield();
776    }
777  }
778}
779
780extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) {
781  SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
782  Release(thr, pc, (uptr)g);
783  atomic_store(g, 1, memory_order_release);
784}
785
786extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) {
787  SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
788  atomic_store(g, 0, memory_order_relaxed);
789}
790
791static void thread_finalize(void *v) {
792  uptr iter = (uptr)v;
793  if (iter > 1) {
794    if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
795      Printf("ThreadSanitizer: failed to set thread key\n");
796      Die();
797    }
798    return;
799  }
800  {
801    ThreadState *thr = cur_thread();
802    ThreadFinish(thr);
803    SignalContext *sctx = thr->signal_ctx;
804    if (sctx) {
805      thr->signal_ctx = 0;
806      UnmapOrDie(sctx, sizeof(*sctx));
807    }
808  }
809}
810
811
812struct ThreadParam {
813  void* (*callback)(void *arg);
814  void *param;
815  atomic_uintptr_t tid;
816};
817
818extern "C" void *__tsan_thread_start_func(void *arg) {
819  ThreadParam *p = (ThreadParam*)arg;
820  void* (*callback)(void *arg) = p->callback;
821  void *param = p->param;
822  int tid = 0;
823  {
824    ThreadState *thr = cur_thread();
825    // Thread-local state is not initialized yet.
826    ScopedIgnoreInterceptors ignore;
827    if (pthread_setspecific(g_thread_finalize_key,
828                            (void *)kPthreadDestructorIterations)) {
829      Printf("ThreadSanitizer: failed to set thread key\n");
830      Die();
831    }
832    while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
833      pthread_yield();
834    atomic_store(&p->tid, 0, memory_order_release);
835    ThreadStart(thr, tid, GetTid());
836  }
837  void *res = callback(param);
838  // Prevent the callback from being tail called,
839  // it mixes up stack traces.
840  volatile int foo = 42;
841  foo++;
842  return res;
843}
844
845TSAN_INTERCEPTOR(int, pthread_create,
846    void *th, void *attr, void *(*callback)(void*), void * param) {
847  SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
848  if (ctx->after_multithreaded_fork) {
849    if (flags()->die_after_fork) {
850      Report("ThreadSanitizer: starting new threads after multi-threaded "
851          "fork is not supported. Dying (set die_after_fork=0 to override)\n");
852      Die();
853    } else {
854      VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
855          "fork is not supported (pid %d). Continuing because of "
856          "die_after_fork=0, but you are on your own\n", internal_getpid());
857    }
858  }
859  __sanitizer_pthread_attr_t myattr;
860  if (attr == 0) {
861    pthread_attr_init(&myattr);
862    attr = &myattr;
863  }
864  int detached = 0;
865  REAL(pthread_attr_getdetachstate)(attr, &detached);
866  AdjustStackSize(attr);
867
868  ThreadParam p;
869  p.callback = callback;
870  p.param = param;
871  atomic_store(&p.tid, 0, memory_order_relaxed);
872  int res = -1;
873  {
874    // Otherwise we see false positives in pthread stack manipulation.
875    ScopedIgnoreInterceptors ignore;
876    ThreadIgnoreBegin(thr, pc);
877    res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
878    ThreadIgnoreEnd(thr, pc);
879  }
880  if (res == 0) {
881    int tid = ThreadCreate(thr, pc, *(uptr*)th, detached);
882    CHECK_NE(tid, 0);
883    atomic_store(&p.tid, tid, memory_order_release);
884    while (atomic_load(&p.tid, memory_order_acquire) != 0)
885      pthread_yield();
886  }
887  if (attr == &myattr)
888    pthread_attr_destroy(&myattr);
889  return res;
890}
891
892TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
893  SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
894  int tid = ThreadTid(thr, pc, (uptr)th);
895  ThreadIgnoreBegin(thr, pc);
896  int res = BLOCK_REAL(pthread_join)(th, ret);
897  ThreadIgnoreEnd(thr, pc);
898  if (res == 0) {
899    ThreadJoin(thr, pc, tid);
900  }
901  return res;
902}
903
904TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
905  SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
906  int tid = ThreadTid(thr, pc, (uptr)th);
907  int res = REAL(pthread_detach)(th);
908  if (res == 0) {
909    ThreadDetach(thr, pc, tid);
910  }
911  return res;
912}
913
914// Problem:
915// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
916// pthread_cond_t has different size in the different versions.
917// If call new REAL functions for old pthread_cond_t, they will corrupt memory
918// after pthread_cond_t (old cond is smaller).
919// If we call old REAL functions for new pthread_cond_t, we will lose  some
920// functionality (e.g. old functions do not support waiting against
921// CLOCK_REALTIME).
922// Proper handling would require to have 2 versions of interceptors as well.
923// But this is messy, in particular requires linker scripts when sanitizer
924// runtime is linked into a shared library.
925// Instead we assume we don't have dynamic libraries built against old
926// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
927// that allows to work with old libraries (but this mode does not support
928// some features, e.g. pthread_condattr_getpshared).
929static void *init_cond(void *c, bool force = false) {
930  // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
931  // So we allocate additional memory on the side large enough to hold
932  // any pthread_cond_t object. Always call new REAL functions, but pass
933  // the aux object to them.
934  // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
935  // first word of pthread_cond_t to zero.
936  // It's all relevant only for linux.
937  if (!common_flags()->legacy_pthread_cond)
938    return c;
939  atomic_uintptr_t *p = (atomic_uintptr_t*)c;
940  uptr cond = atomic_load(p, memory_order_acquire);
941  if (!force && cond != 0)
942    return (void*)cond;
943  void *newcond = WRAP(malloc)(pthread_cond_t_sz);
944  internal_memset(newcond, 0, pthread_cond_t_sz);
945  if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
946      memory_order_acq_rel))
947    return newcond;
948  WRAP(free)(newcond);
949  return (void*)cond;
950}
951
952struct CondMutexUnlockCtx {
953  ThreadState *thr;
954  uptr pc;
955  void *m;
956};
957
958static void cond_mutex_unlock(CondMutexUnlockCtx *arg) {
959  MutexLock(arg->thr, arg->pc, (uptr)arg->m);
960}
961
962INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
963  void *cond = init_cond(c, true);
964  SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a);
965  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
966  return REAL(pthread_cond_init)(cond, a);
967}
968
969INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
970  void *cond = init_cond(c);
971  SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
972  MutexUnlock(thr, pc, (uptr)m);
973  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
974  CondMutexUnlockCtx arg = {thr, pc, m};
975  // This ensures that we handle mutex lock even in case of pthread_cancel.
976  // See test/tsan/cond_cancel.cc.
977  int res = call_pthread_cancel_with_cleanup(
978      (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait),
979      cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg);
980  if (res == errno_EOWNERDEAD)
981    MutexRepair(thr, pc, (uptr)m);
982  MutexLock(thr, pc, (uptr)m);
983  return res;
984}
985
986INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
987  void *cond = init_cond(c);
988  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
989  MutexUnlock(thr, pc, (uptr)m);
990  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
991  CondMutexUnlockCtx arg = {thr, pc, m};
992  // This ensures that we handle mutex lock even in case of pthread_cancel.
993  // See test/tsan/cond_cancel.cc.
994  int res = call_pthread_cancel_with_cleanup(
995      REAL(pthread_cond_timedwait), cond, m, abstime,
996      (void(*)(void *arg))cond_mutex_unlock, &arg);
997  if (res == errno_EOWNERDEAD)
998    MutexRepair(thr, pc, (uptr)m);
999  MutexLock(thr, pc, (uptr)m);
1000  return res;
1001}
1002
1003INTERCEPTOR(int, pthread_cond_signal, void *c) {
1004  void *cond = init_cond(c);
1005  SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond);
1006  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
1007  return REAL(pthread_cond_signal)(cond);
1008}
1009
1010INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
1011  void *cond = init_cond(c);
1012  SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond);
1013  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
1014  return REAL(pthread_cond_broadcast)(cond);
1015}
1016
1017INTERCEPTOR(int, pthread_cond_destroy, void *c) {
1018  void *cond = init_cond(c);
1019  SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond);
1020  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
1021  int res = REAL(pthread_cond_destroy)(cond);
1022  if (common_flags()->legacy_pthread_cond) {
1023    // Free our aux cond and zero the pointer to not leave dangling pointers.
1024    WRAP(free)(cond);
1025    atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
1026  }
1027  return res;
1028}
1029
1030TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
1031  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
1032  int res = REAL(pthread_mutex_init)(m, a);
1033  if (res == 0) {
1034    bool recursive = false;
1035    if (a) {
1036      int type = 0;
1037      if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
1038        recursive = (type == PTHREAD_MUTEX_RECURSIVE
1039            || type == PTHREAD_MUTEX_RECURSIVE_NP);
1040    }
1041    MutexCreate(thr, pc, (uptr)m, false, recursive, false);
1042  }
1043  return res;
1044}
1045
1046TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1047  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
1048  int res = REAL(pthread_mutex_destroy)(m);
1049  if (res == 0 || res == EBUSY) {
1050    MutexDestroy(thr, pc, (uptr)m);
1051  }
1052  return res;
1053}
1054
1055TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1056  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
1057  int res = REAL(pthread_mutex_trylock)(m);
1058  if (res == EOWNERDEAD)
1059    MutexRepair(thr, pc, (uptr)m);
1060  if (res == 0 || res == EOWNERDEAD)
1061    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1062  return res;
1063}
1064
1065TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
1066  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
1067  int res = REAL(pthread_mutex_timedlock)(m, abstime);
1068  if (res == 0) {
1069    MutexLock(thr, pc, (uptr)m);
1070  }
1071  return res;
1072}
1073
1074TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
1075  SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
1076  int res = REAL(pthread_spin_init)(m, pshared);
1077  if (res == 0) {
1078    MutexCreate(thr, pc, (uptr)m, false, false, false);
1079  }
1080  return res;
1081}
1082
1083TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
1084  SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
1085  int res = REAL(pthread_spin_destroy)(m);
1086  if (res == 0) {
1087    MutexDestroy(thr, pc, (uptr)m);
1088  }
1089  return res;
1090}
1091
1092TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1093  SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
1094  int res = REAL(pthread_spin_lock)(m);
1095  if (res == 0) {
1096    MutexLock(thr, pc, (uptr)m);
1097  }
1098  return res;
1099}
1100
1101TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
1102  SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
1103  int res = REAL(pthread_spin_trylock)(m);
1104  if (res == 0) {
1105    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1106  }
1107  return res;
1108}
1109
1110TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
1111  SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
1112  MutexUnlock(thr, pc, (uptr)m);
1113  int res = REAL(pthread_spin_unlock)(m);
1114  return res;
1115}
1116
1117TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
1118  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
1119  int res = REAL(pthread_rwlock_init)(m, a);
1120  if (res == 0) {
1121    MutexCreate(thr, pc, (uptr)m, true, false, false);
1122  }
1123  return res;
1124}
1125
1126TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
1127  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
1128  int res = REAL(pthread_rwlock_destroy)(m);
1129  if (res == 0) {
1130    MutexDestroy(thr, pc, (uptr)m);
1131  }
1132  return res;
1133}
1134
1135TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1136  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
1137  int res = REAL(pthread_rwlock_rdlock)(m);
1138  if (res == 0) {
1139    MutexReadLock(thr, pc, (uptr)m);
1140  }
1141  return res;
1142}
1143
1144TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1145  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
1146  int res = REAL(pthread_rwlock_tryrdlock)(m);
1147  if (res == 0) {
1148    MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true);
1149  }
1150  return res;
1151}
1152
1153TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
1154  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1155  int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
1156  if (res == 0) {
1157    MutexReadLock(thr, pc, (uptr)m);
1158  }
1159  return res;
1160}
1161
1162TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1163  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1164  int res = REAL(pthread_rwlock_wrlock)(m);
1165  if (res == 0) {
1166    MutexLock(thr, pc, (uptr)m);
1167  }
1168  return res;
1169}
1170
1171TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1172  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
1173  int res = REAL(pthread_rwlock_trywrlock)(m);
1174  if (res == 0) {
1175    MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1176  }
1177  return res;
1178}
1179
1180TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
1181  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1182  int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
1183  if (res == 0) {
1184    MutexLock(thr, pc, (uptr)m);
1185  }
1186  return res;
1187}
1188
1189TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1190  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
1191  MutexReadOrWriteUnlock(thr, pc, (uptr)m);
1192  int res = REAL(pthread_rwlock_unlock)(m);
1193  return res;
1194}
1195
1196TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
1197  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1198  MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1199  int res = REAL(pthread_barrier_init)(b, a, count);
1200  return res;
1201}
1202
1203TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1204  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1205  MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1206  int res = REAL(pthread_barrier_destroy)(b);
1207  return res;
1208}
1209
1210TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1211  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1212  Release(thr, pc, (uptr)b);
1213  MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1214  int res = REAL(pthread_barrier_wait)(b);
1215  MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1216  if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
1217    Acquire(thr, pc, (uptr)b);
1218  }
1219  return res;
1220}
1221
1222TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
1223  SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
1224  if (o == 0 || f == 0)
1225    return EINVAL;
1226  atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o);
1227  u32 v = atomic_load(a, memory_order_acquire);
1228  if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
1229                                               memory_order_relaxed)) {
1230    (*f)();
1231    if (!thr->in_ignored_lib)
1232      Release(thr, pc, (uptr)o);
1233    atomic_store(a, 2, memory_order_release);
1234  } else {
1235    while (v != 2) {
1236      pthread_yield();
1237      v = atomic_load(a, memory_order_acquire);
1238    }
1239    if (!thr->in_ignored_lib)
1240      Acquire(thr, pc, (uptr)o);
1241  }
1242  return 0;
1243}
1244
1245TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) {
1246  SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value);
1247  int res = REAL(sem_init)(s, pshared, value);
1248  return res;
1249}
1250
1251TSAN_INTERCEPTOR(int, sem_destroy, void *s) {
1252  SCOPED_TSAN_INTERCEPTOR(sem_destroy, s);
1253  int res = REAL(sem_destroy)(s);
1254  return res;
1255}
1256
1257TSAN_INTERCEPTOR(int, sem_wait, void *s) {
1258  SCOPED_TSAN_INTERCEPTOR(sem_wait, s);
1259  int res = BLOCK_REAL(sem_wait)(s);
1260  if (res == 0) {
1261    Acquire(thr, pc, (uptr)s);
1262  }
1263  return res;
1264}
1265
1266TSAN_INTERCEPTOR(int, sem_trywait, void *s) {
1267  SCOPED_TSAN_INTERCEPTOR(sem_trywait, s);
1268  int res = BLOCK_REAL(sem_trywait)(s);
1269  if (res == 0) {
1270    Acquire(thr, pc, (uptr)s);
1271  }
1272  return res;
1273}
1274
1275TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) {
1276  SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime);
1277  int res = BLOCK_REAL(sem_timedwait)(s, abstime);
1278  if (res == 0) {
1279    Acquire(thr, pc, (uptr)s);
1280  }
1281  return res;
1282}
1283
1284TSAN_INTERCEPTOR(int, sem_post, void *s) {
1285  SCOPED_TSAN_INTERCEPTOR(sem_post, s);
1286  Release(thr, pc, (uptr)s);
1287  int res = REAL(sem_post)(s);
1288  return res;
1289}
1290
1291TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) {
1292  SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval);
1293  int res = REAL(sem_getvalue)(s, sval);
1294  if (res == 0) {
1295    Acquire(thr, pc, (uptr)s);
1296  }
1297  return res;
1298}
1299
1300TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
1301  SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf);
1302  return REAL(__xstat)(version, path, buf);
1303}
1304
1305TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) {
1306  SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf);
1307  return REAL(__xstat)(0, path, buf);
1308}
1309
1310TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) {
1311  SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf);
1312  return REAL(__xstat64)(version, path, buf);
1313}
1314
1315TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) {
1316  SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf);
1317  return REAL(__xstat64)(0, path, buf);
1318}
1319
1320TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) {
1321  SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf);
1322  return REAL(__lxstat)(version, path, buf);
1323}
1324
1325TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) {
1326  SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf);
1327  return REAL(__lxstat)(0, path, buf);
1328}
1329
1330TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
1331  SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf);
1332  return REAL(__lxstat64)(version, path, buf);
1333}
1334
1335TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) {
1336  SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf);
1337  return REAL(__lxstat64)(0, path, buf);
1338}
1339
1340TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1341  SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1342  if (fd > 0)
1343    FdAccess(thr, pc, fd);
1344  return REAL(__fxstat)(version, fd, buf);
1345}
1346
1347TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1348  SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1349  if (fd > 0)
1350    FdAccess(thr, pc, fd);
1351  return REAL(__fxstat)(0, fd, buf);
1352}
1353
1354TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1355  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1356  if (fd > 0)
1357    FdAccess(thr, pc, fd);
1358  return REAL(__fxstat64)(version, fd, buf);
1359}
1360
1361TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1362  SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1363  if (fd > 0)
1364    FdAccess(thr, pc, fd);
1365  return REAL(__fxstat64)(0, fd, buf);
1366}
1367
1368TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
1369  SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
1370  int fd = REAL(open)(name, flags, mode);
1371  if (fd >= 0)
1372    FdFileCreate(thr, pc, fd);
1373  return fd;
1374}
1375
1376TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
1377  SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
1378  int fd = REAL(open64)(name, flags, mode);
1379  if (fd >= 0)
1380    FdFileCreate(thr, pc, fd);
1381  return fd;
1382}
1383
1384TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
1385  SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
1386  int fd = REAL(creat)(name, mode);
1387  if (fd >= 0)
1388    FdFileCreate(thr, pc, fd);
1389  return fd;
1390}
1391
1392TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
1393  SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
1394  int fd = REAL(creat64)(name, mode);
1395  if (fd >= 0)
1396    FdFileCreate(thr, pc, fd);
1397  return fd;
1398}
1399
1400TSAN_INTERCEPTOR(int, dup, int oldfd) {
1401  SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
1402  int newfd = REAL(dup)(oldfd);
1403  if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
1404    FdDup(thr, pc, oldfd, newfd);
1405  return newfd;
1406}
1407
1408TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1409  SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
1410  int newfd2 = REAL(dup2)(oldfd, newfd);
1411  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1412    FdDup(thr, pc, oldfd, newfd2);
1413  return newfd2;
1414}
1415
1416TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1417  SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1418  int newfd2 = REAL(dup3)(oldfd, newfd, flags);
1419  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1420    FdDup(thr, pc, oldfd, newfd2);
1421  return newfd2;
1422}
1423
1424TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1425  SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
1426  int fd = REAL(eventfd)(initval, flags);
1427  if (fd >= 0)
1428    FdEventCreate(thr, pc, fd);
1429  return fd;
1430}
1431
1432TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1433  SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1434  if (fd >= 0)
1435    FdClose(thr, pc, fd);
1436  fd = REAL(signalfd)(fd, mask, flags);
1437  if (fd >= 0)
1438    FdSignalCreate(thr, pc, fd);
1439  return fd;
1440}
1441
1442TSAN_INTERCEPTOR(int, inotify_init, int fake) {
1443  SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
1444  int fd = REAL(inotify_init)(fake);
1445  if (fd >= 0)
1446    FdInotifyCreate(thr, pc, fd);
1447  return fd;
1448}
1449
1450TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
1451  SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
1452  int fd = REAL(inotify_init1)(flags);
1453  if (fd >= 0)
1454    FdInotifyCreate(thr, pc, fd);
1455  return fd;
1456}
1457
1458TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
1459  SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
1460  int fd = REAL(socket)(domain, type, protocol);
1461  if (fd >= 0)
1462    FdSocketCreate(thr, pc, fd);
1463  return fd;
1464}
1465
1466TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
1467  SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
1468  int res = REAL(socketpair)(domain, type, protocol, fd);
1469  if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
1470    FdPipeCreate(thr, pc, fd[0], fd[1]);
1471  return res;
1472}
1473
1474TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
1475  SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
1476  FdSocketConnecting(thr, pc, fd);
1477  int res = REAL(connect)(fd, addr, addrlen);
1478  if (res == 0 && fd >= 0)
1479    FdSocketConnect(thr, pc, fd);
1480  return res;
1481}
1482
1483TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
1484  SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
1485  int res = REAL(bind)(fd, addr, addrlen);
1486  if (fd > 0 && res == 0)
1487    FdAccess(thr, pc, fd);
1488  return res;
1489}
1490
1491TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1492  SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
1493  int res = REAL(listen)(fd, backlog);
1494  if (fd > 0 && res == 0)
1495    FdAccess(thr, pc, fd);
1496  return res;
1497}
1498
1499TSAN_INTERCEPTOR(int, epoll_create, int size) {
1500  SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
1501  int fd = REAL(epoll_create)(size);
1502  if (fd >= 0)
1503    FdPollCreate(thr, pc, fd);
1504  return fd;
1505}
1506
1507TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
1508  SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
1509  int fd = REAL(epoll_create1)(flags);
1510  if (fd >= 0)
1511    FdPollCreate(thr, pc, fd);
1512  return fd;
1513}
1514
1515TSAN_INTERCEPTOR(int, close, int fd) {
1516  SCOPED_TSAN_INTERCEPTOR(close, fd);
1517  if (fd >= 0)
1518    FdClose(thr, pc, fd);
1519  return REAL(close)(fd);
1520}
1521
1522TSAN_INTERCEPTOR(int, __close, int fd) {
1523  SCOPED_TSAN_INTERCEPTOR(__close, fd);
1524  if (fd >= 0)
1525    FdClose(thr, pc, fd);
1526  return REAL(__close)(fd);
1527}
1528
1529// glibc guts
1530TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1531  SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
1532  int fds[64];
1533  int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1534  for (int i = 0; i < cnt; i++) {
1535    if (fds[i] > 0)
1536      FdClose(thr, pc, fds[i]);
1537  }
1538  REAL(__res_iclose)(state, free_addr);
1539}
1540
1541TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1542  SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
1543  int res = REAL(pipe)(pipefd);
1544  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1545    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1546  return res;
1547}
1548
1549TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1550  SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1551  int res = REAL(pipe2)(pipefd, flags);
1552  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1553    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1554  return res;
1555}
1556
1557TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) {
1558  SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags);
1559  if (fd >= 0) {
1560    FdAccess(thr, pc, fd);
1561    FdRelease(thr, pc, fd);
1562  }
1563  int res = REAL(send)(fd, buf, len, flags);
1564  return res;
1565}
1566
1567TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) {
1568  SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags);
1569  if (fd >= 0) {
1570    FdAccess(thr, pc, fd);
1571    FdRelease(thr, pc, fd);
1572  }
1573  int res = REAL(sendmsg)(fd, msg, flags);
1574  return res;
1575}
1576
1577TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) {
1578  SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags);
1579  if (fd >= 0)
1580    FdAccess(thr, pc, fd);
1581  int res = REAL(recv)(fd, buf, len, flags);
1582  if (res >= 0 && fd >= 0) {
1583    FdAcquire(thr, pc, fd);
1584  }
1585  return res;
1586}
1587
1588TSAN_INTERCEPTOR(int, unlink, char *path) {
1589  SCOPED_TSAN_INTERCEPTOR(unlink, path);
1590  Release(thr, pc, File2addr(path));
1591  int res = REAL(unlink)(path);
1592  return res;
1593}
1594
1595TSAN_INTERCEPTOR(void*, tmpfile, int fake) {
1596  SCOPED_TSAN_INTERCEPTOR(tmpfile, fake);
1597  void *res = REAL(tmpfile)(fake);
1598  if (res) {
1599    int fd = fileno_unlocked(res);
1600    if (fd >= 0)
1601      FdFileCreate(thr, pc, fd);
1602  }
1603  return res;
1604}
1605
1606TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
1607  SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake);
1608  void *res = REAL(tmpfile64)(fake);
1609  if (res) {
1610    int fd = fileno_unlocked(res);
1611    if (fd >= 0)
1612      FdFileCreate(thr, pc, fd);
1613  }
1614  return res;
1615}
1616
1617TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
1618  // libc file streams can call user-supplied functions, see fopencookie.
1619  {
1620    SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
1621    MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
1622  }
1623  return REAL(fread)(ptr, size, nmemb, f);
1624}
1625
1626TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
1627  // libc file streams can call user-supplied functions, see fopencookie.
1628  {
1629    SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
1630    MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
1631  }
1632  return REAL(fwrite)(p, size, nmemb, f);
1633}
1634
1635TSAN_INTERCEPTOR(void, abort, int fake) {
1636  SCOPED_TSAN_INTERCEPTOR(abort, fake);
1637  REAL(fflush)(0);
1638  REAL(abort)(fake);
1639}
1640
1641TSAN_INTERCEPTOR(int, puts, const char *s) {
1642  SCOPED_TSAN_INTERCEPTOR(puts, s);
1643  MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false);
1644  return REAL(puts)(s);
1645}
1646
1647TSAN_INTERCEPTOR(int, rmdir, char *path) {
1648  SCOPED_TSAN_INTERCEPTOR(rmdir, path);
1649  Release(thr, pc, Dir2addr(path));
1650  int res = REAL(rmdir)(path);
1651  return res;
1652}
1653
1654TSAN_INTERCEPTOR(void*, opendir, char *path) {
1655  SCOPED_TSAN_INTERCEPTOR(opendir, path);
1656  void *res = REAL(opendir)(path);
1657  if (res != 0)
1658    Acquire(thr, pc, Dir2addr(path));
1659  return res;
1660}
1661
1662TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1663  SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
1664  if (epfd >= 0)
1665    FdAccess(thr, pc, epfd);
1666  if (epfd >= 0 && fd >= 0)
1667    FdAccess(thr, pc, fd);
1668  if (op == EPOLL_CTL_ADD && epfd >= 0)
1669    FdRelease(thr, pc, epfd);
1670  int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1671  return res;
1672}
1673
1674TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
1675  SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
1676  if (epfd >= 0)
1677    FdAccess(thr, pc, epfd);
1678  int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
1679  if (res > 0 && epfd >= 0)
1680    FdAcquire(thr, pc, epfd);
1681  return res;
1682}
1683
1684namespace __tsan {
1685
1686static void CallUserSignalHandler(ThreadState *thr, bool sync, bool sigact,
1687    int sig, my_siginfo_t *info, void *uctx) {
1688  // Ensure that the handler does not spoil errno.
1689  const int saved_errno = errno;
1690  errno = 99;
1691  // Need to remember pc before the call, because the handler can reset it.
1692  uptr pc = sigact ?
1693     (uptr)sigactions[sig].sa_sigaction :
1694     (uptr)sigactions[sig].sa_handler;
1695  pc += 1;  // return address is expected, OutputReport() will undo this
1696  if (sigact)
1697    sigactions[sig].sa_sigaction(sig, info, uctx);
1698  else
1699    sigactions[sig].sa_handler(sig);
1700  // We do not detect errno spoiling for SIGTERM,
1701  // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
1702  // tsan reports false positive in such case.
1703  // It's difficult to properly detect this situation (reraise),
1704  // because in async signal processing case (when handler is called directly
1705  // from rtl_generic_sighandler) we have not yet received the reraised
1706  // signal; and it looks too fragile to intercept all ways to reraise a signal.
1707  if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
1708    __tsan::StackTrace stack;
1709    stack.ObtainCurrent(thr, pc);
1710    ThreadRegistryLock l(ctx->thread_registry);
1711    ScopedReport rep(ReportTypeErrnoInSignal);
1712    if (!IsFiredSuppression(ctx, rep, stack)) {
1713      rep.AddStack(&stack, true);
1714      OutputReport(thr, rep);
1715    }
1716  }
1717  errno = saved_errno;
1718}
1719
1720void ProcessPendingSignals(ThreadState *thr) {
1721  SignalContext *sctx = SigCtx(thr);
1722  if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler)
1723    return;
1724  thr->in_signal_handler = true;
1725  sctx->pending_signal_count = 0;
1726  // These are too big for stack.
1727  static THREADLOCAL __sanitizer_sigset_t emptyset, oldset;
1728  REAL(sigfillset)(&emptyset);
1729  pthread_sigmask(SIG_SETMASK, &emptyset, &oldset);
1730  for (int sig = 0; sig < kSigCount; sig++) {
1731    SignalDesc *signal = &sctx->pending_signals[sig];
1732    if (signal->armed) {
1733      signal->armed = false;
1734      if (sigactions[sig].sa_handler != SIG_DFL
1735          && sigactions[sig].sa_handler != SIG_IGN) {
1736        CallUserSignalHandler(thr, false, signal->sigaction,
1737            sig, &signal->siginfo, &signal->ctx);
1738      }
1739    }
1740  }
1741  pthread_sigmask(SIG_SETMASK, &oldset, 0);
1742  CHECK_EQ(thr->in_signal_handler, true);
1743  thr->in_signal_handler = false;
1744}
1745
1746}  // namespace __tsan
1747
1748static bool is_sync_signal(SignalContext *sctx, int sig) {
1749  return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL ||
1750      sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
1751      // If we are sending signal to ourselves, we must process it now.
1752      (sctx && sig == sctx->int_signal_send);
1753}
1754
1755void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
1756    my_siginfo_t *info, void *ctx) {
1757  ThreadState *thr = cur_thread();
1758  SignalContext *sctx = SigCtx(thr);
1759  if (sig < 0 || sig >= kSigCount) {
1760    VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
1761    return;
1762  }
1763  // Don't mess with synchronous signals.
1764  const bool sync = is_sync_signal(sctx, sig);
1765  if (sync ||
1766      // If we are in blocking function, we can safely process it now
1767      // (but check if we are in a recursive interceptor,
1768      // i.e. pthread_join()->munmap()).
1769      (sctx && sctx->in_blocking_func == 1)) {
1770    CHECK_EQ(thr->in_signal_handler, false);
1771    thr->in_signal_handler = true;
1772    if (sctx && sctx->in_blocking_func == 1) {
1773      // We ignore interceptors in blocking functions,
1774      // temporary enbled them again while we are calling user function.
1775      int const i = thr->ignore_interceptors;
1776      thr->ignore_interceptors = 0;
1777      CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
1778      thr->ignore_interceptors = i;
1779    } else {
1780      CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
1781    }
1782    CHECK_EQ(thr->in_signal_handler, true);
1783    thr->in_signal_handler = false;
1784    return;
1785  }
1786
1787  if (sctx == 0)
1788    return;
1789  SignalDesc *signal = &sctx->pending_signals[sig];
1790  if (signal->armed == false) {
1791    signal->armed = true;
1792    signal->sigaction = sigact;
1793    if (info)
1794      internal_memcpy(&signal->siginfo, info, sizeof(*info));
1795    if (ctx)
1796      internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
1797    sctx->pending_signal_count++;
1798  }
1799}
1800
1801static void rtl_sighandler(int sig) {
1802  rtl_generic_sighandler(false, sig, 0, 0);
1803}
1804
1805static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
1806  rtl_generic_sighandler(true, sig, info, ctx);
1807}
1808
1809TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
1810  SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old);
1811  if (old)
1812    internal_memcpy(old, &sigactions[sig], sizeof(*old));
1813  if (act == 0)
1814    return 0;
1815  internal_memcpy(&sigactions[sig], act, sizeof(*act));
1816  sigaction_t newact;
1817  internal_memcpy(&newact, act, sizeof(newact));
1818  REAL(sigfillset)(&newact.sa_mask);
1819  if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
1820    if (newact.sa_flags & SA_SIGINFO)
1821      newact.sa_sigaction = rtl_sigaction;
1822    else
1823      newact.sa_handler = rtl_sighandler;
1824  }
1825  int res = REAL(sigaction)(sig, &newact, 0);
1826  return res;
1827}
1828
1829TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
1830  sigaction_t act;
1831  act.sa_handler = h;
1832  REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask));
1833  act.sa_flags = 0;
1834  sigaction_t old;
1835  int res = sigaction(sig, &act, &old);
1836  if (res)
1837    return SIG_ERR;
1838  return old.sa_handler;
1839}
1840
1841TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
1842  SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
1843  return REAL(sigsuspend)(mask);
1844}
1845
1846TSAN_INTERCEPTOR(int, raise, int sig) {
1847  SCOPED_TSAN_INTERCEPTOR(raise, sig);
1848  SignalContext *sctx = SigCtx(thr);
1849  CHECK_NE(sctx, 0);
1850  int prev = sctx->int_signal_send;
1851  sctx->int_signal_send = sig;
1852  int res = REAL(raise)(sig);
1853  CHECK_EQ(sctx->int_signal_send, sig);
1854  sctx->int_signal_send = prev;
1855  return res;
1856}
1857
1858TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
1859  SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
1860  SignalContext *sctx = SigCtx(thr);
1861  CHECK_NE(sctx, 0);
1862  int prev = sctx->int_signal_send;
1863  if (pid == (int)internal_getpid()) {
1864    sctx->int_signal_send = sig;
1865  }
1866  int res = REAL(kill)(pid, sig);
1867  if (pid == (int)internal_getpid()) {
1868    CHECK_EQ(sctx->int_signal_send, sig);
1869    sctx->int_signal_send = prev;
1870  }
1871  return res;
1872}
1873
1874TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
1875  SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
1876  SignalContext *sctx = SigCtx(thr);
1877  CHECK_NE(sctx, 0);
1878  int prev = sctx->int_signal_send;
1879  if (tid == pthread_self()) {
1880    sctx->int_signal_send = sig;
1881  }
1882  int res = REAL(pthread_kill)(tid, sig);
1883  if (tid == pthread_self()) {
1884    CHECK_EQ(sctx->int_signal_send, sig);
1885    sctx->int_signal_send = prev;
1886  }
1887  return res;
1888}
1889
1890TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
1891  SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
1892  // It's intercepted merely to process pending signals.
1893  return REAL(gettimeofday)(tv, tz);
1894}
1895
1896TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
1897    void *hints, void *rv) {
1898  SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
1899  // We miss atomic synchronization in getaddrinfo,
1900  // and can report false race between malloc and free
1901  // inside of getaddrinfo. So ignore memory accesses.
1902  ThreadIgnoreBegin(thr, pc);
1903  int res = REAL(getaddrinfo)(node, service, hints, rv);
1904  ThreadIgnoreEnd(thr, pc);
1905  return res;
1906}
1907
1908// Linux kernel has a bug that leads to kernel deadlock if a process
1909// maps TBs of memory and then calls mlock().
1910static void MlockIsUnsupported() {
1911  static atomic_uint8_t printed;
1912  if (atomic_exchange(&printed, 1, memory_order_relaxed))
1913    return;
1914  VPrintf(1, "INFO: ThreadSanitizer ignores mlock/munlock[all]\n");
1915}
1916
1917TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) {
1918  MlockIsUnsupported();
1919  return 0;
1920}
1921
1922TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) {
1923  MlockIsUnsupported();
1924  return 0;
1925}
1926
1927TSAN_INTERCEPTOR(int, mlockall, int flags) {
1928  MlockIsUnsupported();
1929  return 0;
1930}
1931
1932TSAN_INTERCEPTOR(int, munlockall, void) {
1933  MlockIsUnsupported();
1934  return 0;
1935}
1936
1937TSAN_INTERCEPTOR(int, fork, int fake) {
1938  if (cur_thread()->in_symbolizer)
1939    return REAL(fork)(fake);
1940  SCOPED_INTERCEPTOR_RAW(fork, fake);
1941  ForkBefore(thr, pc);
1942  int pid = REAL(fork)(fake);
1943  if (pid == 0) {
1944    // child
1945    ForkChildAfter(thr, pc);
1946    FdOnFork(thr, pc);
1947  } else if (pid > 0) {
1948    // parent
1949    ForkParentAfter(thr, pc);
1950  } else {
1951    // error
1952    ForkParentAfter(thr, pc);
1953  }
1954  return pid;
1955}
1956
1957TSAN_INTERCEPTOR(int, vfork, int fake) {
1958  // Some programs (e.g. openjdk) call close for all file descriptors
1959  // in the child process. Under tsan it leads to false positives, because
1960  // address space is shared, so the parent process also thinks that
1961  // the descriptors are closed (while they are actually not).
1962  // This leads to false positives due to missed synchronization.
1963  // Strictly saying this is undefined behavior, because vfork child is not
1964  // allowed to call any functions other than exec/exit. But this is what
1965  // openjdk does, so we want to handle it.
1966  // We could disable interceptors in the child process. But it's not possible
1967  // to simply intercept and wrap vfork, because vfork child is not allowed
1968  // to return from the function that calls vfork, and that's exactly what
1969  // we would do. So this would require some assembly trickery as well.
1970  // Instead we simply turn vfork into fork.
1971  return WRAP(fork)(fake);
1972}
1973
1974static int OnExit(ThreadState *thr) {
1975  int status = Finalize(thr);
1976  REAL(fflush)(0);
1977  return status;
1978}
1979
1980struct TsanInterceptorContext {
1981  ThreadState *thr;
1982  const uptr caller_pc;
1983  const uptr pc;
1984};
1985
1986static void HandleRecvmsg(ThreadState *thr, uptr pc,
1987    __sanitizer_msghdr *msg) {
1988  int fds[64];
1989  int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
1990  for (int i = 0; i < cnt; i++)
1991    FdEventCreate(thr, pc, fds[i]);
1992}
1993
1994#include "sanitizer_common/sanitizer_platform_interceptors.h"
1995// Causes interceptor recursion (getaddrinfo() and fopen())
1996#undef SANITIZER_INTERCEPT_GETADDRINFO
1997
1998#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
1999
2000#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
2001  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
2002                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2003                    true)
2004
2005#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
2006  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
2007                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2008                    false)
2009
2010#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)      \
2011  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);         \
2012  TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2013  ctx = (void *)&_ctx;                                \
2014  (void) ctx;
2015
2016#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2017  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
2018  TsanInterceptorContext _ctx = {thr, caller_pc, pc};     \
2019  ctx = (void *)&_ctx;                                    \
2020  (void) ctx;
2021
2022#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2023  Acquire(thr, pc, File2addr(path));                  \
2024  if (file) {                                         \
2025    int fd = fileno_unlocked(file);                   \
2026    if (fd >= 0) FdFileCreate(thr, pc, fd);           \
2027  }
2028
2029#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2030  if (file) {                                    \
2031    int fd = fileno_unlocked(file);              \
2032    if (fd >= 0) FdClose(thr, pc, fd);           \
2033  }
2034
2035#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res)  \
2036  libignore()->OnLibraryLoaded(filename)
2037
2038#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2039  libignore()->OnLibraryUnloaded()
2040
2041#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
2042  FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2043
2044#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
2045  FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2046
2047#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
2048  FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2049
2050#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
2051  FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
2052
2053#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2054  ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2055
2056#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2057  __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
2058
2059#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2060
2061#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2062  OnExit(((TsanInterceptorContext *) ctx)->thr)
2063
2064#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
2065  MutexLock(((TsanInterceptorContext *)ctx)->thr, \
2066            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2067
2068#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2069  MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2070            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2071
2072#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2073  MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2074            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2075
2076#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2077  HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2078      ((TsanInterceptorContext *)ctx)->pc, msg)
2079
2080#include "sanitizer_common/sanitizer_common_interceptors.inc"
2081
2082#define TSAN_SYSCALL() \
2083  ThreadState *thr = cur_thread(); \
2084  if (thr->ignore_interceptors) \
2085    return; \
2086  ScopedSyscall scoped_syscall(thr) \
2087/**/
2088
2089struct ScopedSyscall {
2090  ThreadState *thr;
2091
2092  explicit ScopedSyscall(ThreadState *thr)
2093      : thr(thr) {
2094    Initialize(thr);
2095  }
2096
2097  ~ScopedSyscall() {
2098    ProcessPendingSignals(thr);
2099  }
2100};
2101
2102static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2103  TSAN_SYSCALL();
2104  MemoryAccessRange(thr, pc, p, s, write);
2105}
2106
2107static void syscall_acquire(uptr pc, uptr addr) {
2108  TSAN_SYSCALL();
2109  Acquire(thr, pc, addr);
2110  DPrintf("syscall_acquire(%p)\n", addr);
2111}
2112
2113static void syscall_release(uptr pc, uptr addr) {
2114  TSAN_SYSCALL();
2115  DPrintf("syscall_release(%p)\n", addr);
2116  Release(thr, pc, addr);
2117}
2118
2119static void syscall_fd_close(uptr pc, int fd) {
2120  TSAN_SYSCALL();
2121  FdClose(thr, pc, fd);
2122}
2123
2124static USED void syscall_fd_acquire(uptr pc, int fd) {
2125  TSAN_SYSCALL();
2126  FdAcquire(thr, pc, fd);
2127  DPrintf("syscall_fd_acquire(%p)\n", fd);
2128}
2129
2130static USED void syscall_fd_release(uptr pc, int fd) {
2131  TSAN_SYSCALL();
2132  DPrintf("syscall_fd_release(%p)\n", fd);
2133  FdRelease(thr, pc, fd);
2134}
2135
2136static void syscall_pre_fork(uptr pc) {
2137  TSAN_SYSCALL();
2138  ForkBefore(thr, pc);
2139}
2140
2141static void syscall_post_fork(uptr pc, int pid) {
2142  TSAN_SYSCALL();
2143  if (pid == 0) {
2144    // child
2145    ForkChildAfter(thr, pc);
2146    FdOnFork(thr, pc);
2147  } else if (pid > 0) {
2148    // parent
2149    ForkParentAfter(thr, pc);
2150  } else {
2151    // error
2152    ForkParentAfter(thr, pc);
2153  }
2154}
2155
2156#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2157  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2158
2159#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2160  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2161
2162#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2163  do {                                       \
2164    (void)(p);                               \
2165    (void)(s);                               \
2166  } while (false)
2167
2168#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2169  do {                                        \
2170    (void)(p);                                \
2171    (void)(s);                                \
2172  } while (false)
2173
2174#define COMMON_SYSCALL_ACQUIRE(addr) \
2175    syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2176
2177#define COMMON_SYSCALL_RELEASE(addr) \
2178    syscall_release(GET_CALLER_PC(), (uptr)(addr))
2179
2180#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2181
2182#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2183
2184#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2185
2186#define COMMON_SYSCALL_PRE_FORK() \
2187  syscall_pre_fork(GET_CALLER_PC())
2188
2189#define COMMON_SYSCALL_POST_FORK(res) \
2190  syscall_post_fork(GET_CALLER_PC(), res)
2191
2192#include "sanitizer_common/sanitizer_common_syscalls.inc"
2193
2194namespace __tsan {
2195
2196static void finalize(void *arg) {
2197  ThreadState *thr = cur_thread();
2198  uptr pc = 0;
2199  atexit_ctx->exit(thr, pc);
2200  int status = Finalize(thr);
2201  // Make sure the output is not lost.
2202  // Flushing all the streams here may freeze the process if a child thread is
2203  // performing file stream operations at the same time.
2204  REAL(fflush)(stdout);
2205  REAL(fflush)(stderr);
2206  if (status)
2207    REAL(_exit)(status);
2208}
2209
2210static void unreachable() {
2211  Report("FATAL: ThreadSanitizer: unreachable called\n");
2212  Die();
2213}
2214
2215void InitializeInterceptors() {
2216  // We need to setup it early, because functions like dlsym() can call it.
2217  REAL(memset) = internal_memset;
2218  REAL(memcpy) = internal_memcpy;
2219  REAL(memcmp) = internal_memcmp;
2220
2221  // Instruct libc malloc to consume less memory.
2222  mallopt(1, 0);  // M_MXFAST
2223  mallopt(-3, 32*1024);  // M_MMAP_THRESHOLD
2224
2225  InitializeCommonInterceptors();
2226
2227  // We can not use TSAN_INTERCEPT to get setjmp addr,
2228  // because it does &setjmp and setjmp is not present in some versions of libc.
2229  using __interception::GetRealFunctionAddress;
2230  GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0);
2231  GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
2232  GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0);
2233  GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
2234
2235  TSAN_INTERCEPT(longjmp);
2236  TSAN_INTERCEPT(siglongjmp);
2237
2238  TSAN_INTERCEPT(malloc);
2239  TSAN_INTERCEPT(__libc_memalign);
2240  TSAN_INTERCEPT(calloc);
2241  TSAN_INTERCEPT(realloc);
2242  TSAN_INTERCEPT(free);
2243  TSAN_INTERCEPT(cfree);
2244  TSAN_INTERCEPT(mmap);
2245  TSAN_INTERCEPT(mmap64);
2246  TSAN_INTERCEPT(munmap);
2247  TSAN_INTERCEPT(memalign);
2248  TSAN_INTERCEPT(valloc);
2249  TSAN_INTERCEPT(pvalloc);
2250  TSAN_INTERCEPT(posix_memalign);
2251
2252  TSAN_INTERCEPT(strlen);
2253  TSAN_INTERCEPT(memset);
2254  TSAN_INTERCEPT(memcpy);
2255  TSAN_INTERCEPT(memmove);
2256  TSAN_INTERCEPT(memcmp);
2257  TSAN_INTERCEPT(strchr);
2258  TSAN_INTERCEPT(strchrnul);
2259  TSAN_INTERCEPT(strrchr);
2260  TSAN_INTERCEPT(strcpy);  // NOLINT
2261  TSAN_INTERCEPT(strncpy);
2262  TSAN_INTERCEPT(strstr);
2263  TSAN_INTERCEPT(strdup);
2264
2265  TSAN_INTERCEPT(pthread_create);
2266  TSAN_INTERCEPT(pthread_join);
2267  TSAN_INTERCEPT(pthread_detach);
2268
2269  TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2");
2270  TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2");
2271  TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
2272  TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2");
2273  TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
2274  TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2");
2275
2276  TSAN_INTERCEPT(pthread_mutex_init);
2277  TSAN_INTERCEPT(pthread_mutex_destroy);
2278  TSAN_INTERCEPT(pthread_mutex_trylock);
2279  TSAN_INTERCEPT(pthread_mutex_timedlock);
2280
2281  TSAN_INTERCEPT(pthread_spin_init);
2282  TSAN_INTERCEPT(pthread_spin_destroy);
2283  TSAN_INTERCEPT(pthread_spin_lock);
2284  TSAN_INTERCEPT(pthread_spin_trylock);
2285  TSAN_INTERCEPT(pthread_spin_unlock);
2286
2287  TSAN_INTERCEPT(pthread_rwlock_init);
2288  TSAN_INTERCEPT(pthread_rwlock_destroy);
2289  TSAN_INTERCEPT(pthread_rwlock_rdlock);
2290  TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
2291  TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
2292  TSAN_INTERCEPT(pthread_rwlock_wrlock);
2293  TSAN_INTERCEPT(pthread_rwlock_trywrlock);
2294  TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
2295  TSAN_INTERCEPT(pthread_rwlock_unlock);
2296
2297  TSAN_INTERCEPT(pthread_barrier_init);
2298  TSAN_INTERCEPT(pthread_barrier_destroy);
2299  TSAN_INTERCEPT(pthread_barrier_wait);
2300
2301  TSAN_INTERCEPT(pthread_once);
2302
2303  TSAN_INTERCEPT(sem_init);
2304  TSAN_INTERCEPT(sem_destroy);
2305  TSAN_INTERCEPT(sem_wait);
2306  TSAN_INTERCEPT(sem_trywait);
2307  TSAN_INTERCEPT(sem_timedwait);
2308  TSAN_INTERCEPT(sem_post);
2309  TSAN_INTERCEPT(sem_getvalue);
2310
2311  TSAN_INTERCEPT(stat);
2312  TSAN_INTERCEPT(__xstat);
2313  TSAN_INTERCEPT(stat64);
2314  TSAN_INTERCEPT(__xstat64);
2315  TSAN_INTERCEPT(lstat);
2316  TSAN_INTERCEPT(__lxstat);
2317  TSAN_INTERCEPT(lstat64);
2318  TSAN_INTERCEPT(__lxstat64);
2319  TSAN_INTERCEPT(fstat);
2320  TSAN_INTERCEPT(__fxstat);
2321  TSAN_INTERCEPT(fstat64);
2322  TSAN_INTERCEPT(__fxstat64);
2323  TSAN_INTERCEPT(open);
2324  TSAN_INTERCEPT(open64);
2325  TSAN_INTERCEPT(creat);
2326  TSAN_INTERCEPT(creat64);
2327  TSAN_INTERCEPT(dup);
2328  TSAN_INTERCEPT(dup2);
2329  TSAN_INTERCEPT(dup3);
2330  TSAN_INTERCEPT(eventfd);
2331  TSAN_INTERCEPT(signalfd);
2332  TSAN_INTERCEPT(inotify_init);
2333  TSAN_INTERCEPT(inotify_init1);
2334  TSAN_INTERCEPT(socket);
2335  TSAN_INTERCEPT(socketpair);
2336  TSAN_INTERCEPT(connect);
2337  TSAN_INTERCEPT(bind);
2338  TSAN_INTERCEPT(listen);
2339  TSAN_INTERCEPT(epoll_create);
2340  TSAN_INTERCEPT(epoll_create1);
2341  TSAN_INTERCEPT(close);
2342  TSAN_INTERCEPT(__close);
2343  TSAN_INTERCEPT(__res_iclose);
2344  TSAN_INTERCEPT(pipe);
2345  TSAN_INTERCEPT(pipe2);
2346
2347  TSAN_INTERCEPT(send);
2348  TSAN_INTERCEPT(sendmsg);
2349  TSAN_INTERCEPT(recv);
2350
2351  TSAN_INTERCEPT(unlink);
2352  TSAN_INTERCEPT(tmpfile);
2353  TSAN_INTERCEPT(tmpfile64);
2354  TSAN_INTERCEPT(fread);
2355  TSAN_INTERCEPT(fwrite);
2356  TSAN_INTERCEPT(abort);
2357  TSAN_INTERCEPT(puts);
2358  TSAN_INTERCEPT(rmdir);
2359  TSAN_INTERCEPT(opendir);
2360
2361  TSAN_INTERCEPT(epoll_ctl);
2362  TSAN_INTERCEPT(epoll_wait);
2363
2364  TSAN_INTERCEPT(sigaction);
2365  TSAN_INTERCEPT(signal);
2366  TSAN_INTERCEPT(sigsuspend);
2367  TSAN_INTERCEPT(raise);
2368  TSAN_INTERCEPT(kill);
2369  TSAN_INTERCEPT(pthread_kill);
2370  TSAN_INTERCEPT(sleep);
2371  TSAN_INTERCEPT(usleep);
2372  TSAN_INTERCEPT(nanosleep);
2373  TSAN_INTERCEPT(gettimeofday);
2374  TSAN_INTERCEPT(getaddrinfo);
2375
2376  TSAN_INTERCEPT(mlock);
2377  TSAN_INTERCEPT(munlock);
2378  TSAN_INTERCEPT(mlockall);
2379  TSAN_INTERCEPT(munlockall);
2380
2381  TSAN_INTERCEPT(fork);
2382  TSAN_INTERCEPT(vfork);
2383  TSAN_INTERCEPT(on_exit);
2384  TSAN_INTERCEPT(__cxa_atexit);
2385  TSAN_INTERCEPT(_exit);
2386
2387  // Need to setup it, because interceptors check that the function is resolved.
2388  // But atexit is emitted directly into the module, so can't be resolved.
2389  REAL(atexit) = (int(*)(void(*)()))unreachable;
2390  atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext)))
2391      AtExitContext();
2392
2393  if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
2394    Printf("ThreadSanitizer: failed to setup atexit callback\n");
2395    Die();
2396  }
2397
2398  if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
2399    Printf("ThreadSanitizer: failed to create thread key\n");
2400    Die();
2401  }
2402
2403  FdInit();
2404}
2405
2406void *internal_start_thread(void(*func)(void *arg), void *arg) {
2407  // Start the thread with signals blocked, otherwise it can steal user signals.
2408  __sanitizer_sigset_t set, old;
2409  internal_sigfillset(&set);
2410  internal_sigprocmask(SIG_SETMASK, &set, &old);
2411  void *th;
2412  REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
2413  internal_sigprocmask(SIG_SETMASK, &old, 0);
2414  return th;
2415}
2416
2417void internal_join_thread(void *th) {
2418  REAL(pthread_join)(th, 0);
2419}
2420
2421}  // namespace __tsan
2422