tsan_rtl.cc revision 4af0f21c0c98950df1136dbec8824a029ed5bb8e 
1//===-- tsan_rtl.cc -------------------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12// Main file (entry points) for the TSan run-time.
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_atomic.h"
16#include "sanitizer_common/sanitizer_common.h"
17#include "sanitizer_common/sanitizer_libc.h"
18#include "sanitizer_common/sanitizer_stackdepot.h"
19#include "sanitizer_common/sanitizer_placement_new.h"
20#include "sanitizer_common/sanitizer_symbolizer.h"
21#include "tsan_defs.h"
22#include "tsan_platform.h"
23#include "tsan_rtl.h"
24#include "tsan_mman.h"
25#include "tsan_suppressions.h"
26#include "tsan_symbolize.h"
27
28volatile int __tsan_resumed = 0;
29
30extern "C" void __tsan_resume() {
31  __tsan_resumed = 1;
32}
33
34namespace __tsan {
35
36#ifndef TSAN_GO
37THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
38#endif
39static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
40
41// Can be overriden by a front-end.
42bool CPP_WEAK OnFinalize(bool failed) {
43  return failed;
44}
45
46static Context *ctx;
47Context *CTX() {
48  return ctx;
49}
50
51static char thread_registry_placeholder[sizeof(ThreadRegistry)];
52
53static ThreadContextBase *CreateThreadContext(u32 tid) {
54  // Map thread trace when context is created.
55  MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event));
56  MapThreadTrace(GetThreadTraceHeader(tid), sizeof(Trace));
57  new(ThreadTrace(tid)) Trace();
58  void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
59  return new(mem) ThreadContext(tid);
60}
61
62#ifndef TSAN_GO
63static const u32 kThreadQuarantineSize = 16;
64#else
65static const u32 kThreadQuarantineSize = 64;
66#endif
67
68Context::Context()
69  : initialized()
70  , report_mtx(MutexTypeReport, StatMtxReport)
71  , nreported()
72  , nmissed_expected()
73  , thread_registry(new(thread_registry_placeholder) ThreadRegistry(
74      CreateThreadContext, kMaxTid, kThreadQuarantineSize))
75  , racy_stacks(MBlockRacyStacks)
76  , racy_addresses(MBlockRacyAddresses)
77  , fired_suppressions(8) {
78}
79
80// The objects are allocated in TLS, so one may rely on zero-initialization.
81ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
82                         uptr stk_addr, uptr stk_size,
83                         uptr tls_addr, uptr tls_size)
84  : fast_state(tid, epoch)
85  // Do not touch these, rely on zero initialization,
86  // they may be accessed before the ctor.
87  // , ignore_reads_and_writes()
88  // , in_rtl()
89  , shadow_stack_pos(&shadow_stack[0])
90#ifndef TSAN_GO
91  , jmp_bufs(MBlockJmpBuf)
92#endif
93  , tid(tid)
94  , unique_id(unique_id)
95  , stk_addr(stk_addr)
96  , stk_size(stk_size)
97  , tls_addr(tls_addr)
98  , tls_size(tls_size) {
99}
100
101static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
102  uptr n_threads;
103  uptr n_running_threads;
104  ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
105  InternalScopedBuffer<char> buf(4096);
106  internal_snprintf(buf.data(), buf.size(), "%d: nthr=%d nlive=%d\n",
107      i, n_threads, n_running_threads);
108  internal_write(fd, buf.data(), internal_strlen(buf.data()));
109  WriteMemoryProfile(buf.data(), buf.size());
110  internal_write(fd, buf.data(), internal_strlen(buf.data()));
111}
112
113static void BackgroundThread(void *arg) {
114  ScopedInRtl in_rtl;
115  Context *ctx = CTX();
116  const u64 kMs2Ns = 1000 * 1000;
117
118  fd_t mprof_fd = kInvalidFd;
119  if (flags()->profile_memory && flags()->profile_memory[0]) {
120    InternalScopedBuffer<char> filename(4096);
121    internal_snprintf(filename.data(), filename.size(), "%s.%d",
122        flags()->profile_memory, (int)internal_getpid());
123    uptr openrv = OpenFile(filename.data(), true);
124    if (internal_iserror(openrv)) {
125      Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
126          &filename[0]);
127    } else {
128      mprof_fd = openrv;
129    }
130  }
131
132  u64 last_flush = NanoTime();
133  for (int i = 0; ; i++) {
134    SleepForSeconds(1);
135    u64 now = NanoTime();
136
137    // Flush memory if requested.
138    if (flags()->flush_memory_ms) {
139      if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
140        FlushShadowMemory();
141        last_flush = NanoTime();
142      }
143    }
144
145    // Write memory profile if requested.
146    if (mprof_fd != kInvalidFd)
147      MemoryProfiler(ctx, mprof_fd, i);
148
149#ifndef TSAN_GO
150    // Flush symbolizer cache if requested.
151    if (flags()->flush_symbolizer_ms > 0) {
152      u64 last = atomic_load(&ctx->last_symbolize_time_ns,
153                             memory_order_relaxed);
154      if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
155        Lock l(&ctx->report_mtx);
156        SpinMutexLock l2(&CommonSanitizerReportMutex);
157        SymbolizeFlush();
158        atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
159      }
160    }
161#endif
162  }
163}
164
165void DontNeedShadowFor(uptr addr, uptr size) {
166  uptr shadow_beg = MemToShadow(addr);
167  uptr shadow_end = MemToShadow(addr + size);
168  FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg);
169}
170
171void MapShadow(uptr addr, uptr size) {
172  MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier);
173}
174
175void MapThreadTrace(uptr addr, uptr size) {
176  DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
177  CHECK_GE(addr, kTraceMemBegin);
178  CHECK_LE(addr + size, kTraceMemBegin + kTraceMemSize);
179  if (addr != (uptr)MmapFixedNoReserve(addr, size)) {
180    Printf("FATAL: ThreadSanitizer can not mmap thread trace\n");
181    Die();
182  }
183}
184
185void Initialize(ThreadState *thr) {
186  // Thread safe because done before all threads exist.
187  static bool is_initialized = false;
188  if (is_initialized)
189    return;
190  is_initialized = true;
191  SanitizerToolName = "ThreadSanitizer";
192  // Install tool-specific callbacks in sanitizer_common.
193  SetCheckFailedCallback(TsanCheckFailed);
194
195  ScopedInRtl in_rtl;
196#ifndef TSAN_GO
197  InitializeAllocator();
198#endif
199  InitializeInterceptors();
200  const char *env = InitializePlatform();
201  InitializeMutex();
202  InitializeDynamicAnnotations();
203  ctx = new(ctx_placeholder) Context;
204#ifndef TSAN_GO
205  InitializeShadowMemory();
206#endif
207  InitializeFlags(&ctx->flags, env);
208  // Setup correct file descriptor for error reports.
209  if (internal_strcmp(flags()->log_path, "stdout") == 0)
210    __sanitizer_set_report_fd(kStdoutFd);
211  else if (internal_strcmp(flags()->log_path, "stderr") == 0)
212    __sanitizer_set_report_fd(kStderrFd);
213  else
214    __sanitizer_set_report_path(flags()->log_path);
215  InitializeSuppressions();
216#ifndef TSAN_GO
217  InitializeLibIgnore();
218  // Initialize external symbolizer before internal threads are started.
219  const char *external_symbolizer = flags()->external_symbolizer_path;
220  if (external_symbolizer != 0 && external_symbolizer[0] != '\0') {
221    if (!getSymbolizer()->InitializeExternal(external_symbolizer)) {
222      Printf("Failed to start external symbolizer: '%s'\n",
223             external_symbolizer);
224      Die();
225    }
226  }
227#endif
228  internal_start_thread(&BackgroundThread, 0);
229
230  if (ctx->flags.verbosity)
231    Printf("***** Running under ThreadSanitizer v2 (pid %d) *****\n",
232           (int)internal_getpid());
233
234  // Initialize thread 0.
235  int tid = ThreadCreate(thr, 0, 0, true);
236  CHECK_EQ(tid, 0);
237  ThreadStart(thr, tid, internal_getpid());
238  CHECK_EQ(thr->in_rtl, 1);
239  ctx->initialized = true;
240
241  if (flags()->stop_on_start) {
242    Printf("ThreadSanitizer is suspended at startup (pid %d)."
243           " Call __tsan_resume().\n",
244           (int)internal_getpid());
245    while (__tsan_resumed == 0) {}
246  }
247}
248
249int Finalize(ThreadState *thr) {
250  ScopedInRtl in_rtl;
251  Context *ctx = __tsan::ctx;
252  bool failed = false;
253
254  if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
255    SleepForMillis(flags()->atexit_sleep_ms);
256
257  // Wait for pending reports.
258  ctx->report_mtx.Lock();
259  CommonSanitizerReportMutex.Lock();
260  CommonSanitizerReportMutex.Unlock();
261  ctx->report_mtx.Unlock();
262
263#ifndef TSAN_GO
264  if (ctx->flags.verbosity)
265    AllocatorPrintStats();
266#endif
267
268  ThreadFinalize(thr);
269
270  if (ctx->nreported) {
271    failed = true;
272#ifndef TSAN_GO
273    Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
274#else
275    Printf("Found %d data race(s)\n", ctx->nreported);
276#endif
277  }
278
279  if (ctx->nmissed_expected) {
280    failed = true;
281    Printf("ThreadSanitizer: missed %d expected races\n",
282        ctx->nmissed_expected);
283  }
284
285  if (flags()->print_suppressions)
286    PrintMatchedSuppressions();
287#ifndef TSAN_GO
288  if (flags()->print_benign)
289    PrintMatchedBenignRaces();
290#endif
291
292  failed = OnFinalize(failed);
293
294  StatAggregate(ctx->stat, thr->stat);
295  StatOutput(ctx->stat);
296  return failed ? flags()->exitcode : 0;
297}
298
299#ifndef TSAN_GO
300u32 CurrentStackId(ThreadState *thr, uptr pc) {
301  if (thr->shadow_stack_pos == 0)  // May happen during bootstrap.
302    return 0;
303  if (pc) {
304    thr->shadow_stack_pos[0] = pc;
305    thr->shadow_stack_pos++;
306  }
307  u32 id = StackDepotPut(thr->shadow_stack,
308                         thr->shadow_stack_pos - thr->shadow_stack);
309  if (pc)
310    thr->shadow_stack_pos--;
311  return id;
312}
313#endif
314
315void TraceSwitch(ThreadState *thr) {
316  thr->nomalloc++;
317  ScopedInRtl in_rtl;
318  Trace *thr_trace = ThreadTrace(thr->tid);
319  Lock l(&thr_trace->mtx);
320  unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
321  TraceHeader *hdr = &thr_trace->headers[trace];
322  hdr->epoch0 = thr->fast_state.epoch();
323  hdr->stack0.ObtainCurrent(thr, 0);
324  hdr->mset0 = thr->mset;
325  thr->nomalloc--;
326}
327
328Trace *ThreadTrace(int tid) {
329  return (Trace*)GetThreadTraceHeader(tid);
330}
331
332uptr TraceTopPC(ThreadState *thr) {
333  Event *events = (Event*)GetThreadTrace(thr->tid);
334  uptr pc = events[thr->fast_state.GetTracePos()];
335  return pc;
336}
337
338uptr TraceSize() {
339  return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
340}
341
342uptr TraceParts() {
343  return TraceSize() / kTracePartSize;
344}
345
346#ifndef TSAN_GO
347extern "C" void __tsan_trace_switch() {
348  TraceSwitch(cur_thread());
349}
350
351extern "C" void __tsan_report_race() {
352  ReportRace(cur_thread());
353}
354#endif
355
356ALWAYS_INLINE
357Shadow LoadShadow(u64 *p) {
358  u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
359  return Shadow(raw);
360}
361
362ALWAYS_INLINE
363void StoreShadow(u64 *sp, u64 s) {
364  atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
365}
366
367ALWAYS_INLINE
368void StoreIfNotYetStored(u64 *sp, u64 *s) {
369  StoreShadow(sp, *s);
370  *s = 0;
371}
372
373static inline void HandleRace(ThreadState *thr, u64 *shadow_mem,
374                              Shadow cur, Shadow old) {
375  thr->racy_state[0] = cur.raw();
376  thr->racy_state[1] = old.raw();
377  thr->racy_shadow_addr = shadow_mem;
378#ifndef TSAN_GO
379  HACKY_CALL(__tsan_report_race);
380#else
381  ReportRace(thr);
382#endif
383}
384
385static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) {
386  return old.epoch() >= thr->fast_synch_epoch;
387}
388
389static inline bool HappensBefore(Shadow old, ThreadState *thr) {
390  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
391}
392
393ALWAYS_INLINE USED
394void MemoryAccessImpl(ThreadState *thr, uptr addr,
395    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
396    u64 *shadow_mem, Shadow cur) {
397  StatInc(thr, StatMop);
398  StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
399  StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
400
401  // This potentially can live in an MMX/SSE scratch register.
402  // The required intrinsics are:
403  // __m128i _mm_move_epi64(__m128i*);
404  // _mm_storel_epi64(u64*, __m128i);
405  u64 store_word = cur.raw();
406
407  // scan all the shadow values and dispatch to 4 categories:
408  // same, replace, candidate and race (see comments below).
409  // we consider only 3 cases regarding access sizes:
410  // equal, intersect and not intersect. initially I considered
411  // larger and smaller as well, it allowed to replace some
412  // 'candidates' with 'same' or 'replace', but I think
413  // it's just not worth it (performance- and complexity-wise).
414
415  Shadow old(0);
416  if (kShadowCnt == 1) {
417    int idx = 0;
418#include "tsan_update_shadow_word_inl.h"
419  } else if (kShadowCnt == 2) {
420    int idx = 0;
421#include "tsan_update_shadow_word_inl.h"
422    idx = 1;
423#include "tsan_update_shadow_word_inl.h"
424  } else if (kShadowCnt == 4) {
425    int idx = 0;
426#include "tsan_update_shadow_word_inl.h"
427    idx = 1;
428#include "tsan_update_shadow_word_inl.h"
429    idx = 2;
430#include "tsan_update_shadow_word_inl.h"
431    idx = 3;
432#include "tsan_update_shadow_word_inl.h"
433  } else if (kShadowCnt == 8) {
434    int idx = 0;
435#include "tsan_update_shadow_word_inl.h"
436    idx = 1;
437#include "tsan_update_shadow_word_inl.h"
438    idx = 2;
439#include "tsan_update_shadow_word_inl.h"
440    idx = 3;
441#include "tsan_update_shadow_word_inl.h"
442    idx = 4;
443#include "tsan_update_shadow_word_inl.h"
444    idx = 5;
445#include "tsan_update_shadow_word_inl.h"
446    idx = 6;
447#include "tsan_update_shadow_word_inl.h"
448    idx = 7;
449#include "tsan_update_shadow_word_inl.h"
450  } else {
451    CHECK(false);
452  }
453
454  // we did not find any races and had already stored
455  // the current access info, so we are done
456  if (LIKELY(store_word == 0))
457    return;
458  // choose a random candidate slot and replace it
459  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
460  StatInc(thr, StatShadowReplace);
461  return;
462 RACE:
463  HandleRace(thr, shadow_mem, cur, old);
464  return;
465}
466
467void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
468    int size, bool kAccessIsWrite, bool kIsAtomic) {
469  while (size) {
470    int size1 = 1;
471    int kAccessSizeLog = kSizeLog1;
472    if (size >= 8 && (addr & ~7) == ((addr + 8) & ~7)) {
473      size1 = 8;
474      kAccessSizeLog = kSizeLog8;
475    } else if (size >= 4 && (addr & ~7) == ((addr + 4) & ~7)) {
476      size1 = 4;
477      kAccessSizeLog = kSizeLog4;
478    } else if (size >= 2 && (addr & ~7) == ((addr + 2) & ~7)) {
479      size1 = 2;
480      kAccessSizeLog = kSizeLog2;
481    }
482    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
483    addr += size1;
484    size -= size1;
485  }
486}
487
488ALWAYS_INLINE USED
489void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
490    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
491  u64 *shadow_mem = (u64*)MemToShadow(addr);
492  DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
493      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
494      (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
495      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
496      (uptr)shadow_mem[0], (uptr)shadow_mem[1],
497      (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
498#if TSAN_DEBUG
499  if (!IsAppMem(addr)) {
500    Printf("Access to non app mem %zx\n", addr);
501    DCHECK(IsAppMem(addr));
502  }
503  if (!IsShadowMem((uptr)shadow_mem)) {
504    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
505    DCHECK(IsShadowMem((uptr)shadow_mem));
506  }
507#endif
508
509  if (*shadow_mem == kShadowRodata) {
510    // Access to .rodata section, no races here.
511    // Measurements show that it can be 10-20% of all memory accesses.
512    StatInc(thr, StatMop);
513    StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
514    StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
515    StatInc(thr, StatMopRodata);
516    return;
517  }
518
519  FastState fast_state = thr->fast_state;
520  if (fast_state.GetIgnoreBit())
521    return;
522  fast_state.IncrementEpoch();
523  thr->fast_state = fast_state;
524  Shadow cur(fast_state);
525  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
526  cur.SetWrite(kAccessIsWrite);
527  cur.SetAtomic(kIsAtomic);
528
529  // We must not store to the trace if we do not store to the shadow.
530  // That is, this call must be moved somewhere below.
531  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
532
533  MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
534      shadow_mem, cur);
535}
536
537static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
538                           u64 val) {
539  (void)thr;
540  (void)pc;
541  if (size == 0)
542    return;
543  // FIXME: fix me.
544  uptr offset = addr % kShadowCell;
545  if (offset) {
546    offset = kShadowCell - offset;
547    if (size <= offset)
548      return;
549    addr += offset;
550    size -= offset;
551  }
552  DCHECK_EQ(addr % 8, 0);
553  // If a user passes some insane arguments (memset(0)),
554  // let it just crash as usual.
555  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
556    return;
557  // Don't want to touch lots of shadow memory.
558  // If a program maps 10MB stack, there is no need reset the whole range.
559  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
560  // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
561  // so we do it only for C/C++.
562  if (kGoMode || size < 64*1024) {
563    u64 *p = (u64*)MemToShadow(addr);
564    CHECK(IsShadowMem((uptr)p));
565    CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
566    // FIXME: may overwrite a part outside the region
567    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
568      p[i++] = val;
569      for (uptr j = 1; j < kShadowCnt; j++)
570        p[i++] = 0;
571    }
572  } else {
573    // The region is big, reset only beginning and end.
574    const uptr kPageSize = 4096;
575    u64 *begin = (u64*)MemToShadow(addr);
576    u64 *end = begin + size / kShadowCell * kShadowCnt;
577    u64 *p = begin;
578    // Set at least first kPageSize/2 to page boundary.
579    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
580      *p++ = val;
581      for (uptr j = 1; j < kShadowCnt; j++)
582        *p++ = 0;
583    }
584    // Reset middle part.
585    u64 *p1 = p;
586    p = RoundDown(end, kPageSize);
587    UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
588    MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1);
589    // Set the ending.
590    while (p < end) {
591      *p++ = val;
592      for (uptr j = 1; j < kShadowCnt; j++)
593        *p++ = 0;
594    }
595  }
596}
597
598void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
599  MemoryRangeSet(thr, pc, addr, size, 0);
600}
601
602void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
603  // Processing more than 1k (4k of shadow) is expensive,
604  // can cause excessive memory consumption (user does not necessary touch
605  // the whole range) and most likely unnecessary.
606  if (size > 1024)
607    size = 1024;
608  CHECK_EQ(thr->is_freeing, false);
609  thr->is_freeing = true;
610  MemoryAccessRange(thr, pc, addr, size, true);
611  thr->is_freeing = false;
612  thr->fast_state.IncrementEpoch();
613  TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
614  Shadow s(thr->fast_state);
615  s.ClearIgnoreBit();
616  s.MarkAsFreed();
617  s.SetWrite(true);
618  s.SetAddr0AndSizeLog(0, 3);
619  MemoryRangeSet(thr, pc, addr, size, s.raw());
620}
621
622void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
623  thr->fast_state.IncrementEpoch();
624  TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
625  Shadow s(thr->fast_state);
626  s.ClearIgnoreBit();
627  s.SetWrite(true);
628  s.SetAddr0AndSizeLog(0, 3);
629  MemoryRangeSet(thr, pc, addr, size, s.raw());
630}
631
632ALWAYS_INLINE USED
633void FuncEntry(ThreadState *thr, uptr pc) {
634  DCHECK_EQ(thr->in_rtl, 0);
635  StatInc(thr, StatFuncEnter);
636  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
637  thr->fast_state.IncrementEpoch();
638  TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
639
640  // Shadow stack maintenance can be replaced with
641  // stack unwinding during trace switch (which presumably must be faster).
642  DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]);
643#ifndef TSAN_GO
644  DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
645#else
646  if (thr->shadow_stack_pos == thr->shadow_stack_end) {
647    const int sz = thr->shadow_stack_end - thr->shadow_stack;
648    const int newsz = 2 * sz;
649    uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack,
650        newsz * sizeof(uptr));
651    internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
652    internal_free(thr->shadow_stack);
653    thr->shadow_stack = newstack;
654    thr->shadow_stack_pos = newstack + sz;
655    thr->shadow_stack_end = newstack + newsz;
656  }
657#endif
658  thr->shadow_stack_pos[0] = pc;
659  thr->shadow_stack_pos++;
660}
661
662ALWAYS_INLINE USED
663void FuncExit(ThreadState *thr) {
664  DCHECK_EQ(thr->in_rtl, 0);
665  StatInc(thr, StatFuncExit);
666  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
667  thr->fast_state.IncrementEpoch();
668  TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
669
670  DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]);
671#ifndef TSAN_GO
672  DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
673#endif
674  thr->shadow_stack_pos--;
675}
676
677void ThreadIgnoreBegin(ThreadState *thr) {
678  DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
679  thr->ignore_reads_and_writes++;
680  CHECK_GE(thr->ignore_reads_and_writes, 0);
681  thr->fast_state.SetIgnoreBit();
682}
683
684void ThreadIgnoreEnd(ThreadState *thr) {
685  DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
686  thr->ignore_reads_and_writes--;
687  CHECK_GE(thr->ignore_reads_and_writes, 0);
688  if (thr->ignore_reads_and_writes == 0)
689    thr->fast_state.ClearIgnoreBit();
690}
691
692bool MD5Hash::operator==(const MD5Hash &other) const {
693  return hash[0] == other.hash[0] && hash[1] == other.hash[1];
694}
695
696#if TSAN_DEBUG
697void build_consistency_debug() {}
698#else
699void build_consistency_release() {}
700#endif
701
702#if TSAN_COLLECT_STATS
703void build_consistency_stats() {}
704#else
705void build_consistency_nostats() {}
706#endif
707
708#if TSAN_SHADOW_COUNT == 1
709void build_consistency_shadow1() {}
710#elif TSAN_SHADOW_COUNT == 2
711void build_consistency_shadow2() {}
712#elif TSAN_SHADOW_COUNT == 4
713void build_consistency_shadow4() {}
714#else
715void build_consistency_shadow8() {}
716#endif
717
718}  // namespace __tsan
719
720#ifndef TSAN_GO
721// Must be included in this file to make sure everything is inlined.
722#include "tsan_interface_inl.h"
723#endif
724