1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_rtl.cc -------------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Main file (entry points) for the TSan run-time. 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 15fce5bd4cc29fddb5e8f0cb9c12df7c10187a991dDmitry Vyukov#include "sanitizer_common/sanitizer_atomic.h" 160969bcf2c936126b1f6e636b978aade8fc207437Alexey Samsonov#include "sanitizer_common/sanitizer_common.h" 1716e0075746b21ed866ec3be21ef0d1e46f0efed5Kostya Serebryany#include "sanitizer_common/sanitizer_libc.h" 18848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov#include "sanitizer_common/sanitizer_stackdepot.h" 1947b1634df012507799eb39aa17d4022d748ba67bAlexey Samsonov#include "sanitizer_common/sanitizer_placement_new.h" 208cc1f81b2cc1fa0d4cda4f4635d955aed04c09c8Alexey Samsonov#include "sanitizer_common/sanitizer_symbolizer.h" 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_defs.h" 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_platform.h" 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mman.h" 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_suppressions.h" 26a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov#include "tsan_symbolize.h" 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 285d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#ifdef __SSE3__ 295d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines// <emmintrin.h> transitively includes <stdlib.h>, 305d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines// and it's prohibited to include std headers into tsan runtime. 315d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines// So we do this dirty trick. 325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#define _MM_MALLOC_H_INCLUDED 335d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#define __MM_MALLOC_H 345d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#include <emmintrin.h> 355d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinestypedef __m128i m128; 365d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#endif 375d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 38adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukovvolatile int __tsan_resumed = 0; 397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_resume() { 41adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov __tsan_resumed = 1; 427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 46b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 470a4c906dbc8f150657ddd4f19a7192b779f1d605Alexey SamsonovTHREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); 48b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 490a4c906dbc8f150657ddd4f19a7192b779f1d605Alexey Samsonovstatic char ctx_placeholder[sizeof(Context)] ALIGNED(64); 502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesContext *ctx; 517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5222881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov// Can be overriden by a front-end. 536a135be19fa0cc594fd61f9caf3c0be2b7f1466eDmitry Vyukov#ifdef TSAN_EXTERNAL_HOOKS 546a135be19fa0cc594fd61f9caf3c0be2b7f1466eDmitry Vyukovbool OnFinalize(bool failed); 552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid OnInitialize(); 566a135be19fa0cc594fd61f9caf3c0be2b7f1466eDmitry Vyukov#else 572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 586a135be19fa0cc594fd61f9caf3c0be2b7f1466eDmitry Vyukovbool WEAK OnFinalize(bool failed) { 5922881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov return failed; 6022881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov} 612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid WEAK OnInitialize() {} 636a135be19fa0cc594fd61f9caf3c0be2b7f1466eDmitry Vyukov#endif 6422881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov 652bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic char thread_registry_placeholder[sizeof(ThreadRegistry)]; 662bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 672bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic ThreadContextBase *CreateThreadContext(u32 tid) { 682bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov // Map thread trace when context is created. 692bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event)); 709743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov MapThreadTrace(GetThreadTraceHeader(tid), sizeof(Trace)); 719743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov new(ThreadTrace(tid)) Trace(); 729743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); 732bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov return new(mem) ThreadContext(tid); 742bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 752bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 762bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#ifndef TSAN_GO 772bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic const u32 kThreadQuarantineSize = 16; 782bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#else 792bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic const u32 kThreadQuarantineSize = 64; 802bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#endif 812bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 827ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyContext::Context() 837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : initialized() 847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , report_mtx(MutexTypeReport, StatMtxReport) 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , nreported() 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , nmissed_expected() 872bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov , thread_registry(new(thread_registry_placeholder) ThreadRegistry( 882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)) 897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , racy_stacks(MBlockRacyStacks) 90158c6ac3bb46753db217f9c2c73485811a3a1890Dmitry Vyukov , racy_addresses(MBlockRacyAddresses) 910a05e5fa28a7424f8146549057c53b4590f3a251Alexey Samsonov , fired_suppressions(8) { 927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The objects are allocated in TLS, so one may rely on zero-initialization. 95ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry VyukovThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, 962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines unsigned reuse_count, 977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr stk_addr, uptr stk_size, 987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr tls_addr, uptr tls_size) 997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : fast_state(tid, epoch) 1007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Do not touch these, rely on zero initialization, 1017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // they may be accessed before the ctor. 102dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov // , ignore_reads_and_writes() 1032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // , ignore_interceptors() 1042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines , clock(tid, reuse_count) 1058b30c254a63a7421dd81d13dee086a54c4ca134bDmitry Vyukov#ifndef TSAN_GO 1068b30c254a63a7421dd81d13dee086a54c4ca134bDmitry Vyukov , jmp_bufs(MBlockJmpBuf) 1078b30c254a63a7421dd81d13dee086a54c4ca134bDmitry Vyukov#endif 1087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , tid(tid) 109ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov , unique_id(unique_id) 1107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , stk_addr(stk_addr) 1117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , stk_size(stk_size) 1127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , tls_addr(tls_addr) 1132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines , tls_size(tls_size) 1142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 1152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines , last_sleep_clock(tid) 1162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 1172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines{ 1187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 120a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukovstatic void MemoryProfiler(Context *ctx, fd_t fd, int i) { 1214bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov uptr n_threads; 1224bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov uptr n_running_threads; 1234bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads); 124a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov InternalScopedBuffer<char> buf(4096); 1255d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads); 1264bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov internal_write(fd, buf.data(), internal_strlen(buf.data())); 1274bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov} 1284bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov 1294bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukovstatic void BackgroundThread(void *arg) { 1302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 1312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // This is a non-initialized non-user thread, nothing to see here. 1322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // We don't use ScopedIgnoreInterceptors, because we want ignores to be 1332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // enabled even when the thread function exits (e.g. during pthread thread 1342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // shutdown code). 1352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur_thread()->ignore_interceptors++; 1362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 137f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov const u64 kMs2Ns = 1000 * 1000; 1384bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov 1394bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov fd_t mprof_fd = kInvalidFd; 1404bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov if (flags()->profile_memory && flags()->profile_memory[0]) { 1415d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (internal_strcmp(flags()->profile_memory, "stdout") == 0) { 1425d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines mprof_fd = 1; 1435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) { 1445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines mprof_fd = 2; 1459578a3ecfc35a264ede1135033398e2a77a6cad6Peter Collingbourne } else { 1465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines InternalScopedBuffer<char> filename(4096); 1475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines internal_snprintf(filename.data(), filename.size(), "%s.%d", 1485d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines flags()->profile_memory, (int)internal_getpid()); 1495d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines uptr openrv = OpenFile(filename.data(), true); 1505d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (internal_iserror(openrv)) { 1515d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", 1525d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines &filename[0]); 1535d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } else { 1545d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines mprof_fd = openrv; 1555d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 1564bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov } 1574bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov } 1584bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov 1594bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov u64 last_flush = NanoTime(); 1605d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines u64 last_rss_check = NanoTime(); 16192b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov uptr last_rss = 0; 1622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0; 1632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; 1642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines i++) { 1652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SleepForMillis(100); 1664bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov u64 now = NanoTime(); 16726127735454fddae3495794f38189d57dde6510fDmitry Vyukov 168a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov // Flush memory if requested. 16992b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (flags()->flush_memory_ms > 0) { 170f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) { 17192b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (flags()->verbosity > 0) 17292b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov Printf("ThreadSanitizer: periodic memory flush\n"); 1734bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov FlushShadowMemory(); 1744bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov last_flush = NanoTime(); 1754bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov } 1764bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov } 1775d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // GetRSS can be expensive on huge programs, so don't do it every 100ms. 1785d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (flags()->memory_limit_mb > 0 && last_rss_check + 1000 * kMs2Ns < now) { 1795d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines last_rss_check = now; 18092b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov uptr rss = GetRSS(); 18192b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov uptr limit = uptr(flags()->memory_limit_mb) << 20; 18292b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (flags()->verbosity > 0) { 18392b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov Printf("ThreadSanitizer: memory flush check" 18492b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov " RSS=%llu LAST=%llu LIMIT=%llu\n", 18592b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov (u64)rss>>20, (u64)last_rss>>20, (u64)limit>>20); 18692b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov } 18792b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (2 * rss > limit + last_rss) { 18892b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (flags()->verbosity > 0) 18992b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov Printf("ThreadSanitizer: flushing memory due to RSS\n"); 19092b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov FlushShadowMemory(); 19192b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov rss = GetRSS(); 19292b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov if (flags()->verbosity > 0) 19392b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov Printf("ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20); 19492b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov } 19592b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov last_rss = rss; 19692b54796149a8b5995fa49c43f43b709b83c5644Dmitry Vyukov } 1974bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov 198a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov // Write memory profile if requested. 1994bebe7bab966c82f0a8952f797ed3d490624dc62Dmitry Vyukov if (mprof_fd != kInvalidFd) 200a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov MemoryProfiler(ctx, mprof_fd, i); 201a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov 202a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov#ifndef TSAN_GO 203f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov // Flush symbolizer cache if requested. 204f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov if (flags()->flush_symbolizer_ms > 0) { 205f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov u64 last = atomic_load(&ctx->last_symbolize_time_ns, 206f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov memory_order_relaxed); 207f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { 208f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov Lock l(&ctx->report_mtx); 2097ed46ff7af911da0dd2067734d1408c6986c6657Alexey Samsonov SpinMutexLock l2(&CommonSanitizerReportMutex); 210f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov SymbolizeFlush(); 211f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); 212f63dde3594da0dba4c8039f0cb3a4196a76f1affDmitry Vyukov } 213a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov } 214a38e40fde45acccb124f7419ecbe21ef6cfd306bDmitry Vyukov#endif 21526127735454fddae3495794f38189d57dde6510fDmitry Vyukov } 21626127735454fddae3495794f38189d57dde6510fDmitry Vyukov} 21726127735454fddae3495794f38189d57dde6510fDmitry Vyukov 2182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void StartBackgroundThread() { 2192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->background_thread = internal_start_thread(&BackgroundThread, 0); 2202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 2232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void StopBackgroundThread() { 2242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed); 2252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines internal_join_thread(ctx->background_thread); 2262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->background_thread = 0; 2272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 2292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2307ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukovvoid DontNeedShadowFor(uptr addr, uptr size) { 2317ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov uptr shadow_beg = MemToShadow(addr); 2327ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov uptr shadow_end = MemToShadow(addr + size); 2337ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); 2347ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov} 2357ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov 236a05fcc1e3e045097f2f1a20798cbe038bbb1d6a9Dmitry Vyukovvoid MapShadow(uptr addr, uptr size) { 2372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Global data is not 64K aligned, but there are no adjacent mappings, 2382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // so we can get away with unaligned mapping. 2392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment 2406b641c5e63be45a03f96346886d27c4b4135ddafDmitry Vyukov MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier); 2415d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 2425d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // Meta shadow is 2:1, so tread carefully. 2435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines static bool data_mapped = false; 2445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines static uptr mapped_meta_end = 0; 2455d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines uptr meta_begin = (uptr)MemToMeta(addr); 2465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines uptr meta_end = (uptr)MemToMeta(addr + size); 2475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines meta_begin = RoundDownTo(meta_begin, 64 << 10); 2485d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines meta_end = RoundUpTo(meta_end, 64 << 10); 2495d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (!data_mapped) { 2505d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // First call maps data+bss. 2515d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines data_mapped = true; 2525d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines MmapFixedNoReserve(meta_begin, meta_end - meta_begin); 2535d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } else { 2545d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // Mapping continous heap. 2555d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // Windows wants 64K alignment. 2565d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines meta_begin = RoundDownTo(meta_begin, 64 << 10); 2575d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines meta_end = RoundUpTo(meta_end, 64 << 10); 2585d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (meta_end <= mapped_meta_end) 2595d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return; 2605d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (meta_begin < mapped_meta_end) 2615d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines meta_begin = mapped_meta_end; 2625d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines MmapFixedNoReserve(meta_begin, meta_end - meta_begin); 2635d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines mapped_meta_end = meta_end; 2645d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 2655d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n", 2665d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines addr, addr+size, meta_begin, meta_end); 267a05fcc1e3e045097f2f1a20798cbe038bbb1d6a9Dmitry Vyukov} 268a05fcc1e3e045097f2f1a20798cbe038bbb1d6a9Dmitry Vyukov 2696535c31c66569df262791f0b56852c496977c977Dmitry Vyukovvoid MapThreadTrace(uptr addr, uptr size) { 270dae1251f196f9694d428a04f14987b06112ae52cDmitry Vyukov DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size); 2716535c31c66569df262791f0b56852c496977c977Dmitry Vyukov CHECK_GE(addr, kTraceMemBegin); 2726535c31c66569df262791f0b56852c496977c977Dmitry Vyukov CHECK_LE(addr + size, kTraceMemBegin + kTraceMemSize); 2732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment 27401a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov uptr addr1 = (uptr)MmapFixedNoReserve(addr, size); 27501a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov if (addr1 != addr) { 27601a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p->%p)\n", 27701a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov addr, size, addr1); 2786535c31c66569df262791f0b56852c496977c977Dmitry Vyukov Die(); 2796535c31c66569df262791f0b56852c496977c977Dmitry Vyukov } 2806535c31c66569df262791f0b56852c496977c977Dmitry Vyukov} 2816535c31c66569df262791f0b56852c496977c977Dmitry Vyukov 2827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Initialize(ThreadState *thr) { 2837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Thread safe because done before all threads exist. 2847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static bool is_initialized = false; 2857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (is_initialized) 2867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany is_initialized = true; 2882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // We are not ready to handle interceptors yet. 2892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ScopedIgnoreInterceptors ignore; 290859778a4e2dffa4024fa3e13b105fd62eca44b1cKostya Serebryany SanitizerToolName = "ThreadSanitizer"; 291591616d323d73b7ea7cd8fea4eec46cedccda27eAlexey Samsonov // Install tool-specific callbacks in sanitizer_common. 292591616d323d73b7ea7cd8fea4eec46cedccda27eAlexey Samsonov SetCheckFailedCallback(TsanCheckFailed); 293591616d323d73b7ea7cd8fea4eec46cedccda27eAlexey Samsonov 294bbbb20b7212155ebc5b5b4ee1c68c987dcf30320Dmitry Vyukov#ifndef TSAN_GO 2952e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov InitializeAllocator(); 296bbbb20b7212155ebc5b5b4ee1c68c987dcf30320Dmitry Vyukov#endif 2977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeInterceptors(); 2987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const char *env = InitializePlatform(); 2997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeMutex(); 3007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeDynamicAnnotations(); 3017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx = new(ctx_placeholder) Context; 302a05fcc1e3e045097f2f1a20798cbe038bbb1d6a9Dmitry Vyukov#ifndef TSAN_GO 3037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeShadowMemory(); 304a05fcc1e3e045097f2f1a20798cbe038bbb1d6a9Dmitry Vyukov#endif 3057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeFlags(&ctx->flags, env); 306b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Setup correct file descriptor for error reports. 307b48c2b2072c8cc17dc1579a6b20ce6c2a575821dDmitry Vyukov __sanitizer_set_report_path(flags()->log_path); 3087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeSuppressions(); 30985a6dad26739fcd9742eae04b4a539e29889e937Dmitry Vyukov#ifndef TSAN_GO 3104af0f21c0c98950df1136dbec8824a029ed5bb8eDmitry Vyukov InitializeLibIgnore(); 3112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Symbolizer::Init(common_flags()->external_symbolizer_path); 31266d91e3356a0c4d7aff3beaaaff3e87bbaec805cAlexey Samsonov Symbolizer::Get()->AddHooks(EnterSymbolizer, ExitSymbolizer); 31385a6dad26739fcd9742eae04b4a539e29889e937Dmitry Vyukov#endif 3142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines StartBackgroundThread(); 3152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 3162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SetSandboxingCallback(StopBackgroundThread); 3172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 3182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) 3192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd = DDetector::Create(flags()); 3208cc1f81b2cc1fa0d4cda4f4635d955aed04c09c8Alexey Samsonov 3217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->flags.verbosity) 322b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", 3230b694fcab9b2f33bdd6691cbea4e80a5c27191b1Peter Collingbourne (int)internal_getpid()); 3247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Initialize thread 0. 3267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int tid = ThreadCreate(thr, 0, 0, true); 3277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(tid, 0); 3280b694fcab9b2f33bdd6691cbea4e80a5c27191b1Peter Collingbourne ThreadStart(thr, tid, internal_getpid()); 3297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->initialized = true; 3307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 331adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov if (flags()->stop_on_start) { 332b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer is suspended at startup (pid %d)." 333adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov " Call __tsan_resume().\n", 3340b694fcab9b2f33bdd6691cbea4e80a5c27191b1Peter Collingbourne (int)internal_getpid()); 335ba5e99668e3030cc5bab357a04271a1bdbac209cAlexey Samsonov while (__tsan_resumed == 0) {} 3367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines OnInitialize(); 3397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint Finalize(ThreadState *thr) { 3427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Context *ctx = __tsan::ctx; 3437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool failed = false; 3447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 34554e0a9a391616bbd2a4f0fd2d01076291274cb98Dmitry Vyukov if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1) 34654e0a9a391616bbd2a4f0fd2d01076291274cb98Dmitry Vyukov SleepForMillis(flags()->atexit_sleep_ms); 34754e0a9a391616bbd2a4f0fd2d01076291274cb98Dmitry Vyukov 348d0a51c02157e8293ea365ad0d429ef8e73d115deDmitry Vyukov // Wait for pending reports. 349d0a51c02157e8293ea365ad0d429ef8e73d115deDmitry Vyukov ctx->report_mtx.Lock(); 3507ed46ff7af911da0dd2067734d1408c6986c6657Alexey Samsonov CommonSanitizerReportMutex.Lock(); 3517ed46ff7af911da0dd2067734d1408c6986c6657Alexey Samsonov CommonSanitizerReportMutex.Unlock(); 352d0a51c02157e8293ea365ad0d429ef8e73d115deDmitry Vyukov ctx->report_mtx.Unlock(); 353d0a51c02157e8293ea365ad0d429ef8e73d115deDmitry Vyukov 354bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov#ifndef TSAN_GO 355bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov if (ctx->flags.verbosity) 356bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov AllocatorPrintStats(); 357bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov#endif 358bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov 3597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadFinalize(thr); 3607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->nreported) { 3627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany failed = true; 363b3b21231dd9dfeb34f5f302c879f2d11b6312080Dmitry Vyukov#ifndef TSAN_GO 364b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); 365b3b21231dd9dfeb34f5f302c879f2d11b6312080Dmitry Vyukov#else 366b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Found %d data race(s)\n", ctx->nreported); 367b3b21231dd9dfeb34f5f302c879f2d11b6312080Dmitry Vyukov#endif 3687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->nmissed_expected) { 3717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany failed = true; 372b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: missed %d expected races\n", 3737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->nmissed_expected); 3747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 376f754eb501d6bd163fff6747716b7703fe45be4b8Dmitry Vyukov if (flags()->print_suppressions) 377f754eb501d6bd163fff6747716b7703fe45be4b8Dmitry Vyukov PrintMatchedSuppressions(); 3780fd908cf5a555483633e2d9703932bde18009682Dmitry Vyukov#ifndef TSAN_GO 3790fd908cf5a555483633e2d9703932bde18009682Dmitry Vyukov if (flags()->print_benign) 3800fd908cf5a555483633e2d9703932bde18009682Dmitry Vyukov PrintMatchedBenignRaces(); 3810fd908cf5a555483633e2d9703932bde18009682Dmitry Vyukov#endif 382f754eb501d6bd163fff6747716b7703fe45be4b8Dmitry Vyukov 38322881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov failed = OnFinalize(failed); 38422881ec8c8a3c01f9b993b186040444b0b5caa50Dmitry Vyukov 38508adb1815b4958df88f904b9780a055b0d387294Dmitry Vyukov StatAggregate(ctx->stat, thr->stat); 3867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatOutput(ctx->stat); 387b7b6b1cd9df0c954b1f890fcebf373db984923b3Dmitry Vyukov return failed ? flags()->exitcode : 0; 3887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3900ab628c61594eb80612e5389d9c33da0e0d70c66Dmitry Vyukov#ifndef TSAN_GO 3912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ForkBefore(ThreadState *thr, uptr pc) { 3922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->Lock(); 3932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->report_mtx.Lock(); 3942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ForkParentAfter(ThreadState *thr, uptr pc) { 3972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->report_mtx.Unlock(); 3982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->Unlock(); 3992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4012d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ForkChildAfter(ThreadState *thr, uptr pc) { 4022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->report_mtx.Unlock(); 4032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->Unlock(); 4042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr nthread = 0; 4062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */); 4072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines VPrintf(1, "ThreadSanitizer: forked new process with pid %d," 4082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines " parent had %d threads\n", (int)internal_getpid(), (int)nthread); 4092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (nthread == 1) { 4102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines internal_start_thread(&BackgroundThread, 0); 4112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 4122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // We've just forked a multi-threaded process. We cannot reasonably function 4132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // after that (some mutexes may be locked before fork). So just enable 4142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // ignores for everything in the hope that we will exec soon. 4152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->after_multithreaded_fork = true; 4162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->ignore_interceptors++; 4172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadIgnoreBegin(thr, pc); 4182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadIgnoreSyncBegin(thr, pc); 4192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 4202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 4222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4235d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#ifdef TSAN_GO 4245d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesNOINLINE 4255d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesvoid GrowShadowStack(ThreadState *thr) { 4265d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const int sz = thr->shadow_stack_end - thr->shadow_stack; 4275d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const int newsz = 2 * sz; 4285d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack, 4295d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines newsz * sizeof(uptr)); 4305d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr)); 4315d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines internal_free(thr->shadow_stack); 4325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->shadow_stack = newstack; 4335d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->shadow_stack_pos = newstack + sz; 4345d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->shadow_stack_end = newstack + newsz; 4355d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines} 4365d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#endif 4375d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 438848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukovu32 CurrentStackId(ThreadState *thr, uptr pc) { 439848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov if (thr->shadow_stack_pos == 0) // May happen during bootstrap. 440848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov return 0; 4415d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (pc != 0) { 4425d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#ifndef TSAN_GO 4435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 4445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#else 4455d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (thr->shadow_stack_pos == thr->shadow_stack_end) 4465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines GrowShadowStack(thr); 4475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#endif 448848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov thr->shadow_stack_pos[0] = pc; 449848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov thr->shadow_stack_pos++; 450848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov } 451848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov u32 id = StackDepotPut(thr->shadow_stack, 452848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov thr->shadow_stack_pos - thr->shadow_stack); 4535d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (pc != 0) 454848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov thr->shadow_stack_pos--; 455848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov return id; 456848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov} 457848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov 458b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukovvoid TraceSwitch(ThreadState *thr) { 4599ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov thr->nomalloc++; 4609743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov Trace *thr_trace = ThreadTrace(thr->tid); 4619743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov Lock l(&thr_trace->mtx); 4620415ac00935795a70d87ae662ccad58ea0704537Dmitry Vyukov unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); 4639743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov TraceHeader *hdr = &thr_trace->headers[trace]; 4647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany hdr->epoch0 = thr->fast_state.epoch(); 4657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany hdr->stack0.ObtainCurrent(thr, 0); 466ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov hdr->mset0 = thr->mset; 4679ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov thr->nomalloc--; 4687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4709743d74426ae43898e4da55e591b09be18f8aa6eDmitry VyukovTrace *ThreadTrace(int tid) { 4719743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov return (Trace*)GetThreadTraceHeader(tid); 4729743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov} 4739743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov 474385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukovuptr TraceTopPC(ThreadState *thr) { 475385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukov Event *events = (Event*)GetThreadTrace(thr->tid); 476d698edc4f74a17048eef3342a9fa42b3ebba802aDmitry Vyukov uptr pc = events[thr->fast_state.GetTracePos()]; 477385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukov return pc; 478385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukov} 479385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukov 480d698edc4f74a17048eef3342a9fa42b3ebba802aDmitry Vyukovuptr TraceSize() { 481d698edc4f74a17048eef3342a9fa42b3ebba802aDmitry Vyukov return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); 482d698edc4f74a17048eef3342a9fa42b3ebba802aDmitry Vyukov} 483d698edc4f74a17048eef3342a9fa42b3ebba802aDmitry Vyukov 4840415ac00935795a70d87ae662ccad58ea0704537Dmitry Vyukovuptr TraceParts() { 4850415ac00935795a70d87ae662ccad58ea0704537Dmitry Vyukov return TraceSize() / kTracePartSize; 4860415ac00935795a70d87ae662ccad58ea0704537Dmitry Vyukov} 4870415ac00935795a70d87ae662ccad58ea0704537Dmitry Vyukov 488b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 4897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_trace_switch() { 4907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceSwitch(cur_thread()); 4917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_report_race() { 4947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ReportRace(cur_thread()); 4957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 496b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 4977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4987ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 49943c36e4b055f348d6076e6da44f9cd3e4399568fTimur IskhodzhanovShadow LoadShadow(u64 *p) { 5007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); 5017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return Shadow(raw); 5027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5047ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 50543c36e4b055f348d6076e6da44f9cd3e4399568fTimur Iskhodzhanovvoid StoreShadow(u64 *sp, u64 s) { 5067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); 5077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5097ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 51043c36e4b055f348d6076e6da44f9cd3e4399568fTimur Iskhodzhanovvoid StoreIfNotYetStored(u64 *sp, u64 *s) { 5117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreShadow(sp, *s); 5127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany *s = 0; 5137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5155d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE 5165d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesvoid HandleRace(ThreadState *thr, u64 *shadow_mem, 5177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur, Shadow old) { 5187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_state[0] = cur.raw(); 5197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_state[1] = old.raw(); 5207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_shadow_addr = shadow_mem; 521b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 5227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany HACKY_CALL(__tsan_report_race); 523b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#else 524b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov ReportRace(thr); 525b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 5267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool HappensBefore(Shadow old, ThreadState *thr) { 529c8f0a00ede42f83ac79ff63c347ca8ddcda77155Dmitry Vyukov return thr->clock.get(old.TidWithIgnore()) >= old.epoch(); 5307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE 5335d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesvoid MemoryAccessImpl1(ThreadState *thr, uptr addr, 534334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, 5357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem, Shadow cur) { 5367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMop); 5377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 5387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 5397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // This potentially can live in an MMX/SSE scratch register. 5417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The required intrinsics are: 5427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // __m128i _mm_move_epi64(__m128i*); 5437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // _mm_storel_epi64(u64*, __m128i); 5447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 store_word = cur.raw(); 5457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // scan all the shadow values and dispatch to 4 categories: 5477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // same, replace, candidate and race (see comments below). 5487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // we consider only 3 cases regarding access sizes: 5497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // equal, intersect and not intersect. initially I considered 5507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // larger and smaller as well, it allowed to replace some 5517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // 'candidates' with 'same' or 'replace', but I think 5527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // it's just not worth it (performance- and complexity-wise). 5537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 554286c914d37d28df611ae083cac40148cf2622ee7Dmitry Vyukov Shadow old(0); 5557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (kShadowCnt == 1) { 5567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 5577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 2) { 5597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 5607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 5627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 4) { 5647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 5657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 5677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 2; 5697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 3; 5717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 8) { 5737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 5747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 5767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 2; 5787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 3; 5807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 4; 5827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 5; 5847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 6; 5867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 7; 5887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 5897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 5907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(false); 5917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 5927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // we did not find any races and had already stored 5947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // the current access info, so we are done 5957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (LIKELY(store_word == 0)) 5967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 5977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // choose a random candidate slot and replace it 5987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); 5997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowReplace); 6007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 6017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany RACE: 6027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany HandleRace(thr, shadow_mem, cur, old); 6037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 6047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 6057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 6068ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukovvoid UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, 6078ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov int size, bool kAccessIsWrite, bool kIsAtomic) { 6088ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov while (size) { 6098ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov int size1 = 1; 6108ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov int kAccessSizeLog = kSizeLog1; 6118ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov if (size >= 8 && (addr & ~7) == ((addr + 8) & ~7)) { 6128ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov size1 = 8; 6138ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov kAccessSizeLog = kSizeLog8; 6148ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov } else if (size >= 4 && (addr & ~7) == ((addr + 4) & ~7)) { 6158ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov size1 = 4; 6168ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov kAccessSizeLog = kSizeLog4; 6178ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov } else if (size >= 2 && (addr & ~7) == ((addr + 2) & ~7)) { 6188ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov size1 = 2; 6198ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov kAccessSizeLog = kSizeLog2; 6208ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov } 6218ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic); 6228ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov addr += size1; 6238ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov size -= size1; 6248ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov } 6258ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov} 6268ecd0e5d9f389d18653892851c6ffb2f235de4b7Dmitry Vyukov 6275d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE 6285d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesbool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) { 6295d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines Shadow cur(a); 6305d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines for (uptr i = 0; i < kShadowCnt; i++) { 6315d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines Shadow old(LoadShadow(&s[i])); 6325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (Shadow::Addr0AndSizeAreEqual(cur, old) && 6335d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines old.TidWithIgnore() == cur.TidWithIgnore() && 6345d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines old.epoch() > sync_epoch && 6355d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines old.IsAtomic() == cur.IsAtomic() && 6365d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines old.IsRead() <= cur.IsRead()) 6375d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return true; 6385d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 6395d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return false; 6405d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines} 6415d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 6425d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4 6435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \ 6445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \ 6455d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) 6465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE 6475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesbool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) { 6485d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // This is an optimized version of ContainsSameAccessSlow. 6495d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // load current access into access[0:63] 6505d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 access = _mm_cvtsi64_si128(a); 6515d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // duplicate high part of access in addr0: 6525d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr0[0:31] = access[32:63] 6535d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr0[32:63] = access[32:63] 6545d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr0[64:95] = access[32:63] 6555d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr0[96:127] = access[32:63] 6565d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 addr0 = SHUF(access, access, 1, 1, 1, 1); 6575d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // load 4 shadow slots 6585d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 shadow0 = _mm_load_si128((__m128i*)s); 6595d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 shadow1 = _mm_load_si128((__m128i*)s + 1); 6605d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // load high parts of 4 shadow slots into addr_vect: 6615d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr_vect[0:31] = shadow0[32:63] 6625d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr_vect[32:63] = shadow0[96:127] 6635d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr_vect[64:95] = shadow1[32:63] 6645d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr_vect[96:127] = shadow1[96:127] 6655d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3); 6665d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (!is_write) { 6675d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // set IsRead bit in addr_vect 6685d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15); 6695d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0); 6705d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines addr_vect = _mm_or_si128(addr_vect, rw_mask); 6715d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 6725d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr0 == addr_vect? 6735d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect); 6745d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch1[0:63] = sync_epoch 6755d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch); 6765d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch[0:31] = sync_epoch[0:31] 6775d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch[32:63] = sync_epoch[0:31] 6785d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch[64:95] = sync_epoch[0:31] 6795d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch[96:127] = sync_epoch[0:31] 6805d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0); 6815d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // load low parts of shadow cell epochs into epoch_vect: 6825d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch_vect[0:31] = shadow0[0:31] 6835d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch_vect[32:63] = shadow0[64:95] 6845d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch_vect[64:95] = shadow1[0:31] 6855d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch_vect[96:127] = shadow1[64:95] 6865d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2); 6875d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // epoch_vect >= sync_epoch? 6885d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch); 6895d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // addr_res & epoch_res 6905d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const m128 res = _mm_and_si128(addr_res, epoch_res); 6915d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // mask[0] = res[7] 6925d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // mask[1] = res[15] 6935d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // ... 6945d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // mask[15] = res[127] 6955d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines const int mask = _mm_movemask_epi8(res); 6965d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return mask != 0; 6975d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines} 6985d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#endif 6995d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 7005d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE 7015d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesbool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) { 7025d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4 7035d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write); 7045d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write)); 7055d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return res; 7065d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#else 7075d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return ContainsSameAccessSlow(s, a, sync_epoch, is_write); 7085d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#endif 7095d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines} 7105d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 711d475aa8578f4a1955cdb3e3159eda8b229f8c021Kostya SerebryanyALWAYS_INLINE USED 7127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccess(ThreadState *thr, uptr pc, uptr addr, 713334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) { 7147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem = (u64*)MemToShadow(addr); 71568230a12bbd22c9402dd8f9af027fcb2e119f978Dmitry Vyukov DPrintf2("#%d: MemoryAccess: @%p %p size=%d" 716e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", 7177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)thr->fast_state.tid(), (void*)pc, (void*)addr, 7187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, 719e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov (uptr)shadow_mem[0], (uptr)shadow_mem[1], 720e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov (uptr)shadow_mem[2], (uptr)shadow_mem[3]); 7217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 7227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsAppMem(addr)) { 723b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Access to non app mem %zx\n", addr); 7247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsAppMem(addr)); 7257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 7267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsShadowMem((uptr)shadow_mem)) { 727b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); 7287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsShadowMem((uptr)shadow_mem)); 7297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 7307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 7317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 7325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (kCppMode && *shadow_mem == kShadowRodata) { 73382dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov // Access to .rodata section, no races here. 73482dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov // Measurements show that it can be 10-20% of all memory accesses. 73582dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov StatInc(thr, StatMop); 73682dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 73782dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 73882dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov StatInc(thr, StatMopRodata); 73982dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov return; 74082dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov } 74182dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov 7427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FastState fast_state = thr->fast_state; 7435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (fast_state.GetIgnoreBit()) { 7445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMop); 7455d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 7465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 7475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMopIgnored); 7487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 7492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 7502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 7517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur(fast_state); 7527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); 7537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetWrite(kAccessIsWrite); 754334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov cur.SetAtomic(kIsAtomic); 7557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 7565d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), 7575d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->fast_synch_epoch, kAccessIsWrite))) { 7585d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMop); 7595d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 7605d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 7615d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMopSame); 7625d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return; 7635d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 7645d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 7655d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (kCollectHistory) { 7665d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines fast_state.IncrementEpoch(); 7675d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->fast_state = fast_state; 7685d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines TraceAddEvent(thr, fast_state, EventTypeMop, pc); 7695d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines cur.IncrementEpoch(); 7705d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 7715d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 7725d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, 7735d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines shadow_mem, cur); 7745d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines} 7755d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 7765d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines// Called by MemoryAccessRange in tsan_rtl_thread.cc 7775d71de26cedae3dafc17449fe0182045c0bd20e8Stephen HinesALWAYS_INLINE USED 7785d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesvoid MemoryAccessImpl(ThreadState *thr, uptr addr, 7795d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, 7805d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines u64 *shadow_mem, Shadow cur) { 7815d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), 7825d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->fast_synch_epoch, kAccessIsWrite))) { 7835d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMop); 7845d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 7855d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 7865d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines StatInc(thr, StatMopSame); 7875d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return; 7885d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 7895d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines 7905d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, 7917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem, cur); 7927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 7937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 7947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, 7957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 val) { 79674172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov (void)thr; 79774172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov (void)pc; 7987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size == 0) 7997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 8007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // FIXME: fix me. 8017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr offset = addr % kShadowCell; 8027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (offset) { 8037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany offset = kShadowCell - offset; 8047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size <= offset) 8057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 8067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany addr += offset; 8077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany size -= offset; 8087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 809aaac6e206453574d0130f2ae8d743f630d0c0a42Dmitry Vyukov DCHECK_EQ(addr % 8, 0); 810aaac6e206453574d0130f2ae8d743f630d0c0a42Dmitry Vyukov // If a user passes some insane arguments (memset(0)), 811aaac6e206453574d0130f2ae8d743f630d0c0a42Dmitry Vyukov // let it just crash as usual. 812aaac6e206453574d0130f2ae8d743f630d0c0a42Dmitry Vyukov if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) 813aaac6e206453574d0130f2ae8d743f630d0c0a42Dmitry Vyukov return; 81474172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // Don't want to touch lots of shadow memory. 81574172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // If a program maps 10MB stack, there is no need reset the whole range. 81626af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); 8179c4d7a4e5fc1b49757ac733fec72c5e89948fefbDmitry Vyukov // UnmapOrDie/MmapFixedNoReserve does not work on Windows, 8189c4d7a4e5fc1b49757ac733fec72c5e89948fefbDmitry Vyukov // so we do it only for C/C++. 8192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (kGoMode || size < common_flags()->clear_shadow_mmap_threshold) { 82074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov u64 *p = (u64*)MemToShadow(addr); 82174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov CHECK(IsShadowMem((uptr)p)); 82274172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); 82374172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // FIXME: may overwrite a part outside the region 82474172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { 82574172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov p[i++] = val; 82674172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov for (uptr j = 1; j < kShadowCnt; j++) 82774172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov p[i++] = 0; 82874172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov } 82974172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov } else { 83074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // The region is big, reset only beginning and end. 83174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov const uptr kPageSize = 4096; 83274172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov u64 *begin = (u64*)MemToShadow(addr); 83374172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov u64 *end = begin + size / kShadowCell * kShadowCnt; 83474172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov u64 *p = begin; 83574172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // Set at least first kPageSize/2 to page boundary. 83674172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { 83774172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov *p++ = val; 83874172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov for (uptr j = 1; j < kShadowCnt; j++) 83974172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov *p++ = 0; 84074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov } 84174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // Reset middle part. 84274172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov u64 *p1 = p; 84374172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov p = RoundDown(end, kPageSize); 84474172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov UnmapOrDie((void*)p1, (uptr)p - (uptr)p1); 84574172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1); 84674172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // Set the ending. 84774172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov while (p < end) { 84874172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov *p++ = val; 84974172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov for (uptr j = 1; j < kShadowCnt; j++) 85074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov *p++ = 0; 85174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov } 85226af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov } 8537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 8547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 8557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { 8567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MemoryRangeSet(thr, pc, addr, size, 0); 8577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 8587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 8597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { 86074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // Processing more than 1k (4k of shadow) is expensive, 86174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // can cause excessive memory consumption (user does not necessary touch 86274172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov // the whole range) and most likely unnecessary. 86374172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov if (size > 1024) 86474172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov size = 1024; 8653285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov CHECK_EQ(thr->is_freeing, false); 8663285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = true; 8677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MemoryAccessRange(thr, pc, addr, size, true); 8683285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = false; 8692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (kCollectHistory) { 8702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->fast_state.IncrementEpoch(); 8712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); 8722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 873069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov Shadow s(thr->fast_state); 874064c84731c1cf41dcd7195c9380170b9aa6887b6Dmitry Vyukov s.ClearIgnoreBit(); 875069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.MarkAsFreed(); 876069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.SetWrite(true); 877069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.SetAddr0AndSizeLog(0, 3); 878069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov MemoryRangeSet(thr, pc, addr, size, s.raw()); 8797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 8807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 88126af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukovvoid MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { 8822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (kCollectHistory) { 8832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->fast_state.IncrementEpoch(); 8842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); 8852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 88626af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov Shadow s(thr->fast_state); 887064c84731c1cf41dcd7195c9380170b9aa6887b6Dmitry Vyukov s.ClearIgnoreBit(); 88826af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov s.SetWrite(true); 88926af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov s.SetAddr0AndSizeLog(0, 3); 89026af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov MemoryRangeSet(thr, pc, addr, size, s.raw()); 89126af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov} 89226af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov 893d475aa8578f4a1955cdb3e3159eda8b229f8c021Kostya SerebryanyALWAYS_INLINE USED 8947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncEntry(ThreadState *thr, uptr pc) { 8957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatFuncEnter); 89625d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); 8972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (kCollectHistory) { 8982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->fast_state.IncrementEpoch(); 8992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); 9002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 9017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 9027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Shadow stack maintenance can be replaced with 9037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // stack unwinding during trace switch (which presumably must be faster). 90401a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); 90525d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#ifndef TSAN_GO 90601a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 90725d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#else 9085d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (thr->shadow_stack_pos == thr->shadow_stack_end) 9095d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines GrowShadowStack(thr); 91025d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#endif 9117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos[0] = pc; 9127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos++; 9137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 9147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 915d475aa8578f4a1955cdb3e3159eda8b229f8c021Kostya SerebryanyALWAYS_INLINE USED 9167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncExit(ThreadState *thr) { 9177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatFuncExit); 91825d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); 9192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (kCollectHistory) { 9202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->fast_state.IncrementEpoch(); 9212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); 9222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 9237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 92401a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); 92525d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#ifndef TSAN_GO 92601a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 92725d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#endif 9287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos--; 9297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 9307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 9312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ThreadIgnoreBegin(ThreadState *thr, uptr pc) { 932652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid); 933652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov thr->ignore_reads_and_writes++; 934e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov CHECK_GT(thr->ignore_reads_and_writes, 0); 935652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov thr->fast_state.SetIgnoreBit(); 9362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 9372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!ctx->after_multithreaded_fork) 9382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->mop_ignore_set.Add(CurrentStackId(thr, pc)); 9392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 940652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov} 941652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov 9422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ThreadIgnoreEnd(ThreadState *thr, uptr pc) { 943652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid); 944652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov thr->ignore_reads_and_writes--; 945652f78a41de2887aedc1b0314b58bb6b622c80caDmitry Vyukov CHECK_GE(thr->ignore_reads_and_writes, 0); 9462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (thr->ignore_reads_and_writes == 0) { 9477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.ClearIgnoreBit(); 9482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 9492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->mop_ignore_set.Reset(); 9502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 9512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 9527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 9537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 9542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) { 955e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid); 956e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->ignore_sync++; 957e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov CHECK_GT(thr->ignore_sync, 0); 9582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 9592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!ctx->after_multithreaded_fork) 9602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->sync_ignore_set.Add(CurrentStackId(thr, pc)); 9612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 962e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 963e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 9642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) { 965e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid); 966e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->ignore_sync--; 967e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov CHECK_GE(thr->ignore_sync, 0); 9682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 9692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (thr->ignore_sync == 0) 9702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->sync_ignore_set.Reset(); 9712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 972e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 973e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 974b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukovbool MD5Hash::operator==(const MD5Hash &other) const { 975b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov return hash[0] == other.hash[0] && hash[1] == other.hash[1]; 976b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov} 977b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov 9787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 9797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_debug() {} 9807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 9817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_release() {} 9827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 9837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 9847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 9857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_stats() {} 9867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 9877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_nostats() {} 9887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 9897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 9907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_SHADOW_COUNT == 1 9917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow1() {} 9927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#elif TSAN_SHADOW_COUNT == 2 9937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow2() {} 9947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#elif TSAN_SHADOW_COUNT == 4 9957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow4() {} 9967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 9977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow8() {} 9987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 9997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 10007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 10017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1002b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 10037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Must be included in this file to make sure everything is inlined. 10047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_interface_inl.h" 1005b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 1006