1//===-- tsan_platform_linux.cc --------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12// Linux-specific code.
13//===----------------------------------------------------------------------===//
14
15
16#include "sanitizer_common/sanitizer_platform.h"
17#if SANITIZER_LINUX || SANITIZER_FREEBSD
18
19#include "sanitizer_common/sanitizer_common.h"
20#include "sanitizer_common/sanitizer_libc.h"
21#include "sanitizer_common/sanitizer_procmaps.h"
22#include "sanitizer_common/sanitizer_stoptheworld.h"
23#include "tsan_platform.h"
24#include "tsan_rtl.h"
25#include "tsan_flags.h"
26
27#include <fcntl.h>
28#include <pthread.h>
29#include <signal.h>
30#include <stdio.h>
31#include <stdlib.h>
32#include <string.h>
33#include <stdarg.h>
34#include <sys/mman.h>
35#include <sys/syscall.h>
36#include <sys/socket.h>
37#include <sys/time.h>
38#include <sys/types.h>
39#include <sys/resource.h>
40#include <sys/stat.h>
41#include <unistd.h>
42#include <errno.h>
43#include <sched.h>
44#include <dlfcn.h>
45#if SANITIZER_LINUX
46#define __need_res_state
47#include <resolv.h>
48#endif
49
50#ifdef sa_handler
51# undef sa_handler
52#endif
53
54#ifdef sa_sigaction
55# undef sa_sigaction
56#endif
57
58#if SANITIZER_FREEBSD
59extern "C" void *__libc_stack_end;
60void *__libc_stack_end = 0;
61#endif
62
63namespace __tsan {
64
65const uptr kPageSize = 4096;
66
67enum {
68  MemTotal  = 0,
69  MemShadow = 1,
70  MemMeta   = 2,
71  MemFile   = 3,
72  MemMmap   = 4,
73  MemTrace  = 5,
74  MemHeap   = 6,
75  MemOther  = 7,
76  MemCount  = 8,
77};
78
79void FillProfileCallback(uptr start, uptr rss, bool file,
80                         uptr *mem, uptr stats_size) {
81  mem[MemTotal] += rss;
82  start >>= 40;
83  if (start < 0x10)
84    mem[MemShadow] += rss;
85  else if (start >= 0x20 && start < 0x30)
86    mem[file ? MemFile : MemMmap] += rss;
87  else if (start >= 0x30 && start < 0x40)
88    mem[MemMeta] += rss;
89  else if (start >= 0x7e)
90    mem[file ? MemFile : MemMmap] += rss;
91  else if (start >= 0x60 && start < 0x62)
92    mem[MemTrace] += rss;
93  else if (start >= 0x7d && start < 0x7e)
94    mem[MemHeap] += rss;
95  else
96    mem[MemOther] += rss;
97}
98
99void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
100  uptr mem[MemCount] = {};
101  __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
102  internal_snprintf(buf, buf_size,
103      "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
104      " trace:%zd heap:%zd other:%zd nthr=%zd/%zd\n",
105      mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
106      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
107      mem[MemHeap] >> 20, mem[MemOther] >> 20,
108      nlive, nthread);
109}
110
111uptr GetRSS() {
112  uptr mem[7] = {};
113  __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
114  return mem[6];
115}
116
117#if SANITIZER_LINUX
118void FlushShadowMemoryCallback(
119    const SuspendedThreadsList &suspended_threads_list,
120    void *argument) {
121  FlushUnneededShadowMemory(kLinuxShadowBeg, kLinuxShadowEnd - kLinuxShadowBeg);
122}
123#endif
124
125void FlushShadowMemory() {
126#if SANITIZER_LINUX
127  StopTheWorld(FlushShadowMemoryCallback, 0);
128#endif
129}
130
131#ifndef TSAN_GO
132static void ProtectRange(uptr beg, uptr end) {
133  CHECK_LE(beg, end);
134  if (beg == end)
135    return;
136  if (beg != (uptr)Mprotect(beg, end - beg)) {
137    Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
138    Printf("FATAL: Make sure you are not using unlimited stack\n");
139    Die();
140  }
141}
142
143// Mark shadow for .rodata sections with the special kShadowRodata marker.
144// Accesses to .rodata can't race, so this saves time, memory and trace space.
145static void MapRodata() {
146  // First create temp file.
147  const char *tmpdir = GetEnv("TMPDIR");
148  if (tmpdir == 0)
149    tmpdir = GetEnv("TEST_TMPDIR");
150#ifdef P_tmpdir
151  if (tmpdir == 0)
152    tmpdir = P_tmpdir;
153#endif
154  if (tmpdir == 0)
155    return;
156  char name[256];
157  internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
158                    tmpdir, (int)internal_getpid());
159  uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
160  if (internal_iserror(openrv))
161    return;
162  internal_unlink(name);  // Unlink it now, so that we can reuse the buffer.
163  fd_t fd = openrv;
164  // Fill the file with kShadowRodata.
165  const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
166  InternalScopedBuffer<u64> marker(kMarkerSize);
167  // volatile to prevent insertion of memset
168  for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
169    *p = kShadowRodata;
170  internal_write(fd, marker.data(), marker.size());
171  // Map the file into memory.
172  uptr page = internal_mmap(0, kPageSize, PROT_READ | PROT_WRITE,
173                            MAP_PRIVATE | MAP_ANONYMOUS, fd, 0);
174  if (internal_iserror(page)) {
175    internal_close(fd);
176    return;
177  }
178  // Map the file into shadow of .rodata sections.
179  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
180  uptr start, end, offset, prot;
181  // Reusing the buffer 'name'.
182  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) {
183    if (name[0] != 0 && name[0] != '['
184        && (prot & MemoryMappingLayout::kProtectionRead)
185        && (prot & MemoryMappingLayout::kProtectionExecute)
186        && !(prot & MemoryMappingLayout::kProtectionWrite)
187        && IsAppMem(start)) {
188      // Assume it's .rodata
189      char *shadow_start = (char*)MemToShadow(start);
190      char *shadow_end = (char*)MemToShadow(end);
191      for (char *p = shadow_start; p < shadow_end; p += marker.size()) {
192        internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
193                      PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
194      }
195    }
196  }
197  internal_close(fd);
198}
199
200void InitializeShadowMemory() {
201  // Map memory shadow.
202  uptr shadow = (uptr)MmapFixedNoReserve(kLinuxShadowBeg,
203    kLinuxShadowEnd - kLinuxShadowBeg);
204  if (shadow != kLinuxShadowBeg) {
205    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
206    Printf("FATAL: Make sure to compile with -fPIE and "
207               "to link with -pie (%p, %p).\n", shadow, kLinuxShadowBeg);
208    Die();
209  }
210  DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
211      kLinuxShadowBeg, kLinuxShadowEnd,
212      (kLinuxShadowEnd - kLinuxShadowBeg) >> 30);
213
214  // Map meta shadow.
215  if (MemToMeta(kLinuxAppMemBeg) < (u32*)kMetaShadow) {
216    Printf("ThreadSanitizer: bad meta shadow (%p -> %p < %p)\n",
217        kLinuxAppMemBeg, MemToMeta(kLinuxAppMemBeg), kMetaShadow);
218    Die();
219  }
220  if (MemToMeta(kLinuxAppMemEnd) >= (u32*)(kMetaShadow + kMetaSize)) {
221    Printf("ThreadSanitizer: bad meta shadow (%p -> %p >= %p)\n",
222        kLinuxAppMemEnd, MemToMeta(kLinuxAppMemEnd), kMetaShadow + kMetaSize);
223    Die();
224  }
225  uptr meta = (uptr)MmapFixedNoReserve(kMetaShadow, kMetaSize);
226  if (meta != kMetaShadow) {
227    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
228    Printf("FATAL: Make sure to compile with -fPIE and "
229               "to link with -pie (%p, %p).\n", meta, kMetaShadow);
230    Die();
231  }
232  DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
233      kMetaShadow, kMetaShadow + kMetaSize, kMetaSize >> 30);
234
235  // Protect gaps.
236  const uptr kClosedLowBeg  = 0x200000;
237  const uptr kClosedLowEnd  = kLinuxShadowBeg - 1;
238  const uptr kClosedMidBeg = kLinuxShadowEnd + 1;
239  const uptr kClosedMidEnd = min(min(kLinuxAppMemBeg, kTraceMemBegin),
240      kMetaShadow);
241
242  ProtectRange(kClosedLowBeg, kClosedLowEnd);
243  ProtectRange(kClosedMidBeg, kClosedMidEnd);
244  VPrintf(2, "kClosedLow   %zx-%zx (%zuGB)\n",
245      kClosedLowBeg, kClosedLowEnd, (kClosedLowEnd - kClosedLowBeg) >> 30);
246  VPrintf(2, "kClosedMid   %zx-%zx (%zuGB)\n",
247      kClosedMidBeg, kClosedMidEnd, (kClosedMidEnd - kClosedMidBeg) >> 30);
248  VPrintf(2, "app mem: %zx-%zx (%zuGB)\n",
249      kLinuxAppMemBeg, kLinuxAppMemEnd,
250      (kLinuxAppMemEnd - kLinuxAppMemBeg) >> 30);
251  VPrintf(2, "stack: %zx\n", (uptr)&shadow);
252
253  MapRodata();
254}
255#endif
256
257static uptr g_data_start;
258static uptr g_data_end;
259
260#ifndef TSAN_GO
261static void CheckPIE() {
262  // Ensure that the binary is indeed compiled with -pie.
263  MemoryMappingLayout proc_maps(true);
264  uptr start, end;
265  if (proc_maps.Next(&start, &end,
266                     /*offset*/0, /*filename*/0, /*filename_size*/0,
267                     /*protection*/0)) {
268    if ((u64)start < kLinuxAppMemBeg) {
269      Printf("FATAL: ThreadSanitizer can not mmap the shadow memory ("
270             "something is mapped at 0x%zx < 0x%zx)\n",
271             start, kLinuxAppMemBeg);
272      Printf("FATAL: Make sure to compile with -fPIE"
273             " and to link with -pie.\n");
274      Die();
275    }
276  }
277}
278
279static void InitDataSeg() {
280  MemoryMappingLayout proc_maps(true);
281  uptr start, end, offset;
282  char name[128];
283  bool prev_is_data = false;
284  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
285                        /*protection*/ 0)) {
286    DPrintf("%p-%p %p %s\n", start, end, offset, name);
287    bool is_data = offset != 0 && name[0] != 0;
288    // BSS may get merged with [heap] in /proc/self/maps. This is not very
289    // reliable.
290    bool is_bss = offset == 0 &&
291      (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data;
292    if (g_data_start == 0 && is_data)
293      g_data_start = start;
294    if (is_bss)
295      g_data_end = end;
296    prev_is_data = is_data;
297  }
298  DPrintf("guessed data_start=%p data_end=%p\n",  g_data_start, g_data_end);
299  CHECK_LT(g_data_start, g_data_end);
300  CHECK_GE((uptr)&g_data_start, g_data_start);
301  CHECK_LT((uptr)&g_data_start, g_data_end);
302}
303
304#endif  // #ifndef TSAN_GO
305
306static rlim_t getlim(int res) {
307  rlimit rlim;
308  CHECK_EQ(0, getrlimit(res, &rlim));
309  return rlim.rlim_cur;
310}
311
312static void setlim(int res, rlim_t lim) {
313  // The following magic is to prevent clang from replacing it with memset.
314  volatile rlimit rlim;
315  rlim.rlim_cur = lim;
316  rlim.rlim_max = lim;
317  setrlimit(res, (rlimit*)&rlim);
318}
319
320const char *InitializePlatform() {
321  void *p = 0;
322  if (sizeof(p) == 8) {
323    // Disable core dumps, dumping of 16TB usually takes a bit long.
324    setlim(RLIMIT_CORE, 0);
325  }
326
327  // Go maps shadow memory lazily and works fine with limited address space.
328  // Unlimited stack is not a problem as well, because the executable
329  // is not compiled with -pie.
330  if (kCppMode) {
331    bool reexec = false;
332    // TSan doesn't play well with unlimited stack size (as stack
333    // overlaps with shadow memory). If we detect unlimited stack size,
334    // we re-exec the program with limited stack size as a best effort.
335    if (getlim(RLIMIT_STACK) == (rlim_t)-1) {
336      const uptr kMaxStackSize = 32 * 1024 * 1024;
337      VReport(1, "Program is run with unlimited stack size, which wouldn't "
338                 "work with ThreadSanitizer.\n"
339                 "Re-execing with stack size limited to %zd bytes.\n",
340              kMaxStackSize);
341      SetStackSizeLimitInBytes(kMaxStackSize);
342      reexec = true;
343    }
344
345    if (getlim(RLIMIT_AS) != (rlim_t)-1) {
346      Report("WARNING: Program is run with limited virtual address space,"
347             " which wouldn't work with ThreadSanitizer.\n");
348      Report("Re-execing with unlimited virtual address space.\n");
349      setlim(RLIMIT_AS, -1);
350      reexec = true;
351    }
352    if (reexec)
353      ReExec();
354  }
355
356#ifndef TSAN_GO
357  CheckPIE();
358  InitTlsSize();
359  InitDataSeg();
360#endif
361  return GetEnv(kTsanOptionsEnv);
362}
363
364bool IsGlobalVar(uptr addr) {
365  return g_data_start && addr >= g_data_start && addr < g_data_end;
366}
367
368#ifndef TSAN_GO
369// Extract file descriptors passed to glibc internal __res_iclose function.
370// This is required to properly "close" the fds, because we do not see internal
371// closes within glibc. The code is a pure hack.
372int ExtractResolvFDs(void *state, int *fds, int nfd) {
373#if SANITIZER_LINUX
374  int cnt = 0;
375  __res_state *statp = (__res_state*)state;
376  for (int i = 0; i < MAXNS && cnt < nfd; i++) {
377    if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
378      fds[cnt++] = statp->_u._ext.nssocks[i];
379  }
380  return cnt;
381#else
382  return 0;
383#endif
384}
385
386// Extract file descriptors passed via UNIX domain sockets.
387// This is requried to properly handle "open" of these fds.
388// see 'man recvmsg' and 'man 3 cmsg'.
389int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
390  int res = 0;
391  msghdr *msg = (msghdr*)msgp;
392  struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
393  for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
394    if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS)
395      continue;
396    int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
397    for (int i = 0; i < n; i++) {
398      fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
399      if (res == nfd)
400        return res;
401    }
402  }
403  return res;
404}
405
406int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
407    void *abstime), void *c, void *m, void *abstime,
408    void(*cleanup)(void *arg), void *arg) {
409  // pthread_cleanup_push/pop are hardcore macros mess.
410  // We can't intercept nor call them w/o including pthread.h.
411  int res;
412  pthread_cleanup_push(cleanup, arg);
413  res = fn(c, m, abstime);
414  pthread_cleanup_pop(0);
415  return res;
416}
417#endif
418
419}  // namespace __tsan
420
421#endif  // SANITIZER_LINUX
422