1/*
2  This file is part of ThreadSanitizer, a dynamic data race detector
3  based on Valgrind.
4
5  Copyright (C) 2008-2010 Google Inc
6     opensource@google.com
7  Copyright (C) 2007-2008 OpenWorks LLP
8      info@open-works.co.uk
9
10  This program is free software; you can redistribute it and/or
11  modify it under the terms of the GNU General Public License as
12  published by the Free Software Foundation; either version 2 of the
13  License, or (at your option) any later version.
14
15  This program is distributed in the hope that it will be useful, but
16  WITHOUT ANY WARRANTY; without even the implied warranty of
17  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  General Public License for more details.
19
20  You should have received a copy of the GNU General Public License
21  along with this program; if not, write to the Free Software
22  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
23  02111-1307, USA.
24
25  The GNU General Public License is contained in the file COPYING.
26*/
27
28// Author: Konstantin Serebryany.
29// Parts of the code in this file are derived from Helgrind,
30// a data race detector written by Julian Seward.
31// Note that the rest of ThreadSanitizer code is not derived from Helgrind
32// and is published under the BSD license.
33
34#include "ts_valgrind.h"
35#include "valgrind.h"
36#include "ts_valgrind_client_requests.h"
37#include "thread_sanitizer.h"
38#include "ts_trace_info.h"
39#include "ts_race_verifier.h"
40#include "common_util.h"
41
42#include "coregrind/pub_core_basics.h"
43#include "coregrind/pub_core_machine.h"
44#include "coregrind/pub_core_clreq.h"
45#include "coregrind/pub_core_threadstate.h"
46#include "pub_tool_libcproc.h"
47
48
49//---------------------- C++ malloc support -------------- {{{1
50void *operator new (size_t size) {
51  return VG_(malloc)((HChar*)g_malloc_stack.Top(), size);
52}
53void *operator new [](size_t size) {
54  return VG_(malloc)((HChar*)g_malloc_stack.Top(), size);
55}
56void operator delete (void *p) {
57  VG_(free)(p);
58}
59void operator delete [](void *p) {
60  VG_(free)(p);
61}
62
63extern "C" void *malloc(size_t size) {
64  return VG_(malloc)((HChar*)g_malloc_stack.Top(), size);
65}
66
67extern "C" void free(void *ptr) {
68  VG_(free)(ptr);
69}
70
71
72//---------------------- Utils ------------------- {{{1
73
74extern "C" int puts(const char *s) {
75  Printf("%s", s);
76  return 1;
77}
78
79extern "C" void exit(int e) { VG_(exit)(e); }
80
81#ifdef VGO_darwin
82extern "C" void abort() { CHECK(0); }
83#endif
84
85
86// TODO: make this rtn public
87extern "C" {
88  Bool VG_(get_fnname_no_cxx_demangle) ( Addr a, Char* buf, Int nbuf );
89}
90
91
92const int kBuffSize = 1024 * 10 - 1;
93// not thread-safe.
94static char g_buff1[kBuffSize+1];
95static char g_buff2[kBuffSize+1];
96
97string PcToRtnName(uintptr_t pc, bool demangle) {
98  if (demangle) {
99    if(VG_(get_fnname)(pc, (Char*)g_buff1, kBuffSize)) {
100      return g_buff1;
101    }
102  } else {
103    if(VG_(get_fnname_no_cxx_demangle)(pc, (Char*)g_buff1, kBuffSize)) {
104      return g_buff1;
105    }
106  }
107  return "(no symbols)";
108}
109
110void PcToStrings(uintptr_t pc, bool demangle,
111                string *img_name, string *rtn_name,
112                string *file_name, int *line_no) {
113  const int kBuffSize = 1024 * 10 - 1;
114  Bool has_dirname = False;
115
116  if (VG_(get_filename_linenum)
117      (pc, (Char*)g_buff1, kBuffSize, (Char*)g_buff2, kBuffSize,
118       &has_dirname, (UInt*)line_no) &&
119      has_dirname) {
120    *file_name = string(g_buff2) + "/" + g_buff1;
121  } else {
122    VG_(get_linenum)(pc, (UInt *)line_no);
123    if (VG_(get_filename)(pc, (Char*)g_buff1, kBuffSize)) {
124      *file_name = g_buff1;
125    }
126  }
127  *file_name = ConvertToPlatformIndependentPath(*file_name);
128
129  *rtn_name = PcToRtnName(pc, demangle);
130
131  if (VG_(get_objname)(pc, (Char*)g_buff1, kBuffSize)) {
132    *img_name = g_buff1;
133  }
134}
135
136
137
138string Demangle(const char *str) {
139  return str;
140}
141
142extern "C"
143size_t strlen(const char *s) {
144  return VG_(strlen)((const Char*)s);
145}
146
147static inline ThreadId GetVgTid() {
148  extern ThreadId VG_(running_tid); // HACK: avoid calling get_running_tid()
149  ThreadId res = VG_(running_tid);
150  //DCHECK(res == VG_(get_running_tid)());
151  return res;
152}
153
154static inline uintptr_t GetVgPc(ThreadId vg_tid) {
155  Addr pc = VG_(threads)[vg_tid].arch.vex.VG_INSTR_PTR;
156  DCHECK(pc == VG_(get_IP)(vg_tid));
157  return pc;
158  //return (uintptr_t)VG_(get_IP)(vg_tid);
159}
160
161static inline uintptr_t GetVgSp(ThreadId vg_tid) {
162  Addr sp = VG_(threads)[vg_tid].arch.vex.VG_STACK_PTR;
163  DCHECK(sp == VG_(get_SP)(vg_tid));
164  return sp;
165}
166
167#ifdef VGP_arm_linux
168static inline uintptr_t GetVgLr(ThreadId vg_tid) {
169  return (uintptr_t)VG_(threads)[vg_tid].arch.vex.guest_R14;
170}
171#endif
172
173static uintptr_t g_current_pc;
174
175uintptr_t GetPcOfCurrentThread() {
176  return g_current_pc;
177}
178
179void GetThreadStack(int tid, uintptr_t *min_addr, uintptr_t *max_addr) {
180  // tid is not used because we call it from the current thread anyway.
181  uintptr_t stack_max  = VG_(thread_get_stack_max)(GetVgTid());
182  uintptr_t stack_size = VG_(thread_get_stack_size)(GetVgTid());
183  uintptr_t stack_min  = stack_max - stack_size;
184  *min_addr = stack_min;
185  *max_addr = stack_max;
186}
187
188struct CallStackRecord {
189  Addr pc;
190  Addr sp;
191#ifdef VGP_arm_linux
192  // We need to store LR in order to keep the shadow stack consistent.
193  Addr lr;
194#endif
195};
196
197const size_t kMaxMopsPerTrace = 2048;
198
199struct ValgrindThread {
200  int32_t zero_based_uniq_tid;
201  Thread *ts_thread;
202  uint32_t literace_sampling;
203  vector<CallStackRecord> call_stack;
204
205  int ignore_accesses;
206  int ignore_sync;
207  int in_signal_handler;
208
209  // thread-local event buffer (tleb).
210  uintptr_t tleb[kMaxMopsPerTrace];
211  TraceInfo *trace_info;
212
213  // PC (as in trace_info->pc()) of the trace currently being verified.
214  // 0 if outside of the verification sleep loop.
215  // -1 in the last iteration of the loop.
216  uintptr_t verifier_current_pc;
217
218  // End time of the current verification loop.
219  unsigned verifier_wakeup_time_ms;
220
221  ValgrindThread() {
222    Clear();
223  }
224
225  void Clear() {
226    ts_thread = NULL;
227    zero_based_uniq_tid = -1;
228    literace_sampling = G_flags->literace_sampling;  // cache it.
229    ignore_accesses = 0;
230    ignore_sync = 0;
231    in_signal_handler = 0;
232    call_stack.clear();
233    trace_info = NULL;
234    verifier_current_pc = 0;
235    verifier_wakeup_time_ms = 0;
236  }
237};
238
239// If true, ignore all accesses in all threads.
240extern bool global_ignore;
241
242// Array of VG_N_THREADS
243static ValgrindThread *g_valgrind_threads = 0;
244static map<uintptr_t, int> *g_ptid_to_ts_tid;
245
246// maintains a uniq thread id (first thread will have id=0)
247static int32_t g_uniq_thread_id_counter = 0;
248
249static int32_t VgTidToTsTid(ThreadId vg_tid) {
250  DCHECK(vg_tid < VG_N_THREADS);
251  DCHECK(vg_tid >= 1);
252  DCHECK(g_valgrind_threads);
253  DCHECK(g_valgrind_threads[vg_tid].zero_based_uniq_tid >= 0);
254  return g_valgrind_threads[vg_tid].zero_based_uniq_tid;
255}
256
257static vector<string> *g_command_line_options = 0;
258static void InitCommandLineOptions() {
259  if(G_flags == NULL) {
260    G_flags = new FLAGS;
261  }
262  if (g_command_line_options == NULL) {
263    g_command_line_options = new vector<string>;
264  }
265}
266
267Bool ts_process_cmd_line_option (Char* arg) {
268  InitCommandLineOptions();
269  g_command_line_options->push_back((char*)arg);
270  return True;
271}
272
273void ts_print_usage (void) {
274  InitCommandLineOptions();
275  ThreadSanitizerParseFlags(g_command_line_options);
276
277  ThreadSanitizerPrintUsage();
278}
279
280void ts_print_debug_usage(void) {
281  ThreadSanitizerPrintUsage();
282}
283
284extern int VG_(clo_error_exitcode);
285
286void ts_post_clo_init(void) {
287  ScopedMallocCostCenter malloc_cc(__FUNCTION__);
288  InitCommandLineOptions();
289  ThreadSanitizerParseFlags(g_command_line_options);
290
291  // we get num-callers from valgrind flags.
292  G_flags->num_callers = VG_(clo_backtrace_size);
293  if (!G_flags->error_exitcode)
294    G_flags->error_exitcode = VG_(clo_error_exitcode);
295
296  extern Int   VG_(clo_n_suppressions);
297  extern Int   VG_(clo_gen_suppressions);
298  extern Char* VG_(clo_suppressions)[];
299  extern Int   VG_(clo_n_fullpath_after);
300  extern Char* VG_(clo_fullpath_after)[];
301  // get the suppressions from Valgrind
302  for (int i = 0; i < VG_(clo_n_suppressions); i++) {
303    G_flags->suppressions.push_back((char*)VG_(clo_suppressions)[i]);
304  }
305  // get the --fullpath-after prefixes from Valgrind and treat them as
306  // --file-prefix-to-cut arguments.
307  for (int i = 0; i < VG_(clo_n_fullpath_after); i++) {
308    G_flags->file_prefix_to_cut.push_back((char*)VG_(clo_fullpath_after)[i]);
309  }
310  G_flags->generate_suppressions |= VG_(clo_gen_suppressions) >= 1;
311
312  if (G_flags->html) {
313    Report("<pre>\n"
314           "<br id=race0>"
315           "<a href=\"#race1\">Go to first race report</a>\n");
316  }
317  Report("ThreadSanitizerValgrind r%s: %s\n",
318         TS_VERSION,
319         G_flags->pure_happens_before ? "hybrid=no" : "hybrid=yes");
320  if (DEBUG_MODE) {
321    Report("INFO: Debug build\n");
322  }
323  if (G_flags->max_mem_in_mb) {
324    Report("INFO: ThreadSanitizer memory limit: %dMB\n",
325           (int)G_flags->max_mem_in_mb);
326  }
327  ThreadSanitizerInit();
328
329  g_valgrind_threads = new ValgrindThread[VG_N_THREADS];
330  g_ptid_to_ts_tid = new map<uintptr_t, int>;
331
332  if (g_race_verifier_active) {
333    RaceVerifierInit(G_flags->race_verifier, G_flags->race_verifier_extra);
334    global_ignore = true;
335  }
336}
337
338// Remember, valgrind is essentially single-threaded.
339// Each time we switch to another thread, we set the global g_cur_tleb
340// to the tleb of the current thread. This allows to load the tleb in one
341// instruction.
342static uintptr_t *g_cur_tleb;
343static void OnStartClientCode(ThreadId vg_tid, ULong nDisp) {
344  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
345  g_cur_tleb = thr->tleb;
346}
347
348INLINE void FlushMops(ValgrindThread *thr, bool keep_trace_info = false) {
349  DCHECK(!g_race_verifier_active || global_ignore);
350  TraceInfo *t = thr->trace_info;
351  if (!t) return;
352  if (!keep_trace_info) {
353    thr->trace_info = NULL;
354  }
355
356  if (global_ignore || thr->ignore_accesses ||
357       (thr->literace_sampling &&
358        t->LiteRaceSkipTraceRealTid(thr->zero_based_uniq_tid, thr->literace_sampling))) {
359    thr->trace_info = NULL;
360    return;
361  }
362
363  size_t n = t->n_mops();
364  DCHECK(n > 0);
365  uintptr_t *tleb = thr->tleb;
366  DCHECK(thr->ts_thread);
367  ThreadSanitizerHandleTrace(thr->ts_thread, t, tleb);
368}
369
370static void ShowCallStack(ValgrindThread *thr) {
371  size_t n = thr->call_stack.size();
372  Printf("        ");
373  for (size_t i = n - 1; i > n - 10 && i >= 0; i--) {
374    Printf("{pc=%p sp=%p}, ", thr->call_stack[i].pc, thr->call_stack[i].sp);
375  }
376  Printf("\n");
377}
378
379static INLINE void UpdateCallStack(ValgrindThread *thr, uintptr_t sp) {
380  DCHECK(!g_race_verifier_active);
381  if (thr->trace_info) FlushMops(thr, true /* keep_trace_info */);
382  vector<CallStackRecord> &call_stack = thr->call_stack;
383  while (!call_stack.empty()) {
384    CallStackRecord &record = call_stack.back();
385    Addr cur_top = record.sp;
386    if (sp < cur_top) break;
387    call_stack.pop_back();
388    int32_t ts_tid = thr->zero_based_uniq_tid;
389    ThreadSanitizerHandleRtnExit(ts_tid);
390    if (debug_rtn) {
391      Printf("T%d: [%ld]<< pc=%p sp=%p cur_sp=%p %s\n",
392             ts_tid, thr->call_stack.size(), record.pc,
393             record.sp, sp,
394             PcToRtnNameAndFilePos(record.pc).c_str());
395      ShowCallStack(thr);
396    }
397  }
398}
399
400VG_REGPARM(1)
401static void OnTrace(TraceInfo *trace_info) {
402  DCHECK(!g_race_verifier_active);
403  //trace_info->counter()++;
404  if (global_ignore) return;
405  ThreadId vg_tid = GetVgTid();
406  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
407
408  // First, flush the old trace_info.
409  if (thr->trace_info) {
410    FlushMops(thr);
411  }
412
413  UpdateCallStack(thr, GetVgSp(vg_tid));
414
415  // Start the new trace, zero the contents of tleb.
416  size_t n = trace_info->n_mops();
417  uintptr_t *tleb = thr->tleb;
418  for (size_t i = 0; i < n; i++)
419    tleb[i] = 0;
420  thr->trace_info = trace_info;
421  DCHECK(thr->trace_info);
422  DCHECK(thr->trace_info->n_mops() <= kMaxMopsPerTrace);
423}
424
425static inline void Put(EventType type, int32_t tid, uintptr_t pc,
426                       uintptr_t a, uintptr_t info) {
427  if (DEBUG_MODE && G_flags->dry_run >= 1) return;
428  Event event(type, tid, pc, a, info);
429  ThreadSanitizerHandleOneEvent(&event);
430}
431
432static void rtn_call(Addr sp_post_call_insn, Addr pc_post_call_insn,
433                     IGNORE_BELOW_RTN ignore_below) {
434  DCHECK(!g_race_verifier_active);
435  if (global_ignore) return;
436  ThreadId vg_tid = GetVgTid();
437  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
438  int ts_tid = thr->zero_based_uniq_tid;
439  CallStackRecord record;
440  record.pc = pc_post_call_insn;
441  record.sp = sp_post_call_insn + 4;  // sp before call.
442  UpdateCallStack(thr, record.sp);
443#ifdef VGP_arm_linux
444  record.lr = GetVgLr(vg_tid);
445#endif
446  thr->call_stack.push_back(record);
447  // If the shadow stack grows too high this usually means it is not cleaned
448  // properly. Or this may be a very deep recursion.
449  DCHECK(thr->call_stack.size() < 10000);
450  uintptr_t call_pc = GetVgPc(vg_tid);
451  if (thr->trace_info) FlushMops(thr);
452  ThreadSanitizerHandleRtnCall(ts_tid, call_pc, record.pc,
453                               ignore_below);
454
455  if (debug_rtn) {
456    Printf("T%d: [%ld]>> pc=%p sp=%p %s\n",
457           ts_tid, thr->call_stack.size(), (void*)record.pc,
458           (void*)record.sp,
459           PcToRtnNameAndFilePos(record.pc).c_str());
460    ShowCallStack(thr);
461  }
462}
463
464VG_REGPARM(2) void evh__rtn_call_ignore_unknown ( Addr sp, Addr pc) {
465  rtn_call(sp, pc, IGNORE_BELOW_RTN_UNKNOWN);
466}
467VG_REGPARM(2) void evh__rtn_call_ignore_yes ( Addr sp, Addr pc) {
468  rtn_call(sp, pc, IGNORE_BELOW_RTN_YES);
469}
470VG_REGPARM(2) void evh__rtn_call_ignore_no ( Addr sp, Addr pc) {
471  rtn_call(sp, pc, IGNORE_BELOW_RTN_NO);
472}
473
474#ifdef VGP_arm_linux
475// Handle shadow stack frame deletion on ARM.
476// Instrumented code calls this function for each non-call jump out of
477// a superblock. If the |sp_post_call_insn| (the jump target address) is equal
478// to a link register value of one or more frames on top of the shadow stack,
479// those frames are popped out.
480// TODO(glider): there may be problems with optimized recursive functions that
481// don't change PC, SP and LR.
482VG_REGPARM(2)
483void evh__delete_frame ( Addr sp_post_call_insn,
484                         Addr pc_post_call_insn) {
485  DCHECK(!g_race_verifier_active);
486  ThreadId vg_tid = GetVgTid();
487  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
488  if (thr->trace_info) FlushMops(thr);
489  vector<CallStackRecord> &call_stack = thr->call_stack;
490  int32_t ts_tid = VgTidToTsTid(vg_tid);
491  while (!call_stack.empty()) {
492    CallStackRecord &record = call_stack.back();
493    if (record.lr != pc_post_call_insn) break;
494    call_stack.pop_back();
495    ThreadSanitizerHandleRtnExit(ts_tid);
496  }
497}
498#endif
499
500void ts_fini(Int exitcode) {
501  ThreadSanitizerFini();
502  if (g_race_verifier_active) {
503    RaceVerifierFini();
504  }
505  if (G_flags->error_exitcode && GetNumberOfFoundErrors() > 0) {
506    exit(G_flags->error_exitcode);
507  }
508}
509
510
511void evh__pre_thread_ll_create ( ThreadId parent, ThreadId child ) {
512  tl_assert(parent != child);
513  ValgrindThread *thr = &g_valgrind_threads[child];
514  //  Printf("thread_create: %d->%d\n", parent, child);
515  if (thr->zero_based_uniq_tid != -1) {
516    Printf("ThreadSanitizer WARNING: reusing TID %d w/o exiting thread\n",
517           child);
518  }
519  thr->Clear();
520  thr->zero_based_uniq_tid = g_uniq_thread_id_counter++;
521  // Printf("VG: T%d: VG_THR_START: parent=%d\n", VgTidToTsTid(child), VgTidToTsTid(parent));
522  Put(THR_START, VgTidToTsTid(child), 0, 0,
523      parent > 0 ? VgTidToTsTid(parent) : 0);
524  thr->ts_thread = ThreadSanitizerGetThreadByTid(thr->zero_based_uniq_tid);
525  CHECK(thr->ts_thread);
526}
527
528void evh__pre_workq_task_start(ThreadId vg_tid, Addr workitem) {
529  uintptr_t pc = GetVgPc(vg_tid);
530  int32_t ts_tid = VgTidToTsTid(vg_tid);
531  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
532  FlushMops(thr);
533  Put(WAIT, ts_tid, pc, workitem, 0);
534}
535
536void evh__pre_thread_first_insn(const ThreadId vg_tid) {
537  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
538  FlushMops(thr);
539  Put(THR_FIRST_INSN, VgTidToTsTid(vg_tid), GetVgPc(vg_tid), 0, 0);
540}
541
542
543void evh__pre_thread_ll_exit ( ThreadId quit_tid ) {
544//  Printf("thread_exit: %d\n", quit_tid);
545//  Printf("T%d quiting thread; stack size=%ld\n",
546//         VgTidToTsTid(quit_tid),
547//         (int)g_valgrind_threads[quit_tid].call_stack.size());
548  ValgrindThread *thr = &g_valgrind_threads[quit_tid];
549  FlushMops(thr);
550  Put(THR_END, VgTidToTsTid(quit_tid), 0, 0, 0);
551  g_valgrind_threads[quit_tid].zero_based_uniq_tid = -1;
552}
553
554  extern "C" void VG_(show_all_errors)();
555
556// Whether we are currently ignoring sync events for the given thread at the
557// given address.
558static inline Bool ignoring_sync(ThreadId vg_tid, uintptr_t addr) {
559  // We ignore locking events if ignore_sync != 0 and if we are not
560  // inside a signal handler.
561  return (g_valgrind_threads[vg_tid].ignore_sync &&
562          !g_valgrind_threads[vg_tid].in_signal_handler) ||
563      ThreadSanitizerIgnoreForNacl(addr);
564}
565
566Bool ts_handle_client_request(ThreadId vg_tid, UWord* args, UWord* ret) {
567  if (args[0] == VG_USERREQ__NACL_MEM_START) {
568    // This will get truncated on x86-32, but we don't support it with NaCl
569    // anyway.
570    const uintptr_t kFourGig = (uintptr_t)0x100000000ULL;
571    uintptr_t mem_start = args[1];
572    uintptr_t mem_end = mem_start + kFourGig;
573    ThreadSanitizerNaclUntrustedRegion(mem_start, mem_end);
574    return True;
575  }
576  if (!VG_IS_TOOL_USERREQ('T', 'S', args[0]))
577    return False;
578  int32_t ts_tid = VgTidToTsTid(vg_tid);
579  // Ignore almost everything in race verifier mode.
580  if (g_race_verifier_active) {
581    if (args[0] == TSREQ_EXPECT_RACE) {
582      Put(EXPECT_RACE, ts_tid, /*descr=*/args[3],
583          /*p=*/args[1], /*size*/args[2]);
584    }
585    *ret = 0;
586    return True;
587  }
588  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
589  if (thr->trace_info) FlushMops(thr);
590  UpdateCallStack(thr, GetVgSp(vg_tid));
591  *ret = 0;
592  uintptr_t pc = GetVgPc(vg_tid);
593  switch (args[0]) {
594    case TSREQ_SET_MY_PTHREAD_T:
595      (*g_ptid_to_ts_tid)[args[1]] = ts_tid;
596      break;
597    case TSREQ_THR_STACK_TOP:
598      Put(THR_STACK_TOP, ts_tid, pc, args[1], 0);
599      break;
600    case TSREQ_PTHREAD_JOIN_POST:
601      Put(THR_JOIN_AFTER, ts_tid, pc, (*g_ptid_to_ts_tid)[args[1]], 0);
602      break;
603    case TSREQ_CLEAN_MEMORY:
604      Put(MALLOC, ts_tid, pc, /*ptr=*/args[1], /*size=*/args[2]);
605      break;
606    case TSREQ_MAIN_IN:
607      g_has_entered_main = true;
608      // Report("INFO: Entred main(); argc=%d\n", (int)args[1]);
609      break;
610    case TSREQ_MAIN_OUT:
611      g_has_exited_main = true;
612      if (G_flags->exit_after_main) {
613        Report("INFO: Exited main(); ret=%d\n", (int)args[1]);
614        VG_(show_all_errors)();
615        ThreadSanitizerFini();
616        if (g_race_verifier_active) {
617          RaceVerifierFini();
618        }
619        exit((int)args[1]);
620      }
621      break;
622    case TSREQ_MALLOC:
623      // Printf("Malloc: %p %ld\n", args[1], args[2]);
624      Put(MALLOC, ts_tid, pc, /*ptr=*/args[1], /*size=*/args[2]);
625      break;
626    case TSREQ_FREE:
627      // Printf("Free: %p\n", args[1]);
628      Put(FREE, ts_tid, pc, /*ptr=*/args[1], 0);
629      break;
630    case TSREQ_MMAP:
631      Put(MMAP, ts_tid, pc, /*ptr=*/args[1], /*size=*/args[2]);
632      break;
633    case TSREQ_MUNMAP:
634      Put(MUNMAP, ts_tid, pc, /*ptr=*/args[1], /*size=*/args[2]);
635      break;
636    case TSREQ_BENIGN_RACE:
637      Put(BENIGN_RACE, ts_tid, /*descr=*/args[3],
638          /*p=*/args[1], /*size=*/args[2]);
639      break;
640    case TSREQ_EXPECT_RACE:
641      Put(EXPECT_RACE, ts_tid, /*descr=*/args[3],
642          /*p=*/args[1], /*size*/args[2]);
643      break;
644    case TSREQ_FLUSH_EXPECTED_RACES:
645      Put(FLUSH_EXPECTED_RACES, ts_tid, 0, 0, 0);
646      break;
647    case TSREQ_PCQ_CREATE:
648      Put(PCQ_CREATE, ts_tid, pc, /*pcq=*/args[1], 0);
649      break;
650    case TSREQ_PCQ_DESTROY:
651      Put(PCQ_DESTROY, ts_tid, pc, /*pcq=*/args[1], 0);
652      break;
653    case TSREQ_PCQ_PUT:
654      Put(PCQ_PUT, ts_tid, pc, /*pcq=*/args[1], 0);
655      break;
656    case TSREQ_PCQ_GET:
657      Put(PCQ_GET, ts_tid, pc, /*pcq=*/args[1], 0);
658      break;
659    case TSREQ_TRACE_MEM:
660      Put(TRACE_MEM, ts_tid, pc, /*mem=*/args[1], 0);
661      break;
662    case TSREQ_MUTEX_IS_USED_AS_CONDVAR:
663      Put(HB_LOCK, ts_tid, pc, /*lock=*/args[1], 0);
664      break;
665    case TSREQ_MUTEX_IS_NOT_PHB:
666      Put(NON_HB_LOCK, ts_tid, pc, /*lock=*/args[1], 0);
667      break;
668    case TSREQ_GLOBAL_IGNORE_ON:
669      Report("INFO: GLOBAL IGNORE ON\n");
670      global_ignore = true;
671      break;
672    case TSREQ_GLOBAL_IGNORE_OFF:
673      Report("INFO: GLOBAL IGNORE OFF\n");
674      global_ignore = false;
675      break;
676    case TSREQ_IGNORE_READS_BEGIN:
677      Put(IGNORE_READS_BEG, ts_tid, pc, 0, 0);
678      break;
679    case TSREQ_IGNORE_READS_END:
680      Put(IGNORE_READS_END, ts_tid, pc, 0, 0);
681      break;
682    case TSREQ_IGNORE_WRITES_BEGIN:
683      Put(IGNORE_WRITES_BEG, ts_tid, pc, 0, 0);
684      break;
685    case TSREQ_IGNORE_WRITES_END:
686      Put(IGNORE_WRITES_END, ts_tid, pc, 0, 0);
687      break;
688    case TSREQ_SET_THREAD_NAME:
689      Put(SET_THREAD_NAME, ts_tid, pc, /*name=*/args[1], 0);
690      break;
691    case TSREQ_SET_STACKTOP_STACKSIZE:
692      Put(THR_STACK_TOP, ts_tid, pc, /*addr=*/args[1], /*size=*/args[2]);
693      break;
694    case TSREQ_IGNORE_ALL_ACCESSES_BEGIN:
695      g_valgrind_threads[vg_tid].ignore_accesses++;
696      break;
697    case TSREQ_IGNORE_ALL_ACCESSES_END:
698      g_valgrind_threads[vg_tid].ignore_accesses--;
699      CHECK(g_valgrind_threads[vg_tid].ignore_accesses >= 0);
700      break;
701    case TSREQ_IGNORE_ALL_SYNC_BEGIN:
702      g_valgrind_threads[vg_tid].ignore_sync++;
703      break;
704    case TSREQ_IGNORE_ALL_SYNC_END:
705      g_valgrind_threads[vg_tid].ignore_sync--;
706      CHECK(g_valgrind_threads[vg_tid].ignore_sync >= 0);
707      break;
708    case TSREQ_PUBLISH_MEMORY_RANGE:
709      Put(PUBLISH_RANGE, ts_tid, pc, /*mem=*/args[1], /*size=*/args[2]);
710      break;
711    case TSREQ_UNPUBLISH_MEMORY_RANGE:
712      Put(UNPUBLISH_RANGE, ts_tid, pc, /*mem=*/args[1], /*size=*/args[2]);
713      break;
714    case TSREQ_PRINT_MEMORY_USAGE:
715    case TSREQ_PRINT_STATS:
716    case TSREQ_RESET_STATS:
717    case TSREQ_PTH_API_ERROR:
718      break;
719    case TSREQ_PTHREAD_RWLOCK_CREATE_POST:
720      Put(LOCK_CREATE, ts_tid, pc, /*lock=*/args[1], 0);
721      break;
722    case TSREQ_PTHREAD_RWLOCK_DESTROY_PRE:
723      Put(LOCK_DESTROY, ts_tid, pc, /*lock=*/args[1], 0);
724      break;
725    case TSREQ_PTHREAD_RWLOCK_LOCK_POST:
726      if (ignoring_sync(vg_tid, args[1]))
727        break;
728      Put(args[2] ? WRITER_LOCK : READER_LOCK, ts_tid, pc, /*lock=*/args[1], 0);
729      break;
730    case TSREQ_PTHREAD_RWLOCK_UNLOCK_PRE:
731      if (ignoring_sync(vg_tid, args[1]))
732        break;
733      Put(UNLOCK, ts_tid, pc, /*lock=*/args[1], 0);
734      break;
735    case TSREQ_PTHREAD_SPIN_LOCK_INIT_OR_UNLOCK:
736      Put(UNLOCK_OR_INIT, ts_tid, pc, /*lock=*/args[1], 0);
737      break;
738    case TSREQ_POSIX_SEM_INIT_POST:
739    case TSREQ_POSIX_SEM_DESTROY_PRE:
740      break;
741    case TSREQ_SIGNAL:
742      if (ignoring_sync(vg_tid, args[1]))
743        break;
744      Put(SIGNAL, ts_tid, pc, args[1], 0);
745      break;
746    case TSREQ_WAIT:
747      if (ignoring_sync(vg_tid, args[1]))
748        break;
749      Put(WAIT, ts_tid, pc, args[1], 0);
750      break;
751    case TSREQ_CYCLIC_BARRIER_INIT:
752      Put(CYCLIC_BARRIER_INIT, ts_tid, pc, args[1], args[2]);
753      break;
754    case TSREQ_CYCLIC_BARRIER_WAIT_BEFORE:
755      Put(CYCLIC_BARRIER_WAIT_BEFORE, ts_tid, pc, args[1], 0);
756      break;
757    case TSREQ_CYCLIC_BARRIER_WAIT_AFTER:
758      Put(CYCLIC_BARRIER_WAIT_AFTER, ts_tid, pc, args[1], 0);
759      break;
760    case TSREQ_GET_MY_SEGMENT:
761      break;
762    case TSREQ_GET_THREAD_ID:
763      *ret = ts_tid;
764      break;
765    case TSREQ_GET_VG_THREAD_ID:
766      *ret = vg_tid;
767      break;
768    case TSREQ_GET_SEGMENT_ID:
769      break;
770    case TSREQ_THREAD_SANITIZER_QUERY:
771      *ret = (UWord)ThreadSanitizerQuery((const char *)args[1]);
772      break;
773    case TSREQ_FLUSH_STATE:
774      Put(FLUSH_STATE, ts_tid, pc, 0, 0);
775      break;
776    default: CHECK(0);
777  }
778  return True;
779}
780
781static void SignalIn(ThreadId vg_tid, Int sigNo, Bool alt_stack) {
782  g_valgrind_threads[vg_tid].in_signal_handler++;
783  DCHECK(g_valgrind_threads[vg_tid].in_signal_handler == 1);
784//  int32_t ts_tid = VgTidToTsTid(vg_tid);
785//  Printf("T%d %s\n", ts_tid, __FUNCTION__);
786}
787
788static void SignalOut(ThreadId vg_tid, Int sigNo) {
789  g_valgrind_threads[vg_tid].in_signal_handler--;
790  CHECK(g_valgrind_threads[vg_tid].in_signal_handler >= 0);
791  DCHECK(g_valgrind_threads[vg_tid].in_signal_handler == 0);
792//  int32_t ts_tid = VgTidToTsTid(vg_tid);
793//  Printf("T%d %s\n", ts_tid, __FUNCTION__);
794}
795
796
797// ---------------------------- RaceVerifier    ---------------------------{{{1
798
799/**
800 * In race verifier mode _every_ IRSB is instrumented with a sleep loop at the
801 * beginning (but, of course, in most cases it is not executed).
802 * Its code logically looks like
803 *  irsb_start:
804 *   bool need_sleep = OnTraceVerify1();
805 *   if (need_sleep) {
806 *     sched_yield();
807 *     goto irsb_start;
808 *   }
809 *   OnTraceVerify2(trace_info);
810 *
811 * This loop verifies mops from the _previous_ trace_info and sets up the new
812 * trace info in OnTraceVerify2. Only IRSBs with "interesting" mops have
813 * non-zero trace_info.
814 */
815
816/**
817 * Race verification loop.
818 * On the first pass (for a trace_info), if there are mops to be verified,
819 * register them with RaceVerifier and calculate the wake up time.
820 * On the following passes, check the wake up time against the clock.
821 * The loop state is kept in ValgrindThread.
822 * Returns true if need to sleep more, false if the loop must be ended.
823 */
824VG_REGPARM(1)
825static uint32_t OnTraceVerify1() {
826  DCHECK(g_race_verifier_active);
827  ThreadId vg_tid = GetVgTid();
828
829  // First, flush the old trace_info.
830  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
831
832  // thr->trace_info is the trace info for the previous superblock.
833  if (!thr->trace_info)
834    // Nothing to do here.
835    return 0;
836
837  if (!thr->verifier_current_pc) {
838    // This is the first iteration of the sleep loop.
839    // Register memory accesses.
840    int sleep_time_ms = RaceVerifierGetSleepTime(thr->trace_info->pc());
841    if (!sleep_time_ms) {
842      thr->trace_info = NULL;
843      return 0;
844    }
845    size_t n = thr->trace_info->n_mops();
846    uintptr_t* tleb = thr->tleb;
847    int need_sleep = 0;
848    for (size_t i = 0; i < n; ++i) {
849      uintptr_t addr = tleb[i];
850      if (addr) {
851        MopInfo *mop = thr->trace_info->GetMop(i);
852        need_sleep += RaceVerifierStartAccess(thr->zero_based_uniq_tid, addr,
853            mop->pc(), mop->is_write());
854      }
855    }
856    // Setup the sleep timer.
857    thr->verifier_current_pc = thr->trace_info->pc();
858    if (need_sleep) {
859      unsigned now = VG_(read_millisecond_timer)();
860      thr->verifier_wakeup_time_ms = now + sleep_time_ms;
861      return 1;
862    } else {
863      thr->verifier_current_pc = (unsigned)-1;
864      return 0;
865    }
866  } else {
867    // Continuation of the sleep loop.
868    DCHECK(thr->verifier_current_pc == thr->trace_info->pc());
869    unsigned now = VG_(read_millisecond_timer)();
870    if (now < thr->verifier_wakeup_time_ms) {
871      // sleep more
872      return 1;
873    } else {
874      // done, go straight to OnTraceVerify2
875      thr->verifier_current_pc = (unsigned)-1;
876      return 0;
877    }
878  }
879}
880
881/**
882 * Race verification loop exit.
883 * Unregisters mops with the RaceVerifier.
884 * Sets up the new trace_info.
885 */
886VG_REGPARM(1)
887static void OnTraceVerify2(TraceInfo *trace_info) {
888  DCHECK(g_race_verifier_active);
889  ThreadId vg_tid = GetVgTid();
890  ValgrindThread *thr = &g_valgrind_threads[vg_tid];
891
892  DCHECK(!thr->trace_info || thr->verifier_current_pc == (unsigned)-1);
893  thr->verifier_current_pc = 0;
894  thr->verifier_wakeup_time_ms = 0;
895
896  if (thr->trace_info) {
897    // Unregister accesses from the old trace_info.
898    size_t n = thr->trace_info->n_mops();
899    uintptr_t* tleb = thr->tleb;
900    for (size_t i = 0; i < n; ++i) {
901      uintptr_t addr = tleb[i];
902      if (addr) {
903        MopInfo *mop = thr->trace_info->GetMop(i);
904        RaceVerifierEndAccess(thr->zero_based_uniq_tid, addr,
905            mop->pc(), mop->is_write());
906      }
907    }
908  }
909
910  // Start the new trace, zero the contents of tleb.
911  thr->trace_info = trace_info;
912  if (trace_info) {
913    size_t n = trace_info->n_mops();
914    uintptr_t *tleb = thr->tleb;
915    for (size_t i = 0; i < n; i++)
916      tleb[i] = 0;
917    DCHECK(thr->trace_info->n_mops() <= kMaxMopsPerTrace);
918  }
919}
920
921/**
922 * Add a race verification preamble to the IRSB.
923 */
924static void ts_instrument_trace_entry_verify(IRSB *bbOut,
925    VexGuestLayout* layout, TraceInfo *trace_info, uintptr_t cur_pc) {
926   HChar*   hName = (HChar*)"OnTraceVerify1";
927   void *callback = (void*)OnTraceVerify1;
928   IRExpr **args = mkIRExprVec_0();
929   IRTemp need_sleep = newIRTemp(bbOut->tyenv, Ity_I32);
930   IRDirty* di = unsafeIRDirty_1_N(need_sleep, 0, hName,
931       VG_(fnptr_to_fnentry)(callback), args);
932   addStmtToIRSB( bbOut, IRStmt_Dirty(di));
933
934   IRTemp need_sleep_i1 = newIRTemp(bbOut->tyenv, Ity_I1);
935   IRStmt* cmp_stmt = IRStmt_WrTmp(need_sleep_i1,
936       IRExpr_Binop(Iop_CmpNE32,
937           IRExpr_RdTmp(need_sleep),
938           IRExpr_Const(IRConst_U32(0))));
939   addStmtToIRSB(bbOut, cmp_stmt);
940
941   IRConst* exit_dst = layout->sizeof_IP == 8 ?
942       IRConst_U64(cur_pc) : IRConst_U32(cur_pc);
943   IRStmt* exit_stmt = IRStmt_Exit(IRExpr_RdTmp(need_sleep_i1),
944       Ijk_YieldNoRedir, exit_dst);
945   addStmtToIRSB(bbOut, exit_stmt);
946
947   hName = (HChar*)"OnTraceVerify2";
948   callback = (void*)OnTraceVerify2;
949   args = mkIRExprVec_1(mkIRExpr_HWord((HWord)trace_info));
950   di = unsafeIRDirty_0_N(1, hName, VG_(fnptr_to_fnentry)(callback), args);
951   addStmtToIRSB( bbOut, IRStmt_Dirty(di));
952}
953
954
955// ---------------------------- Instrumentation ---------------------------{{{1
956
957static IRTemp gen_Get_SP ( IRSB*           bbOut,
958                           VexGuestLayout* layout,
959                           Int             hWordTy_szB )
960{
961  IRExpr* sp_expr;
962  IRTemp  sp_temp;
963  IRType  sp_type;
964  /* This in effect forces the host and guest word sizes to be the
965     same. */
966  tl_assert(hWordTy_szB == layout->sizeof_SP);
967  sp_type = layout->sizeof_SP == 8 ? Ity_I64 : Ity_I32;
968  sp_expr = IRExpr_Get( layout->offset_SP, sp_type );
969  sp_temp = newIRTemp( bbOut->tyenv, sp_type );
970  addStmtToIRSB( bbOut, IRStmt_WrTmp( sp_temp, sp_expr ) );
971  return sp_temp;
972}
973
974static void ts_instrument_trace_entry(IRSB *bbOut, TraceInfo *trace_info) {
975   CHECK(trace_info);
976   HChar*   hName = (HChar*)"OnTrace";
977   void *callback = (void*)OnTrace;
978   IRExpr **args = mkIRExprVec_1(mkIRExpr_HWord((HWord)trace_info));
979   IRDirty* di = unsafeIRDirty_0_N( 1,
980                           hName,
981                           VG_(fnptr_to_fnentry)(callback),
982                           args);
983   addStmtToIRSB( bbOut, IRStmt_Dirty(di));
984}
985
986static void ts_instrument_final_jump (
987                                /*MOD*/IRSB* sbOut,
988                                IRExpr* next,
989                                IRJumpKind jumpkind,
990                                VexGuestLayout* layout,
991                                IRType gWordTy, IRType hWordTy ) {
992
993#ifndef VGP_arm_linux
994  // On non-ARM systems we instrument only function calls.
995  if (jumpkind != Ijk_Call) return;
996#else
997  if (jumpkind != Ijk_Call) {
998    // On an ARM system a non-call jump may possibly exit a function.
999    IRTemp sp_post_call_insn
1000        = gen_Get_SP( sbOut, layout, sizeofIRType(hWordTy) );
1001    IRExpr **args = mkIRExprVec_2(
1002        IRExpr_RdTmp(sp_post_call_insn),
1003        next
1004        );
1005    IRDirty* di = unsafeIRDirty_0_N(
1006        2/*regparms*/,
1007        (char*)"evh__delete_frame",
1008        VG_(fnptr_to_fnentry)((void*) &evh__delete_frame ),
1009        args );
1010    addStmtToIRSB( sbOut, IRStmt_Dirty(di) );
1011    return;  // do not fall through
1012  }
1013#endif
1014  {
1015    const char *fn_name = "evh__rtn_call_ignore_unknown";
1016    void *fn = (void*)&evh__rtn_call_ignore_unknown;
1017    // Instrument the call instruction to keep the shadow stack consistent.
1018    IRTemp sp_post_call_insn
1019        = gen_Get_SP( sbOut, layout, sizeofIRType(hWordTy) );
1020    IRExpr **args = mkIRExprVec_2(
1021        IRExpr_RdTmp(sp_post_call_insn),
1022        next
1023        );
1024    if (next->tag == Iex_Const) {
1025      IRConst *con = next->Iex.Const.con;
1026      uintptr_t target = 0;
1027      if (con->tag == Ico_U32 || con->tag == Ico_U64) {
1028        target = con->tag == Ico_U32 ? con->Ico.U32 : con->Ico.U64;
1029        bool ignore = ThreadSanitizerIgnoreAccessesBelowFunction(target);
1030        if (ignore) {
1031          fn_name = "evh__rtn_call_ignore_yes";
1032          fn = (void*)&evh__rtn_call_ignore_yes;
1033        } else {
1034          fn_name = "evh__rtn_call_ignore_no";
1035          fn = (void*)&evh__rtn_call_ignore_no;
1036        }
1037      }
1038    }
1039    IRDirty* di = unsafeIRDirty_0_N(
1040        2/*regparms*/,
1041        (char*)fn_name,
1042        VG_(fnptr_to_fnentry)(fn),
1043        args );
1044    addStmtToIRSB( sbOut, IRStmt_Dirty(di) );
1045  }
1046}
1047
1048// Generate exprs/stmts that make g_cur_tleb[idx] = x.
1049static void gen_store_to_tleb(IRSB *bbOut, IRTemp tleb_temp,
1050                              uintptr_t idx, IRExpr *x, IRType tyAddr) {
1051  CHECK(tleb_temp != IRTemp_INVALID);
1052  IRExpr *idx_expr  = mkIRExpr_HWord(idx * sizeof(uintptr_t));
1053  IRExpr *tleb_plus_idx_expr = IRExpr_Binop(
1054      sizeof(uintptr_t) == 8 ? Iop_Add64 : Iop_Add32,
1055      IRExpr_RdTmp(tleb_temp), idx_expr);
1056  IRTemp temp = newIRTemp(bbOut->tyenv, tyAddr);
1057  IRStmt *temp_stmt = IRStmt_WrTmp(temp, tleb_plus_idx_expr);
1058  IRStmt *store_stmt = IRStmt_Store(Iend_LE, IRExpr_RdTmp(temp), x);
1059
1060  addStmtToIRSB(bbOut, temp_stmt);
1061  addStmtToIRSB(bbOut, store_stmt);
1062}
1063
1064static void instrument_mem_access ( TraceInfo *trace_info,
1065                                    IRTemp tleb_temp,
1066                                    uintptr_t pc,
1067                                    size_t  *trace_idx,
1068                                    IRSB*   bbOut,
1069                                    IRStmt* st,
1070                                    IRExpr* addr,
1071                                    Int     szB,
1072                                    Bool    isStore,
1073                                    Bool    dtor_head,
1074                                    Int     hWordTy_szB ) {
1075  IRType   tyAddr   = Ity_INVALID;
1076
1077  tl_assert(isIRAtom(addr));
1078  tl_assert(hWordTy_szB == 4 || hWordTy_szB == 8);
1079
1080  tyAddr = typeOfIRExpr( bbOut->tyenv, addr );
1081  tl_assert(tyAddr == Ity_I32 || tyAddr == Ity_I64);
1082
1083  if (szB == 28) {
1084    // Ignore weird-sized accesses for now.
1085    // See http://code.google.com/p/data-race-test/issues/detail?id=36
1086    return;
1087  }
1088
1089  bool check_ident_store = false;
1090
1091  if (st->tag == Ist_Store && dtor_head &&
1092      typeOfIRExpr(bbOut->tyenv, st->Ist.Store.data) == tyAddr) {
1093    check_ident_store = true;
1094  }
1095
1096  size_t next_trace_idx = *trace_idx + 1;
1097
1098  if (next_trace_idx > kMaxMopsPerTrace) {
1099    if (next_trace_idx == kMaxMopsPerTrace) {
1100      Report("INFO: too many mops in trace: %p %s\n", pc,
1101             PcToRtnName(pc, true).c_str());
1102    }
1103    return;
1104  }
1105
1106  if (!trace_info) {
1107    // not instrumenting yet.
1108    *trace_idx = next_trace_idx;
1109    return;
1110  }
1111
1112  IRExpr *expr_to_store = NULL;
1113
1114  if (check_ident_store) {
1115    int is_64 = (sizeof(void*) == 8);
1116    // generate expression (*addr == new_value ? 0 : addr):
1117
1118    // old_value = *addr
1119    IRExpr *addr_load_expr = IRExpr_Load(Iend_LE, tyAddr, addr);
1120    IRTemp star_addr = newIRTemp(bbOut->tyenv, tyAddr);
1121    IRStmt *star_addr_stmt = IRStmt_WrTmp(star_addr, addr_load_expr);
1122    addStmtToIRSB(bbOut, star_addr_stmt);
1123    // sub = (old_value - new_value)
1124    IRTemp sub = newIRTemp(bbOut->tyenv, tyAddr);
1125    IRExpr *sub_expr = IRExpr_Binop((IROp)(Iop_Sub32 + is_64),
1126                                    IRExpr_RdTmp(star_addr),
1127                                    st->Ist.Store.data);
1128    IRStmt *sub_stmt = IRStmt_WrTmp(sub, sub_expr);
1129    addStmtToIRSB(bbOut, sub_stmt);
1130    // mask = (sub==0) ? 0 : -1
1131    IRTemp mask = newIRTemp(bbOut->tyenv, tyAddr);
1132    IRExpr *mask_expr = IRExpr_Unop((IROp)(Iop_CmpwNEZ32 + is_64),
1133                                    IRExpr_RdTmp(sub));
1134    IRStmt *mask_stmt = IRStmt_WrTmp(mask, mask_expr);
1135    addStmtToIRSB(bbOut, mask_stmt);
1136
1137    // res = mask & addr
1138    IRTemp and_tmp = newIRTemp(bbOut->tyenv, tyAddr);
1139    IRExpr *and_expr = IRExpr_Binop((IROp)(Iop_And32 + is_64),
1140                                    IRExpr_RdTmp(mask), addr);
1141    IRStmt *and_stmt = IRStmt_WrTmp(and_tmp, and_expr);
1142    addStmtToIRSB(bbOut, and_stmt);
1143
1144    expr_to_store = IRExpr_RdTmp(and_tmp);
1145  } else {
1146    expr_to_store = addr;
1147  }
1148
1149  // OnMop: g_cur_tleb[idx] = expr_to_store
1150  gen_store_to_tleb(bbOut, tleb_temp, *trace_idx, expr_to_store, tyAddr);
1151  // Create a mop {pc, size, is_write}
1152  MopInfo *mop = trace_info->GetMop(*trace_idx);
1153  new (mop) MopInfo(pc, szB, isStore, false);
1154  (*trace_idx)++;
1155
1156  CHECK(*trace_idx == next_trace_idx);
1157}
1158
1159void instrument_statement (IRStmt* st, IRSB* bbIn, IRSB* bbOut, IRType hWordTy,
1160                           TraceInfo *trace_info, IRTemp tleb_temp,
1161                           size_t *idx, uintptr_t *cur_pc, bool dtor_head) {
1162  switch (st->tag) {
1163    case Ist_NoOp:
1164    case Ist_AbiHint:
1165    case Ist_Put:
1166    case Ist_PutI:
1167    case Ist_Exit:
1168      /* None of these can contain any memory references. */
1169      break;
1170
1171    case Ist_IMark:
1172      *cur_pc = st->Ist.IMark.addr;
1173      break;
1174
1175    case Ist_MBE:
1176      //instrument_memory_bus_event( bbOut, st->Ist.MBE.event );
1177      switch (st->Ist.MBE.event) {
1178        case Imbe_Fence:
1179          break; /* not interesting */
1180        default:
1181          ppIRStmt(st);
1182          tl_assert(0);
1183      }
1184      break;
1185
1186    case Ist_CAS:
1187      break;
1188
1189    case Ist_Store:
1190      instrument_mem_access(trace_info, tleb_temp, *cur_pc, idx,
1191        bbOut, st,
1192        st->Ist.Store.addr,
1193        sizeofIRType(typeOfIRExpr(bbIn->tyenv, st->Ist.Store.data)),
1194        True/*isStore*/, dtor_head,
1195        sizeofIRType(hWordTy)
1196      );
1197      break;
1198
1199    case Ist_WrTmp: {
1200      IRExpr* data = st->Ist.WrTmp.data;
1201      if (data->tag == Iex_Load) {
1202        instrument_mem_access(trace_info, tleb_temp, *cur_pc, idx,
1203            bbOut, st,
1204            data->Iex.Load.addr,
1205            sizeofIRType(data->Iex.Load.ty),
1206            False/*!isStore*/, dtor_head,
1207            sizeofIRType(hWordTy)
1208            );
1209      }
1210      break;
1211    }
1212
1213    case Ist_LLSC: {
1214      /* Ignore load-linked's and store-conditionals. */
1215      break;
1216    }
1217
1218    case Ist_Dirty: {
1219      Int      dataSize;
1220      IRDirty* d = st->Ist.Dirty.details;
1221      if (d->mFx != Ifx_None) {
1222        /* This dirty helper accesses memory.  Collect the
1223           details. */
1224        tl_assert(d->mAddr != NULL);
1225        tl_assert(d->mSize != 0);
1226        dataSize = d->mSize;
1227        if (d->mFx == Ifx_Read || d->mFx == Ifx_Modify) {
1228          instrument_mem_access(trace_info, tleb_temp, *cur_pc, idx,
1229            bbOut, st, d->mAddr, dataSize, False/*!isStore*/, dtor_head,
1230            sizeofIRType(hWordTy)
1231          );
1232        }
1233        if (d->mFx == Ifx_Write || d->mFx == Ifx_Modify) {
1234          instrument_mem_access(trace_info, tleb_temp, *cur_pc, idx,
1235            bbOut, st, d->mAddr, dataSize, True/*isStore*/, dtor_head,
1236            sizeofIRType(hWordTy)
1237          );
1238        }
1239      } else {
1240        tl_assert(d->mAddr == NULL);
1241        tl_assert(d->mSize == 0);
1242      }
1243      break;
1244    }
1245
1246    default:
1247      ppIRStmt(st);
1248      tl_assert(0);
1249  } /* switch (st->tag) */
1250}
1251
1252static IRSB* ts_instrument ( VgCallbackClosure* closure,
1253                             IRSB* bbIn,
1254                             VexGuestLayout* layout,
1255                             VexGuestExtents* vge,
1256                             IRType gWordTy, IRType hWordTy) {
1257  if (G_flags->dry_run >= 2) return bbIn;
1258  Int   i;
1259  IRSB* bbOut;
1260  uintptr_t pc = closure->readdr;
1261
1262  char objname[kBuffSize];
1263  if (VG_(get_objname)(pc, (Char*)objname, kBuffSize)) {
1264    if (StringMatch("*/ld-2*", objname)) {
1265      // we want to completely ignore ld-so.
1266      return bbIn;
1267    }
1268  }
1269
1270  bool instrument_memory = ThreadSanitizerWantToInstrumentSblock(pc);
1271
1272  if (gWordTy != hWordTy) {
1273    /* We don't currently support this case. */
1274    VG_(tool_panic)((Char*)"host/guest word size mismatch");
1275  }
1276
1277  /* Set up BB */
1278  bbOut           = emptyIRSB();
1279  bbOut->tyenv    = deepCopyIRTypeEnv(bbIn->tyenv);
1280  bbOut->next     = deepCopyIRExpr(bbIn->next);
1281  bbOut->jumpkind = bbIn->jumpkind;
1282
1283  // Copy verbatim any IR preamble preceding the first IMark
1284  i = 0;
1285  while (i < bbIn->stmts_used && bbIn->stmts[i]->tag != Ist_IMark) {
1286    addStmtToIRSB( bbOut, bbIn->stmts[i] );
1287    i++;
1288  }
1289  int first = i;
1290  size_t n_mops = 0;
1291  uintptr_t cur_pc = pc;
1292
1293  IRTemp tleb_temp = IRTemp_INVALID;
1294
1295  bool dtor_head = false;
1296  char buff[1000];
1297  // get_fnname_w_offset returns demangled name with optional "+offset" prefix.
1298  // If we have "::~" and don't have "+", this SB is the first in this dtor.
1299  // We do all this stuff to avoid benign races on vptr:
1300  // http://code.google.com/p/data-race-test/wiki/PopularDataRaces#Data_race_on_vptr
1301  if (VG_(get_fnname_w_offset)(pc, (Char*)buff, sizeof(buff)) &&
1302      VG_(strstr)((Char*)buff, (Char*)"::~") != NULL) {
1303    char *offset_str = (char*)VG_(strchr)((Char*)buff, '+');
1304    if (offset_str == NULL) {
1305      // we are in the first BB of DTOR.
1306      dtor_head = true;
1307    } else {
1308      // We are not in the first BB.
1309      // On x86_64 (it seems like) the vfptr is updated only in the first BB.
1310      // On x86 with -fPIC, the vfptr may be updated in the second BB
1311      // (because -fPIC adds a call which splits the first BB).
1312      // See http://code.google.com/p/chromium/issues/detail?id=61199
1313#ifdef VGA_x86
1314      char *end;
1315      size_t offset = my_strtol(offset_str + 1, &end, 10);
1316      if (offset <= 32) {
1317        dtor_head = true;
1318      }
1319#endif
1320    }
1321  }
1322
1323
1324  uintptr_t instrument_pc = 0; // if != 0, instrument only the instruction at this address
1325  if (g_race_verifier_active) {
1326    uintptr_t min_pc = vge->base[0];
1327    uintptr_t max_pc = min_pc + vge->len[0];
1328    bool verify_trace = RaceVerifierGetAddresses(min_pc, max_pc, &instrument_pc);
1329    if (!verify_trace)
1330      instrument_memory = false;
1331  }
1332
1333  // count mops
1334  if (instrument_memory) {
1335    for (i = first; i < bbIn->stmts_used; i++) {
1336      IRStmt* st = bbIn->stmts[i];
1337      tl_assert(st);
1338      tl_assert(isFlatIRStmt(st));
1339      if (st->tag == Ist_IMark)
1340        cur_pc = st->Ist.IMark.addr;
1341      if (!instrument_pc || cur_pc == instrument_pc)
1342        instrument_statement(st, bbIn, bbOut, hWordTy,
1343            NULL, tleb_temp, &n_mops, &cur_pc, dtor_head);
1344    } /* iterate over bbIn->stmts */
1345  }
1346  TraceInfo *trace_info = NULL;
1347  if (n_mops > 0) {
1348    trace_info = TraceInfo::NewTraceInfo(n_mops, pc);
1349  }
1350  size_t n_mops_done = 0;
1351  bool need_to_insert_on_trace = n_mops > 0 || g_race_verifier_active;
1352  // instrument mops and copy the rest of BB to the new one.
1353  for (i = first; i < bbIn->stmts_used; i++) {
1354    IRStmt* st = bbIn->stmts[i];
1355    tl_assert(st);
1356    tl_assert(isFlatIRStmt(st));
1357    if (st->tag != Ist_IMark && need_to_insert_on_trace) {
1358      if (g_race_verifier_active) {
1359        ts_instrument_trace_entry_verify(bbOut, layout, trace_info,
1360            closure->readdr);
1361      } else {
1362        ts_instrument_trace_entry(bbOut, trace_info);
1363      }
1364      need_to_insert_on_trace = false;
1365      // Generate temp for *g_cur_tleb.
1366      IRType   tyAddr = sizeof(uintptr_t) == 8 ?  Ity_I64 : Ity_I32;
1367      IRExpr *tleb_ptr_expr = mkIRExpr_HWord((HWord)&g_cur_tleb);
1368      IRExpr *tleb_expr = IRExpr_Load(Iend_LE, tyAddr, tleb_ptr_expr);
1369      tleb_temp = newIRTemp(bbOut->tyenv, tyAddr);
1370      IRStmt *stmt = IRStmt_WrTmp(tleb_temp, tleb_expr);
1371      addStmtToIRSB(bbOut, stmt);
1372    }
1373    if (instrument_memory) {
1374      if (st->tag == Ist_IMark)
1375        cur_pc = st->Ist.IMark.addr;
1376      if (!instrument_pc || cur_pc == instrument_pc)
1377        instrument_statement(st, bbIn, bbOut, hWordTy,
1378            trace_info, tleb_temp, &n_mops_done, &cur_pc, dtor_head);
1379    }
1380    addStmtToIRSB( bbOut, st );
1381  } /* iterate over bbIn->stmts */
1382  CHECK(n_mops == n_mops_done);
1383  if (!g_race_verifier_active)
1384    ts_instrument_final_jump(bbOut, bbIn->next, bbIn->jumpkind, layout, gWordTy, hWordTy);
1385  return bbOut;
1386}
1387
1388extern "C"
1389void ts_pre_clo_init(void) {
1390  VG_(details_name)            ((Char*)"ThreadSanitizer");
1391  VG_(details_version)         ((Char*)NULL);
1392  VG_(details_description)     ((Char*)"a data race detector");
1393  VG_(details_copyright_author)(
1394      (Char*)"Copyright (C) 2008-2010, and GNU GPL'd, by Google Inc.");
1395  VG_(details_bug_reports_to)  ((Char*)"data-race-test@googlegroups.com");
1396
1397  VG_(basic_tool_funcs)        (ts_post_clo_init,
1398                                ts_instrument,
1399                                ts_fini);
1400
1401  VG_(needs_client_requests)     (ts_handle_client_request);
1402
1403  VG_(needs_command_line_options)(ts_process_cmd_line_option,
1404                                  ts_print_usage,
1405                                  ts_print_debug_usage);
1406   VG_(track_pre_thread_ll_create)( evh__pre_thread_ll_create );
1407   VG_(track_pre_thread_ll_exit)  ( evh__pre_thread_ll_exit );
1408
1409   if (!g_race_verifier_active) {
1410     VG_(track_workq_task_start)( evh__pre_workq_task_start );
1411     VG_(track_pre_thread_first_insn)( evh__pre_thread_first_insn );
1412   }
1413
1414   VG_(clo_vex_control).iropt_unroll_thresh = 0;
1415   VG_(clo_vex_control).guest_chase_thresh = 0;
1416
1417   VG_(track_pre_deliver_signal) (&SignalIn);
1418   VG_(track_post_deliver_signal)(&SignalOut);
1419
1420   VG_(track_start_client_code)( OnStartClientCode );
1421}
1422
1423VG_DETERMINE_INTERFACE_VERSION(ts_pre_clo_init)
1424
1425// {{{1 end
1426// vim:shiftwidth=2:softtabstop=2:expandtab
1427