1/*--------------------------------------------------------------------*/
2/*--- Client-space code for DRD.          drd_pthread_intercepts.c ---*/
3/*--------------------------------------------------------------------*/
4
5/*
6  This file is part of DRD, a thread error detector.
7
8  Copyright (C) 2006-2013 Bart Van Assche <bvanassche@acm.org>.
9
10  This program is free software; you can redistribute it and/or
11  modify it under the terms of the GNU General Public License as
12  published by the Free Software Foundation; either version 2 of the
13  License, or (at your option) any later version.
14
15  This program is distributed in the hope that it will be useful, but
16  WITHOUT ANY WARRANTY; without even the implied warranty of
17  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  General Public License for more details.
19
20  You should have received a copy of the GNU General Public License
21  along with this program; if not, write to the Free Software
22  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
23  02111-1307, USA.
24
25  The GNU General Public License is contained in the file COPYING.
26*/
27
28/* ---------------------------------------------------------------------
29   ALL THE CODE IN THIS FILE RUNS ON THE SIMULATED CPU.
30
31   These functions are not called directly - they're the targets of code
32   redirection or load notifications (see pub_core_redir.h for info).
33   They're named weirdly so that the intercept code can find them when the
34   shared object is initially loaded.
35
36   Note that this filename has the "drd_" prefix because it can appear
37   in stack traces, and the "drd_" makes it a little clearer that it
38   originates from Valgrind.
39   ------------------------------------------------------------------ */
40
41/*
42 * Define _GNU_SOURCE to make sure that pthread_spinlock_t is available when
43 * compiling with older glibc versions (2.3 or before).
44 */
45#ifndef _GNU_SOURCE
46#define _GNU_SOURCE
47#endif
48
49#include <assert.h>         /* assert() */
50#include <errno.h>
51#include <pthread.h>        /* pthread_mutex_t */
52#include <semaphore.h>      /* sem_t */
53#include <stdint.h>         /* uintptr_t */
54#include <stdio.h>          /* fprintf() */
55#include <stdlib.h>         /* malloc(), free() */
56#include <unistd.h>         /* confstr() */
57#include "config.h"         /* HAVE_PTHREAD_MUTEX_ADAPTIVE_NP etc. */
58#ifdef HAVE_USABLE_LINUX_FUTEX_H
59#include <asm/unistd.h>     /* __NR_futex */
60#include <linux/futex.h>    /* FUTEX_WAIT */
61#ifndef FUTEX_PRIVATE_FLAG
62#define FUTEX_PRIVATE_FLAG 0
63#endif
64#endif
65#include "drd_basics.h"     /* DRD_() */
66#include "drd_clientreq.h"
67#include "pub_tool_redir.h" /* VG_WRAP_FUNCTION_ZZ() */
68
69
70/*
71 * Notes regarding thread creation:
72 * - sg_init() runs on the context of the created thread and copies the vector
73 *   clock of the creator thread. This only works reliably if the creator
74 *   thread waits until this copy has been performed.
75 * - DRD_(thread_compute_minimum_vc)() does not take the vector clocks into
76 *   account that are involved in thread creation and for which the
77 *   corresponding thread has not yet been created. So not waiting until the
78 *   created thread has been started would make it possible that segments get
79 *   discarded that should not yet be discarded. Or: some data races are not
80 *   detected.
81 */
82
83/**
84 * Macro for generating a Valgrind interception function.
85 * @param[in] ret_ty Return type of the function to be generated.
86 * @param[in] zf Z-encoded name of the interception function.
87 * @param[in] implf Name of the function that implements the intercept.
88 * @param[in] arg_decl Argument declaration list enclosed in parentheses.
89 * @param[in] argl Argument list enclosed in parentheses.
90 */
91#ifdef VGO_darwin
92static int never_true;
93#define PTH_FUNC(ret_ty, zf, implf, argl_decl, argl)                    \
94   ret_ty VG_WRAP_FUNCTION_ZZ(VG_Z_LIBPTHREAD_SONAME,zf) argl_decl;     \
95   ret_ty VG_WRAP_FUNCTION_ZZ(VG_Z_LIBPTHREAD_SONAME,zf) argl_decl      \
96   {									\
97      ret_ty pth_func_result = implf argl;				\
98      /* Apparently inserting a function call in wrapper functions */   \
99      /* is sufficient to avoid misaligned stack errors.           */	\
100      if (never_true)							\
101	 fflush(stdout);						\
102      return pth_func_result;						\
103   }
104#else
105#define PTH_FUNC(ret_ty, zf, implf, argl_decl, argl)                    \
106   ret_ty VG_WRAP_FUNCTION_ZZ(VG_Z_LIBPTHREAD_SONAME,zf) argl_decl;     \
107   ret_ty VG_WRAP_FUNCTION_ZZ(VG_Z_LIBPTHREAD_SONAME,zf) argl_decl      \
108   { return implf argl; }
109#endif
110
111/**
112 * Macro for generating three Valgrind interception functions: one with the
113 * Z-encoded name zf, one with ZAZa ("@*") appended to the name zf and one
114 * with ZDZa ("$*") appended to the name zf. The second generated interception
115 * function will intercept versioned symbols on Linux, and the third will
116 * intercept versioned symbols on Darwin.
117 */
118#define PTH_FUNCS(ret_ty, zf, implf, argl_decl, argl)           \
119   PTH_FUNC(ret_ty, zf, implf, argl_decl, argl);                \
120   PTH_FUNC(ret_ty, zf ## ZAZa, implf, argl_decl, argl);        \
121   PTH_FUNC(ret_ty, zf ## ZDZa, implf, argl_decl, argl);
122
123/*
124 * Not inlining one of the intercept functions will cause the regression
125 * tests to fail because this would cause an additional stackfram to appear
126 * in the output. The __always_inline macro guarantees that inlining will
127 * happen, even when compiling with optimization disabled.
128 */
129#undef __always_inline /* since already defined in <cdefs.h> */
130#if __GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 2
131#define __always_inline __inline__ __attribute__((always_inline))
132#else
133#define __always_inline __inline__
134#endif
135
136/* Local data structures. */
137
138typedef struct {
139   pthread_mutex_t mutex;
140   int counter;
141   int waiters;
142} DrdSema;
143
144typedef struct
145{
146   void* (*start)(void*);
147   void* arg;
148   int   detachstate;
149   DrdSema* wrapper_started;
150} DrdPosixThreadArgs;
151
152
153/* Local function declarations. */
154
155static void DRD_(init)(void) __attribute__((constructor));
156static void DRD_(check_threading_library)(void);
157static void DRD_(set_main_thread_state)(void);
158static void DRD_(sema_init)(DrdSema* sema);
159static void DRD_(sema_destroy)(DrdSema* sema);
160static void DRD_(sema_down)(DrdSema* sema);
161static void DRD_(sema_up)(DrdSema* sema);
162
163
164/* Function definitions. */
165
166/**
167 * Shared library initialization function. The function init() is called after
168 * dlopen() has loaded the shared library with DRD client intercepts because
169 * the constructor attribute was specified in the declaration of this function.
170 * Note: do specify the -nostdlib option to gcc when linking this code into a
171 * shared library because doing so would cancel the effect of the constructor
172 * attribute ! Using the gcc option -nodefaultlibs is fine because this last
173 * option preserves the shared library initialization code that calls
174 * constructor and destructor functions.
175 */
176static void DRD_(init)(void)
177{
178   DRD_(check_threading_library)();
179   DRD_(set_main_thread_state)();
180}
181
182static void DRD_(sema_init)(DrdSema* sema)
183{
184   DRD_IGNORE_VAR(sema->counter);
185   pthread_mutex_init(&sema->mutex, NULL);
186   sema->counter = 0;
187   sema->waiters = 0;
188}
189
190static void DRD_(sema_destroy)(DrdSema* sema)
191{
192   pthread_mutex_destroy(&sema->mutex);
193}
194
195static void DRD_(sema_down)(DrdSema* sema)
196{
197   int res = ENOSYS;
198
199   pthread_mutex_lock(&sema->mutex);
200   if (sema->counter == 0) {
201      sema->waiters++;
202      while (sema->counter == 0) {
203         pthread_mutex_unlock(&sema->mutex);
204#ifdef HAVE_USABLE_LINUX_FUTEX_H
205         if (syscall(__NR_futex, (UWord)&sema->counter,
206                     FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0) == 0)
207            res = 0;
208         else
209            res = errno;
210#endif
211         /*
212          * Invoke sched_yield() on non-Linux systems, if the futex syscall has
213          * not been invoked or if this code has been built on a Linux system
214          * where __NR_futex is defined and is run on a Linux system that does
215          * not support the futex syscall.
216          */
217         if (res != 0 && res != EWOULDBLOCK)
218            sched_yield();
219         pthread_mutex_lock(&sema->mutex);
220      }
221      sema->waiters--;
222   }
223   sema->counter--;
224   pthread_mutex_unlock(&sema->mutex);
225}
226
227static void DRD_(sema_up)(DrdSema* sema)
228{
229   pthread_mutex_lock(&sema->mutex);
230   sema->counter++;
231#ifdef HAVE_USABLE_LINUX_FUTEX_H
232   if (sema->waiters > 0)
233      syscall(__NR_futex, (UWord)&sema->counter,
234              FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1);
235#endif
236   pthread_mutex_unlock(&sema->mutex);
237}
238
239/**
240 * POSIX threads and DRD each have their own mutex type identification.
241 * Convert POSIX threads' mutex type to DRD's mutex type. In the code below
242 * if-statements are used to test the value of 'kind' instead of a switch
243 * statement because some of the PTHREAD_MUTEX_ macro's may have the same
244 * value.
245 */
246static MutexT DRD_(pthread_to_drd_mutex_type)(int kind)
247{
248   /*
249    * See also PTHREAD_MUTEX_KIND_MASK_NP in glibc source file
250    * <nptl/pthreadP.h>.
251    */
252   kind &= PTHREAD_MUTEX_RECURSIVE | PTHREAD_MUTEX_ERRORCHECK |
253      PTHREAD_MUTEX_NORMAL | PTHREAD_MUTEX_DEFAULT;
254
255   if (kind == PTHREAD_MUTEX_RECURSIVE)
256      return mutex_type_recursive_mutex;
257   else if (kind == PTHREAD_MUTEX_ERRORCHECK)
258      return mutex_type_errorcheck_mutex;
259   else if (kind == PTHREAD_MUTEX_NORMAL)
260      return mutex_type_default_mutex;
261   else if (kind == PTHREAD_MUTEX_DEFAULT)
262      return mutex_type_default_mutex;
263#if defined(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
264   else if (kind == PTHREAD_MUTEX_ADAPTIVE_NP)
265      return mutex_type_default_mutex;
266#endif
267   else
268      return mutex_type_invalid_mutex;
269}
270
271#define IS_ALIGNED(p) (((uintptr_t)(p) & (sizeof(*(p)) - 1)) == 0)
272
273/**
274 * Read the mutex type stored in the client memory used for the mutex
275 * implementation.
276 *
277 * @note This function depends on the implementation of the POSIX threads
278 *   library -- the POSIX standard does not define the name of the member in
279 *   which the mutex type is stored.
280 * @note The function mutex_type() has been declared inline in order
281 *   to avoid that it shows up in call stacks (drd/tests/...exp* files).
282 * @note glibc stores the mutex type in the lowest two bits, and uses the
283 *   higher bits for flags like PTHREAD_MUTEXATTR_FLAG_ROBUST and
284 *   PTHREAD_MUTEXATTR_FLAG_PSHARED.
285 */
286static __always_inline MutexT DRD_(mutex_type)(pthread_mutex_t* mutex)
287{
288#if defined(HAVE_PTHREAD_MUTEX_T__M_KIND)
289   /* glibc + LinuxThreads. */
290   if (IS_ALIGNED(&mutex->__m_kind))
291   {
292      const int kind = mutex->__m_kind & 3;
293      return DRD_(pthread_to_drd_mutex_type)(kind);
294   }
295#elif defined(HAVE_PTHREAD_MUTEX_T__DATA__KIND)
296   /* glibc + NPTL. */
297   if (IS_ALIGNED(&mutex->__data.__kind))
298   {
299      const int kind = mutex->__data.__kind & 3;
300      return DRD_(pthread_to_drd_mutex_type)(kind);
301   }
302#else
303   /*
304    * Another POSIX threads implementation. The mutex type won't be printed
305    * when enabling --trace-mutex=yes.
306    */
307#endif
308   return mutex_type_unknown;
309}
310
311/**
312 * Tell DRD whether 'tid' is a joinable thread or a detached thread.
313 */
314static void DRD_(set_joinable)(const pthread_t tid, const int joinable)
315{
316   assert(joinable == 0 || joinable == 1);
317   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__SET_JOINABLE,
318                                   tid, joinable, 0, 0, 0);
319}
320
321/** Tell DRD that the calling thread is about to enter pthread_create(). */
322static __always_inline void DRD_(entering_pthread_create)(void)
323{
324   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__ENTERING_PTHREAD_CREATE,
325                                   0, 0, 0, 0, 0);
326}
327
328/** Tell DRD that the calling thread has left pthread_create(). */
329static __always_inline void DRD_(left_pthread_create)(void)
330{
331   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LEFT_PTHREAD_CREATE,
332                                   0, 0, 0, 0, 0);
333}
334
335/**
336 * Entry point for newly created threads. This function is called from the
337 * thread created by pthread_create().
338 */
339static void* DRD_(thread_wrapper)(void* arg)
340{
341   DrdPosixThreadArgs* arg_ptr;
342   DrdPosixThreadArgs arg_copy;
343
344   arg_ptr = (DrdPosixThreadArgs*)arg;
345   arg_copy = *arg_ptr;
346
347   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__SET_PTHREADID,
348                                   pthread_self(), 0, 0, 0, 0);
349
350   DRD_(set_joinable)(pthread_self(),
351                      arg_copy.detachstate == PTHREAD_CREATE_JOINABLE);
352
353   /*
354    * Only set 'wrapper_started' after VG_USERREQ__SET_PTHREADID and
355    * DRD_(set_joinable)() have been invoked to avoid a race with
356    * a pthread_detach() invocation for this thread from another thread.
357    */
358   DRD_(sema_up)(arg_copy.wrapper_started);
359
360   return (arg_copy.start)(arg_copy.arg);
361}
362
363/**
364 * Return 1 if the LinuxThreads implementation of POSIX Threads has been
365 * detected, and 0 otherwise.
366 *
367 * @see For more information about the confstr() function, see also
368 * http://www.opengroup.org/onlinepubs/009695399/functions/confstr.html
369 */
370static int DRD_(detected_linuxthreads)(void)
371{
372#if defined(linux)
373#if defined(_CS_GNU_LIBPTHREAD_VERSION)
374   /* Linux with a recent glibc. */
375   HChar buffer[256];
376   unsigned len;
377   len = confstr(_CS_GNU_LIBPTHREAD_VERSION, buffer, sizeof(buffer));
378   assert(len <= sizeof(buffer));
379   return len > 0 && buffer[0] == 'l';
380#else
381   /* Linux without _CS_GNU_LIBPTHREAD_VERSION: most likely LinuxThreads. */
382   return 1;
383#endif
384#else
385   /* Another OS than Linux, hence no LinuxThreads. */
386   return 0;
387#endif
388}
389
390/**
391 * Stop and print an error message in case a non-supported threading
392 * library implementation (LinuxThreads) has been detected.
393 */
394static void DRD_(check_threading_library)(void)
395{
396   if (DRD_(detected_linuxthreads)())
397   {
398      if (getenv("LD_ASSUME_KERNEL"))
399      {
400         fprintf(stderr,
401"Detected the LinuxThreads threading library. Sorry, but DRD only supports\n"
402"the newer NPTL (Native POSIX Threads Library). Please try to rerun DRD\n"
403"after having unset the environment variable LD_ASSUME_KERNEL. Giving up.\n"
404);
405      }
406      else
407      {
408         fprintf(stderr,
409"Detected the LinuxThreads threading library. Sorry, but DRD only supports\n"
410"the newer NPTL (Native POSIX Threads Library). Please try to rerun DRD\n"
411"after having upgraded to a newer version of your Linux distribution.\n"
412"Giving up.\n"
413);
414      }
415      abort();
416   }
417}
418
419/**
420 * The main thread is the only thread not created by pthread_create().
421 * Update DRD's state information about the main thread.
422 */
423static void DRD_(set_main_thread_state)(void)
424{
425   // Make sure that DRD knows about the main thread's POSIX thread ID.
426   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__SET_PTHREADID,
427                                   pthread_self(), 0, 0, 0, 0);
428}
429
430/*
431 * Note: as of today there exist three different versions of pthread_create
432 * in Linux:
433 * - pthread_create@GLIBC_2.0
434 * - pthread_create@@GLIBC_2.1
435 * - pthread_create@@GLIBC_2.2.5
436 * As an example, in libpthread-2.3.4 both pthread_create@GLIBC_2.0 and
437 * pthread_create@@GLIBC_2.1 are defined, while in libpthread-2.9 all three
438 * versions have been implemented. In any glibc version where more than one
439 * pthread_create function has been implemented, older versions call the
440 * newer versions. Or: the pthread_create* wrapper defined below can be
441 * called recursively. Any code in this wrapper should take this in account.
442 * As an example, it is not safe to invoke the DRD_STOP_RECORDING
443 * / DRD_START_RECORDING client requests from the pthread_create wrapper.
444 * See also the implementation of pthread_create@GLIBC_2.0 in
445 * glibc-2.9/nptl/pthread_create.c.
446 */
447
448static __always_inline
449int pthread_create_intercept(pthread_t* thread, const pthread_attr_t* attr,
450                             void* (*start)(void*), void* arg)
451{
452   int    ret;
453   OrigFn fn;
454   DrdSema wrapper_started;
455   DrdPosixThreadArgs thread_args;
456
457   VALGRIND_GET_ORIG_FN(fn);
458
459   DRD_(sema_init)(&wrapper_started);
460   thread_args.start           = start;
461   thread_args.arg             = arg;
462   thread_args.wrapper_started = &wrapper_started;
463   /*
464    * Find out whether the thread will be started as a joinable thread
465    * or as a detached thread. If no thread attributes have been specified,
466    * this means that the new thread will be started as a joinable thread.
467    */
468   thread_args.detachstate = PTHREAD_CREATE_JOINABLE;
469   if (attr)
470   {
471      if (pthread_attr_getdetachstate(attr, &thread_args.detachstate) != 0)
472         assert(0);
473   }
474   assert(thread_args.detachstate == PTHREAD_CREATE_JOINABLE
475          || thread_args.detachstate == PTHREAD_CREATE_DETACHED);
476
477   DRD_(entering_pthread_create)();
478   CALL_FN_W_WWWW(ret, fn, thread, attr, DRD_(thread_wrapper), &thread_args);
479   DRD_(left_pthread_create)();
480
481   if (ret == 0) {
482      /* Wait until the thread wrapper started. */
483      DRD_(sema_down)(&wrapper_started);
484   }
485
486   DRD_(sema_destroy)(&wrapper_started);
487
488   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DRD_START_NEW_SEGMENT,
489                                   pthread_self(), 0, 0, 0, 0);
490
491   return ret;
492}
493
494PTH_FUNCS(int, pthreadZucreate, pthread_create_intercept,
495          (pthread_t *thread, const pthread_attr_t *attr,
496           void *(*start) (void *), void *arg),
497          (thread, attr, start, arg));
498
499static __always_inline
500int pthread_join_intercept(pthread_t pt_joinee, void **thread_return)
501{
502   int      ret;
503   OrigFn   fn;
504
505   VALGRIND_GET_ORIG_FN(fn);
506   /*
507    * Avoid that the sys_futex(td->tid) call invoked by the NPTL pthread_join()
508    * implementation triggers a (false positive) race report.
509    */
510   ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
511   CALL_FN_W_WW(ret, fn, pt_joinee, thread_return);
512   if (ret == 0)
513   {
514      VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_THREAD_JOIN,
515                                      pt_joinee, 0, 0, 0, 0);
516   }
517   ANNOTATE_IGNORE_READS_AND_WRITES_END();
518   return ret;
519}
520
521PTH_FUNCS(int, pthreadZujoin, pthread_join_intercept,
522          (pthread_t pt_joinee, void **thread_return),
523          (pt_joinee, thread_return));
524
525static __always_inline
526int pthread_detach_intercept(pthread_t pt_thread)
527{
528   int ret;
529   OrigFn fn;
530
531   VALGRIND_GET_ORIG_FN(fn);
532   CALL_FN_W_W(ret, fn, pt_thread);
533   DRD_(set_joinable)(pt_thread, 0);
534
535   return ret;
536}
537
538PTH_FUNCS(int, pthreadZudetach, pthread_detach_intercept,
539          (pthread_t thread), (thread));
540
541// NOTE: be careful to intercept only pthread_cancel() and not
542// pthread_cancel_init() on Linux.
543
544static __always_inline
545int pthread_cancel_intercept(pthread_t pt_thread)
546{
547   int ret;
548   OrigFn fn;
549   VALGRIND_GET_ORIG_FN(fn);
550   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_THREAD_CANCEL,
551                                   pt_thread, 0, 0, 0, 0);
552   CALL_FN_W_W(ret, fn, pt_thread);
553   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_THREAD_CANCEL,
554                                   pt_thread, ret==0, 0, 0, 0);
555   return ret;
556}
557
558PTH_FUNCS(int, pthreadZucancel, pthread_cancel_intercept,
559          (pthread_t thread), (thread))
560
561static __always_inline
562int pthread_once_intercept(pthread_once_t *once_control,
563                           void (*init_routine)(void))
564{
565   int ret;
566   OrigFn fn;
567   VALGRIND_GET_ORIG_FN(fn);
568   /*
569    * Ignore any data races triggered by the implementation of pthread_once().
570    * Necessary for Darwin. This is not necessary for Linux but doesn't have
571    * any known adverse effects.
572    */
573   DRD_IGNORE_VAR(*once_control);
574   ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
575   CALL_FN_W_WW(ret, fn, once_control, init_routine);
576   ANNOTATE_IGNORE_READS_AND_WRITES_END();
577   DRD_STOP_IGNORING_VAR(*once_control);
578   return ret;
579}
580
581PTH_FUNCS(int, pthreadZuonce, pthread_once_intercept,
582          (pthread_once_t *once_control, void (*init_routine)(void)),
583          (once_control, init_routine));
584
585static __always_inline
586int pthread_mutex_init_intercept(pthread_mutex_t *mutex,
587                                 const pthread_mutexattr_t* attr)
588{
589   int ret;
590   OrigFn fn;
591   int mt;
592   VALGRIND_GET_ORIG_FN(fn);
593   mt = PTHREAD_MUTEX_DEFAULT;
594   if (attr)
595      pthread_mutexattr_gettype(attr, &mt);
596   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_INIT,
597                                   mutex, DRD_(pthread_to_drd_mutex_type)(mt),
598                                   0, 0, 0);
599   CALL_FN_W_WW(ret, fn, mutex, attr);
600   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_INIT,
601                                   mutex, 0, 0, 0, 0);
602   return ret;
603}
604
605PTH_FUNCS(int, pthreadZumutexZuinit, pthread_mutex_init_intercept,
606          (pthread_mutex_t *mutex, const pthread_mutexattr_t* attr),
607          (mutex, attr));
608
609static __always_inline
610int pthread_mutex_destroy_intercept(pthread_mutex_t* mutex)
611{
612   int ret;
613   OrigFn fn;
614   VALGRIND_GET_ORIG_FN(fn);
615   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_DESTROY,
616                                   mutex, 0, 0, 0, 0);
617   CALL_FN_W_W(ret, fn, mutex);
618   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_DESTROY,
619                                   mutex, DRD_(mutex_type)(mutex), 0, 0, 0);
620   return ret;
621}
622
623PTH_FUNCS(int, pthreadZumutexZudestroy, pthread_mutex_destroy_intercept,
624          (pthread_mutex_t *mutex), (mutex));
625
626static __always_inline
627int pthread_mutex_lock_intercept(pthread_mutex_t* mutex)
628{
629   int   ret;
630   OrigFn fn;
631   VALGRIND_GET_ORIG_FN(fn);
632   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_LOCK,
633                                   mutex, DRD_(mutex_type)(mutex), 0, 0, 0);
634   CALL_FN_W_W(ret, fn, mutex);
635   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_LOCK,
636                                   mutex, ret == 0, 0, 0, 0);
637   return ret;
638}
639
640PTH_FUNCS(int, pthreadZumutexZulock, pthread_mutex_lock_intercept,
641          (pthread_mutex_t *mutex), (mutex));
642
643static __always_inline
644int pthread_mutex_trylock_intercept(pthread_mutex_t* mutex)
645{
646   int   ret;
647   OrigFn fn;
648   VALGRIND_GET_ORIG_FN(fn);
649   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_LOCK,
650                                   mutex, DRD_(mutex_type)(mutex), 1, 0, 0);
651   CALL_FN_W_W(ret, fn, mutex);
652   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_LOCK,
653                                   mutex, ret == 0, 0, 0, 0);
654   return ret;
655}
656
657PTH_FUNCS(int, pthreadZumutexZutrylock, pthread_mutex_trylock_intercept,
658          (pthread_mutex_t *mutex), (mutex));
659
660static __always_inline
661int pthread_mutex_timedlock_intercept(pthread_mutex_t *mutex,
662                                      const struct timespec *abs_timeout)
663{
664   int   ret;
665   OrigFn fn;
666   VALGRIND_GET_ORIG_FN(fn);
667   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_LOCK,
668                                   mutex, DRD_(mutex_type)(mutex), 0, 0, 0);
669   CALL_FN_W_WW(ret, fn, mutex, abs_timeout);
670   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_LOCK,
671                                   mutex, ret == 0, 0, 0, 0);
672   return ret;
673}
674
675PTH_FUNCS(int, pthreadZumutexZutimedlock, pthread_mutex_timedlock_intercept,
676          (pthread_mutex_t *mutex, const struct timespec *abs_timeout),
677          (mutex, abs_timeout));
678
679static __always_inline
680int pthread_mutex_unlock_intercept(pthread_mutex_t *mutex)
681{
682   int ret;
683   OrigFn fn;
684   VALGRIND_GET_ORIG_FN(fn);
685   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_UNLOCK,
686                                   mutex, DRD_(mutex_type)(mutex), 0, 0, 0);
687   CALL_FN_W_W(ret, fn, mutex);
688   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_UNLOCK,
689                                   mutex, 0, 0, 0, 0);
690   return ret;
691}
692
693PTH_FUNCS(int, pthreadZumutexZuunlock, pthread_mutex_unlock_intercept,
694          (pthread_mutex_t *mutex), (mutex));
695
696static __always_inline
697int pthread_cond_init_intercept(pthread_cond_t* cond,
698                                const pthread_condattr_t* attr)
699{
700   int ret;
701   OrigFn fn;
702   VALGRIND_GET_ORIG_FN(fn);
703   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_INIT,
704                                   cond, 0, 0, 0, 0);
705   CALL_FN_W_WW(ret, fn, cond, attr);
706   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_INIT,
707                                   cond, 0, 0, 0, 0);
708   return ret;
709}
710
711PTH_FUNCS(int, pthreadZucondZuinit, pthread_cond_init_intercept,
712          (pthread_cond_t* cond, const pthread_condattr_t* attr),
713          (cond, attr));
714
715static __always_inline
716int pthread_cond_destroy_intercept(pthread_cond_t* cond)
717{
718   int ret;
719   OrigFn fn;
720   VALGRIND_GET_ORIG_FN(fn);
721   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_DESTROY,
722                                   cond, 0, 0, 0, 0);
723   CALL_FN_W_W(ret, fn, cond);
724   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_DESTROY,
725                                   cond, ret==0, 0, 0, 0);
726   return ret;
727}
728
729PTH_FUNCS(int, pthreadZucondZudestroy, pthread_cond_destroy_intercept,
730          (pthread_cond_t* cond), (cond));
731
732static __always_inline
733int pthread_cond_wait_intercept(pthread_cond_t *cond, pthread_mutex_t *mutex)
734{
735   int   ret;
736   OrigFn fn;
737   VALGRIND_GET_ORIG_FN(fn);
738   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_WAIT,
739                                   cond, mutex, DRD_(mutex_type)(mutex), 0, 0);
740   CALL_FN_W_WW(ret, fn, cond, mutex);
741   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_WAIT,
742                                   cond, mutex, 1, 0, 0);
743   return ret;
744}
745
746PTH_FUNCS(int, pthreadZucondZuwait, pthread_cond_wait_intercept,
747          (pthread_cond_t *cond, pthread_mutex_t *mutex),
748          (cond, mutex));
749
750static __always_inline
751int pthread_cond_timedwait_intercept(pthread_cond_t *cond,
752                                     pthread_mutex_t *mutex,
753                                     const struct timespec* abstime)
754{
755   int   ret;
756   OrigFn fn;
757   VALGRIND_GET_ORIG_FN(fn);
758   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_WAIT,
759                                   cond, mutex, DRD_(mutex_type)(mutex), 0, 0);
760   CALL_FN_W_WWW(ret, fn, cond, mutex, abstime);
761   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_WAIT,
762                                   cond, mutex, 1, 0, 0);
763   return ret;
764}
765
766PTH_FUNCS(int, pthreadZucondZutimedwait, pthread_cond_timedwait_intercept,
767          (pthread_cond_t *cond, pthread_mutex_t *mutex,
768           const struct timespec* abstime),
769          (cond, mutex, abstime));
770
771// NOTE: be careful to intercept only pthread_cond_signal() and not Darwin's
772// pthread_cond_signal_thread_np(). The former accepts one argument; the latter
773// two. Intercepting all pthread_cond_signal* functions will cause only one
774// argument to be passed to pthread_cond_signal_np() and hence will cause this
775// last function to crash.
776
777static __always_inline
778int pthread_cond_signal_intercept(pthread_cond_t* cond)
779{
780   int   ret;
781   OrigFn fn;
782   VALGRIND_GET_ORIG_FN(fn);
783   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_SIGNAL,
784                                   cond, 0, 0, 0, 0);
785   CALL_FN_W_W(ret, fn, cond);
786   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_SIGNAL,
787                                   cond, 0, 0, 0, 0);
788   return ret;
789}
790
791PTH_FUNCS(int, pthreadZucondZusignal, pthread_cond_signal_intercept,
792          (pthread_cond_t* cond), (cond));
793
794static __always_inline
795int pthread_cond_broadcast_intercept(pthread_cond_t* cond)
796{
797   int   ret;
798   OrigFn fn;
799   VALGRIND_GET_ORIG_FN(fn);
800   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_COND_BROADCAST,
801                                   cond, 0, 0, 0, 0);
802   CALL_FN_W_W(ret, fn, cond);
803   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_COND_BROADCAST,
804                                   cond, 0, 0, 0, 0);
805   return ret;
806}
807
808PTH_FUNCS(int, pthreadZucondZubroadcast, pthread_cond_broadcast_intercept,
809          (pthread_cond_t* cond), (cond));
810
811#if defined(HAVE_PTHREAD_SPIN_LOCK) \
812    && !defined(DISABLE_PTHREAD_SPINLOCK_INTERCEPT)
813static __always_inline
814int pthread_spin_init_intercept(pthread_spinlock_t *spinlock, int pshared)
815{
816   int ret;
817   OrigFn fn;
818   VALGRIND_GET_ORIG_FN(fn);
819   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SPIN_INIT_OR_UNLOCK,
820                                   spinlock, 0, 0, 0, 0);
821   CALL_FN_W_WW(ret, fn, spinlock, pshared);
822   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SPIN_INIT_OR_UNLOCK,
823                                   spinlock, 0, 0, 0, 0);
824   return ret;
825}
826
827PTH_FUNCS(int, pthreadZuspinZuinit, pthread_spin_init_intercept,
828          (pthread_spinlock_t *spinlock, int pshared), (spinlock, pshared));
829
830static __always_inline
831int pthread_spin_destroy_intercept(pthread_spinlock_t *spinlock)
832{
833   int ret;
834   OrigFn fn;
835   VALGRIND_GET_ORIG_FN(fn);
836   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_DESTROY,
837                                   spinlock, 0, 0, 0, 0);
838   CALL_FN_W_W(ret, fn, spinlock);
839   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_DESTROY,
840                                   spinlock, mutex_type_spinlock, 0, 0, 0);
841   return ret;
842}
843
844PTH_FUNCS(int, pthreadZuspinZudestroy, pthread_spin_destroy_intercept,
845          (pthread_spinlock_t *spinlock), (spinlock));
846
847static __always_inline
848int pthread_spin_lock_intercept(pthread_spinlock_t *spinlock)
849{
850   int   ret;
851   OrigFn fn;
852   VALGRIND_GET_ORIG_FN(fn);
853   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_LOCK,
854                                   spinlock, mutex_type_spinlock, 0, 0, 0);
855   CALL_FN_W_W(ret, fn, spinlock);
856   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_LOCK,
857                                   spinlock, ret == 0, 0, 0, 0);
858   return ret;
859}
860
861PTH_FUNCS(int, pthreadZuspinZulock, pthread_spin_lock_intercept,
862          (pthread_spinlock_t *spinlock), (spinlock));
863
864static __always_inline
865int pthread_spin_trylock_intercept(pthread_spinlock_t *spinlock)
866{
867   int   ret;
868   OrigFn fn;
869   VALGRIND_GET_ORIG_FN(fn);
870   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_MUTEX_LOCK,
871                                   spinlock, mutex_type_spinlock, 0, 0, 0);
872   CALL_FN_W_W(ret, fn, spinlock);
873   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_MUTEX_LOCK,
874                                   spinlock, ret == 0, 0, 0, 0);
875   return ret;
876}
877
878PTH_FUNCS(int, pthreadZuspinZutrylock, pthread_spin_trylock_intercept,
879          (pthread_spinlock_t *spinlock), (spinlock));
880
881static __always_inline
882int pthread_spin_unlock_intercept(pthread_spinlock_t *spinlock)
883{
884   int   ret;
885   OrigFn fn;
886   VALGRIND_GET_ORIG_FN(fn);
887   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SPIN_INIT_OR_UNLOCK,
888                                   spinlock, mutex_type_spinlock, 0, 0, 0);
889   CALL_FN_W_W(ret, fn, spinlock);
890   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SPIN_INIT_OR_UNLOCK,
891                                   spinlock, 0, 0, 0, 0);
892   return ret;
893}
894
895PTH_FUNCS(int, pthreadZuspinZuunlock, pthread_spin_unlock_intercept,
896          (pthread_spinlock_t *spinlock), (spinlock));
897#endif   // HAVE_PTHREAD_SPIN_LOCK
898
899
900#if defined(HAVE_PTHREAD_BARRIER_INIT)
901static __always_inline
902int pthread_barrier_init_intercept(pthread_barrier_t* barrier,
903                                   const pthread_barrierattr_t* attr,
904                                   unsigned count)
905{
906   int   ret;
907   OrigFn fn;
908   VALGRIND_GET_ORIG_FN(fn);
909   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_BARRIER_INIT,
910                                   barrier, pthread_barrier, count, 0, 0);
911   CALL_FN_W_WWW(ret, fn, barrier, attr, count);
912   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_BARRIER_INIT,
913                                   barrier, pthread_barrier, 0, 0, 0);
914   return ret;
915}
916
917PTH_FUNCS(int, pthreadZubarrierZuinit, pthread_barrier_init_intercept,
918          (pthread_barrier_t* barrier, const pthread_barrierattr_t* attr,
919           unsigned count), (barrier, attr, count));
920
921static __always_inline
922int pthread_barrier_destroy_intercept(pthread_barrier_t* barrier)
923{
924   int   ret;
925   OrigFn fn;
926   VALGRIND_GET_ORIG_FN(fn);
927   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_BARRIER_DESTROY,
928                                   barrier, pthread_barrier, 0, 0, 0);
929   CALL_FN_W_W(ret, fn, barrier);
930   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_BARRIER_DESTROY,
931                                   barrier, pthread_barrier, 0, 0, 0);
932   return ret;
933}
934
935PTH_FUNCS(int, pthreadZubarrierZudestroy, pthread_barrier_destroy_intercept,
936          (pthread_barrier_t* barrier), (barrier));
937
938static __always_inline
939int pthread_barrier_wait_intercept(pthread_barrier_t* barrier)
940{
941   int   ret;
942   OrigFn fn;
943   VALGRIND_GET_ORIG_FN(fn);
944   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_BARRIER_WAIT,
945                                   barrier, pthread_barrier, 0, 0, 0);
946   CALL_FN_W_W(ret, fn, barrier);
947   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_BARRIER_WAIT,
948                              barrier, pthread_barrier,
949                              ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD,
950                              ret == PTHREAD_BARRIER_SERIAL_THREAD, 0);
951   return ret;
952}
953
954PTH_FUNCS(int, pthreadZubarrierZuwait, pthread_barrier_wait_intercept,
955          (pthread_barrier_t* barrier), (barrier));
956#endif   // HAVE_PTHREAD_BARRIER_INIT
957
958
959static __always_inline
960int sem_init_intercept(sem_t *sem, int pshared, unsigned int value)
961{
962   int   ret;
963   OrigFn fn;
964   VALGRIND_GET_ORIG_FN(fn);
965   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_INIT,
966                                   sem, pshared, value, 0, 0);
967   CALL_FN_W_WWW(ret, fn, sem, pshared, value);
968   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_INIT,
969                                   sem, 0, 0, 0, 0);
970   return ret;
971}
972
973PTH_FUNCS(int, semZuinit, sem_init_intercept,
974          (sem_t *sem, int pshared, unsigned int value), (sem, pshared, value));
975
976static __always_inline
977int sem_destroy_intercept(sem_t *sem)
978{
979   int   ret;
980   OrigFn fn;
981   VALGRIND_GET_ORIG_FN(fn);
982   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_DESTROY,
983                                   sem, 0, 0, 0, 0);
984   CALL_FN_W_W(ret, fn, sem);
985   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_DESTROY,
986                                   sem, 0, 0, 0, 0);
987   return ret;
988}
989
990PTH_FUNCS(int, semZudestroy, sem_destroy_intercept, (sem_t *sem), (sem));
991
992static __always_inline
993sem_t* sem_open_intercept(const char *name, int oflag, mode_t mode,
994                          unsigned int value)
995{
996   sem_t *ret;
997   OrigFn fn;
998   VALGRIND_GET_ORIG_FN(fn);
999   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_OPEN,
1000                                   name, oflag, mode, value, 0);
1001   CALL_FN_W_WWWW(ret, fn, name, oflag, mode, value);
1002   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_OPEN,
1003                                   ret != SEM_FAILED ? ret : 0,
1004                                   name, oflag, mode, value);
1005   return ret;
1006}
1007
1008PTH_FUNCS(sem_t *, semZuopen, sem_open_intercept,
1009          (const char *name, int oflag, mode_t mode, unsigned int value),
1010          (name, oflag, mode, value));
1011
1012static __always_inline int sem_close_intercept(sem_t *sem)
1013{
1014   int   ret;
1015   OrigFn fn;
1016   VALGRIND_GET_ORIG_FN(fn);
1017   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_CLOSE,
1018                                   sem, 0, 0, 0, 0);
1019   CALL_FN_W_W(ret, fn, sem);
1020   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_CLOSE,
1021                                   sem, 0, 0, 0, 0);
1022   return ret;
1023}
1024
1025PTH_FUNCS(int, semZuclose, sem_close_intercept, (sem_t *sem), (sem));
1026
1027static __always_inline int sem_wait_intercept(sem_t *sem)
1028{
1029   int   ret;
1030   OrigFn fn;
1031   VALGRIND_GET_ORIG_FN(fn);
1032   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_WAIT,
1033                                   sem, 0, 0, 0, 0);
1034   CALL_FN_W_W(ret, fn, sem);
1035   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_WAIT,
1036                                   sem, ret == 0, 0, 0, 0);
1037   return ret;
1038}
1039
1040PTH_FUNCS(int, semZuwait, sem_wait_intercept, (sem_t *sem), (sem));
1041
1042static __always_inline int sem_trywait_intercept(sem_t *sem)
1043{
1044   int   ret;
1045   OrigFn fn;
1046   VALGRIND_GET_ORIG_FN(fn);
1047   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_WAIT,
1048                                   sem, 0, 0, 0, 0);
1049   CALL_FN_W_W(ret, fn, sem);
1050   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_WAIT,
1051                                   sem, ret == 0, 0, 0, 0);
1052   return ret;
1053}
1054
1055PTH_FUNCS(int, semZutrywait, sem_trywait_intercept, (sem_t *sem), (sem));
1056
1057static __always_inline
1058int sem_timedwait_intercept(sem_t *sem, const struct timespec *abs_timeout)
1059{
1060   int   ret;
1061   OrigFn fn;
1062   VALGRIND_GET_ORIG_FN(fn);
1063   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_WAIT,
1064                                   sem, 0, 0, 0, 0);
1065   CALL_FN_W_WW(ret, fn, sem, abs_timeout);
1066   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_WAIT,
1067                                   sem, ret == 0, 0, 0, 0);
1068   return ret;
1069}
1070
1071PTH_FUNCS(int, semZutimedwait, sem_timedwait_intercept,
1072          (sem_t *sem, const struct timespec *abs_timeout),
1073          (sem, abs_timeout));
1074
1075static __always_inline int sem_post_intercept(sem_t *sem)
1076{
1077   int   ret;
1078   OrigFn fn;
1079   VALGRIND_GET_ORIG_FN(fn);
1080   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_SEM_POST,
1081                                   sem, 0, 0, 0, 0);
1082   CALL_FN_W_W(ret, fn, sem);
1083   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_SEM_POST,
1084                                   sem, ret == 0, 0, 0, 0);
1085   return ret;
1086}
1087
1088PTH_FUNCS(int, semZupost, sem_post_intercept, (sem_t *sem), (sem));
1089
1090/* Android's pthread.h doesn't say anything about rwlocks, hence these
1091   functions have to be conditionally compiled. */
1092#if defined(HAVE_PTHREAD_RWLOCK_T)
1093
1094static __always_inline
1095int pthread_rwlock_init_intercept(pthread_rwlock_t* rwlock,
1096                                  const pthread_rwlockattr_t* attr)
1097{
1098   int   ret;
1099   OrigFn fn;
1100   VALGRIND_GET_ORIG_FN(fn);
1101   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_INIT,
1102                                   rwlock, 0, 0, 0, 0);
1103   CALL_FN_W_WW(ret, fn, rwlock, attr);
1104   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_INIT,
1105                                   rwlock, 0, 0, 0, 0);
1106   return ret;
1107}
1108
1109PTH_FUNCS(int,
1110          pthreadZurwlockZuinit, pthread_rwlock_init_intercept,
1111          (pthread_rwlock_t* rwlock, const pthread_rwlockattr_t* attr),
1112          (rwlock, attr));
1113
1114static __always_inline
1115int pthread_rwlock_destroy_intercept(pthread_rwlock_t* rwlock)
1116{
1117   int   ret;
1118   OrigFn fn;
1119   VALGRIND_GET_ORIG_FN(fn);
1120   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_DESTROY,
1121                                   rwlock, 0, 0, 0, 0);
1122   CALL_FN_W_W(ret, fn, rwlock);
1123   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_DESTROY,
1124                                   rwlock, 0, 0, 0, 0);
1125   return ret;
1126}
1127
1128PTH_FUNCS(int,
1129          pthreadZurwlockZudestroy, pthread_rwlock_destroy_intercept,
1130          (pthread_rwlock_t* rwlock), (rwlock));
1131
1132static __always_inline
1133int pthread_rwlock_rdlock_intercept(pthread_rwlock_t* rwlock)
1134{
1135   int   ret;
1136   OrigFn fn;
1137   VALGRIND_GET_ORIG_FN(fn);
1138   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_RDLOCK,
1139                                   rwlock, 0, 0, 0, 0);
1140   CALL_FN_W_W(ret, fn, rwlock);
1141   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_RDLOCK,
1142                                   rwlock, ret == 0, 0, 0, 0);
1143   return ret;
1144}
1145
1146PTH_FUNCS(int,
1147          pthreadZurwlockZurdlock, pthread_rwlock_rdlock_intercept,
1148          (pthread_rwlock_t* rwlock), (rwlock));
1149
1150static __always_inline
1151int pthread_rwlock_wrlock_intercept(pthread_rwlock_t* rwlock)
1152{
1153   int   ret;
1154   OrigFn fn;
1155   VALGRIND_GET_ORIG_FN(fn);
1156   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_WRLOCK,
1157                                   rwlock, 0, 0, 0, 0);
1158   CALL_FN_W_W(ret, fn, rwlock);
1159   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_WRLOCK,
1160                                   rwlock, ret == 0, 0, 0, 0);
1161   return ret;
1162}
1163
1164PTH_FUNCS(int,
1165          pthreadZurwlockZuwrlock, pthread_rwlock_wrlock_intercept,
1166          (pthread_rwlock_t* rwlock), (rwlock));
1167
1168static __always_inline
1169int pthread_rwlock_timedrdlock_intercept(pthread_rwlock_t* rwlock,
1170                                         const struct timespec *timeout)
1171{
1172   int   ret;
1173   OrigFn fn;
1174   VALGRIND_GET_ORIG_FN(fn);
1175   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_RDLOCK,
1176                                   rwlock, 0, 0, 0, 0);
1177   CALL_FN_W_WW(ret, fn, rwlock, timeout);
1178   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_RDLOCK,
1179                                   rwlock, ret == 0, 0, 0, 0);
1180   return ret;
1181}
1182
1183PTH_FUNCS(int,
1184          pthreadZurwlockZutimedrdlock, pthread_rwlock_timedrdlock_intercept,
1185          (pthread_rwlock_t* rwlock, const struct timespec *timeout),
1186          (rwlock, timeout));
1187
1188static __always_inline
1189int pthread_rwlock_timedwrlock_intercept(pthread_rwlock_t* rwlock,
1190                                         const struct timespec *timeout)
1191{
1192   int   ret;
1193   OrigFn fn;
1194   VALGRIND_GET_ORIG_FN(fn);
1195   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_WRLOCK,
1196                                   rwlock, 0, 0, 0, 0);
1197   CALL_FN_W_WW(ret, fn, rwlock, timeout);
1198   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_WRLOCK,
1199                                   rwlock, ret == 0, 0, 0, 0);
1200   return ret;
1201}
1202
1203PTH_FUNCS(int,
1204          pthreadZurwlockZutimedwrlock, pthread_rwlock_timedwrlock_intercept,
1205          (pthread_rwlock_t* rwlock, const struct timespec *timeout),
1206          (rwlock, timeout));
1207
1208static __always_inline
1209int pthread_rwlock_tryrdlock_intercept(pthread_rwlock_t* rwlock)
1210{
1211   int   ret;
1212   OrigFn fn;
1213   VALGRIND_GET_ORIG_FN(fn);
1214   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_RDLOCK,
1215                                   rwlock, 0, 0, 0, 0);
1216   CALL_FN_W_W(ret, fn, rwlock);
1217   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_RDLOCK,
1218                                   rwlock, ret == 0, 0, 0, 0);
1219   return ret;
1220}
1221
1222PTH_FUNCS(int,
1223          pthreadZurwlockZutryrdlock, pthread_rwlock_tryrdlock_intercept,
1224          (pthread_rwlock_t* rwlock), (rwlock));
1225
1226static __always_inline
1227int pthread_rwlock_trywrlock_intercept(pthread_rwlock_t* rwlock)
1228{
1229   int   ret;
1230   OrigFn fn;
1231   VALGRIND_GET_ORIG_FN(fn);
1232   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_WRLOCK,
1233                                   rwlock, 0, 0, 0, 0);
1234   CALL_FN_W_W(ret, fn, rwlock);
1235   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_WRLOCK,
1236                                   rwlock, ret == 0, 0, 0, 0);
1237   return ret;
1238}
1239
1240PTH_FUNCS(int,
1241          pthreadZurwlockZutrywrlock, pthread_rwlock_trywrlock_intercept,
1242          (pthread_rwlock_t* rwlock), (rwlock));
1243
1244static __always_inline
1245int pthread_rwlock_unlock_intercept(pthread_rwlock_t* rwlock)
1246{
1247   int   ret;
1248   OrigFn fn;
1249   VALGRIND_GET_ORIG_FN(fn);
1250   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__PRE_RWLOCK_UNLOCK,
1251                                   rwlock, 0, 0, 0, 0);
1252   CALL_FN_W_W(ret, fn, rwlock);
1253   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__POST_RWLOCK_UNLOCK,
1254                                   rwlock, ret == 0, 0, 0, 0);
1255   return ret;
1256}
1257
1258PTH_FUNCS(int,
1259          pthreadZurwlockZuunlock, pthread_rwlock_unlock_intercept,
1260          (pthread_rwlock_t* rwlock), (rwlock));
1261
1262#endif /* defined(HAVE_PTHREAD_RWLOCK_T) */
1263