1799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar//===-- tsan_libdispatch_mac.cc -------------------------------------------===// 2799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// 3799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// The LLVM Compiler Infrastructure 4799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// 5799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// This file is distributed under the University of Illinois Open Source 6799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// License. See LICENSE.TXT for details. 7799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// 8799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar//===----------------------------------------------------------------------===// 9799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// 10799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// This file is a part of ThreadSanitizer (TSan), a race detector. 11799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// 12799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// Mac-specific libdispatch (GCD) support. 13799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar//===----------------------------------------------------------------------===// 14799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 15799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "sanitizer_common/sanitizer_platform.h" 16799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#if SANITIZER_MAC 17799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 18799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "sanitizer_common/sanitizer_common.h" 19799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "interception/interception.h" 20799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "tsan_interceptors.h" 21799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "tsan_platform.h" 22799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include "tsan_rtl.h" 23799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 24799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include <Block.h> 25799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include <dispatch/dispatch.h> 26799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#include <pthread.h> 27799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 28799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainartypedef long long_t; // NOLINT 29799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 30799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarnamespace __tsan { 31799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 32799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainartypedef struct { 33799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t queue; 34799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar void *orig_context; 35799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_function_t orig_work; 36c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar bool free_context_in_callback; 37c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar bool submitted_synchronously; 38c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar bool is_barrier_block; 39c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr non_queue_sync_object; 40799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} tsan_block_context_t; 41799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 42799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// The offsets of different fields of the dispatch_queue_t structure, exported 43799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// by libdispatch.dylib. 44799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarextern "C" struct dispatch_queue_offsets_s { 45799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_version; 46799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_label; 47799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_label_size; 48799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_flags; 49799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_flags_size; 50799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_serialnum; 51799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_serialnum_size; 52799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_width; 53799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_width_size; 54799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_running; 55799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_running_size; 56799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_suspend_cnt; 57799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_suspend_cnt_size; 58799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_target_queue; 59799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_target_queue_size; 60799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_priority; 61799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uint16_t dqo_priority_size; 62799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} dispatch_queue_offsets; 63799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 64799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarstatic bool IsQueueSerial(dispatch_queue_t q) { 65799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 66799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 67799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar CHECK_NE(width, 0); 68799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return width == 1; 69799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 70799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 71c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarstatic dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 72c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 73c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t target_queue = 74c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar *(dispatch_queue_t *)(((uptr)source) + 75c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_offsets.dqo_target_queue); 76c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar CHECK_NE(target_queue, 0); 77c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return target_queue; 78c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 79c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 80799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarstatic tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, 81799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t queue, 82799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar void *orig_context, 83799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_function_t orig_work) { 84799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar tsan_block_context_t *new_context = 85799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); 86799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar new_context->queue = queue; 87799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar new_context->orig_context = orig_context; 88799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar new_context->orig_work = orig_work; 89c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->free_context_in_callback = true; 90c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->submitted_synchronously = false; 91c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->is_barrier_block = false; 92799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return new_context; 93799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 94799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 95c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarstatic void dispatch_callback_wrap(void *param) { 96c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 97799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar tsan_block_context_t *context = (tsan_block_context_t *)param; 98c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar bool is_queue_serial = context->queue && IsQueueSerial(context->queue); 99c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object; 100799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 101c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr serial_sync = (uptr)sync_ptr; 102c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr); 103c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)context; 104c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar bool serial_task = context->is_barrier_block || is_queue_serial; 105c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 106c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, submit_sync); 107c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, serial_sync); 108c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (serial_task) Acquire(thr, pc, concurrent_sync); 109799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 110799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 111799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar context->orig_work(context->orig_context); 112799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 113c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 114c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, serial_task ? serial_sync : concurrent_sync); 115c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (context->submitted_synchronously) Release(thr, pc, submit_sync); 116c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 117c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (context->free_context_in_callback) user_free(thr, pc, context); 118c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 119c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 120c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarstatic void invoke_block(void *param) { 121c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t block = (dispatch_block_t)param; 122c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar block(); 123799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 124799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 125799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarstatic void invoke_and_release_block(void *param) { 126799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_block_t block = (dispatch_block_t)param; 127799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar block(); 128799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Block_release(block); 129799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 130799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 131c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar#define DISPATCH_INTERCEPT_B(name, barrier) \ 132799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 133799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 134c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 135799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_block_t heap_block = Block_copy(block); \ 136c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 137799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar tsan_block_context_t *new_context = \ 138799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 139c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->is_barrier_block = barrier; \ 140799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Release(thr, pc, (uptr)new_context); \ 141c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 142c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 143c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 144c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 145c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 146c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 147c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 148c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 149c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 150c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t heap_block = Block_copy(block); \ 151c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 152c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar tsan_block_context_t new_context = { \ 153c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \ 154c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, (uptr)&new_context); \ 155c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 156c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 157c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 158c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)&new_context); \ 159799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } 160799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 161c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar#define DISPATCH_INTERCEPT_F(name, barrier) \ 162799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 163799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_function_t work) { \ 164799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 165799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar tsan_block_context_t *new_context = \ 166799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar AllocContext(thr, pc, q, context, work); \ 167c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->is_barrier_block = barrier; \ 168799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Release(thr, pc, (uptr)new_context); \ 169c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 170c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(name)(q, new_context, dispatch_callback_wrap); \ 171c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 172c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 173c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 174c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 175c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 176c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_function_t work) { \ 177c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 178c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar tsan_block_context_t new_context = { \ 179c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, context, work, false, true, barrier, 0}; \ 180c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, (uptr)&new_context); \ 181c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 182c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(name)(q, &new_context, dispatch_callback_wrap); \ 183c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 184c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)&new_context); \ 185799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } 186799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 187799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// We wrap dispatch_async, dispatch_sync and friends where we allocate a new 188799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// context, which is used to synchronize (we release the context before 189799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// submitting, and the callback acquires it before executing the original 190799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// callback). 191c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_B(dispatch_async, false) 192c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_B(dispatch_barrier_async, true) 193c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_F(dispatch_async_f, false) 194c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) 195c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) 196c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) 197c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) 198c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) 199c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 200c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 201c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t queue, dispatch_block_t block) { 202c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 203c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 204c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t heap_block = Block_copy(block); 205c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 206c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar tsan_block_context_t *new_context = 207c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 208c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, (uptr)new_context); 209c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 210c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 211c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 212c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 213c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 214c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 215c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t queue, void *context, 216c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_function_t work) { 217c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 218c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_after)(when, queue, ^(void) { 219c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar work(context); 220c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 221c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 222799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 223799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// GCD's dispatch_once implementation has a fast path that contains a racy read 224799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// and it's inlined into user's code. Furthermore, this fast path doesn't 225799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// establish a proper happens-before relations between the initialization and 226799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// code following the call to dispatch_once. We could deal with this in 227799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// instrumented code, but there's not much we can do about it in system 228799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// libraries. Let's disable the fast path (by never storing the value ~0 to 229799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// predicate), so the interceptor is always called, and let's add proper release 230799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// and acquire semantics. Since TSan does not see its own atomic stores, the 231799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// race on predicate won't be reported - the only accesses to it that TSan sees 232799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 233799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// both a macro and a real function, we want to intercept the function, so we 234799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar// need to undefine the macro. 235799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#undef dispatch_once 236799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 237799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_block_t block) { 238c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 239799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 240799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar u32 v = atomic_load(a, memory_order_acquire); 241799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (v == 0 && 242799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 243799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 244799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar block(); 245799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 246799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Release(thr, pc, (uptr)a); 247799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar atomic_store(a, 2, memory_order_release); 248799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } else { 249799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar while (v != 2) { 250799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar internal_sched_yield(); 251799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar v = atomic_load(a, memory_order_acquire); 252799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } 253799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Acquire(thr, pc, (uptr)a); 254799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } 255799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 256799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 257799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#undef dispatch_once_f 258799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 259799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar void *context, dispatch_function_t function) { 260c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 261799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 262799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar WRAP(dispatch_once)(predicate, ^(void) { 263799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar function(context); 264799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar }); 265799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 266799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 267799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 268799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 269799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_semaphore_t dsema) { 270799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 271799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Release(thr, pc, (uptr)dsema); 272799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return REAL(dispatch_semaphore_signal)(dsema); 273799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 274799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 275799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 276799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_time_t timeout) { 277799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 278799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 279799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (result == 0) Acquire(thr, pc, (uptr)dsema); 280799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return result; 281799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 282799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 283799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 284799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_time_t timeout) { 285799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 286799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar long_t result = REAL(dispatch_group_wait)(group, timeout); 287799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (result == 0) Acquire(thr, pc, (uptr)group); 288799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return result; 289799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 290799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 291799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 292799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 293c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. 294799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar Release(thr, pc, (uptr)group); 295799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar REAL(dispatch_group_leave)(group); 296799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 297799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 298799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 299799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t queue, dispatch_block_t block) { 300799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 301799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_retain(group); 302799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_group_enter(group); 303c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); 304799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar WRAP(dispatch_async)(queue, ^(void) { 305c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar block_copy(); 306c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar _Block_release(block_copy); 307799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar WRAP(dispatch_group_leave)(group); 308799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_release(group); 309799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar }); 310799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 311799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 312799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 313799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t queue, void *context, 314799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_function_t work) { 315799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 316799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_retain(group); 317799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_group_enter(group); 318799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar WRAP(dispatch_async)(queue, ^(void) { 319799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar work(context); 320799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar WRAP(dispatch_group_leave)(group); 321799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_release(group); 322799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar }); 323799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 324799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 325799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 326799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t q, dispatch_block_t block) { 327799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 328c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 329c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // To make sure the group is still available in the callback (otherwise 330c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // it can be already destroyed). Will be released in the callback. 331c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_retain(group); 332c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 333799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 334c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t heap_block = Block_copy(^(void) { 335c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar { 336c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 337c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // Released when leaving the group (dispatch_group_leave). 338c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)group); 339c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 340c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_release(group); 341c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar block(); 342c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 343799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 344799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar tsan_block_context_t *new_context = 345799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 346c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context->is_barrier_block = true; 347c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, (uptr)new_context); 348c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 349799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 350799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 351799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 352799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar dispatch_queue_t q, void *context, dispatch_function_t work) { 353c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 354c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 355799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 356c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 357c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_block_t handler) { 358c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 359c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 360c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_event_handler)(source, nullptr); 361c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t q = GetTargetQueueFromSource(source); 362c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 363c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, handler, &invoke_block, false, false, false, 0 }; 364c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t new_handler = Block_copy(^(void) { 365c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = handler; // To explicitly capture "handler". 366c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 367c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 368c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 369c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 370c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_source_set_event_handler)(source, new_handler); 371c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_handler); 372c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 373799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 374c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 375c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_function_t handler) { 376c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 377c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 378c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_event_handler)(source, nullptr); 379c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t block = ^(void) { 380c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler(dispatch_get_context(source)); 381c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 382c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_source_set_event_handler)(source, block); 383c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 384c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 385c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 386c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_block_t handler) { 387c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 388c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 389c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 390c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t q = GetTargetQueueFromSource(source); 391c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 392c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, handler, &invoke_block, false, false, false, 0}; 393c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t new_handler = Block_copy(^(void) { 394c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = handler; // To explicitly capture "handler". 395c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 396c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 397c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 398c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 399c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_source_set_cancel_handler)(source, new_handler); 400c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_handler); 401c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 402c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 403c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 404c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_function_t handler) { 405c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 406c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler); 407c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 408c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 409c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t block = ^(void) { 410c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler(dispatch_get_context(source)); 411c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 412c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_source_set_cancel_handler)(source, block); 413c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 414c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 415c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 416c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_block_t handler) { 417c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 418c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler); 419c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 420c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_registration_handler)(source, nullptr); 421c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t q = GetTargetQueueFromSource(source); 422c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 423c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, handler, &invoke_block, false, false, false, 0}; 424c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t new_handler = Block_copy(^(void) { 425c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = handler; // To explicitly capture "handler". 426c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 427c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 428c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 429c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 430c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_source_set_registration_handler)(source, new_handler); 431c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_handler); 432c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 433c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 434c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 435c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_source_t source, dispatch_function_t handler) { 436c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 437c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler); 438c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (handler == nullptr) 439c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_source_set_registration_handler)(source, nullptr); 440c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t block = ^(void) { 441c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar handler(dispatch_get_context(source)); 442c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 443c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_source_set_registration_handler)(source, block); 444c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 445c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 446c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 447c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t queue, void (^block)(size_t)) { 448c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 449c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 450c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void *parent_to_child_sync = nullptr; 451c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; 452c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void *child_to_parent_sync = nullptr; 453c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; 454c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 455c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, parent_to_child_sync_uptr); 456c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void (^new_block)(size_t) = ^(size_t iteration) { 457c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_apply); 458c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, parent_to_child_sync_uptr); 459c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 460c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar block(iteration); 461c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 462c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, child_to_parent_sync_uptr); 463c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 464c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 465c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_apply)(iterations, queue, new_block); 466c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 467c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, child_to_parent_sync_uptr); 468c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 469c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 470c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 471c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t queue, void *context, 472c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void (*work)(void *, size_t)) { 473c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 474c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void (^new_block)(size_t) = ^(size_t iteration) { 475c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar work(context, iteration); 476c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 477c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar WRAP(dispatch_apply)(iterations, queue, new_block); 478c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 479c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 480c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 481c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarDECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 482c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 483c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 484c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 485c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 486c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 487c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_data_create)(buffer, size, q, destructor); 488c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 489c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 490c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar destructor = ^(void) { WRAP(free)((void *)buffer); }; 491c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 492c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar destructor = ^(void) { WRAP(munmap)((void *)buffer, size); }; 493c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 494c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 495c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t heap_block = Block_copy(destructor); 496c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 497c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar tsan_block_context_t *new_context = 498c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 499c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)new_context; 500c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 501c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 502c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(new_context); 503c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 504c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 505c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 506c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainartypedef void (^fd_handler_t)(dispatch_data_t data, int error); 507c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainartypedef void (^cleanup_handler_t)(int error); 508c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 509c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 510c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t q, fd_handler_t h) { 511c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 512c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 513c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 514c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 515c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 516c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(data, error); 517c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 518c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 519c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 520c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 521c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 522c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_read)(fd, length, q, new_h); 523c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 524c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 525c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 526c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 527c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_queue_t q, fd_handler_t h) { 528c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 529c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 530c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 531c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 532c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 533c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(data, error); 534c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 535c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 536c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 537c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 538c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 539c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_write)(fd, data, q, new_h); 540c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 541c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 542c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 543c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 544c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 545c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 546c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 547c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 548c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_handler_t new_h = 549c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_copy(^(bool done, dispatch_data_t data, int error) { 550c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 551c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(done, data, error); 552c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 553c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 554c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 555c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 556c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 557c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_io_read)(channel, offset, length, q, new_h); 558c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 559c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 560c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 561c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 562c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_data_t data, dispatch_queue_t q, 563c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_handler_t h) { 564c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 565c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 566c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 567c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_handler_t new_h = 568c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_copy(^(bool done, dispatch_data_t data, int error) { 569c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 570c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(done, data, error); 571c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 572c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 573c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 574c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 575c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 576c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_io_write)(channel, offset, data, q, new_h); 577c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 578c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 579c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 580c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 581c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t barrier) { 582c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 583c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 584c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar nullptr, nullptr, &invoke_block, false, false, false, 0}; 585c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.non_queue_sync_object = (uptr)channel; 586c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.is_barrier_block = true; 587c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_block_t new_block = Block_copy(^(void) { 588c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 589c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar barrier(); 590c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 591c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 592c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 593c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 594c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 595c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_io_barrier)(channel, new_block); 596c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_block); 597c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 598c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 599c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 600c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 601c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 602c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block dispatch_io_t new_channel = nullptr; 603c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 604c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 605c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar cleanup_handler_t new_h = Block_copy(^(int error) { 606c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar { 607c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 608c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 609c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 610c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 611c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(error); 612c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 613c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 614c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 615c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 616c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 617c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 618c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 619c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return new_channel; 620c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 621c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 622c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 623c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_type_t type, const char *path, int oflag, 624c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 625c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 626c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, h); 627c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block dispatch_io_t new_channel = nullptr; 628c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 629c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 630c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar cleanup_handler_t new_h = Block_copy(^(int error) { 631c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar { 632c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 633c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 634c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 635c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 636c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(error); 637c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 638c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 639c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 640c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 641c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 642c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_channel = 643c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 644c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 645c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return new_channel; 646c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 647c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 648c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 649c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 650c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar cleanup_handler_t h) { 651c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 652c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block dispatch_io_t new_channel = nullptr; 653c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar __block tsan_block_context_t new_context = { 654c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar q, nullptr, &invoke_block, false, false, false, 0}; 655c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar cleanup_handler_t new_h = Block_copy(^(int error) { 656c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar { 657c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 658c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 659c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 660c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_context.orig_context = ^(void) { 661c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar h(error); 662c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }; 663c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_callback_wrap(&new_context); 664c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar }); 665c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr submit_sync = (uptr)&new_context; 666c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, submit_sync); 667c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 668c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Block_release(new_h); 669c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return new_channel; 670c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 671c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 672c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarTSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 673c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar dispatch_io_close_flags_t flags) { 674c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 675c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 676c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return REAL(dispatch_io_close)(channel, flags); 677799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} 678799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 679799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar} // namespace __tsan 680799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar 681799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar#endif // SANITIZER_MAC 682