1//===-- tsan_libdispatch_mac.cc -------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer (TSan), a race detector. 11// 12// Mac-specific libdispatch (GCD) support. 13//===----------------------------------------------------------------------===// 14 15#include "sanitizer_common/sanitizer_platform.h" 16#if SANITIZER_MAC 17 18#include "sanitizer_common/sanitizer_common.h" 19#include "interception/interception.h" 20#include "tsan_interceptors.h" 21#include "tsan_platform.h" 22#include "tsan_rtl.h" 23 24#include <Block.h> 25#include <dispatch/dispatch.h> 26#include <pthread.h> 27 28typedef long long_t; // NOLINT 29 30namespace __tsan { 31 32typedef struct { 33 dispatch_queue_t queue; 34 void *orig_context; 35 dispatch_function_t orig_work; 36 bool free_context_in_callback; 37 bool submitted_synchronously; 38 bool is_barrier_block; 39 uptr non_queue_sync_object; 40} tsan_block_context_t; 41 42// The offsets of different fields of the dispatch_queue_t structure, exported 43// by libdispatch.dylib. 44extern "C" struct dispatch_queue_offsets_s { 45 const uint16_t dqo_version; 46 const uint16_t dqo_label; 47 const uint16_t dqo_label_size; 48 const uint16_t dqo_flags; 49 const uint16_t dqo_flags_size; 50 const uint16_t dqo_serialnum; 51 const uint16_t dqo_serialnum_size; 52 const uint16_t dqo_width; 53 const uint16_t dqo_width_size; 54 const uint16_t dqo_running; 55 const uint16_t dqo_running_size; 56 const uint16_t dqo_suspend_cnt; 57 const uint16_t dqo_suspend_cnt_size; 58 const uint16_t dqo_target_queue; 59 const uint16_t dqo_target_queue_size; 60 const uint16_t dqo_priority; 61 const uint16_t dqo_priority_size; 62} dispatch_queue_offsets; 63 64static bool IsQueueSerial(dispatch_queue_t q) { 65 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 66 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 67 CHECK_NE(width, 0); 68 return width == 1; 69} 70 71static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 72 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 73 dispatch_queue_t target_queue = 74 *(dispatch_queue_t *)(((uptr)source) + 75 dispatch_queue_offsets.dqo_target_queue); 76 CHECK_NE(target_queue, 0); 77 return target_queue; 78} 79 80static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, 81 dispatch_queue_t queue, 82 void *orig_context, 83 dispatch_function_t orig_work) { 84 tsan_block_context_t *new_context = 85 (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); 86 new_context->queue = queue; 87 new_context->orig_context = orig_context; 88 new_context->orig_work = orig_work; 89 new_context->free_context_in_callback = true; 90 new_context->submitted_synchronously = false; 91 new_context->is_barrier_block = false; 92 return new_context; 93} 94 95static void dispatch_callback_wrap(void *param) { 96 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 97 tsan_block_context_t *context = (tsan_block_context_t *)param; 98 bool is_queue_serial = context->queue && IsQueueSerial(context->queue); 99 uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object; 100 101 uptr serial_sync = (uptr)sync_ptr; 102 uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr); 103 uptr submit_sync = (uptr)context; 104 bool serial_task = context->is_barrier_block || is_queue_serial; 105 106 Acquire(thr, pc, submit_sync); 107 Acquire(thr, pc, serial_sync); 108 if (serial_task) Acquire(thr, pc, concurrent_sync); 109 110 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 111 context->orig_work(context->orig_context); 112 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 113 114 Release(thr, pc, serial_task ? serial_sync : concurrent_sync); 115 if (context->submitted_synchronously) Release(thr, pc, submit_sync); 116 117 if (context->free_context_in_callback) user_free(thr, pc, context); 118} 119 120static void invoke_block(void *param) { 121 dispatch_block_t block = (dispatch_block_t)param; 122 block(); 123} 124 125static void invoke_and_release_block(void *param) { 126 dispatch_block_t block = (dispatch_block_t)param; 127 block(); 128 Block_release(block); 129} 130 131#define DISPATCH_INTERCEPT_B(name, barrier) \ 132 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 133 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 134 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 135 dispatch_block_t heap_block = Block_copy(block); \ 136 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 137 tsan_block_context_t *new_context = \ 138 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 139 new_context->is_barrier_block = barrier; \ 140 Release(thr, pc, (uptr)new_context); \ 141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 142 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 143 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 144 } 145 146#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 147 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 148 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 150 dispatch_block_t heap_block = Block_copy(block); \ 151 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 152 tsan_block_context_t new_context = { \ 153 q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \ 154 Release(thr, pc, (uptr)&new_context); \ 155 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 156 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 157 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 158 Acquire(thr, pc, (uptr)&new_context); \ 159 } 160 161#define DISPATCH_INTERCEPT_F(name, barrier) \ 162 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 163 dispatch_function_t work) { \ 164 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 165 tsan_block_context_t *new_context = \ 166 AllocContext(thr, pc, q, context, work); \ 167 new_context->is_barrier_block = barrier; \ 168 Release(thr, pc, (uptr)new_context); \ 169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 170 REAL(name)(q, new_context, dispatch_callback_wrap); \ 171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 172 } 173 174#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 176 dispatch_function_t work) { \ 177 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 178 tsan_block_context_t new_context = { \ 179 q, context, work, false, true, barrier, 0}; \ 180 Release(thr, pc, (uptr)&new_context); \ 181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 182 REAL(name)(q, &new_context, dispatch_callback_wrap); \ 183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 184 Acquire(thr, pc, (uptr)&new_context); \ 185 } 186 187// We wrap dispatch_async, dispatch_sync and friends where we allocate a new 188// context, which is used to synchronize (we release the context before 189// submitting, and the callback acquires it before executing the original 190// callback). 191DISPATCH_INTERCEPT_B(dispatch_async, false) 192DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) 193DISPATCH_INTERCEPT_F(dispatch_async_f, false) 194DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) 195DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) 196DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) 197DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) 198DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) 199 200TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 201 dispatch_queue_t queue, dispatch_block_t block) { 202 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 204 dispatch_block_t heap_block = Block_copy(block); 205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 206 tsan_block_context_t *new_context = 207 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 208 Release(thr, pc, (uptr)new_context); 209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 210 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 211 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 212} 213 214TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 215 dispatch_queue_t queue, void *context, 216 dispatch_function_t work) { 217 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 218 WRAP(dispatch_after)(when, queue, ^(void) { 219 work(context); 220 }); 221} 222 223// GCD's dispatch_once implementation has a fast path that contains a racy read 224// and it's inlined into user's code. Furthermore, this fast path doesn't 225// establish a proper happens-before relations between the initialization and 226// code following the call to dispatch_once. We could deal with this in 227// instrumented code, but there's not much we can do about it in system 228// libraries. Let's disable the fast path (by never storing the value ~0 to 229// predicate), so the interceptor is always called, and let's add proper release 230// and acquire semantics. Since TSan does not see its own atomic stores, the 231// race on predicate won't be reported - the only accesses to it that TSan sees 232// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 233// both a macro and a real function, we want to intercept the function, so we 234// need to undefine the macro. 235#undef dispatch_once 236TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 237 dispatch_block_t block) { 238 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 239 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 240 u32 v = atomic_load(a, memory_order_acquire); 241 if (v == 0 && 242 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 243 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 244 block(); 245 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 246 Release(thr, pc, (uptr)a); 247 atomic_store(a, 2, memory_order_release); 248 } else { 249 while (v != 2) { 250 internal_sched_yield(); 251 v = atomic_load(a, memory_order_acquire); 252 } 253 Acquire(thr, pc, (uptr)a); 254 } 255} 256 257#undef dispatch_once_f 258TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 259 void *context, dispatch_function_t function) { 260 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 261 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 262 WRAP(dispatch_once)(predicate, ^(void) { 263 function(context); 264 }); 265 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 266} 267 268TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 269 dispatch_semaphore_t dsema) { 270 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 271 Release(thr, pc, (uptr)dsema); 272 return REAL(dispatch_semaphore_signal)(dsema); 273} 274 275TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 276 dispatch_time_t timeout) { 277 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 278 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 279 if (result == 0) Acquire(thr, pc, (uptr)dsema); 280 return result; 281} 282 283TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 284 dispatch_time_t timeout) { 285 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 286 long_t result = REAL(dispatch_group_wait)(group, timeout); 287 if (result == 0) Acquire(thr, pc, (uptr)group); 288 return result; 289} 290 291TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 292 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 293 // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. 294 Release(thr, pc, (uptr)group); 295 REAL(dispatch_group_leave)(group); 296} 297 298TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 299 dispatch_queue_t queue, dispatch_block_t block) { 300 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 301 dispatch_retain(group); 302 dispatch_group_enter(group); 303 __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); 304 WRAP(dispatch_async)(queue, ^(void) { 305 block_copy(); 306 _Block_release(block_copy); 307 WRAP(dispatch_group_leave)(group); 308 dispatch_release(group); 309 }); 310} 311 312TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 313 dispatch_queue_t queue, void *context, 314 dispatch_function_t work) { 315 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 316 dispatch_retain(group); 317 dispatch_group_enter(group); 318 WRAP(dispatch_async)(queue, ^(void) { 319 work(context); 320 WRAP(dispatch_group_leave)(group); 321 dispatch_release(group); 322 }); 323} 324 325TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 326 dispatch_queue_t q, dispatch_block_t block) { 327 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 328 329 // To make sure the group is still available in the callback (otherwise 330 // it can be already destroyed). Will be released in the callback. 331 dispatch_retain(group); 332 333 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 334 dispatch_block_t heap_block = Block_copy(^(void) { 335 { 336 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 337 // Released when leaving the group (dispatch_group_leave). 338 Acquire(thr, pc, (uptr)group); 339 } 340 dispatch_release(group); 341 block(); 342 }); 343 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 344 tsan_block_context_t *new_context = 345 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 346 new_context->is_barrier_block = true; 347 Release(thr, pc, (uptr)new_context); 348 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 349} 350 351TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 352 dispatch_queue_t q, void *context, dispatch_function_t work) { 353 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 354} 355 356TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 357 dispatch_source_t source, dispatch_block_t handler) { 358 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 359 if (handler == nullptr) 360 return REAL(dispatch_source_set_event_handler)(source, nullptr); 361 dispatch_queue_t q = GetTargetQueueFromSource(source); 362 __block tsan_block_context_t new_context = { 363 q, handler, &invoke_block, false, false, false, 0 }; 364 dispatch_block_t new_handler = Block_copy(^(void) { 365 new_context.orig_context = handler; // To explicitly capture "handler". 366 dispatch_callback_wrap(&new_context); 367 }); 368 uptr submit_sync = (uptr)&new_context; 369 Release(thr, pc, submit_sync); 370 REAL(dispatch_source_set_event_handler)(source, new_handler); 371 Block_release(new_handler); 372} 373 374TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 375 dispatch_source_t source, dispatch_function_t handler) { 376 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 377 if (handler == nullptr) 378 return REAL(dispatch_source_set_event_handler)(source, nullptr); 379 dispatch_block_t block = ^(void) { 380 handler(dispatch_get_context(source)); 381 }; 382 WRAP(dispatch_source_set_event_handler)(source, block); 383} 384 385TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 386 dispatch_source_t source, dispatch_block_t handler) { 387 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 388 if (handler == nullptr) 389 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 390 dispatch_queue_t q = GetTargetQueueFromSource(source); 391 __block tsan_block_context_t new_context = { 392 q, handler, &invoke_block, false, false, false, 0}; 393 dispatch_block_t new_handler = Block_copy(^(void) { 394 new_context.orig_context = handler; // To explicitly capture "handler". 395 dispatch_callback_wrap(&new_context); 396 }); 397 uptr submit_sync = (uptr)&new_context; 398 Release(thr, pc, submit_sync); 399 REAL(dispatch_source_set_cancel_handler)(source, new_handler); 400 Block_release(new_handler); 401} 402 403TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 404 dispatch_source_t source, dispatch_function_t handler) { 405 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 406 handler); 407 if (handler == nullptr) 408 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 409 dispatch_block_t block = ^(void) { 410 handler(dispatch_get_context(source)); 411 }; 412 WRAP(dispatch_source_set_cancel_handler)(source, block); 413} 414 415TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 416 dispatch_source_t source, dispatch_block_t handler) { 417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 418 handler); 419 if (handler == nullptr) 420 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 421 dispatch_queue_t q = GetTargetQueueFromSource(source); 422 __block tsan_block_context_t new_context = { 423 q, handler, &invoke_block, false, false, false, 0}; 424 dispatch_block_t new_handler = Block_copy(^(void) { 425 new_context.orig_context = handler; // To explicitly capture "handler". 426 dispatch_callback_wrap(&new_context); 427 }); 428 uptr submit_sync = (uptr)&new_context; 429 Release(thr, pc, submit_sync); 430 REAL(dispatch_source_set_registration_handler)(source, new_handler); 431 Block_release(new_handler); 432} 433 434TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 435 dispatch_source_t source, dispatch_function_t handler) { 436 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 437 handler); 438 if (handler == nullptr) 439 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 440 dispatch_block_t block = ^(void) { 441 handler(dispatch_get_context(source)); 442 }; 443 WRAP(dispatch_source_set_registration_handler)(source, block); 444} 445 446TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 447 dispatch_queue_t queue, void (^block)(size_t)) { 448 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 449 450 void *parent_to_child_sync = nullptr; 451 uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; 452 void *child_to_parent_sync = nullptr; 453 uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; 454 455 Release(thr, pc, parent_to_child_sync_uptr); 456 void (^new_block)(size_t) = ^(size_t iteration) { 457 SCOPED_INTERCEPTOR_RAW(dispatch_apply); 458 Acquire(thr, pc, parent_to_child_sync_uptr); 459 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 460 block(iteration); 461 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 462 Release(thr, pc, child_to_parent_sync_uptr); 463 }; 464 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 465 REAL(dispatch_apply)(iterations, queue, new_block); 466 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 467 Acquire(thr, pc, child_to_parent_sync_uptr); 468} 469 470TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 471 dispatch_queue_t queue, void *context, 472 void (*work)(void *, size_t)) { 473 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 474 void (^new_block)(size_t) = ^(size_t iteration) { 475 work(context, iteration); 476 }; 477 WRAP(dispatch_apply)(iterations, queue, new_block); 478} 479 480DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 481DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 482 483TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 484 size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 485 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 486 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 487 return REAL(dispatch_data_create)(buffer, size, q, destructor); 488 489 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 490 destructor = ^(void) { WRAP(free)((void *)buffer); }; 491 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 492 destructor = ^(void) { WRAP(munmap)((void *)buffer, size); }; 493 494 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 495 dispatch_block_t heap_block = Block_copy(destructor); 496 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 497 tsan_block_context_t *new_context = 498 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 499 uptr submit_sync = (uptr)new_context; 500 Release(thr, pc, submit_sync); 501 return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 502 dispatch_callback_wrap(new_context); 503 }); 504} 505 506typedef void (^fd_handler_t)(dispatch_data_t data, int error); 507typedef void (^cleanup_handler_t)(int error); 508 509TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 510 dispatch_queue_t q, fd_handler_t h) { 511 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 512 __block tsan_block_context_t new_context = { 513 q, nullptr, &invoke_block, false, false, false, 0}; 514 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 515 new_context.orig_context = ^(void) { 516 h(data, error); 517 }; 518 dispatch_callback_wrap(&new_context); 519 }); 520 uptr submit_sync = (uptr)&new_context; 521 Release(thr, pc, submit_sync); 522 REAL(dispatch_read)(fd, length, q, new_h); 523 Block_release(new_h); 524} 525 526TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 527 dispatch_queue_t q, fd_handler_t h) { 528 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 529 __block tsan_block_context_t new_context = { 530 q, nullptr, &invoke_block, false, false, false, 0}; 531 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 532 new_context.orig_context = ^(void) { 533 h(data, error); 534 }; 535 dispatch_callback_wrap(&new_context); 536 }); 537 uptr submit_sync = (uptr)&new_context; 538 Release(thr, pc, submit_sync); 539 REAL(dispatch_write)(fd, data, q, new_h); 540 Block_release(new_h); 541} 542 543TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 544 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 545 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 546 __block tsan_block_context_t new_context = { 547 q, nullptr, &invoke_block, false, false, false, 0}; 548 dispatch_io_handler_t new_h = 549 Block_copy(^(bool done, dispatch_data_t data, int error) { 550 new_context.orig_context = ^(void) { 551 h(done, data, error); 552 }; 553 dispatch_callback_wrap(&new_context); 554 }); 555 uptr submit_sync = (uptr)&new_context; 556 Release(thr, pc, submit_sync); 557 REAL(dispatch_io_read)(channel, offset, length, q, new_h); 558 Block_release(new_h); 559} 560 561TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 562 dispatch_data_t data, dispatch_queue_t q, 563 dispatch_io_handler_t h) { 564 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 565 __block tsan_block_context_t new_context = { 566 q, nullptr, &invoke_block, false, false, false, 0}; 567 dispatch_io_handler_t new_h = 568 Block_copy(^(bool done, dispatch_data_t data, int error) { 569 new_context.orig_context = ^(void) { 570 h(done, data, error); 571 }; 572 dispatch_callback_wrap(&new_context); 573 }); 574 uptr submit_sync = (uptr)&new_context; 575 Release(thr, pc, submit_sync); 576 REAL(dispatch_io_write)(channel, offset, data, q, new_h); 577 Block_release(new_h); 578} 579 580TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 581 dispatch_block_t barrier) { 582 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 583 __block tsan_block_context_t new_context = { 584 nullptr, nullptr, &invoke_block, false, false, false, 0}; 585 new_context.non_queue_sync_object = (uptr)channel; 586 new_context.is_barrier_block = true; 587 dispatch_block_t new_block = Block_copy(^(void) { 588 new_context.orig_context = ^(void) { 589 barrier(); 590 }; 591 dispatch_callback_wrap(&new_context); 592 }); 593 uptr submit_sync = (uptr)&new_context; 594 Release(thr, pc, submit_sync); 595 REAL(dispatch_io_barrier)(channel, new_block); 596 Block_release(new_block); 597} 598 599TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 600 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 601 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 602 __block dispatch_io_t new_channel = nullptr; 603 __block tsan_block_context_t new_context = { 604 q, nullptr, &invoke_block, false, false, false, 0}; 605 cleanup_handler_t new_h = Block_copy(^(int error) { 606 { 607 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 608 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 609 } 610 new_context.orig_context = ^(void) { 611 h(error); 612 }; 613 dispatch_callback_wrap(&new_context); 614 }); 615 uptr submit_sync = (uptr)&new_context; 616 Release(thr, pc, submit_sync); 617 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 618 Block_release(new_h); 619 return new_channel; 620} 621 622TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 623 dispatch_io_type_t type, const char *path, int oflag, 624 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 626 q, h); 627 __block dispatch_io_t new_channel = nullptr; 628 __block tsan_block_context_t new_context = { 629 q, nullptr, &invoke_block, false, false, false, 0}; 630 cleanup_handler_t new_h = Block_copy(^(int error) { 631 { 632 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 633 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 634 } 635 new_context.orig_context = ^(void) { 636 h(error); 637 }; 638 dispatch_callback_wrap(&new_context); 639 }); 640 uptr submit_sync = (uptr)&new_context; 641 Release(thr, pc, submit_sync); 642 new_channel = 643 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 644 Block_release(new_h); 645 return new_channel; 646} 647 648TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 649 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 650 cleanup_handler_t h) { 651 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 652 __block dispatch_io_t new_channel = nullptr; 653 __block tsan_block_context_t new_context = { 654 q, nullptr, &invoke_block, false, false, false, 0}; 655 cleanup_handler_t new_h = Block_copy(^(int error) { 656 { 657 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 658 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 659 } 660 new_context.orig_context = ^(void) { 661 h(error); 662 }; 663 dispatch_callback_wrap(&new_context); 664 }); 665 uptr submit_sync = (uptr)&new_context; 666 Release(thr, pc, submit_sync); 667 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 668 Block_release(new_h); 669 return new_channel; 670} 671 672TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 673 dispatch_io_close_flags_t flags) { 674 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 675 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 676 return REAL(dispatch_io_close)(channel, flags); 677} 678 679} // namespace __tsan 680 681#endif // SANITIZER_MAC 682