11dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project/* 21dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * Copyright (C) 2008 The Android Open Source Project 31dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * All rights reserved. 41dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 51dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * Redistribution and use in source and binary forms, with or without 61dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * modification, are permitted provided that the following conditions 71dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * are met: 81dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * * Redistributions of source code must retain the above copyright 91dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * notice, this list of conditions and the following disclaimer. 101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * * Redistributions in binary form must reproduce the above copyright 111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * notice, this list of conditions and the following disclaimer in 121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * the documentation and/or other materials provided with the 131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * distribution. 141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * SUCH DAMAGE. 271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project */ 28d0c884d3595ecca03c3e70de9909c090cd5f9caePierre Peiffer 293e898476c7230b60a0f76968e64ff25f475b48c0Elliott Hughes#include <pthread.h> 303e898476c7230b60a0f76968e64ff25f475b48c0Elliott Hughes 31d0c884d3595ecca03c3e70de9909c090cd5f9caePierre Peiffer#include <errno.h> 32d0c884d3595ecca03c3e70de9909c090cd5f9caePierre Peiffer#include <limits.h> 3386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui#include <stdatomic.h> 345a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui#include <stdlib.h> 3517393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#include <string.h> 3686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui#include <sys/cdefs.h> 3784114c8dd5b17efecf7988f263ce431208d7be5aElliott Hughes#include <sys/mman.h> 38d0c884d3595ecca03c3e70de9909c090cd5f9caePierre Peiffer#include <unistd.h> 39d0c884d3595ecca03c3e70de9909c090cd5f9caePierre Peiffer 401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#include "pthread_internal.h" 41eb847bc8666842a3cfc9c06e8458ad1abebebaf0Elliott Hughes 4204303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_constants.h" 439e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui#include "private/bionic_fortify.h" 44eb847bc8666842a3cfc9c06e8458ad1abebebaf0Elliott Hughes#include "private/bionic_futex.h" 459e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui#include "private/bionic_sdk_version.h" 4686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui#include "private/bionic_systrace.h" 4704303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_time_conversions.h" 48eb847bc8666842a3cfc9c06e8458ad1abebebaf0Elliott Hughes#include "private/bionic_tls.h" 491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 50e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui/* a mutex attribute holds the following fields 51e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * 52e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * bits: name description 53e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * 0-3 type type of mutex 54e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * 4 shared process-shared flag 556b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui * 5 protocol whether it is a priority inherit mutex. 56e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui */ 57e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui#define MUTEXATTR_TYPE_MASK 0x000f 58e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui#define MUTEXATTR_SHARED_MASK 0x0010 596b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui#define MUTEXATTR_PROTOCOL_MASK 0x0020 606b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 616b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui#define MUTEXATTR_PROTOCOL_SHIFT 5 62e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 63e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_init(pthread_mutexattr_t *attr) 64e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui{ 65e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *attr = PTHREAD_MUTEX_DEFAULT; 66e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 67e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 68e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 69e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_destroy(pthread_mutexattr_t *attr) 70e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui{ 71e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *attr = -1; 72e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 73e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 74e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 75e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type_p) 76e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui{ 77e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui int type = (*attr & MUTEXATTR_TYPE_MASK); 78e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 79e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK) { 80e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return EINVAL; 81e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui } 82e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 83e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *type_p = type; 84e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 85e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 86e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 87e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) 88e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui{ 89e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK ) { 90e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return EINVAL; 91e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui } 92e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 93e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type; 94e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 95e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 96e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 97e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui/* process-shared mutexes are not supported at the moment */ 98e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 99e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) 100e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui{ 101e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui switch (pshared) { 102e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui case PTHREAD_PROCESS_PRIVATE: 103e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *attr &= ~MUTEXATTR_SHARED_MASK; 104e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 105e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 106e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui case PTHREAD_PROCESS_SHARED: 107e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui /* our current implementation of pthread actually supports shared 108e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * mutexes but won't cleanup if a process dies with the mutex held. 109e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * Nevertheless, it's better than nothing. Shared mutexes are used 110e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * by surfaceflinger and audioflinger. 111e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui */ 112e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *attr |= MUTEXATTR_SHARED_MASK; 113e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 114e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui } 115e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return EINVAL; 116e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 117e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 118e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cuiint pthread_mutexattr_getpshared(const pthread_mutexattr_t* attr, int* pshared) { 119e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; 120e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui return 0; 121e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} 122e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 1236b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuiint pthread_mutexattr_setprotocol(pthread_mutexattr_t* attr, int protocol) { 1246b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (protocol != PTHREAD_PRIO_NONE && protocol != PTHREAD_PRIO_INHERIT) { 1256b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EINVAL; 1266b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1276b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui *attr = (*attr & ~MUTEXATTR_PROTOCOL_MASK) | (protocol << MUTEXATTR_PROTOCOL_SHIFT); 1286b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 1296b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 1306b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 1316b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuiint pthread_mutexattr_getprotocol(const pthread_mutexattr_t* attr, int* protocol) { 1326b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui *protocol = (*attr & MUTEXATTR_PROTOCOL_MASK) >> MUTEXATTR_PROTOCOL_SHIFT; 1336b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 1346b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 1356b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 1366b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Priority Inheritance mutex implementation 1376b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistruct PIMutex { 1386b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // mutex type, can be 0 (normal), 1 (recursive), 2 (errorcheck), constant during lifetime 1396b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint8_t type; 1406b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // process-shared flag, constant during lifetime 1416b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui bool shared; 1426b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // <number of times a thread holding a recursive PI mutex> - 1 1436b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t counter; 1446b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // owner_tid is read/written by both userspace code and kernel code. It includes three fields: 1456b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // FUTEX_WAITERS, FUTEX_OWNER_DIED and FUTEX_TID_MASK. 1466b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui atomic_int owner_tid; 1476b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui}; 1486b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 1496b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline int PIMutexTryLock(PIMutex& mutex) { 1506b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui pid_t tid = __get_thread()->tid; 1516b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // Handle common case first. 1526b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui int old_owner = 0; 1536b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, 1546b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui &old_owner, tid, 1556b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_acquire, 1566b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_relaxed))) { 1576b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 1586b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1596b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (tid == (old_owner & FUTEX_TID_MASK)) { 1606b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // We already own this mutex. 1616b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (mutex.type == PTHREAD_MUTEX_NORMAL) { 1626b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EBUSY; 1636b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1646b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (mutex.type == PTHREAD_MUTEX_ERRORCHECK) { 1656b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EDEADLK; 1666b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1676b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (mutex.counter == 0xffff) { 1686b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EAGAIN; 1696b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1706b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui mutex.counter++; 1716b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 1726b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1736b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EBUSY; 1746b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 1756b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 1769e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui// Inlining this function in pthread_mutex_lock() adds the cost of stack frame instructions on 1775a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// ARM/ARM64, which increases at most 20 percent overhead. So make it noinline. 1785a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic int __attribute__((noinline)) PIMutexTimedLock(PIMutex& mutex, 17976f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry bool use_realtime_clock, 1805a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui const timespec* abs_timeout) { 1816b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui int ret = PIMutexTryLock(mutex); 1826b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(ret == 0)) { 1836b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 1846b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1856b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (ret == EBUSY) { 1865a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui ScopedTrace trace("Contending for pthread mutex"); 18776f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry ret = -__futex_pi_lock_ex(&mutex.owner_tid, mutex.shared, use_realtime_clock, abs_timeout); 1886b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 1896b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return ret; 1906b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 1916b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 1926b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic int PIMutexUnlock(PIMutex& mutex) { 1936b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui pid_t tid = __get_thread()->tid; 1946b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui int old_owner = tid; 1956b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // Handle common case first. 1966b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(mutex.type == PTHREAD_MUTEX_NORMAL)) { 1976b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, 1986b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui &old_owner, 0, 1996b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_release, 2006b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_relaxed))) { 2016b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 2026b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2036b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2046b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 2056b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (tid != (old_owner & FUTEX_TID_MASK)) { 2066b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // The mutex can only be unlocked by the thread who owns it. 2076b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EPERM; 2086b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2096b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (mutex.type == PTHREAD_MUTEX_RECURSIVE) { 2106b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (mutex.counter != 0u) { 2116b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui --mutex.counter; 2126b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 2136b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2146b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2156b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (old_owner == tid) { 2166b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // No thread is waiting. 2176b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, 2186b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui &old_owner, 0, 2196b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_release, 2206b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_relaxed))) { 2216b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 2226b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2236b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2246b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return -__futex_pi_unlock(&mutex.owner_tid, mutex.shared); 2256b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 2266b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 2276b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic int PIMutexDestroy(PIMutex& mutex) { 2286b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // The mutex should be in unlocked state (owner_tid == 0) when destroyed. 2296b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // Store 0xffffffff to make the mutex unusable. 2306b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui int old_owner = 0; 2316b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (atomic_compare_exchange_strong_explicit(&mutex.owner_tid, &old_owner, 0xffffffff, 2326b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui memory_order_relaxed, memory_order_relaxed)) { 2336b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 2346b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 2356b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return EBUSY; 2366b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} 2375a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2385a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui#if !defined(__LP64__) 2395a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2405a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuinamespace PIMutexAllocator { 2415a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// pthread_mutex_t has only 4 bytes in 32-bit programs, which are not enough to hold PIMutex. 2425a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// So we use malloc to allocate PIMutexes and use 16-bit of pthread_mutex_t as indexes to find 2435a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// the allocated PIMutexes. This allows at most 65536 PI mutexes. 2445a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// When calling operations like pthread_mutex_lock/unlock, the 16-bit index is mapped to the 2455a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// corresponding PIMutex. To make the map operation fast, we use a lockless mapping method: 2465a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// Once a PIMutex is allocated, all the data used to map index to the PIMutex isn't changed until 2475a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// it is destroyed. 2485a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// Below are the data structures: 2495a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// // struct Node contains a PIMutex. 2505a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// typedef Node NodeArray[256]; 2515a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// typedef NodeArray* NodeArrayP; 2525a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// NodeArrayP nodes[256]; 2535a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// 2545a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// A 16-bit index is mapped to Node as below: 2555a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// (*nodes[index >> 8])[index & 0xff] 2565a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// 2575a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// Also use a free list to allow O(1) finding recycled PIMutexes. 2585a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2595a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuiunion Node { 2605a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex mutex; 2615a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int next_free_id; // If not -1, refer to the next node in the free PIMutex list. 2625a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui}; 2635a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuitypedef Node NodeArray[256]; 2645a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuitypedef NodeArray* NodeArrayP; 2655a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2665a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// lock_ protects below items. 2675a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic Lock lock; 2685a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic NodeArrayP* nodes; 2695a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic int next_to_alloc_id; 2705a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic int first_free_id = -1; // If not -1, refer to the first node in the free PIMutex list. 2715a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2725a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic inline __always_inline Node& IdToNode(int id) { 2735a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return (*nodes[id >> 8])[id & 0xff]; 2745a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} 2755a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2765a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic inline __always_inline PIMutex& IdToPIMutex(int id) { 2775a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return IdToNode(id).mutex; 2785a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} 2795a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 2805a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic int AllocIdLocked() { 2815a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (first_free_id != -1) { 2825a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int result = first_free_id; 2835a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui first_free_id = IdToNode(result).next_free_id; 2845a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return result; 2855a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 2865a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (next_to_alloc_id >= 0x10000) { 2875a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return -1; 2885a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 2895a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int array_pos = next_to_alloc_id >> 8; 2905a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int node_pos = next_to_alloc_id & 0xff; 2915a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (node_pos == 0) { 2925a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (array_pos == 0) { 2935a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui nodes = static_cast<NodeArray**>(calloc(256, sizeof(NodeArray*))); 2945a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (nodes == nullptr) { 2955a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return -1; 2965a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 2975a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 2985a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui nodes[array_pos] = static_cast<NodeArray*>(malloc(sizeof(NodeArray))); 2995a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (nodes[array_pos] == nullptr) { 3005a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return -1; 3015a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 3025a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 3035a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return next_to_alloc_id++; 3045a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} 3055a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 3065a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// If succeed, return an id referring to a PIMutex, otherwise return -1. 3075a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// A valid id is in range [0, 0xffff]. 3085a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic int AllocId() { 3095a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui lock.lock(); 3105a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int result = AllocIdLocked(); 3115a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui lock.unlock(); 3125a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (result != -1) { 3135a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui memset(&IdToPIMutex(result), 0, sizeof(PIMutex)); 3145a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 3155a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return result; 3165a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} 3175a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 3185a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistatic void FreeId(int id) { 3195a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui lock.lock(); 3205a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui IdToNode(id).next_free_id = first_free_id; 3215a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui first_free_id = id; 3225a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui lock.unlock(); 3235a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} 3245a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 3255a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} // namespace PIMutexAllocator 3265a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 3275a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui#endif // !defined(__LP64__) 3286b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 3291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 330e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Convenience macro, creates a mask of 'bits' bits that starts from 331e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * the 'shift'-th least significant bit in a 32-bit word. 332e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 333e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * Examples: FIELD_MASK(0,4) -> 0xf 334e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * FIELD_MASK(16,9) -> 0x1ff0000 335e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner */ 336e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define FIELD_MASK(shift,bits) (((1 << (bits))-1) << (shift)) 337e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 338e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* This one is used to create a bit pattern from a given field value */ 339e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define FIELD_TO_BITS(val,shift,bits) (((val) & ((1 << (bits))-1)) << (shift)) 340e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 341e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* And this one does the opposite, i.e. extract a field's value from a bit pattern */ 342e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define FIELD_FROM_BITS(val,shift,bits) (((val) >> (shift)) & ((1 << (bits))-1)) 343e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 344e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui/* Convenience macros. 345e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * 346e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui * These are used to form or modify the bit pattern of a given mutex value 347e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui */ 348e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui 349e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Mutex state: 350e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 351e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 0 for unlocked 352e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 1 for locked, no waiters 353e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 2 for locked, maybe waiters 354e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner */ 355e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_SHIFT 0 356e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_LEN 2 357e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 358e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_MASK FIELD_MASK(MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 359e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_FROM_BITS(v) FIELD_FROM_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 360e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_TO_BITS(v) FIELD_TO_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN) 361e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 36217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#define MUTEX_STATE_UNLOCKED 0 /* must be 0 to match PTHREAD_MUTEX_INITIALIZER */ 363e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_LOCKED_UNCONTENDED 1 /* must be 1 due to atomic dec in unlock operation */ 364e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_LOCKED_CONTENDED 2 /* must be 1 + LOCKED_UNCONTENDED due to atomic dec */ 365e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 366e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_BITS_UNLOCKED MUTEX_STATE_TO_BITS(MUTEX_STATE_UNLOCKED) 367e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_BITS_LOCKED_UNCONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_UNCONTENDED) 368e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_BITS_LOCKED_CONTENDED MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_CONTENDED) 369e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 3700307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui// Return true iff the mutex is unlocked. 3710307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui#define MUTEX_STATE_BITS_IS_UNLOCKED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_UNLOCKED) 372e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 3730307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui// Return true iff the mutex is locked with no waiters. 3740307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui#define MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_UNCONTENDED) 3750307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui 3760307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui// return true iff the mutex is locked with maybe waiters. 3770307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui#define MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_CONTENDED) 378e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 379e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* used to flip from LOCKED_UNCONTENDED to LOCKED_CONTENDED */ 380e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_STATE_BITS_FLIP_CONTENTION(v) ((v) ^ (MUTEX_STATE_BITS_LOCKED_CONTENDED ^ MUTEX_STATE_BITS_LOCKED_UNCONTENDED)) 381e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 382e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Mutex counter: 383e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 384e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * We need to check for overflow before incrementing, and we also need to 385e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * detect when the counter is 0 386e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner */ 387e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_COUNTER_SHIFT 2 388e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_COUNTER_LEN 11 389e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_COUNTER_MASK FIELD_MASK(MUTEX_COUNTER_SHIFT, MUTEX_COUNTER_LEN) 390e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 391e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_COUNTER_BITS_WILL_OVERFLOW(v) (((v) & MUTEX_COUNTER_MASK) == MUTEX_COUNTER_MASK) 392e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_COUNTER_BITS_IS_ZERO(v) (((v) & MUTEX_COUNTER_MASK) == 0) 393e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 394e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Used to increment the counter directly after overflow has been checked */ 39586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui#define MUTEX_COUNTER_BITS_ONE FIELD_TO_BITS(1, MUTEX_COUNTER_SHIFT,MUTEX_COUNTER_LEN) 3961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 397e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Mutex shared bit flag 398e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * 399e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * This flag is set to indicate that the mutex is shared among processes. 400e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * This changes the futex opcode we use for futex wait/wake operations 401e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * (non-shared operations are much faster). 402e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner */ 403e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_SHARED_SHIFT 13 404e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_SHARED_MASK FIELD_MASK(MUTEX_SHARED_SHIFT,1) 405e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 406e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner/* Mutex type: 407e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner * We support normal, recursive and errorcheck mutexes. 408e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner */ 409e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_TYPE_SHIFT 14 410e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_TYPE_LEN 2 411e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_TYPE_MASK FIELD_MASK(MUTEX_TYPE_SHIFT,MUTEX_TYPE_LEN) 412022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner 413e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner#define MUTEX_TYPE_TO_BITS(t) FIELD_TO_BITS(t, MUTEX_TYPE_SHIFT, MUTEX_TYPE_LEN) 414e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 41517393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#define MUTEX_TYPE_BITS_NORMAL MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_NORMAL) 41617393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#define MUTEX_TYPE_BITS_RECURSIVE MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_RECURSIVE) 41717393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#define MUTEX_TYPE_BITS_ERRORCHECK MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_ERRORCHECK) 4186b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Use a special mutex type to mark priority inheritance mutexes. 4199e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui#define PI_MUTEX_STATE MUTEX_TYPE_TO_BITS(3) 4206b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 4216b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// For a PI mutex, it includes below fields: 4226b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Atomic(uint16_t) state; 4235a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// PIMutex pi_mutex; // uint16_t pi_mutex_id in 32-bit programs 4246b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4256b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// state holds the following fields: 4266b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4276b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// bits: name description 4286b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 15-14 type mutex type, should be 3 4299e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui// 13-0 padding should be 0 4306b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4316b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// pi_mutex holds the state of a PI mutex. 4325a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui// pi_mutex_id holds an integer to find the state of a PI mutex. 4336b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4346b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// For a Non-PI mutex, it includes below fields: 4356b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Atomic(uint16_t) state; 4366b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// atomic_int owner_tid; // Atomic(uint16_t) in 32-bit programs 4376b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4386b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// state holds the following fields: 4396b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4406b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// bits: name description 4416b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 15-14 type mutex type, can be 0 (normal), 1 (recursive), 2 (errorcheck) 4426b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 13 shared process-shared flag 4436b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 12-2 counter <number of times a thread holding a recursive Non-PI mutex> - 1 4446b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 1-0 state lock state (0, 1 or 2) 4456b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4466b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// bits 15-13 are constant during the lifetime of the mutex. 4476b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4486b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// owner_tid is used only in recursive and errorcheck Non-PI mutexes to hold the mutex owner 4496b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// thread id. 4506b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// 4516b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// PI mutexes and Non-PI mutexes are distinguished by checking type field in state. 45217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui#if defined(__LP64__) 4535a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistruct pthread_mutex_internal_t { 4545a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui _Atomic(uint16_t) state; 4555a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui uint16_t __pad; 4565a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui union { 4575a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui atomic_int owner_tid; 4585a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex pi_mutex; 4595a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui }; 4605a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui char __reserved[28]; 4615a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 4625a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex& ToPIMutex() { 4635a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return pi_mutex; 4645a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 4655a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 4665a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui void FreePIMutex() { 4675a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 4685a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui} __attribute__((aligned(4))); 4695a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 470e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui#else 4715a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cuistruct pthread_mutex_internal_t { 4725a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui _Atomic(uint16_t) state; 4735a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui union { 4745a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui _Atomic(uint16_t) owner_tid; 4755a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui uint16_t pi_mutex_id; 4765a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui }; 4775a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 4785a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex& ToPIMutex() { 4795a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return PIMutexAllocator::IdToPIMutex(pi_mutex_id); 4805a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 4815a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui 4825a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui void FreePIMutex() { 4835a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutexAllocator::FreeId(pi_mutex_id); 4845a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 485e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui} __attribute__((aligned(4))); 4865a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui#endif 48786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 48817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuistatic_assert(sizeof(pthread_mutex_t) == sizeof(pthread_mutex_internal_t), 48917393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui "pthread_mutex_t should actually be pthread_mutex_internal_t in implementation."); 49017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui 49117393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui// For binary compatibility with old version of pthread_mutex_t, we can't use more strict alignment 49217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui// than 4-byte alignment. 49317393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuistatic_assert(alignof(pthread_mutex_t) == 4, 49417393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui "pthread_mutex_t should fulfill the alignment of pthread_mutex_internal_t."); 49517393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui 49617393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuistatic inline pthread_mutex_internal_t* __get_internal_mutex(pthread_mutex_t* mutex_interface) { 49717393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui return reinterpret_cast<pthread_mutex_internal_t*>(mutex_interface); 49886fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui} 49986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 50017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiint pthread_mutex_init(pthread_mutex_t* mutex_interface, const pthread_mutexattr_t* attr) { 50117393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 50217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui 50317393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui memset(mutex, 0, sizeof(pthread_mutex_internal_t)); 50486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 505d4e753fea9345c24dcce89531dff9019bef14eb3Elliott Hughes if (__predict_true(attr == NULL)) { 50617393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui atomic_init(&mutex->state, MUTEX_TYPE_BITS_NORMAL); 50788f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner return 0; 508ba9c6f0989ae94778ba2b9f597adc827c9dc81e8David 'Digit' Turner } 50988f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner 510e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t state = 0; 511dff7203ee99ccac446b9a1c4371753a5216c6db4Elliott Hughes if ((*attr & MUTEXATTR_SHARED_MASK) != 0) { 51217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui state |= MUTEX_SHARED_MASK; 513dff7203ee99ccac446b9a1c4371753a5216c6db4Elliott Hughes } 51488f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner 51588f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner switch (*attr & MUTEXATTR_TYPE_MASK) { 51688f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner case PTHREAD_MUTEX_NORMAL: 517e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui state |= MUTEX_TYPE_BITS_NORMAL; 518e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui break; 51988f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner case PTHREAD_MUTEX_RECURSIVE: 520e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui state |= MUTEX_TYPE_BITS_RECURSIVE; 521e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui break; 52288f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner case PTHREAD_MUTEX_ERRORCHECK: 523e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui state |= MUTEX_TYPE_BITS_ERRORCHECK; 524e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui break; 52588f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner default: 52688f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner return EINVAL; 52788f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner } 52888f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner 5296b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (((*attr & MUTEXATTR_PROTOCOL_MASK) >> MUTEXATTR_PROTOCOL_SHIFT) == PTHREAD_PRIO_INHERIT) { 5305a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui#if !defined(__LP64__) 5315a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (state & MUTEX_SHARED_MASK) { 5325a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return EINVAL; 5335a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 5345a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int id = PIMutexAllocator::AllocId(); 5355a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (id == -1) { 5365a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return ENOMEM; 5375a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 5385a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui mutex->pi_mutex_id = id; 5396b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui#endif 5409e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui atomic_init(&mutex->state, PI_MUTEX_STATE); 5415a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex& pi_mutex = mutex->ToPIMutex(); 5425a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui pi_mutex.type = *attr & MUTEXATTR_TYPE_MASK; 5435a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui pi_mutex.shared = (*attr & MUTEXATTR_SHARED_MASK) != 0; 5446b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } else { 5456b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui atomic_init(&mutex->state, state); 5466b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui atomic_init(&mutex->owner_tid, 0); 5476b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 54888f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner return 0; 5491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 5501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5516b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// namespace for Non-PI mutex routines. 5526b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuinamespace NonPI { 5536b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 5546b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline int NormalMutexTryLock(pthread_mutex_internal_t* mutex, 5556b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared) { 556e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 557e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 5585b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui 559e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t old_state = unlocked; 56017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 56117393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui locked_uncontended, memory_order_acquire, memory_order_relaxed))) { 5625b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return 0; 5635b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 5645b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return EBUSY; 5655b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui} 5661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project/* 5686b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui * Lock a normal Non-PI mutex. 5691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 5701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * As noted above, there are three states: 5711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 0 (unlocked, no contention) 5721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 1 (locked, no contention) 5731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 2 (locked, contention) 5741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * 5751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * Non-recursive mutexes don't use the thread-id or counter fields, and the 5761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * "type" value is zero, so the only bits that will be set are the ones in 5771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * the lock state field. 5781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project */ 5796b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline int NormalMutexLock(pthread_mutex_internal_t* mutex, 5806b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared, 5816b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui bool use_realtime_clock, 5826b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui const timespec* abs_timeout_or_null) { 5836b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(NormalMutexTryLock(mutex, shared) == 0)) { 5845b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return 0; 58586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui } 586dd586f2ebd0c42904e699f3634568a38c97d4da7Elliott Hughes int result = check_timespec(abs_timeout_or_null, true); 587c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui if (result != 0) { 588c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui return result; 589c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui } 590a406ee6d5f616192e9a13afad6ac6a9969814fc1Brigid Smith 59186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui ScopedTrace trace("Contending for pthread mutex"); 592a406ee6d5f616192e9a13afad6ac6a9969814fc1Brigid Smith 593e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 594e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 5955b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui 59686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // We want to go to sleep until the mutex is available, which requires 59786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // promoting it to locked_contended. We need to swap in the new state 59817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui // and then wait until somebody wakes us up. 59986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // An atomic_exchange is used to compete with other threads for the lock. 60086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // If it returns unlocked, we have acquired the lock, otherwise another 60186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // thread still holds the lock and we should wait again. 60286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // If lock is acquired, an acquire fence is needed to make all memory accesses 6035b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // made by other threads visible to the current CPU. 60417393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui while (atomic_exchange_explicit(&mutex->state, locked_contended, 60586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_acquire) != unlocked) { 606c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock, 607c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui abs_timeout_or_null) == -ETIMEDOUT) { 6085b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return ETIMEDOUT; 6095b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 6101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project } 6115b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return 0; 6121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 6131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project/* 6156b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui * Release a normal Non-PI mutex. The caller is responsible for determining 6161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project * that we are in fact the owner of this lock. 6171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project */ 6186b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline void NormalMutexUnlock(pthread_mutex_internal_t* mutex, 6196b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared) { 620e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; 621e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 62286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 62386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // We use an atomic_exchange to release the lock. If locked_contended state 62486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // is returned, some threads is waiting for the lock and we need to wake up 62586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // one of them. 62686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // A release fence is required to make previous stores visible to next 62786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // lock owner threads. 62817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (atomic_exchange_explicit(&mutex->state, unlocked, 62986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_release) == locked_contended) { 63086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Wake up one waiting thread. We don't know which thread will be 63186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // woken or when it'll start executing -- futexes make no guarantees 63286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // here. There may not even be a thread waiting. 63386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // 63486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // The newly-woken thread will replace the unlocked state we just set above 63586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // with locked_contended state, which means that when it eventually releases 63686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // the mutex it will also call FUTEX_WAKE. This results in one extra wake 63786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // call whenever a lock is contended, but let us avoid forgetting anyone 63886fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // without requiring us to track the number of sleepers. 63986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // 64086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // It's possible for another thread to sneak in and grab the lock between 64186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // the exchange above and the wake call below. If the new thread is "slow" 64286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // and holds the lock for a while, we'll wake up a sleeper, which will swap 64386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // in locked_uncontended state and then go back to sleep since the lock is 64486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // still held. If the new thread is "fast", running to completion before 64586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // we call wake, the thread we eventually wake will find an unlocked mutex 64686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // and will execute. Either way we have correct behavior and nobody is 64786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // orphaned on the wait queue. 64817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui __futex_wake_ex(&mutex->state, shared, 1); 6491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project } 6501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 6511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6526b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui/* This common inlined function is used to increment the counter of a recursive Non-PI mutex. 653022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner * 6545b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui * If the counter overflows, it will return EAGAIN. 6555b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui * Otherwise, it atomically increments the counter and returns 0. 656022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner * 657022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner */ 6586b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline int RecursiveIncrement(pthread_mutex_internal_t* mutex, 6596b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t old_state) { 66086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Detect recursive lock overflow and return EAGAIN. 66186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // This is safe because only the owner thread can modify the 66286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // counter bits in the mutex value. 66317393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(old_state)) { 664022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner return EAGAIN; 665022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner } 666022d303116f742cd337852d37547e2ea24d97a25David 'Digit' Turner 667e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui // Other threads are able to change the lower bits (e.g. promoting it to "contended"), 668e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui // but the mutex counter will not overflow. So we use atomic_fetch_add operation here. 6696b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // The mutex is already locked by current thread, so we don't need an acquire fence. 67017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui atomic_fetch_add_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); 67186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui return 0; 6721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 6731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6746b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Wait on a recursive or errorcheck Non-PI mutex. 6756b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic inline __always_inline int RecursiveOrErrorcheckMutexWait(pthread_mutex_internal_t* mutex, 6766b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared, 6776b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t old_state, 6786b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui bool use_realtime_clock, 6796b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui const timespec* abs_timeout) { 680f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui// __futex_wait always waits on a 32-bit value. But state is 16-bit. For a normal mutex, the owner_tid 681f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui// field in mutex is not used. On 64-bit devices, the __pad field in mutex is not used. 682f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui// But when a recursive or errorcheck mutex is used on 32-bit devices, we need to add the 683f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui// owner_tid value in the value argument for __futex_wait, otherwise we may always get EAGAIN error. 684f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui 685f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui#if defined(__LP64__) 686c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui return __futex_wait_ex(&mutex->state, shared, old_state, use_realtime_clock, abs_timeout); 687f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui 688f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui#else 689f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui // This implementation works only when the layout of pthread_mutex_internal_t matches below expectation. 690f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui // And it is based on the assumption that Android is always in little-endian devices. 691f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui static_assert(offsetof(pthread_mutex_internal_t, state) == 0, ""); 692f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui static_assert(offsetof(pthread_mutex_internal_t, owner_tid) == 2, ""); 693f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui 694f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui uint32_t owner_tid = atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed); 695c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui return __futex_wait_ex(&mutex->state, shared, (owner_tid << 16) | old_state, 696c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui use_realtime_clock, abs_timeout); 697f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui#endif 698f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui} 699f796985923e2d8308e00ed9567f36546dafb98d7Yabin Cui 7006b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui// Lock a Non-PI mutex. 7016b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cuistatic int MutexLockWithTimeout(pthread_mutex_internal_t* mutex, bool use_realtime_clock, 7026b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui const timespec* abs_timeout_or_null) { 703e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 704e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 705e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 70640e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 70786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Handle common case first. 708d4e753fea9345c24dcce89531dff9019bef14eb3Elliott Hughes if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) { 7096b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return NormalMutexLock(mutex, shared, use_realtime_clock, abs_timeout_or_null); 71040e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner } 71140e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 71286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Do we already own this recursive or error-check mutex? 713e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui pid_t tid = __get_thread()->tid; 714e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { 7155b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { 7165b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return EDEADLK; 7175b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 7186b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return RecursiveIncrement(mutex, old_state); 7195b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 72040e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 721e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 722e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 723e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; 72488f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner 72586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // First, if the mutex is unlocked, try to quickly acquire it. 72686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // In the optimistic case where this works, set the state to locked_uncontended. 72717393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (old_state == unlocked) { 7285b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // If exchanged successfully, an acquire fence is required to make 7295b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // all memory accesses made by other threads visible to the current CPU. 73017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 731e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui locked_uncontended, memory_order_acquire, memory_order_relaxed))) { 732e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 733e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner return 0; 73440e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner } 735e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner } 73640e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 737a406ee6d5f616192e9a13afad6ac6a9969814fc1Brigid Smith ScopedTrace trace("Contending for pthread mutex"); 738a406ee6d5f616192e9a13afad6ac6a9969814fc1Brigid Smith 73986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui while (true) { 74017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (old_state == unlocked) { 74186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // NOTE: We put the state to locked_contended since we _know_ there 74286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // is contention when we are in this loop. This ensures all waiters 74386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // will be unlocked. 74486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 7455b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // If exchanged successfully, an acquire fence is required to make 7465b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // all memory accesses made by other threads visible to the current CPU. 74717393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, 748e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui &old_state, locked_contended, 74986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_acquire, 75086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_relaxed))) { 751e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 75286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui return 0; 753e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner } 75486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui continue; 75517393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) { 7565b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // We should set it to locked_contended beforing going to sleep. This can make 75786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // sure waiters will be woken up eventually. 75886fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 75917393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state); 76017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, 76117393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui &old_state, new_state, 76286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_relaxed, 76386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_relaxed))) { 764e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner continue; 765e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner } 76617393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui old_state = new_state; 767e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner } 768e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 769dd586f2ebd0c42904e699f3634568a38c97d4da7Elliott Hughes int result = check_timespec(abs_timeout_or_null, true); 770c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui if (result != 0) { 771c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui return result; 7725b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 773c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui // We are in locked_contended state, sleep until someone wakes us up. 7746b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (RecursiveOrErrorcheckMutexWait(mutex, shared, old_state, use_realtime_clock, 7756b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui abs_timeout_or_null) == -ETIMEDOUT) { 7765b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui return ETIMEDOUT; 7775b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui } 77817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 77940e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner } 7801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 7811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 7826b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui} // namespace NonPI 7836b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 7849e989f12d1186231d97dac6d038db7955acebdf3Yabin Cuistatic inline __always_inline bool IsMutexDestroyed(uint16_t mutex_state) { 7859e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return mutex_state == 0xffff; 7869e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui} 7879e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui 7889e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui// Inlining this function in pthread_mutex_lock() adds the cost of stack frame instructions on 7899e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui// ARM64. So make it noinline. 7909e989f12d1186231d97dac6d038db7955acebdf3Yabin Cuistatic int __attribute__((noinline)) HandleUsingDestroyedMutex(pthread_mutex_t* mutex, 7919e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui const char* function_name) { 7929e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (bionic_get_application_target_sdk_version() >= __ANDROID_API_P__) { 7939e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui __fortify_fatal("%s called on a destroyed mutex (%p)", function_name, mutex); 7949e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 7959e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return EBUSY; 7969e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui} 7979e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui 79817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiint pthread_mutex_lock(pthread_mutex_t* mutex_interface) { 799511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris#if !defined(__LP64__) 800baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // Some apps depend on being able to pass NULL as a mutex and get EINVAL 801baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // back. Don't need to worry about it for LP64 since the ABI is brand new, 802baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // but keep compatibility for LP32. http://b/19995172. 803511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris if (mutex_interface == NULL) { 804511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris return EINVAL; 805511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris } 806511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris#endif 807511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris 80817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 809e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 810e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 8115b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // Avoid slowing down fast path of normal mutex lock operation. 8125b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 8135a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 8145a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (__predict_true(NonPI::NormalMutexTryLock(mutex, shared) == 0)) { 8155a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return 0; 8165a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 8179e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 8189e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (old_state == PI_MUTEX_STATE) { 8195a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui PIMutex& m = mutex->ToPIMutex(); 8205a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui // Handle common case first. 8215a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (__predict_true(PIMutexTryLock(m) == 0)) { 8225a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return 0; 8235a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 82476f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return PIMutexTimedLock(mutex->ToPIMutex(), false, nullptr); 8256b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 8269e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (__predict_false(IsMutexDestroyed(old_state))) { 8279e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return HandleUsingDestroyedMutex(mutex_interface, __FUNCTION__); 8289e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 8296b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return NonPI::MutexLockWithTimeout(mutex, false, nullptr); 8305b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui} 8315b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui 83217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiint pthread_mutex_unlock(pthread_mutex_t* mutex_interface) { 833511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris#if !defined(__LP64__) 834baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // Some apps depend on being able to pass NULL as a mutex and get EINVAL 835baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // back. Don't need to worry about it for LP64 since the ABI is brand new, 836baa2a973bd776a51bb05a8590ab05d86eea7b321Dan Albert // but keep compatibility for LP32. http://b/19995172. 837511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris if (mutex_interface == NULL) { 838511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris return EINVAL; 839511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris } 840511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris#endif 841511cfd9dc8cb41bca4920687c7d816ee916ee8e5Christopher Ferris 84217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 843e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 844e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 845e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 84640e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 84786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Handle common case first. 848d4e753fea9345c24dcce89531dff9019bef14eb3Elliott Hughes if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 8496b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui NonPI::NormalMutexUnlock(mutex, shared); 8508641833b62e3b319796dc80ea16eb1592c05edf6Fabrice Di Meglio return 0; 851ba9c6f0989ae94778ba2b9f597adc827c9dc81e8David 'Digit' Turner } 8529e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (old_state == PI_MUTEX_STATE) { 8535a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return PIMutexUnlock(mutex->ToPIMutex()); 8546b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 8559e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (__predict_false(IsMutexDestroyed(old_state))) { 8569e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return HandleUsingDestroyedMutex(mutex_interface, __FUNCTION__); 8579e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 85840e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 85986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Do we already own this recursive or error-check mutex? 860e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui pid_t tid = __get_thread()->tid; 861e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if ( tid != atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed) ) { 86240e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner return EPERM; 863e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui } 86440e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 86586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // If the counter is > 0, we can simply decrement it atomically. 86686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Since other threads can mutate the lower state bits (and only the 86786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // lower state bits), use a compare_exchange loop to do it. 86817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (!MUTEX_COUNTER_BITS_IS_ZERO(old_state)) { 86986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // We still own the mutex, so a release fence is not needed. 87017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui atomic_fetch_sub_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); 87186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui return 0; 87240e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner } 873e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner 87486fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // The counter is 0, so we'are going to unlock the mutex by resetting its 87586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // state to unlocked, we need to perform a atomic_exchange inorder to read 87686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // the current state, which will be locked_contended if there may have waiters 87786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // to awake. 87886fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // A release fence is required to make previous stores visible to next 87986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // lock owner threads. 880e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui atomic_store_explicit(&mutex->owner_tid, 0, memory_order_relaxed); 881e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 88217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui old_state = atomic_exchange_explicit(&mutex->state, unlocked, memory_order_release); 88317393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(old_state)) { 88417393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui __futex_wake_ex(&mutex->state, shared, 1); 88588f06cd84a70f8a5212cb03272ec2c7cf0017afaDavid 'Digit' Turner } 88686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 88740e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner return 0; 8881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 8891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 89017393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiint pthread_mutex_trylock(pthread_mutex_t* mutex_interface) { 89117393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 89286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui 893e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 894e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 8955b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui 8965b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes // Handle common case first. 8975b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 8986b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 8996b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return NonPI::NormalMutexTryLock(mutex, shared); 90040e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner } 9019e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (old_state == PI_MUTEX_STATE) { 9025a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return PIMutexTryLock(mutex->ToPIMutex()); 9036b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 9049e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (__predict_false(IsMutexDestroyed(old_state))) { 9059e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return HandleUsingDestroyedMutex(mutex_interface, __FUNCTION__); 9069e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 90740e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 9085b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes // Do we already own this recursive or error-check mutex? 9095b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes pid_t tid = __get_thread()->tid; 910e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { 9115b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { 9125b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes return EBUSY; 9135b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes } 9146b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return NonPI::RecursiveIncrement(mutex, old_state); 9155b1111a6949b6751ce72bd0b034b7bbe6246a6b6Elliott Hughes } 91640e6b822866ee59f7823000384321bb899416cb1David 'Digit' Turner 9176b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 9186b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; 9196b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; 9206b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui 92186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // Same as pthread_mutex_lock, except that we don't want to wait, and 92286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // the only operation that can succeed is a single compare_exchange to acquire the 92386fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui // lock if it is released / not owned by anyone. No need for a complex loop. 9245b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // If exchanged successfully, an acquire fence is required to make 9255b8e7cd957f9380e93c3aee84962d157fe0bc526Yabin Cui // all memory accesses made by other threads visible to the current CPU. 92617393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cui old_state = unlocked; 927e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 928e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui locked_uncontended, 92986fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_acquire, 93086fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui memory_order_relaxed))) { 931e69c24543db577d8b219ab74b0ba7566e0f13b38Yabin Cui atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); 932e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner return 0; 933e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner } 934e1414aa96bb62057b1a25c6a9ea1797dd38dce59David 'Digit' Turner return EBUSY; 9351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project} 9361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9370e714a5b41451e84c5ded93a42c9a4b0a9440691Elliott Hughes#if !defined(__LP64__) 93817393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiextern "C" int pthread_mutex_lock_timeout_np(pthread_mutex_t* mutex_interface, unsigned ms) { 939c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui timespec ts; 940c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui timespec_from_ms(ts, ms); 94186fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui timespec abs_timeout; 942c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui absolute_timespec_from_timespec(abs_timeout, ts, CLOCK_MONOTONIC); 9436b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui int error = NonPI::MutexLockWithTimeout(__get_internal_mutex(mutex_interface), false, 9446b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui &abs_timeout); 94586fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui if (error == ETIMEDOUT) { 94686fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui error = EBUSY; 94786fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui } 94886fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui return error; 9490e714a5b41451e84c5ded93a42c9a4b0a9440691Elliott Hughes} 9500e714a5b41451e84c5ded93a42c9a4b0a9440691Elliott Hughes#endif 9510e714a5b41451e84c5ded93a42c9a4b0a9440691Elliott Hughes 95276f78fc70953959086f8e37a483fe9107c1e0236Tom Cherrystatic int __pthread_mutex_timedlock(pthread_mutex_t* mutex_interface, bool use_realtime_clock, 95376f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry const timespec* abs_timeout, const char* function) { 9546b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 9556b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 9566b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t mtype = (old_state & MUTEX_TYPE_MASK); 9576b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui // Handle common case first. 9586b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { 9596b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui uint16_t shared = (old_state & MUTEX_SHARED_MASK); 9606b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui if (__predict_true(NonPI::NormalMutexTryLock(mutex, shared) == 0)) { 9616b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui return 0; 9626b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 9636b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 9649e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (old_state == PI_MUTEX_STATE) { 96576f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return PIMutexTimedLock(mutex->ToPIMutex(), use_realtime_clock, abs_timeout); 9666b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 9679e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (__predict_false(IsMutexDestroyed(old_state))) { 96876f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return HandleUsingDestroyedMutex(mutex_interface, function); 9699e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui } 97076f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return NonPI::MutexLockWithTimeout(mutex, use_realtime_clock, abs_timeout); 97176f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry} 97276f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry 97376f78fc70953959086f8e37a483fe9107c1e0236Tom Cherryint pthread_mutex_timedlock(pthread_mutex_t* mutex_interface, const struct timespec* abs_timeout) { 97476f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return __pthread_mutex_timedlock(mutex_interface, true, abs_timeout, __FUNCTION__); 97576f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry} 97676f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry 97776f78fc70953959086f8e37a483fe9107c1e0236Tom Cherryint pthread_mutex_timedlock_monotonic_np(pthread_mutex_t* mutex_interface, 97876f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry const struct timespec* abs_timeout) { 97976f78fc70953959086f8e37a483fe9107c1e0236Tom Cherry return __pthread_mutex_timedlock(mutex_interface, false, abs_timeout, __FUNCTION__); 9807c0c3793722aea293c45921ef50e4adcdf9645ceMathias Agopian} 9817c0c3793722aea293c45921ef50e4adcdf9645ceMathias Agopian 98217393b06bab9cb3e95d0f466a56c746de19b8eeeYabin Cuiint pthread_mutex_destroy(pthread_mutex_t* mutex_interface) { 9830307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); 9840307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); 9859e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (__predict_false(IsMutexDestroyed(old_state))) { 9869e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui return HandleUsingDestroyedMutex(mutex_interface, __FUNCTION__); 9872dec3d7021fca57721ca7cd973d63ef817557aeeYabin Cui } 9889e989f12d1186231d97dac6d038db7955acebdf3Yabin Cui if (old_state == PI_MUTEX_STATE) { 9895a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui int result = PIMutexDestroy(mutex->ToPIMutex()); 9905a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui if (result == 0) { 9915a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui mutex->FreePIMutex(); 9922dec3d7021fca57721ca7cd973d63ef817557aeeYabin Cui atomic_store(&mutex->state, 0xffff); 9935a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui } 9945a00ba7c1c5d563b58255257898fe0a5903933ebYabin Cui return result; 9956b9c85b36d2e69b45d780e6a0b27d64909311a7eYabin Cui } 9960307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui // Store 0xffff to make the mutex unusable. Although POSIX standard says it is undefined 9970307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui // behavior to destroy a locked mutex, we prefer not to change mutex->state in that situation. 9980307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui if (MUTEX_STATE_BITS_IS_UNLOCKED(old_state) && 9990307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 0xffff, 10000307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui memory_order_relaxed, memory_order_relaxed)) { 10010307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui return 0; 100286fc96f73311f43980df770f4ff8022f1e9b296aYabin Cui } 10030307eee293e90e8584104a3c55bf2f270b1149b6Yabin Cui return EBUSY; 10047c0c3793722aea293c45921ef50e4adcdf9645ceMathias Agopian} 1005