1/*
2 * Copyright (C) 2008 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *  * Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 *  * Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in
12 *    the documentation and/or other materials provided with the
13 *    distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <pthread.h>
30
31#include <errno.h>
32#include <limits.h>
33#include <stdatomic.h>
34#include <string.h>
35#include <sys/cdefs.h>
36#include <sys/mman.h>
37#include <unistd.h>
38
39#include "pthread_internal.h"
40
41#include "private/bionic_constants.h"
42#include "private/bionic_futex.h"
43#include "private/bionic_systrace.h"
44#include "private/bionic_time_conversions.h"
45#include "private/bionic_tls.h"
46
47/* a mutex attribute holds the following fields
48 *
49 * bits:     name       description
50 * 0-3       type       type of mutex
51 * 4         shared     process-shared flag
52 */
53#define  MUTEXATTR_TYPE_MASK   0x000f
54#define  MUTEXATTR_SHARED_MASK 0x0010
55
56int pthread_mutexattr_init(pthread_mutexattr_t *attr)
57{
58    *attr = PTHREAD_MUTEX_DEFAULT;
59    return 0;
60}
61
62int pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
63{
64    *attr = -1;
65    return 0;
66}
67
68int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type_p)
69{
70    int type = (*attr & MUTEXATTR_TYPE_MASK);
71
72    if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK) {
73        return EINVAL;
74    }
75
76    *type_p = type;
77    return 0;
78}
79
80int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
81{
82    if (type < PTHREAD_MUTEX_NORMAL || type > PTHREAD_MUTEX_ERRORCHECK ) {
83        return EINVAL;
84    }
85
86    *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type;
87    return 0;
88}
89
90/* process-shared mutexes are not supported at the moment */
91
92int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int  pshared)
93{
94    switch (pshared) {
95    case PTHREAD_PROCESS_PRIVATE:
96        *attr &= ~MUTEXATTR_SHARED_MASK;
97        return 0;
98
99    case PTHREAD_PROCESS_SHARED:
100        /* our current implementation of pthread actually supports shared
101         * mutexes but won't cleanup if a process dies with the mutex held.
102         * Nevertheless, it's better than nothing. Shared mutexes are used
103         * by surfaceflinger and audioflinger.
104         */
105        *attr |= MUTEXATTR_SHARED_MASK;
106        return 0;
107    }
108    return EINVAL;
109}
110
111int pthread_mutexattr_getpshared(const pthread_mutexattr_t* attr, int* pshared) {
112    *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE;
113    return 0;
114}
115
116/* a mutex contains a state value and a owner_tid.
117 * The value is implemented as a 16-bit integer holding the following fields:
118 *
119 * bits:     name     description
120 * 15-14     type     mutex type
121 * 13        shared   process-shared flag
122 * 12-2      counter  counter of recursive mutexes
123 * 1-0       state    lock state (0, 1 or 2)
124 *
125 * The owner_tid is used only in recursive and errorcheck mutex to hold the mutex owner thread tid.
126 */
127
128/* Convenience macro, creates a mask of 'bits' bits that starts from
129 * the 'shift'-th least significant bit in a 32-bit word.
130 *
131 * Examples: FIELD_MASK(0,4)  -> 0xf
132 *           FIELD_MASK(16,9) -> 0x1ff0000
133 */
134#define  FIELD_MASK(shift,bits)           (((1 << (bits))-1) << (shift))
135
136/* This one is used to create a bit pattern from a given field value */
137#define  FIELD_TO_BITS(val,shift,bits)    (((val) & ((1 << (bits))-1)) << (shift))
138
139/* And this one does the opposite, i.e. extract a field's value from a bit pattern */
140#define  FIELD_FROM_BITS(val,shift,bits)  (((val) >> (shift)) & ((1 << (bits))-1))
141
142
143/* Convenience macros.
144 *
145 * These are used to form or modify the bit pattern of a given mutex value
146 */
147
148/* Mutex state:
149 *
150 * 0 for unlocked
151 * 1 for locked, no waiters
152 * 2 for locked, maybe waiters
153 */
154#define  MUTEX_STATE_SHIFT      0
155#define  MUTEX_STATE_LEN        2
156
157#define  MUTEX_STATE_MASK           FIELD_MASK(MUTEX_STATE_SHIFT, MUTEX_STATE_LEN)
158#define  MUTEX_STATE_FROM_BITS(v)   FIELD_FROM_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN)
159#define  MUTEX_STATE_TO_BITS(v)     FIELD_TO_BITS(v, MUTEX_STATE_SHIFT, MUTEX_STATE_LEN)
160
161#define  MUTEX_STATE_UNLOCKED            0   /* must be 0 to match PTHREAD_MUTEX_INITIALIZER */
162#define  MUTEX_STATE_LOCKED_UNCONTENDED  1   /* must be 1 due to atomic dec in unlock operation */
163#define  MUTEX_STATE_LOCKED_CONTENDED    2   /* must be 1 + LOCKED_UNCONTENDED due to atomic dec */
164
165#define  MUTEX_STATE_BITS_UNLOCKED            MUTEX_STATE_TO_BITS(MUTEX_STATE_UNLOCKED)
166#define  MUTEX_STATE_BITS_LOCKED_UNCONTENDED  MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_UNCONTENDED)
167#define  MUTEX_STATE_BITS_LOCKED_CONTENDED    MUTEX_STATE_TO_BITS(MUTEX_STATE_LOCKED_CONTENDED)
168
169// Return true iff the mutex is unlocked.
170#define MUTEX_STATE_BITS_IS_UNLOCKED(v) (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_UNLOCKED)
171
172// Return true iff the mutex is locked with no waiters.
173#define MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(v)  (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_UNCONTENDED)
174
175// return true iff the mutex is locked with maybe waiters.
176#define MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(v)   (((v) & MUTEX_STATE_MASK) == MUTEX_STATE_BITS_LOCKED_CONTENDED)
177
178/* used to flip from LOCKED_UNCONTENDED to LOCKED_CONTENDED */
179#define  MUTEX_STATE_BITS_FLIP_CONTENTION(v)      ((v) ^ (MUTEX_STATE_BITS_LOCKED_CONTENDED ^ MUTEX_STATE_BITS_LOCKED_UNCONTENDED))
180
181/* Mutex counter:
182 *
183 * We need to check for overflow before incrementing, and we also need to
184 * detect when the counter is 0
185 */
186#define  MUTEX_COUNTER_SHIFT         2
187#define  MUTEX_COUNTER_LEN           11
188#define  MUTEX_COUNTER_MASK          FIELD_MASK(MUTEX_COUNTER_SHIFT, MUTEX_COUNTER_LEN)
189
190#define  MUTEX_COUNTER_BITS_WILL_OVERFLOW(v)    (((v) & MUTEX_COUNTER_MASK) == MUTEX_COUNTER_MASK)
191#define  MUTEX_COUNTER_BITS_IS_ZERO(v)          (((v) & MUTEX_COUNTER_MASK) == 0)
192
193/* Used to increment the counter directly after overflow has been checked */
194#define  MUTEX_COUNTER_BITS_ONE      FIELD_TO_BITS(1, MUTEX_COUNTER_SHIFT,MUTEX_COUNTER_LEN)
195
196/* Mutex shared bit flag
197 *
198 * This flag is set to indicate that the mutex is shared among processes.
199 * This changes the futex opcode we use for futex wait/wake operations
200 * (non-shared operations are much faster).
201 */
202#define  MUTEX_SHARED_SHIFT    13
203#define  MUTEX_SHARED_MASK     FIELD_MASK(MUTEX_SHARED_SHIFT,1)
204
205/* Mutex type:
206 * We support normal, recursive and errorcheck mutexes.
207 */
208#define  MUTEX_TYPE_SHIFT      14
209#define  MUTEX_TYPE_LEN        2
210#define  MUTEX_TYPE_MASK       FIELD_MASK(MUTEX_TYPE_SHIFT,MUTEX_TYPE_LEN)
211
212#define  MUTEX_TYPE_TO_BITS(t)       FIELD_TO_BITS(t, MUTEX_TYPE_SHIFT, MUTEX_TYPE_LEN)
213
214#define  MUTEX_TYPE_BITS_NORMAL      MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_NORMAL)
215#define  MUTEX_TYPE_BITS_RECURSIVE   MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_RECURSIVE)
216#define  MUTEX_TYPE_BITS_ERRORCHECK  MUTEX_TYPE_TO_BITS(PTHREAD_MUTEX_ERRORCHECK)
217
218struct pthread_mutex_internal_t {
219  _Atomic(uint16_t) state;
220#if defined(__LP64__)
221  uint16_t __pad;
222  atomic_int owner_tid;
223  char __reserved[32];
224#else
225  _Atomic(uint16_t) owner_tid;
226#endif
227} __attribute__((aligned(4)));
228
229static_assert(sizeof(pthread_mutex_t) == sizeof(pthread_mutex_internal_t),
230              "pthread_mutex_t should actually be pthread_mutex_internal_t in implementation.");
231
232// For binary compatibility with old version of pthread_mutex_t, we can't use more strict alignment
233// than 4-byte alignment.
234static_assert(alignof(pthread_mutex_t) == 4,
235              "pthread_mutex_t should fulfill the alignment of pthread_mutex_internal_t.");
236
237static inline pthread_mutex_internal_t* __get_internal_mutex(pthread_mutex_t* mutex_interface) {
238  return reinterpret_cast<pthread_mutex_internal_t*>(mutex_interface);
239}
240
241int pthread_mutex_init(pthread_mutex_t* mutex_interface, const pthread_mutexattr_t* attr) {
242    pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface);
243
244    memset(mutex, 0, sizeof(pthread_mutex_internal_t));
245
246    if (__predict_true(attr == NULL)) {
247        atomic_init(&mutex->state, MUTEX_TYPE_BITS_NORMAL);
248        return 0;
249    }
250
251    uint16_t state = 0;
252    if ((*attr & MUTEXATTR_SHARED_MASK) != 0) {
253        state |= MUTEX_SHARED_MASK;
254    }
255
256    switch (*attr & MUTEXATTR_TYPE_MASK) {
257    case PTHREAD_MUTEX_NORMAL:
258      state |= MUTEX_TYPE_BITS_NORMAL;
259      break;
260    case PTHREAD_MUTEX_RECURSIVE:
261      state |= MUTEX_TYPE_BITS_RECURSIVE;
262      break;
263    case PTHREAD_MUTEX_ERRORCHECK:
264      state |= MUTEX_TYPE_BITS_ERRORCHECK;
265      break;
266    default:
267        return EINVAL;
268    }
269
270    atomic_init(&mutex->state, state);
271    atomic_init(&mutex->owner_tid, 0);
272    return 0;
273}
274
275static inline __always_inline int __pthread_normal_mutex_trylock(pthread_mutex_internal_t* mutex,
276                                                                 uint16_t shared) {
277    const uint16_t unlocked           = shared | MUTEX_STATE_BITS_UNLOCKED;
278    const uint16_t locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
279
280    uint16_t old_state = unlocked;
281    if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state,
282                         locked_uncontended, memory_order_acquire, memory_order_relaxed))) {
283        return 0;
284    }
285    return EBUSY;
286}
287
288/*
289 * Lock a mutex of type NORMAL.
290 *
291 * As noted above, there are three states:
292 *   0 (unlocked, no contention)
293 *   1 (locked, no contention)
294 *   2 (locked, contention)
295 *
296 * Non-recursive mutexes don't use the thread-id or counter fields, and the
297 * "type" value is zero, so the only bits that will be set are the ones in
298 * the lock state field.
299 */
300static inline __always_inline int __pthread_normal_mutex_lock(pthread_mutex_internal_t* mutex,
301                                                              uint16_t shared,
302                                                              bool use_realtime_clock,
303                                                              const timespec* abs_timeout_or_null) {
304    if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) {
305        return 0;
306    }
307    int result = check_timespec(abs_timeout_or_null, true);
308    if (result != 0) {
309        return result;
310    }
311
312    ScopedTrace trace("Contending for pthread mutex");
313
314    const uint16_t unlocked           = shared | MUTEX_STATE_BITS_UNLOCKED;
315    const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
316
317    // We want to go to sleep until the mutex is available, which requires
318    // promoting it to locked_contended. We need to swap in the new state
319    // and then wait until somebody wakes us up.
320    // An atomic_exchange is used to compete with other threads for the lock.
321    // If it returns unlocked, we have acquired the lock, otherwise another
322    // thread still holds the lock and we should wait again.
323    // If lock is acquired, an acquire fence is needed to make all memory accesses
324    // made by other threads visible to the current CPU.
325    while (atomic_exchange_explicit(&mutex->state, locked_contended,
326                                    memory_order_acquire) != unlocked) {
327        if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock,
328                            abs_timeout_or_null) == -ETIMEDOUT) {
329            return ETIMEDOUT;
330        }
331    }
332    return 0;
333}
334
335/*
336 * Release a normal mutex.  The caller is responsible for determining
337 * that we are in fact the owner of this lock.
338 */
339static inline __always_inline void __pthread_normal_mutex_unlock(pthread_mutex_internal_t* mutex,
340                                                                 uint16_t shared) {
341    const uint16_t unlocked         = shared | MUTEX_STATE_BITS_UNLOCKED;
342    const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
343
344    // We use an atomic_exchange to release the lock. If locked_contended state
345    // is returned, some threads is waiting for the lock and we need to wake up
346    // one of them.
347    // A release fence is required to make previous stores visible to next
348    // lock owner threads.
349    if (atomic_exchange_explicit(&mutex->state, unlocked,
350                                 memory_order_release) == locked_contended) {
351        // Wake up one waiting thread. We don't know which thread will be
352        // woken or when it'll start executing -- futexes make no guarantees
353        // here. There may not even be a thread waiting.
354        //
355        // The newly-woken thread will replace the unlocked state we just set above
356        // with locked_contended state, which means that when it eventually releases
357        // the mutex it will also call FUTEX_WAKE. This results in one extra wake
358        // call whenever a lock is contended, but let us avoid forgetting anyone
359        // without requiring us to track the number of sleepers.
360        //
361        // It's possible for another thread to sneak in and grab the lock between
362        // the exchange above and the wake call below. If the new thread is "slow"
363        // and holds the lock for a while, we'll wake up a sleeper, which will swap
364        // in locked_uncontended state and then go back to sleep since the lock is
365        // still held. If the new thread is "fast", running to completion before
366        // we call wake, the thread we eventually wake will find an unlocked mutex
367        // and will execute. Either way we have correct behavior and nobody is
368        // orphaned on the wait queue.
369        __futex_wake_ex(&mutex->state, shared, 1);
370    }
371}
372
373/* This common inlined function is used to increment the counter of a recursive mutex.
374 *
375 * If the counter overflows, it will return EAGAIN.
376 * Otherwise, it atomically increments the counter and returns 0.
377 *
378 */
379static inline __always_inline int __recursive_increment(pthread_mutex_internal_t* mutex,
380                                                        uint16_t old_state) {
381    // Detect recursive lock overflow and return EAGAIN.
382    // This is safe because only the owner thread can modify the
383    // counter bits in the mutex value.
384    if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(old_state)) {
385        return EAGAIN;
386    }
387
388    // Other threads are able to change the lower bits (e.g. promoting it to "contended"),
389    // but the mutex counter will not overflow. So we use atomic_fetch_add operation here.
390    // The mutex is still locked by current thread, so we don't need a release fence.
391    atomic_fetch_add_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed);
392    return 0;
393}
394
395static inline __always_inline int __recursive_or_errorcheck_mutex_wait(
396                                                      pthread_mutex_internal_t* mutex,
397                                                      uint16_t shared,
398                                                      uint16_t old_state,
399                                                      bool use_realtime_clock,
400                                                      const timespec* abs_timeout) {
401// __futex_wait always waits on a 32-bit value. But state is 16-bit. For a normal mutex, the owner_tid
402// field in mutex is not used. On 64-bit devices, the __pad field in mutex is not used.
403// But when a recursive or errorcheck mutex is used on 32-bit devices, we need to add the
404// owner_tid value in the value argument for __futex_wait, otherwise we may always get EAGAIN error.
405
406#if defined(__LP64__)
407  return __futex_wait_ex(&mutex->state, shared, old_state, use_realtime_clock, abs_timeout);
408
409#else
410  // This implementation works only when the layout of pthread_mutex_internal_t matches below expectation.
411  // And it is based on the assumption that Android is always in little-endian devices.
412  static_assert(offsetof(pthread_mutex_internal_t, state) == 0, "");
413  static_assert(offsetof(pthread_mutex_internal_t, owner_tid) == 2, "");
414
415  uint32_t owner_tid = atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed);
416  return __futex_wait_ex(&mutex->state, shared, (owner_tid << 16) | old_state,
417                         use_realtime_clock, abs_timeout);
418#endif
419}
420
421static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex,
422                                             bool use_realtime_clock,
423                                             const timespec* abs_timeout_or_null) {
424    uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
425    uint16_t mtype = (old_state & MUTEX_TYPE_MASK);
426    uint16_t shared = (old_state & MUTEX_SHARED_MASK);
427
428    // Handle common case first.
429    if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) {
430        return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null);
431    }
432
433    // Do we already own this recursive or error-check mutex?
434    pid_t tid = __get_thread()->tid;
435    if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) {
436        if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) {
437            return EDEADLK;
438        }
439        return __recursive_increment(mutex, old_state);
440    }
441
442    const uint16_t unlocked           = mtype | shared | MUTEX_STATE_BITS_UNLOCKED;
443    const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
444    const uint16_t locked_contended   = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
445
446    // First, if the mutex is unlocked, try to quickly acquire it.
447    // In the optimistic case where this works, set the state to locked_uncontended.
448    if (old_state == unlocked) {
449        // If exchanged successfully, an acquire fence is required to make
450        // all memory accesses made by other threads visible to the current CPU.
451        if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state,
452                             locked_uncontended, memory_order_acquire, memory_order_relaxed))) {
453            atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
454            return 0;
455        }
456    }
457
458    ScopedTrace trace("Contending for pthread mutex");
459
460    while (true) {
461        if (old_state == unlocked) {
462            // NOTE: We put the state to locked_contended since we _know_ there
463            // is contention when we are in this loop. This ensures all waiters
464            // will be unlocked.
465
466            // If exchanged successfully, an acquire fence is required to make
467            // all memory accesses made by other threads visible to the current CPU.
468            if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state,
469                                                                     &old_state, locked_contended,
470                                                                     memory_order_acquire,
471                                                                     memory_order_relaxed))) {
472                atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
473                return 0;
474            }
475            continue;
476        } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) {
477            // We should set it to locked_contended beforing going to sleep. This can make
478            // sure waiters will be woken up eventually.
479
480            int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state);
481            if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state,
482                                                                       &old_state, new_state,
483                                                                       memory_order_relaxed,
484                                                                       memory_order_relaxed))) {
485                continue;
486            }
487            old_state = new_state;
488        }
489
490        int result = check_timespec(abs_timeout_or_null, true);
491        if (result != 0) {
492            return result;
493        }
494        // We are in locked_contended state, sleep until someone wakes us up.
495        if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock,
496                                                 abs_timeout_or_null) == -ETIMEDOUT) {
497            return ETIMEDOUT;
498        }
499        old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
500    }
501}
502
503int pthread_mutex_lock(pthread_mutex_t* mutex_interface) {
504#if !defined(__LP64__)
505    // Some apps depend on being able to pass NULL as a mutex and get EINVAL
506    // back. Don't need to worry about it for LP64 since the ABI is brand new,
507    // but keep compatibility for LP32. http://b/19995172.
508    if (mutex_interface == NULL) {
509        return EINVAL;
510    }
511#endif
512
513    pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface);
514
515    uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
516    uint16_t mtype = (old_state & MUTEX_TYPE_MASK);
517    uint16_t shared = (old_state & MUTEX_SHARED_MASK);
518    // Avoid slowing down fast path of normal mutex lock operation.
519    if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) {
520      if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) {
521        return 0;
522      }
523    }
524    return __pthread_mutex_lock_with_timeout(mutex, false, nullptr);
525}
526
527int pthread_mutex_unlock(pthread_mutex_t* mutex_interface) {
528#if !defined(__LP64__)
529    // Some apps depend on being able to pass NULL as a mutex and get EINVAL
530    // back. Don't need to worry about it for LP64 since the ABI is brand new,
531    // but keep compatibility for LP32. http://b/19995172.
532    if (mutex_interface == NULL) {
533        return EINVAL;
534    }
535#endif
536
537    pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface);
538
539    uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
540    uint16_t mtype  = (old_state & MUTEX_TYPE_MASK);
541    uint16_t shared = (old_state & MUTEX_SHARED_MASK);
542
543    // Handle common case first.
544    if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) {
545        __pthread_normal_mutex_unlock(mutex, shared);
546        return 0;
547    }
548
549    // Do we already own this recursive or error-check mutex?
550    pid_t tid = __get_thread()->tid;
551    if ( tid != atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed) ) {
552        return EPERM;
553    }
554
555    // If the counter is > 0, we can simply decrement it atomically.
556    // Since other threads can mutate the lower state bits (and only the
557    // lower state bits), use a compare_exchange loop to do it.
558    if (!MUTEX_COUNTER_BITS_IS_ZERO(old_state)) {
559        // We still own the mutex, so a release fence is not needed.
560        atomic_fetch_sub_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed);
561        return 0;
562    }
563
564    // The counter is 0, so we'are going to unlock the mutex by resetting its
565    // state to unlocked, we need to perform a atomic_exchange inorder to read
566    // the current state, which will be locked_contended if there may have waiters
567    // to awake.
568    // A release fence is required to make previous stores visible to next
569    // lock owner threads.
570    atomic_store_explicit(&mutex->owner_tid, 0, memory_order_relaxed);
571    const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED;
572    old_state = atomic_exchange_explicit(&mutex->state, unlocked, memory_order_release);
573    if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(old_state)) {
574        __futex_wake_ex(&mutex->state, shared, 1);
575    }
576
577    return 0;
578}
579
580int pthread_mutex_trylock(pthread_mutex_t* mutex_interface) {
581    pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface);
582
583    uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
584    uint16_t mtype  = (old_state & MUTEX_TYPE_MASK);
585    uint16_t shared = (old_state & MUTEX_SHARED_MASK);
586
587    const uint16_t unlocked           = mtype | shared | MUTEX_STATE_BITS_UNLOCKED;
588    const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
589
590    // Handle common case first.
591    if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) {
592        return __pthread_normal_mutex_trylock(mutex, shared);
593    }
594
595    // Do we already own this recursive or error-check mutex?
596    pid_t tid = __get_thread()->tid;
597    if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) {
598        if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) {
599            return EBUSY;
600        }
601        return __recursive_increment(mutex, old_state);
602    }
603
604    // Same as pthread_mutex_lock, except that we don't want to wait, and
605    // the only operation that can succeed is a single compare_exchange to acquire the
606    // lock if it is released / not owned by anyone. No need for a complex loop.
607    // If exchanged successfully, an acquire fence is required to make
608    // all memory accesses made by other threads visible to the current CPU.
609    old_state = unlocked;
610    if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state,
611                                                               locked_uncontended,
612                                                               memory_order_acquire,
613                                                               memory_order_relaxed))) {
614        atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
615        return 0;
616    }
617    return EBUSY;
618}
619
620#if !defined(__LP64__)
621extern "C" int pthread_mutex_lock_timeout_np(pthread_mutex_t* mutex_interface, unsigned ms) {
622    timespec ts;
623    timespec_from_ms(ts, ms);
624    timespec abs_timeout;
625    absolute_timespec_from_timespec(abs_timeout, ts, CLOCK_MONOTONIC);
626    int error = __pthread_mutex_lock_with_timeout(__get_internal_mutex(mutex_interface),
627                                                  false, &abs_timeout);
628    if (error == ETIMEDOUT) {
629        error = EBUSY;
630    }
631    return error;
632}
633#endif
634
635int pthread_mutex_timedlock(pthread_mutex_t* mutex_interface, const timespec* abs_timeout) {
636    return __pthread_mutex_lock_with_timeout(__get_internal_mutex(mutex_interface),
637                                             true, abs_timeout);
638}
639
640int pthread_mutex_destroy(pthread_mutex_t* mutex_interface) {
641    pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface);
642    uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
643    // Store 0xffff to make the mutex unusable. Although POSIX standard says it is undefined
644    // behavior to destroy a locked mutex, we prefer not to change mutex->state in that situation.
645    if (MUTEX_STATE_BITS_IS_UNLOCKED(old_state) &&
646        atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 0xffff,
647                                                memory_order_relaxed, memory_order_relaxed)) {
648      return 0;
649    }
650    return EBUSY;
651}
652