1a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/*
2a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Copyright (C) 2010 The Android Open Source Project
3a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * All rights reserved.
4a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
5a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Redistribution and use in source and binary forms, with or without
6a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * modification, are permitted provided that the following conditions
7a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * are met:
8a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  * Redistributions of source code must retain the above copyright
9a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *    notice, this list of conditions and the following disclaimer.
10a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  * Redistributions in binary form must reproduce the above copyright
11a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *    notice, this list of conditions and the following disclaimer in
12a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *    the documentation and/or other materials provided with the
13a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *    distribution.
14a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
15a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * SUCH DAMAGE.
27a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */
28a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
29a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner#include <errno.h>
3008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui#include <stdatomic.h>
3176615dae93c18ac890e167c547a08c0228709a33Yabin Cui#include <string.h>
3276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle
3376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "pthread_internal.h"
3476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "private/bionic_futex.h"
3576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#include "private/bionic_lock.h"
3604303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_time_conversions.h"
37a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
38a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* Technical note:
39a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
40a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Possible states of a read/write lock:
41a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
42a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  - no readers and no writer (unlocked)
43a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  - one or more readers sharing the lock at the same time (read-locked)
44a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  - one writer holding the lock (write-lock)
45a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
46a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Additionally:
47a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  - trying to get the write-lock while there are any readers blocks
48a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *  - trying to get the read-lock while there is a writer blocks
4976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *  - a single thread can acquire the lock multiple times in read mode
5076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *
5176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *  - Posix states that behavior is undefined (may deadlock) if a thread tries
5276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *    to acquire the lock
5376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *      - in write mode while already holding the lock (whether in read or write mode)
5476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *      - in read mode while already holding the lock in write mode.
5576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *  - This implementation will return EDEADLK in "write after write" and "read after
5676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *    write" cases and will deadlock in write after read case.
57a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner *
58a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */
59a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
6076615dae93c18ac890e167c547a08c0228709a33Yabin Cui// A rwlockattr is implemented as a 32-bit integer which has following fields:
6176615dae93c18ac890e167c547a08c0228709a33Yabin Cui//  bits    name              description
6276615dae93c18ac890e167c547a08c0228709a33Yabin Cui//   1     rwlock_kind       have rwlock preference like PTHREAD_RWLOCK_PREFER_READER_NP.
6376615dae93c18ac890e167c547a08c0228709a33Yabin Cui//   0      process_shared    set to 1 if the rwlock is shared between processes.
6476615dae93c18ac890e167c547a08c0228709a33Yabin Cui
6576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_PSHARED_SHIFT 0
6676615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_KIND_SHIFT    1
6776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
6876615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_PSHARED_MASK  1
6976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_KIND_MASK     2
7076615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_RESERVED_MASK (~3)
7176615dae93c18ac890e167c547a08c0228709a33Yabin Cui
7276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline __always_inline bool __rwlockattr_getpshared(const pthread_rwlockattr_t* attr) {
7376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return (*attr & RWLOCKATTR_PSHARED_MASK) >> RWLOCKATTR_PSHARED_SHIFT;
7476615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
7576615dae93c18ac890e167c547a08c0228709a33Yabin Cui
7676615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline __always_inline void __rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) {
7776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  *attr = (*attr & ~RWLOCKATTR_PSHARED_MASK) | (pshared << RWLOCKATTR_PSHARED_SHIFT);
7876615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
7976615dae93c18ac890e167c547a08c0228709a33Yabin Cui
8076615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __rwlockattr_getkind(const pthread_rwlockattr_t* attr) {
8176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return (*attr & RWLOCKATTR_KIND_MASK) >> RWLOCKATTR_KIND_SHIFT;
8276615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
8376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
8476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline void __rwlockattr_setkind(pthread_rwlockattr_t* attr, int kind) {
8576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  *attr = (*attr & ~RWLOCKATTR_KIND_MASK) | (kind << RWLOCKATTR_KIND_SHIFT);
8676615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
87a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
88a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
8992687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_init(pthread_rwlockattr_t* attr) {
9076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  *attr = 0;
9176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
92a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
93a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
9492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_destroy(pthread_rwlockattr_t* attr) {
9576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  *attr = -1;
9676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
97a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
98a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
9976615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_getpshared(const pthread_rwlockattr_t* attr, int* pshared) {
10076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (__rwlockattr_getpshared(attr)) {
10176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    *pshared = PTHREAD_PROCESS_SHARED;
10276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  } else {
10376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    *pshared = PTHREAD_PROCESS_PRIVATE;
10476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  }
10576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return 0;
10676615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
10776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
10892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) {
10976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  switch (pshared) {
110a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    case PTHREAD_PROCESS_PRIVATE:
11176615dae93c18ac890e167c547a08c0228709a33Yabin Cui      __rwlockattr_setpshared(attr, 0);
11276615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return 0;
113a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    case PTHREAD_PROCESS_SHARED:
11476615dae93c18ac890e167c547a08c0228709a33Yabin Cui      __rwlockattr_setpshared(attr, 1);
11576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle      return 0;
116a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    default:
11776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle      return EINVAL;
11876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  }
119a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
120a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
12176615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t* attr, int* pref) {
12276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  *pref = __rwlockattr_getkind(attr);
12376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
124a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
125a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
12676615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_setkind_np(pthread_rwlockattr_t* attr, int pref) {
12776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  switch (pref) {
12876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    case PTHREAD_RWLOCK_PREFER_READER_NP:   // Fall through.
12976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    case PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP:
13076615dae93c18ac890e167c547a08c0228709a33Yabin Cui      __rwlockattr_setkind(attr, pref);
13176615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return 0;
13276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    default:
13376615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return EINVAL;
1342fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  }
13576615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
13676615dae93c18ac890e167c547a08c0228709a33Yabin Cui
13776615dae93c18ac890e167c547a08c0228709a33Yabin Cui// A rwlock state is implemented as a 32-bit integer which has following rules:
13876615dae93c18ac890e167c547a08c0228709a33Yabin Cui//  bits      name                              description
13976615dae93c18ac890e167c547a08c0228709a33Yabin Cui//   31      owned_by_writer_flag              set to 1 if the lock is owned by a writer now.
14076615dae93c18ac890e167c547a08c0228709a33Yabin Cui//  30-2     reader_count                      the count of readers holding the lock.
14176615dae93c18ac890e167c547a08c0228709a33Yabin Cui//   1       have_pending_writers              set to 1 if having pending writers.
14276615dae93c18ac890e167c547a08c0228709a33Yabin Cui//   0       have_pending_readers              set to 1 if having pending readers.
14376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
14476615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_SHIFT    0
14576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_WRITERS_SHIFT    1
14676615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_READER_COUNT_SHIFT            2
14776615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_OWNED_BY_WRITER_SHIFT        31
14876615dae93c18ac890e167c547a08c0228709a33Yabin Cui
14976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_FLAG     (1 << STATE_HAVE_PENDING_READERS_SHIFT)
15076615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_WRITERS_FLAG     (1 << STATE_HAVE_PENDING_WRITERS_SHIFT)
15176615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_READER_COUNT_CHANGE_STEP  (1 << STATE_READER_COUNT_SHIFT)
15276615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_OWNED_BY_WRITER_FLAG      (1 << STATE_OWNED_BY_WRITER_SHIFT)
15376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
15476615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_OR_WRITERS_FLAG \
15576615dae93c18ac890e167c547a08c0228709a33Yabin Cui          (STATE_HAVE_PENDING_READERS_FLAG | STATE_HAVE_PENDING_WRITERS_FLAG)
15676615dae93c18ac890e167c547a08c0228709a33Yabin Cui
15776615dae93c18ac890e167c547a08c0228709a33Yabin Cuistruct pthread_rwlock_internal_t {
15876615dae93c18ac890e167c547a08c0228709a33Yabin Cui  atomic_int state;
15976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  atomic_int writer_tid;
16076615dae93c18ac890e167c547a08c0228709a33Yabin Cui
16176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  bool pshared;
16276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  bool writer_nonrecursive_preferred;
16376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  uint16_t __pad;
16476615dae93c18ac890e167c547a08c0228709a33Yabin Cui
16576615dae93c18ac890e167c547a08c0228709a33Yabin Cui// When a reader thread plans to suspend on the rwlock, it will add STATE_HAVE_PENDING_READERS_FLAG
16676615dae93c18ac890e167c547a08c0228709a33Yabin Cui// in state, increase pending_reader_count, and wait on pending_reader_wakeup_serial. After woken
16776615dae93c18ac890e167c547a08c0228709a33Yabin Cui// up, the reader thread decreases pending_reader_count, and the last pending reader thread should
16876615dae93c18ac890e167c547a08c0228709a33Yabin Cui// remove STATE_HAVE_PENDING_READERS_FLAG in state. A pending writer thread works in a similar way,
16976615dae93c18ac890e167c547a08c0228709a33Yabin Cui// except that it uses flag and members for writer threads.
17076615dae93c18ac890e167c547a08c0228709a33Yabin Cui
17176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  Lock pending_lock;  // All pending members below are protected by pending_lock.
17276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  uint32_t pending_reader_count;  // Count of pending reader threads.
17376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  uint32_t pending_writer_count;  // Count of pending writer threads.
17476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  uint32_t pending_reader_wakeup_serial;  // Pending reader threads wait on this address by futex_wait.
17576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  uint32_t pending_writer_wakeup_serial;  // Pending writer threads wait on this address by futex_wait.
17608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui
1772fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#if defined(__LP64__)
1782fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  char __reserved[20];
17976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#else
18076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  char __reserved[4];
1812fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#endif
1822fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui};
1832fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
18476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_writer(int state) {
18576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state < 0;
18676615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
18776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
18876615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_readers(int state) {
18976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // If state >= 0, the owned_by_writer_flag is not set.
19076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // And if state >= STATE_READER_COUNT_CHANGE_STEP, the reader_count field is not empty.
19176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state >= STATE_READER_COUNT_CHANGE_STEP;
19276615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
19376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
19476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_readers_or_writer(int state) {
19576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state < 0 || state >= STATE_READER_COUNT_CHANGE_STEP;
19676615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
19776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
19876615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __state_add_writer_flag(int state) {
19976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state | STATE_OWNED_BY_WRITER_FLAG;
20076615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
20176615dae93c18ac890e167c547a08c0228709a33Yabin Cui
20276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_is_last_reader(int state) {
20376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return (state >> STATE_READER_COUNT_SHIFT) == 1;
20476615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
20576615dae93c18ac890e167c547a08c0228709a33Yabin Cui
20676615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_have_pending_writers(int state) {
20776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state & STATE_HAVE_PENDING_WRITERS_FLAG;
20876615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
20976615dae93c18ac890e167c547a08c0228709a33Yabin Cui
21076615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_have_pending_readers_or_writers(int state) {
21176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return state & STATE_HAVE_PENDING_READERS_OR_WRITERS_FLAG;
21276615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
21376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
214b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(sizeof(pthread_rwlock_t) == sizeof(pthread_rwlock_internal_t),
215b58457221364eaad039c2c49a42626b725e980d5Yabin Cui              "pthread_rwlock_t should actually be pthread_rwlock_internal_t in implementation.");
216b58457221364eaad039c2c49a42626b725e980d5Yabin Cui
217b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// For binary compatibility with old version of pthread_rwlock_t, we can't use more strict
218b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// alignment than 4-byte alignment.
219b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(alignof(pthread_rwlock_t) == 4,
220b58457221364eaad039c2c49a42626b725e980d5Yabin Cui             "pthread_rwlock_t should fulfill the alignment requirement of pthread_rwlock_internal_t.");
221b58457221364eaad039c2c49a42626b725e980d5Yabin Cui
22276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline pthread_rwlock_internal_t* __get_internal_rwlock(pthread_rwlock_t* rwlock_interface) {
2232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  return reinterpret_cast<pthread_rwlock_internal_t*>(rwlock_interface);
22408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui}
22508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui
2262fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_init(pthread_rwlock_t* rwlock_interface, const pthread_rwlockattr_t* attr) {
2272fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
22808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui
22976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  memset(rwlock, 0, sizeof(pthread_rwlock_internal_t));
23076615dae93c18ac890e167c547a08c0228709a33Yabin Cui
23176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (__predict_false(attr != NULL)) {
23276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pshared = __rwlockattr_getpshared(attr);
23376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    int kind = __rwlockattr_getkind(attr);
23476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    switch (kind) {
23576615dae93c18ac890e167c547a08c0228709a33Yabin Cui      case PTHREAD_RWLOCK_PREFER_READER_NP:
23676615dae93c18ac890e167c547a08c0228709a33Yabin Cui        rwlock->writer_nonrecursive_preferred = false;
23776615dae93c18ac890e167c547a08c0228709a33Yabin Cui        break;
23876615dae93c18ac890e167c547a08c0228709a33Yabin Cui      case PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP:
23976615dae93c18ac890e167c547a08c0228709a33Yabin Cui        rwlock->writer_nonrecursive_preferred = true;
24076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle        break;
24176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle      default:
24276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle        return EINVAL;
243a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    }
24476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if ((*attr & RWLOCKATTR_RESERVED_MASK) != 0) {
24576615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return EINVAL;
24676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
24776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  }
248a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
2492fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  atomic_init(&rwlock->state, 0);
25076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  rwlock->pending_lock.init(rwlock->pshared);
25176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
252a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
253a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
2542fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_destroy(pthread_rwlock_t* rwlock_interface) {
2552fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
2562fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
2572fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  if (atomic_load_explicit(&rwlock->state, memory_order_relaxed) != 0) {
25876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle    return EBUSY;
25976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  }
26076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
261a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
262a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
26376615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __can_acquire_read_lock(int old_state,
26476615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                                             bool writer_nonrecursive_preferred) {
26576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // If writer is preferred with nonrecursive reader, we prevent further readers from acquiring
26676615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // the lock when there are writers waiting for the lock.
26776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  bool cannot_apply = __state_owned_by_writer(old_state) ||
26876615dae93c18ac890e167c547a08c0228709a33Yabin Cui                      (writer_nonrecursive_preferred && __state_have_pending_writers(old_state));
26976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return !cannot_apply;
27076615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
27176615dae93c18ac890e167c547a08c0228709a33Yabin Cui
27276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __pthread_rwlock_tryrdlock(pthread_rwlock_internal_t* rwlock) {
27376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
27476615dae93c18ac890e167c547a08c0228709a33Yabin Cui
27576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  while (__predict_true(__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred))) {
27676615dae93c18ac890e167c547a08c0228709a33Yabin Cui
27776615dae93c18ac890e167c547a08c0228709a33Yabin Cui    int new_state = old_state + STATE_READER_COUNT_CHANGE_STEP;
27876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (__predict_false(!__state_owned_by_readers(new_state))) { // Happens when reader count overflows.
27976615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return EAGAIN;
28076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
28176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, new_state,
28276615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                              memory_order_acquire, memory_order_relaxed))) {
28376615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return 0;
28476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
28576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  }
28676615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return EBUSY;
28776615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
28876615dae93c18ac890e167c547a08c0228709a33Yabin Cui
2892fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock,
2902fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui                                        const timespec* abs_timeout_or_null) {
2912fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
29276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) {
29376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle    return EDEADLK;
29476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  }
295c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes
29608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  while (true) {
297c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    int result = __pthread_rwlock_tryrdlock(rwlock);
298c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (result == 0 || result == EAGAIN) {
299c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      return result;
300c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    }
301dd586f2ebd0c42904e699f3634568a38c97d4da7Elliott Hughes    result = check_timespec(abs_timeout_or_null, true);
302c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (result != 0) {
303c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      return result;
30476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
30576615dae93c18ac890e167c547a08c0228709a33Yabin Cui
3062fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui    int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
30776615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) {
30876615dae93c18ac890e167c547a08c0228709a33Yabin Cui      continue;
30976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
31076615dae93c18ac890e167c547a08c0228709a33Yabin Cui
31176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.lock();
31276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_reader_count++;
3132fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
31476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    // We rely on the fact that all atomic exchange operations on the same object (here it is
31576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    // rwlock->state) always appear to occur in a single total order. If the pending flag is added
31676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    // before unlocking, the unlocking thread will wakeup the waiter. Otherwise, we will see the
31776615dae93c18ac890e167c547a08c0228709a33Yabin Cui    // state is unlocked and will not wait anymore.
31876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_READERS_FLAG,
31976615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                         memory_order_relaxed);
3202fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
32176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    int old_serial = rwlock->pending_reader_wakeup_serial;
32276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
3232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
324c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    int futex_result = 0;
32576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (!__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) {
326c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      futex_result = __futex_wait_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared,
327c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui                                  old_serial, true, abs_timeout_or_null);
32876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
3292fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
33076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.lock();
33176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_reader_count--;
33276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (rwlock->pending_reader_count == 0) {
33376615dae93c18ac890e167c547a08c0228709a33Yabin Cui      atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_READERS_FLAG,
33476615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                memory_order_relaxed);
33576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
33676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
33776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
338c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (futex_result == -ETIMEDOUT) {
33976615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return ETIMEDOUT;
34076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
34176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  }
34276615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
34376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
34476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __can_acquire_write_lock(int old_state) {
34576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return !__state_owned_by_readers_or_writer(old_state);
34676615dae93c18ac890e167c547a08c0228709a33Yabin Cui}
34776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
34876615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __pthread_rwlock_trywrlock(pthread_rwlock_internal_t* rwlock) {
34976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
35076615dae93c18ac890e167c547a08c0228709a33Yabin Cui
35176615dae93c18ac890e167c547a08c0228709a33Yabin Cui  while (__predict_true(__can_acquire_write_lock(old_state))) {
35276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state,
35376615dae93c18ac890e167c547a08c0228709a33Yabin Cui          __state_add_writer_flag(old_state), memory_order_acquire, memory_order_relaxed))) {
35476615dae93c18ac890e167c547a08c0228709a33Yabin Cui
35576615dae93c18ac890e167c547a08c0228709a33Yabin Cui      atomic_store_explicit(&rwlock->writer_tid, __get_thread()->tid, memory_order_relaxed);
35676615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return 0;
357c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes    }
35808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  }
35976615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return EBUSY;
360c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes}
361a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
3622fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock,
3632fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui                                        const timespec* abs_timeout_or_null) {
3642fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
36576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) {
36676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle    return EDEADLK;
367c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes  }
36808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  while (true) {
369c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    int result = __pthread_rwlock_trywrlock(rwlock);
370c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (result == 0) {
371c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      return result;
372c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    }
373dd586f2ebd0c42904e699f3634568a38c97d4da7Elliott Hughes    result = check_timespec(abs_timeout_or_null, true);
374c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (result != 0) {
375c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      return result;
37676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
37776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
3782fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui    int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
37976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (__can_acquire_write_lock(old_state)) {
38076615dae93c18ac890e167c547a08c0228709a33Yabin Cui      continue;
38176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
38276615dae93c18ac890e167c547a08c0228709a33Yabin Cui
38376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.lock();
38476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_writer_count++;
3852fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
38676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_WRITERS_FLAG,
38776615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                         memory_order_relaxed);
3882fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
38976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    int old_serial = rwlock->pending_writer_wakeup_serial;
39076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
3912fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
392c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    int futex_result = 0;
39376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (!__can_acquire_write_lock(old_state)) {
394c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui      futex_result = __futex_wait_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared,
395c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui                                  old_serial, true, abs_timeout_or_null);
39676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
3972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
39876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.lock();
39976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_writer_count--;
40076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (rwlock->pending_writer_count == 0) {
40176615dae93c18ac890e167c547a08c0228709a33Yabin Cui      atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_WRITERS_FLAG,
40276615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                memory_order_relaxed);
40376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    }
40476615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
40576615dae93c18ac890e167c547a08c0228709a33Yabin Cui
406c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui    if (futex_result == -ETIMEDOUT) {
40776615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return ETIMEDOUT;
40876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle    }
40908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  }
410c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes}
411c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes
4122fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_rdlock(pthread_rwlock_t* rwlock_interface) {
4132fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
41476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // Avoid slowing down fast path of rdlock.
41576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (__predict_true(__pthread_rwlock_tryrdlock(rwlock) == 0)) {
41676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    return 0;
41776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  }
418c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui  return __pthread_rwlock_timedrdlock(rwlock, nullptr);
419a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
420a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
4212fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedrdlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) {
4222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
4232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
42492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle  return __pthread_rwlock_timedrdlock(rwlock, abs_timeout);
42592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle}
42692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle
4272fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) {
42876615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return __pthread_rwlock_tryrdlock(__get_internal_rwlock(rwlock_interface));
429a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
430a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
4312fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_wrlock(pthread_rwlock_t* rwlock_interface) {
4322fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
43376615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // Avoid slowing down fast path of wrlock.
43476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (__predict_true(__pthread_rwlock_trywrlock(rwlock) == 0)) {
43576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    return 0;
43676615dae93c18ac890e167c547a08c0228709a33Yabin Cui  }
437c9a659c57b256001fd63f9825bde69e660c2655bYabin Cui  return __pthread_rwlock_timedwrlock(rwlock, nullptr);
438a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
439a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
4402fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedwrlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) {
4412fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
4422fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
44392687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle  return __pthread_rwlock_timedwrlock(rwlock, abs_timeout);
44492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle}
44592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle
4462fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) {
44776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  return __pthread_rwlock_trywrlock(__get_internal_rwlock(rwlock_interface));
448a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
449a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner
4502fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) {
4512fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
45208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui
4532fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui  int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
45476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (__state_owned_by_writer(old_state)) {
45576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) != __get_thread()->tid) {
45676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle      return EPERM;
457a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    }
45876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    atomic_store_explicit(&rwlock->writer_tid, 0, memory_order_relaxed);
45976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    old_state = atomic_fetch_and_explicit(&rwlock->state, ~STATE_OWNED_BY_WRITER_FLAG,
46076615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                          memory_order_release);
46176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (!__state_have_pending_readers_or_writers(old_state)) {
46276615dae93c18ac890e167c547a08c0228709a33Yabin Cui      return 0;
463a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner    }
4642fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui
46576615dae93c18ac890e167c547a08c0228709a33Yabin Cui  } else if (__state_owned_by_readers(old_state)) {
46676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    old_state = atomic_fetch_sub_explicit(&rwlock->state, STATE_READER_COUNT_CHANGE_STEP,
46776615dae93c18ac890e167c547a08c0228709a33Yabin Cui                                          memory_order_release);
46876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    if (!__state_is_last_reader(old_state) || !__state_have_pending_readers_or_writers(old_state)) {
4692fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui      return 0;
47008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui    }
47176615dae93c18ac890e167c547a08c0228709a33Yabin Cui
47276615dae93c18ac890e167c547a08c0228709a33Yabin Cui  } else {
47376615dae93c18ac890e167c547a08c0228709a33Yabin Cui    return EPERM;
47408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  }
47576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle
47676615dae93c18ac890e167c547a08c0228709a33Yabin Cui  // Wake up pending readers or writers.
47776615dae93c18ac890e167c547a08c0228709a33Yabin Cui  rwlock->pending_lock.lock();
47876615dae93c18ac890e167c547a08c0228709a33Yabin Cui  if (rwlock->pending_writer_count != 0) {
47976615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_writer_wakeup_serial++;
48076615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
48176615dae93c18ac890e167c547a08c0228709a33Yabin Cui
48276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    __futex_wake_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared, 1);
48376615dae93c18ac890e167c547a08c0228709a33Yabin Cui
48476615dae93c18ac890e167c547a08c0228709a33Yabin Cui  } else if (rwlock->pending_reader_count != 0) {
48576615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_reader_wakeup_serial++;
48676615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
48776615dae93c18ac890e167c547a08c0228709a33Yabin Cui
48876615dae93c18ac890e167c547a08c0228709a33Yabin Cui    __futex_wake_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared, INT_MAX);
48976615dae93c18ac890e167c547a08c0228709a33Yabin Cui
49076615dae93c18ac890e167c547a08c0228709a33Yabin Cui  } else {
49176615dae93c18ac890e167c547a08c0228709a33Yabin Cui    // It happens when waiters are woken up by timeout.
49276615dae93c18ac890e167c547a08c0228709a33Yabin Cui    rwlock->pending_lock.unlock();
49308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui  }
49476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle  return 0;
495a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner}
496