pthread_rwlock.cpp revision 76615dae93c18ac890e167c547a08c0228709a33
1a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* 2a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Copyright (C) 2010 The Android Open Source Project 3a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * All rights reserved. 4a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 5a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Redistribution and use in source and binary forms, with or without 6a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * modification, are permitted provided that the following conditions 7a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * are met: 8a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions of source code must retain the above copyright 9a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer. 10a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions in binary form must reproduce the above copyright 11a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer in 12a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * the documentation and/or other materials provided with the 13a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * distribution. 14a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 15a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * SUCH DAMAGE. 27a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 28a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 29a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner#include <errno.h> 3008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui#include <stdatomic.h> 3176615dae93c18ac890e167c547a08c0228709a33Yabin Cui#include <string.h> 3276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 3376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "pthread_internal.h" 3476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "private/bionic_futex.h" 3576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#include "private/bionic_lock.h" 3604303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_time_conversions.h" 37a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 38a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* Technical note: 39a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 40a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Possible states of a read/write lock: 41a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 42a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - no readers and no writer (unlocked) 43a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one or more readers sharing the lock at the same time (read-locked) 44a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one writer holding the lock (write-lock) 45a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 46a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Additionally: 47a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the write-lock while there are any readers blocks 48a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the read-lock while there is a writer blocks 4976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - a single thread can acquire the lock multiple times in read mode 5076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * 5176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - Posix states that behavior is undefined (may deadlock) if a thread tries 5276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * to acquire the lock 5376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in write mode while already holding the lock (whether in read or write mode) 5476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in read mode while already holding the lock in write mode. 5576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - This implementation will return EDEADLK in "write after write" and "read after 5676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * write" cases and will deadlock in write after read case. 57a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 58a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 59a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 6076615dae93c18ac890e167c547a08c0228709a33Yabin Cui// A rwlockattr is implemented as a 32-bit integer which has following fields: 6176615dae93c18ac890e167c547a08c0228709a33Yabin Cui// bits name description 6276615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 1 rwlock_kind have rwlock preference like PTHREAD_RWLOCK_PREFER_READER_NP. 6376615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 0 process_shared set to 1 if the rwlock is shared between processes. 6476615dae93c18ac890e167c547a08c0228709a33Yabin Cui 6576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_PSHARED_SHIFT 0 6676615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_KIND_SHIFT 1 6776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 6876615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_PSHARED_MASK 1 6976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_KIND_MASK 2 7076615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define RWLOCKATTR_RESERVED_MASK (~3) 7176615dae93c18ac890e167c547a08c0228709a33Yabin Cui 7276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline __always_inline bool __rwlockattr_getpshared(const pthread_rwlockattr_t* attr) { 7376615dae93c18ac890e167c547a08c0228709a33Yabin Cui return (*attr & RWLOCKATTR_PSHARED_MASK) >> RWLOCKATTR_PSHARED_SHIFT; 7476615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 7576615dae93c18ac890e167c547a08c0228709a33Yabin Cui 7676615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline __always_inline void __rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) { 7776615dae93c18ac890e167c547a08c0228709a33Yabin Cui *attr = (*attr & ~RWLOCKATTR_PSHARED_MASK) | (pshared << RWLOCKATTR_PSHARED_SHIFT); 7876615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 7976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 8076615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __rwlockattr_getkind(const pthread_rwlockattr_t* attr) { 8176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return (*attr & RWLOCKATTR_KIND_MASK) >> RWLOCKATTR_KIND_SHIFT; 8276615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 8376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 8476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline void __rwlockattr_setkind(pthread_rwlockattr_t* attr, int kind) { 8576615dae93c18ac890e167c547a08c0228709a33Yabin Cui *attr = (*attr & ~RWLOCKATTR_KIND_MASK) | (kind << RWLOCKATTR_KIND_SHIFT); 8676615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 87a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 88a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 8992687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_init(pthread_rwlockattr_t* attr) { 9076615dae93c18ac890e167c547a08c0228709a33Yabin Cui *attr = 0; 9176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 92a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 93a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 9492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_destroy(pthread_rwlockattr_t* attr) { 9576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = -1; 9676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 97a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 98a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 9976615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_getpshared(const pthread_rwlockattr_t* attr, int* pshared) { 10076615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__rwlockattr_getpshared(attr)) { 10176615dae93c18ac890e167c547a08c0228709a33Yabin Cui *pshared = PTHREAD_PROCESS_SHARED; 10276615dae93c18ac890e167c547a08c0228709a33Yabin Cui } else { 10376615dae93c18ac890e167c547a08c0228709a33Yabin Cui *pshared = PTHREAD_PROCESS_PRIVATE; 10476615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 10576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 10676615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 10776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 10892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) { 10976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle switch (pshared) { 110a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_PRIVATE: 11176615dae93c18ac890e167c547a08c0228709a33Yabin Cui __rwlockattr_setpshared(attr, 0); 11276615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 113a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_SHARED: 11476615dae93c18ac890e167c547a08c0228709a33Yabin Cui __rwlockattr_setpshared(attr, 1); 11576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 116a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner default: 11776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 11876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 119a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 120a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 12176615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t* attr, int* pref) { 12276615dae93c18ac890e167c547a08c0228709a33Yabin Cui *pref = __rwlockattr_getkind(attr); 12376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 124a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 125a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 12676615dae93c18ac890e167c547a08c0228709a33Yabin Cuiint pthread_rwlockattr_setkind_np(pthread_rwlockattr_t* attr, int pref) { 12776615dae93c18ac890e167c547a08c0228709a33Yabin Cui switch (pref) { 12876615dae93c18ac890e167c547a08c0228709a33Yabin Cui case PTHREAD_RWLOCK_PREFER_READER_NP: // Fall through. 12976615dae93c18ac890e167c547a08c0228709a33Yabin Cui case PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP: 13076615dae93c18ac890e167c547a08c0228709a33Yabin Cui __rwlockattr_setkind(attr, pref); 13176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 13276615dae93c18ac890e167c547a08c0228709a33Yabin Cui default: 13376615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EINVAL; 1342fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 13576615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 13676615dae93c18ac890e167c547a08c0228709a33Yabin Cui 13776615dae93c18ac890e167c547a08c0228709a33Yabin Cui// A rwlock state is implemented as a 32-bit integer which has following rules: 13876615dae93c18ac890e167c547a08c0228709a33Yabin Cui// bits name description 13976615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 31 owned_by_writer_flag set to 1 if the lock is owned by a writer now. 14076615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 30-2 reader_count the count of readers holding the lock. 14176615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 1 have_pending_writers set to 1 if having pending writers. 14276615dae93c18ac890e167c547a08c0228709a33Yabin Cui// 0 have_pending_readers set to 1 if having pending readers. 14376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 14476615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_SHIFT 0 14576615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_WRITERS_SHIFT 1 14676615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_READER_COUNT_SHIFT 2 14776615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_OWNED_BY_WRITER_SHIFT 31 14876615dae93c18ac890e167c547a08c0228709a33Yabin Cui 14976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_FLAG (1 << STATE_HAVE_PENDING_READERS_SHIFT) 15076615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_WRITERS_FLAG (1 << STATE_HAVE_PENDING_WRITERS_SHIFT) 15176615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_READER_COUNT_CHANGE_STEP (1 << STATE_READER_COUNT_SHIFT) 15276615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_OWNED_BY_WRITER_FLAG (1 << STATE_OWNED_BY_WRITER_SHIFT) 15376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 15476615dae93c18ac890e167c547a08c0228709a33Yabin Cui#define STATE_HAVE_PENDING_READERS_OR_WRITERS_FLAG \ 15576615dae93c18ac890e167c547a08c0228709a33Yabin Cui (STATE_HAVE_PENDING_READERS_FLAG | STATE_HAVE_PENDING_WRITERS_FLAG) 15676615dae93c18ac890e167c547a08c0228709a33Yabin Cui 15776615dae93c18ac890e167c547a08c0228709a33Yabin Cuistruct pthread_rwlock_internal_t { 15876615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_int state; 15976615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_int writer_tid; 16076615dae93c18ac890e167c547a08c0228709a33Yabin Cui 16176615dae93c18ac890e167c547a08c0228709a33Yabin Cui bool pshared; 16276615dae93c18ac890e167c547a08c0228709a33Yabin Cui bool writer_nonrecursive_preferred; 16376615dae93c18ac890e167c547a08c0228709a33Yabin Cui uint16_t __pad; 16476615dae93c18ac890e167c547a08c0228709a33Yabin Cui 16576615dae93c18ac890e167c547a08c0228709a33Yabin Cui// When a reader thread plans to suspend on the rwlock, it will add STATE_HAVE_PENDING_READERS_FLAG 16676615dae93c18ac890e167c547a08c0228709a33Yabin Cui// in state, increase pending_reader_count, and wait on pending_reader_wakeup_serial. After woken 16776615dae93c18ac890e167c547a08c0228709a33Yabin Cui// up, the reader thread decreases pending_reader_count, and the last pending reader thread should 16876615dae93c18ac890e167c547a08c0228709a33Yabin Cui// remove STATE_HAVE_PENDING_READERS_FLAG in state. A pending writer thread works in a similar way, 16976615dae93c18ac890e167c547a08c0228709a33Yabin Cui// except that it uses flag and members for writer threads. 17076615dae93c18ac890e167c547a08c0228709a33Yabin Cui 17176615dae93c18ac890e167c547a08c0228709a33Yabin Cui Lock pending_lock; // All pending members below are protected by pending_lock. 17276615dae93c18ac890e167c547a08c0228709a33Yabin Cui uint32_t pending_reader_count; // Count of pending reader threads. 17376615dae93c18ac890e167c547a08c0228709a33Yabin Cui uint32_t pending_writer_count; // Count of pending writer threads. 17476615dae93c18ac890e167c547a08c0228709a33Yabin Cui uint32_t pending_reader_wakeup_serial; // Pending reader threads wait on this address by futex_wait. 17576615dae93c18ac890e167c547a08c0228709a33Yabin Cui uint32_t pending_writer_wakeup_serial; // Pending writer threads wait on this address by futex_wait. 17608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 1772fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#if defined(__LP64__) 1782fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui char __reserved[20]; 17976615dae93c18ac890e167c547a08c0228709a33Yabin Cui#else 18076615dae93c18ac890e167c547a08c0228709a33Yabin Cui char __reserved[4]; 1812fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#endif 1822fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui}; 1832fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 18476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_writer(int state) { 18576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state < 0; 18676615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 18776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 18876615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_readers(int state) { 18976615dae93c18ac890e167c547a08c0228709a33Yabin Cui // If state >= 0, the owned_by_writer_flag is not set. 19076615dae93c18ac890e167c547a08c0228709a33Yabin Cui // And if state >= STATE_READER_COUNT_CHANGE_STEP, the reader_count field is not empty. 19176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state >= STATE_READER_COUNT_CHANGE_STEP; 19276615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 19376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 19476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_owned_by_readers_or_writer(int state) { 19576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state < 0 || state >= STATE_READER_COUNT_CHANGE_STEP; 19676615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 19776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 19876615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __state_add_writer_flag(int state) { 19976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state | STATE_OWNED_BY_WRITER_FLAG; 20076615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 20176615dae93c18ac890e167c547a08c0228709a33Yabin Cui 20276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_is_last_reader(int state) { 20376615dae93c18ac890e167c547a08c0228709a33Yabin Cui return (state >> STATE_READER_COUNT_SHIFT) == 1; 20476615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 20576615dae93c18ac890e167c547a08c0228709a33Yabin Cui 20676615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_have_pending_writers(int state) { 20776615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state & STATE_HAVE_PENDING_WRITERS_FLAG; 20876615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 20976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 21076615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __state_have_pending_readers_or_writers(int state) { 21176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return state & STATE_HAVE_PENDING_READERS_OR_WRITERS_FLAG; 21276615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 21376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 214b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(sizeof(pthread_rwlock_t) == sizeof(pthread_rwlock_internal_t), 215b58457221364eaad039c2c49a42626b725e980d5Yabin Cui "pthread_rwlock_t should actually be pthread_rwlock_internal_t in implementation."); 216b58457221364eaad039c2c49a42626b725e980d5Yabin Cui 217b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// For binary compatibility with old version of pthread_rwlock_t, we can't use more strict 218b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// alignment than 4-byte alignment. 219b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(alignof(pthread_rwlock_t) == 4, 220b58457221364eaad039c2c49a42626b725e980d5Yabin Cui "pthread_rwlock_t should fulfill the alignment requirement of pthread_rwlock_internal_t."); 221b58457221364eaad039c2c49a42626b725e980d5Yabin Cui 22276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline pthread_rwlock_internal_t* __get_internal_rwlock(pthread_rwlock_t* rwlock_interface) { 2232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return reinterpret_cast<pthread_rwlock_internal_t*>(rwlock_interface); 22408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui} 22508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2262fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_init(pthread_rwlock_t* rwlock_interface, const pthread_rwlockattr_t* attr) { 2272fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 22808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 22976615dae93c18ac890e167c547a08c0228709a33Yabin Cui memset(rwlock, 0, sizeof(pthread_rwlock_internal_t)); 23076615dae93c18ac890e167c547a08c0228709a33Yabin Cui 23176615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_false(attr != NULL)) { 23276615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pshared = __rwlockattr_getpshared(attr); 23376615dae93c18ac890e167c547a08c0228709a33Yabin Cui int kind = __rwlockattr_getkind(attr); 23476615dae93c18ac890e167c547a08c0228709a33Yabin Cui switch (kind) { 23576615dae93c18ac890e167c547a08c0228709a33Yabin Cui case PTHREAD_RWLOCK_PREFER_READER_NP: 23676615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->writer_nonrecursive_preferred = false; 23776615dae93c18ac890e167c547a08c0228709a33Yabin Cui break; 23876615dae93c18ac890e167c547a08c0228709a33Yabin Cui case PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP: 23976615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->writer_nonrecursive_preferred = true; 24076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle break; 24176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle default: 24276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 243a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 24476615dae93c18ac890e167c547a08c0228709a33Yabin Cui if ((*attr & RWLOCKATTR_RESERVED_MASK) != 0) { 24576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EINVAL; 24676615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 24776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 248a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2492fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->state, 0); 25076615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.init(rwlock->pshared); 25176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 252a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 253a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2542fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_destroy(pthread_rwlock_t* rwlock_interface) { 2552fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2562fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2572fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_load_explicit(&rwlock->state, memory_order_relaxed) != 0) { 25876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EBUSY; 25976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 26076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 261a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 262a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 26376615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __can_acquire_read_lock(int old_state, 26476615dae93c18ac890e167c547a08c0228709a33Yabin Cui bool writer_nonrecursive_preferred) { 26576615dae93c18ac890e167c547a08c0228709a33Yabin Cui // If writer is preferred with nonrecursive reader, we prevent further readers from acquiring 26676615dae93c18ac890e167c547a08c0228709a33Yabin Cui // the lock when there are writers waiting for the lock. 26776615dae93c18ac890e167c547a08c0228709a33Yabin Cui bool cannot_apply = __state_owned_by_writer(old_state) || 26876615dae93c18ac890e167c547a08c0228709a33Yabin Cui (writer_nonrecursive_preferred && __state_have_pending_writers(old_state)); 26976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return !cannot_apply; 27076615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 27176615dae93c18ac890e167c547a08c0228709a33Yabin Cui 27276615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __pthread_rwlock_tryrdlock(pthread_rwlock_internal_t* rwlock) { 27376615dae93c18ac890e167c547a08c0228709a33Yabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 27476615dae93c18ac890e167c547a08c0228709a33Yabin Cui 27576615dae93c18ac890e167c547a08c0228709a33Yabin Cui while (__predict_true(__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred))) { 27676615dae93c18ac890e167c547a08c0228709a33Yabin Cui 27776615dae93c18ac890e167c547a08c0228709a33Yabin Cui int new_state = old_state + STATE_READER_COUNT_CHANGE_STEP; 27876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_false(!__state_owned_by_readers(new_state))) { // Happens when reader count overflows. 27976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EAGAIN; 28076615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 28176615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, new_state, 28276615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_acquire, memory_order_relaxed))) { 28376615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 28476615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 28576615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 28676615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EBUSY; 28776615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 28876615dae93c18ac890e167c547a08c0228709a33Yabin Cui 2892fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock, 2902fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 2912fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 29276615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) { 29376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 29476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 295c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 29608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 29776615dae93c18ac890e167c547a08c0228709a33Yabin Cui int ret = __pthread_rwlock_tryrdlock(rwlock); 29876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (ret == 0 || ret == EAGAIN) { 29976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ret; 30076615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 30176615dae93c18ac890e167c547a08c0228709a33Yabin Cui 3022fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 30376615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) { 30476615dae93c18ac890e167c547a08c0228709a33Yabin Cui continue; 30576615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 30676615dae93c18ac890e167c547a08c0228709a33Yabin Cui 30776615dae93c18ac890e167c547a08c0228709a33Yabin Cui timespec ts; 30876615dae93c18ac890e167c547a08c0228709a33Yabin Cui timespec* rel_timeout = NULL; 30976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 31076615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (abs_timeout_or_null != NULL) { 31176615dae93c18ac890e167c547a08c0228709a33Yabin Cui rel_timeout = &ts; 31276615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 31376615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ETIMEDOUT; 31476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 31576615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 31608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 31776615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.lock(); 31876615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_reader_count++; 3192fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 32076615dae93c18ac890e167c547a08c0228709a33Yabin Cui // We rely on the fact that all atomic exchange operations on the same object (here it is 32176615dae93c18ac890e167c547a08c0228709a33Yabin Cui // rwlock->state) always appear to occur in a single total order. If the pending flag is added 32276615dae93c18ac890e167c547a08c0228709a33Yabin Cui // before unlocking, the unlocking thread will wakeup the waiter. Otherwise, we will see the 32376615dae93c18ac890e167c547a08c0228709a33Yabin Cui // state is unlocked and will not wait anymore. 32476615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_READERS_FLAG, 32576615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_relaxed); 3262fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 32776615dae93c18ac890e167c547a08c0228709a33Yabin Cui int old_serial = rwlock->pending_reader_wakeup_serial; 32876615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 3292fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 33076615dae93c18ac890e167c547a08c0228709a33Yabin Cui int futex_ret = 0; 33176615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) { 33276615dae93c18ac890e167c547a08c0228709a33Yabin Cui futex_ret = __futex_wait_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared, 33376615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_serial, rel_timeout); 33476615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 3352fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 33676615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.lock(); 33776615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_reader_count--; 33876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (rwlock->pending_reader_count == 0) { 33976615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_READERS_FLAG, 34076615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_relaxed); 34176615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 34276615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 34376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 34476615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (futex_ret == -ETIMEDOUT) { 34576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ETIMEDOUT; 34676615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 34776615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 34876615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 34976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 35076615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline bool __can_acquire_write_lock(int old_state) { 35176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return !__state_owned_by_readers_or_writer(old_state); 35276615dae93c18ac890e167c547a08c0228709a33Yabin Cui} 35376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 35476615dae93c18ac890e167c547a08c0228709a33Yabin Cuistatic inline __always_inline int __pthread_rwlock_trywrlock(pthread_rwlock_internal_t* rwlock) { 35576615dae93c18ac890e167c547a08c0228709a33Yabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 35676615dae93c18ac890e167c547a08c0228709a33Yabin Cui 35776615dae93c18ac890e167c547a08c0228709a33Yabin Cui while (__predict_true(__can_acquire_write_lock(old_state))) { 35876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, 35976615dae93c18ac890e167c547a08c0228709a33Yabin Cui __state_add_writer_flag(old_state), memory_order_acquire, memory_order_relaxed))) { 36076615dae93c18ac890e167c547a08c0228709a33Yabin Cui 36176615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_store_explicit(&rwlock->writer_tid, __get_thread()->tid, memory_order_relaxed); 36276615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 363c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 36408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 36576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EBUSY; 366c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 367a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 3682fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock, 3692fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 3702fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 37176615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) { 37276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 373c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 37408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 37576615dae93c18ac890e167c547a08c0228709a33Yabin Cui int ret = __pthread_rwlock_trywrlock(rwlock); 37676615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (ret == 0) { 37776615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ret; 37876615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 37976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 3802fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 38176615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__can_acquire_write_lock(old_state)) { 38276615dae93c18ac890e167c547a08c0228709a33Yabin Cui continue; 38376615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 38476615dae93c18ac890e167c547a08c0228709a33Yabin Cui 38576615dae93c18ac890e167c547a08c0228709a33Yabin Cui timespec ts; 38676615dae93c18ac890e167c547a08c0228709a33Yabin Cui timespec* rel_timeout = NULL; 38776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 38876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (abs_timeout_or_null != NULL) { 38976615dae93c18ac890e167c547a08c0228709a33Yabin Cui rel_timeout = &ts; 39076615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 39176615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ETIMEDOUT; 39276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 39376615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 39408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 39576615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.lock(); 39676615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_writer_count++; 3972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 39876615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_WRITERS_FLAG, 39976615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_relaxed); 4002fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 40176615dae93c18ac890e167c547a08c0228709a33Yabin Cui int old_serial = rwlock->pending_writer_wakeup_serial; 40276615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 4032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 40476615dae93c18ac890e167c547a08c0228709a33Yabin Cui int futex_ret = 0; 40576615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!__can_acquire_write_lock(old_state)) { 40676615dae93c18ac890e167c547a08c0228709a33Yabin Cui futex_ret = __futex_wait_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared, 40776615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_serial, rel_timeout); 40876615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 4092fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 41076615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.lock(); 41176615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_writer_count--; 41276615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (rwlock->pending_writer_count == 0) { 41376615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_WRITERS_FLAG, 41476615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_relaxed); 41576615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 41676615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 41776615dae93c18ac890e167c547a08c0228709a33Yabin Cui 41876615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (futex_ret == -ETIMEDOUT) { 41976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return ETIMEDOUT; 42076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 42108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 422c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 423c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 4242fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_rdlock(pthread_rwlock_t* rwlock_interface) { 4252fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 42676615dae93c18ac890e167c547a08c0228709a33Yabin Cui // Avoid slowing down fast path of rdlock. 42776615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_true(__pthread_rwlock_tryrdlock(rwlock) == 0)) { 42876615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 42976615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 430c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedrdlock(rwlock, NULL); 431a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 432a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 4332fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedrdlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 4342fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 4352fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 43692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedrdlock(rwlock, abs_timeout); 43792687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 43892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 4392fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) { 44076615dae93c18ac890e167c547a08c0228709a33Yabin Cui return __pthread_rwlock_tryrdlock(__get_internal_rwlock(rwlock_interface)); 441a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 442a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 4432fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_wrlock(pthread_rwlock_t* rwlock_interface) { 4442fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 44576615dae93c18ac890e167c547a08c0228709a33Yabin Cui // Avoid slowing down fast path of wrlock. 44676615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__predict_true(__pthread_rwlock_trywrlock(rwlock) == 0)) { 44776615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 44876615dae93c18ac890e167c547a08c0228709a33Yabin Cui } 449c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedwrlock(rwlock, NULL); 450a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 451a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 4522fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedwrlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 4532fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 4542fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 45592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedwrlock(rwlock, abs_timeout); 45692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 45792687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 4582fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) { 45976615dae93c18ac890e167c547a08c0228709a33Yabin Cui return __pthread_rwlock_trywrlock(__get_internal_rwlock(rwlock_interface)); 460a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 461a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 4622fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) { 4632fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 46408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 4652fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 46676615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (__state_owned_by_writer(old_state)) { 46776615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) != __get_thread()->tid) { 46876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EPERM; 469a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 47076615dae93c18ac890e167c547a08c0228709a33Yabin Cui atomic_store_explicit(&rwlock->writer_tid, 0, memory_order_relaxed); 47176615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_state = atomic_fetch_and_explicit(&rwlock->state, ~STATE_OWNED_BY_WRITER_FLAG, 47276615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_release); 47376615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!__state_have_pending_readers_or_writers(old_state)) { 47476615dae93c18ac890e167c547a08c0228709a33Yabin Cui return 0; 475a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 4762fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 47776615dae93c18ac890e167c547a08c0228709a33Yabin Cui } else if (__state_owned_by_readers(old_state)) { 47876615dae93c18ac890e167c547a08c0228709a33Yabin Cui old_state = atomic_fetch_sub_explicit(&rwlock->state, STATE_READER_COUNT_CHANGE_STEP, 47976615dae93c18ac890e167c547a08c0228709a33Yabin Cui memory_order_release); 48076615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (!__state_is_last_reader(old_state) || !__state_have_pending_readers_or_writers(old_state)) { 4812fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return 0; 48208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 48376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 48476615dae93c18ac890e167c547a08c0228709a33Yabin Cui } else { 48576615dae93c18ac890e167c547a08c0228709a33Yabin Cui return EPERM; 48608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 48776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 48876615dae93c18ac890e167c547a08c0228709a33Yabin Cui // Wake up pending readers or writers. 48976615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.lock(); 49076615dae93c18ac890e167c547a08c0228709a33Yabin Cui if (rwlock->pending_writer_count != 0) { 49176615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_writer_wakeup_serial++; 49276615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 49376615dae93c18ac890e167c547a08c0228709a33Yabin Cui 49476615dae93c18ac890e167c547a08c0228709a33Yabin Cui __futex_wake_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared, 1); 49576615dae93c18ac890e167c547a08c0228709a33Yabin Cui 49676615dae93c18ac890e167c547a08c0228709a33Yabin Cui } else if (rwlock->pending_reader_count != 0) { 49776615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_reader_wakeup_serial++; 49876615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 49976615dae93c18ac890e167c547a08c0228709a33Yabin Cui 50076615dae93c18ac890e167c547a08c0228709a33Yabin Cui __futex_wake_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared, INT_MAX); 50176615dae93c18ac890e167c547a08c0228709a33Yabin Cui 50276615dae93c18ac890e167c547a08c0228709a33Yabin Cui } else { 50376615dae93c18ac890e167c547a08c0228709a33Yabin Cui // It happens when waiters are woken up by timeout. 50476615dae93c18ac890e167c547a08c0228709a33Yabin Cui rwlock->pending_lock.unlock(); 50508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 50676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 507a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 508