pthread_rwlock.cpp revision b58457221364eaad039c2c49a42626b725e980d5
1a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* 2a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Copyright (C) 2010 The Android Open Source Project 3a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * All rights reserved. 4a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 5a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Redistribution and use in source and binary forms, with or without 6a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * modification, are permitted provided that the following conditions 7a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * are met: 8a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions of source code must retain the above copyright 9a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer. 10a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions in binary form must reproduce the above copyright 11a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer in 12a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * the documentation and/or other materials provided with the 13a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * distribution. 14a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 15a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * SUCH DAMAGE. 27a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 28a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 29a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner#include <errno.h> 3008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui#include <stdatomic.h> 3176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 3276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "pthread_internal.h" 3376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "private/bionic_futex.h" 3404303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_time_conversions.h" 35a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 36a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* Technical note: 37a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 38a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Possible states of a read/write lock: 39a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 40a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - no readers and no writer (unlocked) 41a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one or more readers sharing the lock at the same time (read-locked) 42a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one writer holding the lock (write-lock) 43a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 44a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Additionally: 45a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the write-lock while there are any readers blocks 46a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the read-lock while there is a writer blocks 4776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - a single thread can acquire the lock multiple times in read mode 4876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * 4976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - Posix states that behavior is undefined (may deadlock) if a thread tries 5076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * to acquire the lock 5176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in write mode while already holding the lock (whether in read or write mode) 5276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in read mode while already holding the lock in write mode. 5376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - This implementation will return EDEADLK in "write after write" and "read after 5476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * write" cases and will deadlock in write after read case. 55a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 5692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle * TODO: As it stands now, pending_readers and pending_writers could be merged into a 5776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * a single waiters variable. Keeping them separate adds a bit of clarity and keeps 5876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * the door open for a writer-biased implementation. 59a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 60a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 61a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 6292687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle#define RWLOCKATTR_DEFAULT 0 6392687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle#define RWLOCKATTR_SHARED_MASK 0x0010 64a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 65a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 6692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_init(pthread_rwlockattr_t* attr) { 6776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = PTHREAD_PROCESS_PRIVATE; 6876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 69a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 70a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 7192687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_destroy(pthread_rwlockattr_t* attr) { 7276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = -1; 7376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 74a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 75a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 7692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) { 7776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle switch (pshared) { 78a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_PRIVATE: 79a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_SHARED: 8076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = pshared; 8176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 82a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner default: 8376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 8476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 85a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 86a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 87c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughesint pthread_rwlockattr_getpshared(const pthread_rwlockattr_t* attr, int* pshared) { 8876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *pshared = *attr; 8976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 90a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 91a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 922fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistruct pthread_rwlock_internal_t { 932fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_int state; // 0=unlock, -1=writer lock, +n=reader lock 942fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_int writer_thread_id; 952fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_uint pending_readers; 962fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_uint pending_writers; 972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int32_t attr; 9808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 992fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui bool process_shared() const { 1002fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return attr == PTHREAD_PROCESS_SHARED; 1012fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 10208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 1032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#if defined(__LP64__) 1042fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui char __reserved[36]; 1052fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#else 1062fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui char __reserved[20]; 1072fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#endif 1082fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui}; 1092fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 110b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(sizeof(pthread_rwlock_t) == sizeof(pthread_rwlock_internal_t), 111b58457221364eaad039c2c49a42626b725e980d5Yabin Cui "pthread_rwlock_t should actually be pthread_rwlock_internal_t in implementation."); 112b58457221364eaad039c2c49a42626b725e980d5Yabin Cui 113b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// For binary compatibility with old version of pthread_rwlock_t, we can't use more strict 114b58457221364eaad039c2c49a42626b725e980d5Yabin Cui// alignment than 4-byte alignment. 115b58457221364eaad039c2c49a42626b725e980d5Yabin Cuistatic_assert(alignof(pthread_rwlock_t) == 4, 116b58457221364eaad039c2c49a42626b725e980d5Yabin Cui "pthread_rwlock_t should fulfill the alignment requirement of pthread_rwlock_internal_t."); 117b58457221364eaad039c2c49a42626b725e980d5Yabin Cui 1182fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic inline pthread_rwlock_internal_t* __get_internal_rwlock(pthread_rwlock_t* rwlock_interface) { 1192fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return reinterpret_cast<pthread_rwlock_internal_t*>(rwlock_interface); 12008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui} 12108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 1222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_init(pthread_rwlock_t* rwlock_interface, const pthread_rwlockattr_t* attr) { 1232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 12408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 12508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui if (__predict_true(attr == NULL)) { 12608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui rwlock->attr = 0; 12708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } else { 12876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle switch (*attr) { 12976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle case PTHREAD_PROCESS_SHARED: 13076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle case PTHREAD_PROCESS_PRIVATE: 13176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle rwlock->attr= *attr; 13276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle break; 13376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle default: 13476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 135a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 13676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 137a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1382fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->state, 0); 1392fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->writer_thread_id, 0); 1402fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->pending_readers, 0); 1412fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->pending_writers, 0); 142a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 14376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 144a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 145a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1462fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_destroy(pthread_rwlock_t* rwlock_interface) { 1472fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 1482fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1492fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_load_explicit(&rwlock->state, memory_order_relaxed) != 0) { 15076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EBUSY; 15176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 15276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 153a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 154a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1552fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock, 1562fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 1572fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1582fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id, 1592fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui memory_order_relaxed))) { 16076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 16176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 162c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 16308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 1642fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 1652fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_true(old_state >= 0)) { 1662fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, old_state + 1, 16708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 16808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return 0; 16908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 17076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } else { 1712fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec ts; 1722fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec* rel_timeout = NULL; 1732fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1742fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (abs_timeout_or_null != NULL) { 1752fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout = &ts; 1762fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 1772fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return ETIMEDOUT; 1782fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 17976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 18008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 18108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the pending_readers increment should be observed before 18208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used 18308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // here. Because only a seq_cst fence can ensure sequential consistency for non-atomic 18408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // operations in futex_wait. 1852fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_add_explicit(&rwlock->pending_readers, 1, memory_order_relaxed); 1862fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 18708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 1882fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1892fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state, 1902fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout); 1912fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1922fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_sub_explicit(&rwlock->pending_readers, 1, memory_order_relaxed); 1932fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 19492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle if (ret == -ETIMEDOUT) { 19592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return ETIMEDOUT; 19676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 197c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 19808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 199c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 200a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2012fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock, 2022fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 2032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2042fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id, 2052fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui memory_order_relaxed))) { 20676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 207c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 20876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 20908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 2102fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2112fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_true(old_state == 0)) { 2122fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, 21308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 21408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // writer_thread_id is protected by rwlock and can only be modified in rwlock write 21508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // owner thread. Other threads may read it for EDEADLK error checking, atomic operation 21608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // is safe enough for it. 2172fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); 21808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return 0; 21908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 22076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } else { 2212fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec ts; 2222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec* rel_timeout = NULL; 2232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2242fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (abs_timeout_or_null != NULL) { 2252fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout = &ts; 2262fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 2272fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return ETIMEDOUT; 2282fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 22976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 23008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 23108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the pending_writers increment should be observed before 23208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used 23308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // here. Because only a seq_cst fence can ensure sequential consistency for non-atomic 23408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // operations in futex_wait. 2352fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_add_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); 2362fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 23708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 2382fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2392fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state, 2402fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout); 2412fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2422fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_sub_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); 2432fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 24492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle if (ret == -ETIMEDOUT) { 24592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return ETIMEDOUT; 24676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 24776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 24808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 249c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 250c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 2512fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_rdlock(pthread_rwlock_t* rwlock_interface) { 2522fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2532fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 254c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedrdlock(rwlock, NULL); 255a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 256a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2572fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedrdlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 2582fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2592fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 26092687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedrdlock(rwlock, abs_timeout); 26192687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 26292687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 2632fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) { 2642fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 26508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2662fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2672fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2682fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state >= 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, 2692fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui old_state + 1, memory_order_acquire, memory_order_relaxed)) { 27076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 2712fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return (old_state >= 0) ? 0 : EBUSY; 272a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 273a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2742fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_wrlock(pthread_rwlock_t* rwlock_interface) { 2752fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2762fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 277c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedwrlock(rwlock, NULL); 278a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 279a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2802fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedwrlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 2812fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2822fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 28392687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedwrlock(rwlock, abs_timeout); 28492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 28592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 2862fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) { 2872fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 28808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2892fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2902fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2912fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state == 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, 29208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 2932fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 2942fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (old_state == 0) { 2952fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); 2962fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return 0; 29776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 2981b676ea5fba4af0f3a11ca0c31a40825f2157601Calin Juravle return EBUSY; 299a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 300a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 301a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 3022fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) { 3032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 30408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 3052fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 3062fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(old_state == 0)) { 30708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return EPERM; 3082fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else if (old_state == -1) { 3092fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_load_explicit(&rwlock->writer_thread_id, memory_order_relaxed) != __get_thread()->tid) { 31076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EPERM; 311a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 31208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // We're no longer the owner. 3132fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, 0, memory_order_relaxed); 31408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // Change state from -1 to 0. 3152fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->state, 0, memory_order_release); 31608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 3172fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else { // old_state > 0 31808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // Reduce state by 1. 3192fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state > 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, 3202fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui old_state - 1, memory_order_release, memory_order_relaxed)) { 321a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 3222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 3232fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (old_state <= 0) { 3242fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return EPERM; 3252fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else if (old_state > 1) { 3262fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return 0; 32708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 3282fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui // old_state = 1, which means the last reader calling unlock. It has to wake up waiters. 32908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 33076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 3312fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui // If having waiters, wake up them. 33208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the update of state should be observed before reading 33308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // pending_readers/pending_writers by all threads. Use read locking as an example: 33408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // read locking thread unlocking thread 33508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // pending_readers++; state = 0; 33608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // seq_cst fence seq_cst fence 33708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // read state for futex_wait read pending_readers for futex_wake 33808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // 33908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // So when locking and unlocking threads are running in parallel, we will not get 34008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // in a situation that the locking thread reads state as negative and needs to wait, 34108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // while the unlocking thread reads pending_readers as zero and doesn't need to wake up waiters. 34208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 3432fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(atomic_load_explicit(&rwlock->pending_readers, memory_order_relaxed) > 0 || 3442fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_load_explicit(&rwlock->pending_writers, memory_order_relaxed) > 0)) { 3452fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui __futex_wake_ex(&rwlock->state, rwlock->process_shared(), INT_MAX); 34608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 34776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 348a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 349