pthread_rwlock.cpp revision 2fabea47ac9475bcc52aff0715819d18aa5bdf1d
1a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* 2a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Copyright (C) 2010 The Android Open Source Project 3a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * All rights reserved. 4a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 5a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Redistribution and use in source and binary forms, with or without 6a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * modification, are permitted provided that the following conditions 7a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * are met: 8a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions of source code must retain the above copyright 9a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer. 10a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * * Redistributions in binary form must reproduce the above copyright 11a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * notice, this list of conditions and the following disclaimer in 12a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * the documentation and/or other materials provided with the 13a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * distribution. 14a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 15a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * SUCH DAMAGE. 27a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 28a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 29a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner#include <errno.h> 3008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui#include <stdatomic.h> 3176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 3276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "pthread_internal.h" 3376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle#include "private/bionic_futex.h" 3404303f5a8ab9a992f3671d46b6ee2171582cbd61Elliott Hughes#include "private/bionic_time_conversions.h" 35a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 36a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner/* Technical note: 37a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 38a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Possible states of a read/write lock: 39a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 40a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - no readers and no writer (unlocked) 41a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one or more readers sharing the lock at the same time (read-locked) 42a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - one writer holding the lock (write-lock) 43a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 44a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * Additionally: 45a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the write-lock while there are any readers blocks 46a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * - trying to get the read-lock while there is a writer blocks 4776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - a single thread can acquire the lock multiple times in read mode 4876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * 4976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - Posix states that behavior is undefined (may deadlock) if a thread tries 5076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * to acquire the lock 5176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in write mode while already holding the lock (whether in read or write mode) 5276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - in read mode while already holding the lock in write mode. 5376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * - This implementation will return EDEADLK in "write after write" and "read after 5476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * write" cases and will deadlock in write after read case. 55a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 5692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle * TODO: As it stands now, pending_readers and pending_writers could be merged into a 5776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * a single waiters variable. Keeping them separate adds a bit of clarity and keeps 5876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle * the door open for a writer-biased implementation. 59a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner * 60a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner */ 61a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 6292687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle#define RWLOCKATTR_DEFAULT 0 6392687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle#define RWLOCKATTR_SHARED_MASK 0x0010 64a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 65a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 6692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_init(pthread_rwlockattr_t* attr) { 6776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = PTHREAD_PROCESS_PRIVATE; 6876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 69a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 70a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 7192687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_destroy(pthread_rwlockattr_t* attr) { 7276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = -1; 7376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 74a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 75a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 7692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravleint pthread_rwlockattr_setpshared(pthread_rwlockattr_t* attr, int pshared) { 7776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle switch (pshared) { 78a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_PRIVATE: 79a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner case PTHREAD_PROCESS_SHARED: 8076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *attr = pshared; 8176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 82a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner default: 8376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 8476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 85a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 86a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 87c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughesint pthread_rwlockattr_getpshared(const pthread_rwlockattr_t* attr, int* pshared) { 8876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle *pshared = *attr; 8976f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 90a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 91a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 922fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistruct pthread_rwlock_internal_t { 932fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_int state; // 0=unlock, -1=writer lock, +n=reader lock 942fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_int writer_thread_id; 952fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_uint pending_readers; 962fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_uint pending_writers; 972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int32_t attr; 9808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 992fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui bool process_shared() const { 1002fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return attr == PTHREAD_PROCESS_SHARED; 1012fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 10208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 1032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#if defined(__LP64__) 1042fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui char __reserved[36]; 1052fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#else 1062fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui char __reserved[20]; 1072fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui#endif 1082fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui}; 1092fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1102fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic inline pthread_rwlock_internal_t* __get_internal_rwlock(pthread_rwlock_t* rwlock_interface) { 1112fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui static_assert(sizeof(pthread_rwlock_t) == sizeof(pthread_rwlock_internal_t), 1122fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui "pthread_rwlock_t should actually be pthread_rwlock_internal_t in implementation."); 1132fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return reinterpret_cast<pthread_rwlock_internal_t*>(rwlock_interface); 11408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui} 11508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 1162fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_init(pthread_rwlock_t* rwlock_interface, const pthread_rwlockattr_t* attr) { 1172fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 11808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 11908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui if (__predict_true(attr == NULL)) { 12008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui rwlock->attr = 0; 12108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } else { 12276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle switch (*attr) { 12376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle case PTHREAD_PROCESS_SHARED: 12476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle case PTHREAD_PROCESS_PRIVATE: 12576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle rwlock->attr= *attr; 12676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle break; 12776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle default: 12876f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EINVAL; 129a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 13076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 131a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1322fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->state, 0); 1332fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->writer_thread_id, 0); 1342fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->pending_readers, 0); 1352fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_init(&rwlock->pending_writers, 0); 136a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 13776f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 138a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 139a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1402fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_destroy(pthread_rwlock_t* rwlock_interface) { 1412fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 1422fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1432fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_load_explicit(&rwlock->state, memory_order_relaxed) != 0) { 14476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EBUSY; 14576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 14676f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 147a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 148a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1492fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock, 1502fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 1512fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1522fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id, 1532fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui memory_order_relaxed))) { 15476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 15576f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 156c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 15708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 1582fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 1592fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_true(old_state >= 0)) { 1602fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, old_state + 1, 16108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 16208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return 0; 16308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 16476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } else { 1652fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec ts; 1662fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec* rel_timeout = NULL; 1672fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1682fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (abs_timeout_or_null != NULL) { 1692fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout = &ts; 1702fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 1712fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return ETIMEDOUT; 1722fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 17376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 17408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 17508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the pending_readers increment should be observed before 17608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used 17708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // here. Because only a seq_cst fence can ensure sequential consistency for non-atomic 17808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // operations in futex_wait. 1792fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_add_explicit(&rwlock->pending_readers, 1, memory_order_relaxed); 1802fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 18108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 1822fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1832fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state, 1842fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout); 1852fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1862fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_sub_explicit(&rwlock->pending_readers, 1, memory_order_relaxed); 1872fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 18892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle if (ret == -ETIMEDOUT) { 18992687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return ETIMEDOUT; 19076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 191c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 19208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 193c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 194a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 1952fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuistatic int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock, 1962fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui const timespec* abs_timeout_or_null) { 1972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 1982fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id, 1992fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui memory_order_relaxed))) { 20076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EDEADLK; 201c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes } 20276f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 20308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui while (true) { 2042fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2052fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_true(old_state == 0)) { 2062fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, 20708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 20808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // writer_thread_id is protected by rwlock and can only be modified in rwlock write 20908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // owner thread. Other threads may read it for EDEADLK error checking, atomic operation 21008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // is safe enough for it. 2112fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); 21208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return 0; 21308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 21476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } else { 2152fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec ts; 2162fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui timespec* rel_timeout = NULL; 2172fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2182fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (abs_timeout_or_null != NULL) { 2192fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout = &ts; 2202fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { 2212fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return ETIMEDOUT; 2222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 22376f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 22408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 22508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the pending_writers increment should be observed before 22608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used 22708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // here. Because only a seq_cst fence can ensure sequential consistency for non-atomic 22808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // operations in futex_wait. 2292fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_add_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); 2302fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 23108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 2322fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2332fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state, 2342fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui rel_timeout); 2352fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2362fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_fetch_sub_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); 2372fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 23892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle if (ret == -ETIMEDOUT) { 23992687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return ETIMEDOUT; 24076f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 24176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 24208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 243c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes} 244c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes 2452fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_rdlock(pthread_rwlock_t* rwlock_interface) { 2462fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2472fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 248c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedrdlock(rwlock, NULL); 249a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 250a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2512fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedrdlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 2522fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2532fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 25492687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedrdlock(rwlock, abs_timeout); 25592687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 25692687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 2572fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) { 2582fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 25908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2602fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2612fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2622fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state >= 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, 2632fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui old_state + 1, memory_order_acquire, memory_order_relaxed)) { 26476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 2652fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return (old_state >= 0) ? 0 : EBUSY; 266a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 267a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2682fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_wrlock(pthread_rwlock_t* rwlock_interface) { 2692fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2702fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 271c3f114037dbf028896310609fd28cf2b3da99c4dElliott Hughes return __pthread_rwlock_timedwrlock(rwlock, NULL); 272a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 273a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2742fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_timedwrlock(pthread_rwlock_t* rwlock_interface, const timespec* abs_timeout) { 2752fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 2762fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 27792687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle return __pthread_rwlock_timedwrlock(rwlock, abs_timeout); 27892687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle} 27992687e41bcf108957944dafa80a9bfda219bfb0fCalin Juravle 2802fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) { 2812fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 28208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2832fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 2842fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 2852fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state == 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, 28608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui memory_order_acquire, memory_order_relaxed)) { 2872fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } 2882fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (old_state == 0) { 2892fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); 2902fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return 0; 29176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle } 2921b676ea5fba4af0f3a11ca0c31a40825f2157601Calin Juravle return EBUSY; 293a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 294a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 295a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner 2962fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cuiint pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) { 2972fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); 29808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 2992fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); 3002fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(old_state == 0)) { 30108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui return EPERM; 3022fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else if (old_state == -1) { 3032fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (atomic_load_explicit(&rwlock->writer_thread_id, memory_order_relaxed) != __get_thread()->tid) { 30476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return EPERM; 305a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 30608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // We're no longer the owner. 3072fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->writer_thread_id, 0, memory_order_relaxed); 30808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // Change state from -1 to 0. 3092fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_store_explicit(&rwlock->state, 0, memory_order_release); 31008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui 3112fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else { // old_state > 0 31208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // Reduce state by 1. 3132fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui while (old_state > 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, 3142fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui old_state - 1, memory_order_release, memory_order_relaxed)) { 315a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner } 3162fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui 3172fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (old_state <= 0) { 3182fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return EPERM; 3192fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui } else if (old_state > 1) { 3202fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui return 0; 32108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 3222fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui // old_state = 1, which means the last reader calling unlock. It has to wake up waiters. 32308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 32476f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle 3252fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui // If having waiters, wake up them. 32608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // To avoid losing wake ups, the update of state should be observed before reading 32708ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // pending_readers/pending_writers by all threads. Use read locking as an example: 32808ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // read locking thread unlocking thread 32908ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // pending_readers++; state = 0; 33008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // seq_cst fence seq_cst fence 33108ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // read state for futex_wait read pending_readers for futex_wake 33208ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // 33308ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // So when locking and unlocking threads are running in parallel, we will not get 33408ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // in a situation that the locking thread reads state as negative and needs to wait, 33508ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui // while the unlocking thread reads pending_readers as zero and doesn't need to wake up waiters. 33608ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui atomic_thread_fence(memory_order_seq_cst); 3372fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui if (__predict_false(atomic_load_explicit(&rwlock->pending_readers, memory_order_relaxed) > 0 || 3382fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui atomic_load_explicit(&rwlock->pending_writers, memory_order_relaxed) > 0)) { 3392fabea47ac9475bcc52aff0715819d18aa5bdf1dYabin Cui __futex_wake_ex(&rwlock->state, rwlock->process_shared(), INT_MAX); 34008ee8d2030fbc73c4c144e819dd68806b0351cbeYabin Cui } 34176f352eec12d8938101e5ae33429c72797c3aa23Calin Juravle return 0; 342a418c3b8370cae1c80fbe9a06e7e53025da5d6f0David 'Digit' Turner} 343