11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* rwsem.c: R/W semaphores: contention handling functions 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Written by David Howells (dhowells@redhat.com). 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Derived from arch/i386/kernel/semaphore.c 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/rwsem.h> 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 114ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar/* 124ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar * Initialize an rwsem: 134ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar */ 144ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnarvoid __init_rwsem(struct rw_semaphore *sem, const char *name, 154ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar struct lock_class_key *key) 164ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar{ 174ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar#ifdef CONFIG_DEBUG_LOCK_ALLOC 184ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar /* 194ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar * Make sure we are not reinitializing a held semaphore: 204ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar */ 214ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar debug_check_no_locks_freed((void *)sem, sizeof(*sem)); 224dfbb9d8c6cbfc32faa5c71145bd2a43e1f8237cPeter Zijlstra lockdep_init_map(&sem->dep_map, name, key, 0); 234ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar#endif 244ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar sem->count = RWSEM_UNLOCKED_VALUE; 25ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_lock_init(&sem->wait_lock); 264ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar INIT_LIST_HEAD(&sem->wait_list); 274ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar} 284ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar 294ea2176dfa714882e88180b474e4cbcd888b70afIngo MolnarEXPORT_SYMBOL(__init_rwsem); 304ea2176dfa714882e88180b474e4cbcd888b70afIngo Molnar 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct rwsem_waiter { 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct list_head list; 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *task; 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int flags; 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RWSEM_WAITING_FOR_READ 0x00000001 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define RWSEM_WAITING_FOR_WRITE 0x00000002 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3970bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse/* Wake types for __rwsem_do_wake(). Note that RWSEM_WAKE_NO_ACTIVE and 4070bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * RWSEM_WAKE_READ_OWNED imply that the spinlock must have been kept held 4170bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * since the rwsem value was observed. 4270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse */ 4370bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse#define RWSEM_WAKE_ANY 0 /* Wake whatever's at head of wait list */ 4470bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse#define RWSEM_WAKE_NO_ACTIVE 1 /* rwsem was observed with no active thread */ 4570bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse#define RWSEM_WAKE_READ_OWNED 2 /* rwsem was observed to be read owned */ 4670bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * handle the lock release when processes blocked on it that can now run 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - if we come here from up_xxxx(), then: 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - the 'active part' of count (&0x0000ffff) reached 0 (but may have changed) 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - the 'waiting part' of count (&0xffff0000) is -ve (and will still be so) 52345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse * - there must be someone on the queue 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - the spinlock must be held by the caller 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - woken process blocks are discarded from the list after having task zeroed 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - writers are only woken if downgrading is false 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 5770bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinassestatic struct rw_semaphore * 5870bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse__rwsem_do_wake(struct rw_semaphore *sem, int wake_type) 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct rwsem_waiter *waiter; 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *tsk; 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct list_head *next; 63fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse signed long oldcount, woken, loop, adjustment; 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 65345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); 66345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE)) 67345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse goto readers_only; 68345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse 6970bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse if (wake_type == RWSEM_WAKE_READ_OWNED) 70424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse /* Another active reader was observed, so wakeup is not 71424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * likely to succeed. Save the atomic op. 72424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse */ 73345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse goto out; 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 75345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse /* There's a writer at the front of the queue - try to grant it the 76345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse * write lock. However, we only wake this writer if we can transition 77345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse * the active part of the count from 0 -> 1 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 79fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse adjustment = RWSEM_ACTIVE_WRITE_BIAS; 80fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse if (waiter->list.next == &sem->wait_list) 81fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse adjustment -= RWSEM_WAITING_BIAS; 82fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse 83345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse try_again_write: 84fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse oldcount = rwsem_atomic_update(adjustment, sem) - adjustment; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (oldcount & RWSEM_ACTIVE_MASK) 86345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse /* Someone grabbed the sem already */ 87345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse goto undo_write; 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* We must be careful not to touch 'waiter' after we set ->task = NULL. 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * It is an allocated on the waiter's stack and may become invalid at 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * any time after that point (due to a wakeup from another source). 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_del(&waiter->list); 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tsk = waiter->task; 95d59dd4620fb8d6422555a9e2b82a707718e68327Andrew Morton smp_mb(); 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds waiter->task = NULL; 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(tsk); 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds put_task_struct(tsk); 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 101345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse readers_only: 10270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse /* If we come here from up_xxxx(), another thread might have reached 10370bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * rwsem_down_failed_common() before we acquired the spinlock and 10470bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * woken up a waiter, making it now active. We prefer to check for 10570bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * this first in order to not spend too much time with the spinlock 10670bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * held if we're not going to be able to wake up readers in the end. 10770bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * 10870bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * Note that we do not need to update the rwsem count: any writer 10970bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * trying to acquire rwsem will run rwsem_down_write_failed() due 11070bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * to the waiting threads and block trying to acquire the spinlock. 11170bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * 11270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * We use a dummy atomic update in order to acquire the cache line 11370bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * exclusively since we expect to succeed and run the final rwsem 11470bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse * count adjustment pretty soon. 11570bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse */ 11670bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse if (wake_type == RWSEM_WAKE_ANY && 117424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse rwsem_atomic_update(0, sem) < RWSEM_WAITING_BIAS) 118424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse /* Someone grabbed the sem for write already */ 11970bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse goto out; 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 121345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse /* Grant an infinite number of read locks to the readers at the front 122345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse * of the queue. Note we increment the 'active part' of the count by 123345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse * the number of readers before waking any processes up. 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds woken = 0; 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds woken++; 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (waiter->list.next == &sem->wait_list) 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds waiter = list_entry(waiter->list.next, 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct rwsem_waiter, list); 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (waiter->flags & RWSEM_WAITING_FOR_READ); 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 137fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse adjustment = woken * RWSEM_ACTIVE_READ_BIAS; 138fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse if (waiter->flags & RWSEM_WAITING_FOR_READ) 139fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse /* hit end of list above */ 140fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse adjustment -= RWSEM_WAITING_BIAS; 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 142fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse rwsem_atomic_add(adjustment, sem); 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds next = sem->wait_list.next; 145fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse for (loop = woken; loop > 0; loop--) { 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds waiter = list_entry(next, struct rwsem_waiter, list); 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds next = waiter->list.next; 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tsk = waiter->task; 149d59dd4620fb8d6422555a9e2b82a707718e68327Andrew Morton smp_mb(); 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds waiter->task = NULL; 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(tsk); 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds put_task_struct(tsk); 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sem->wait_list.next = next; 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds next->prev = &sem->wait_list; 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds out: 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sem; 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16191af70814105f4c05e6e11b51c3269907b71794bMichel Lespinasse /* undo the change to the active count, but check for a transition 16291af70814105f4c05e6e11b51c3269907b71794bMichel Lespinasse * 1->0 */ 163345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse undo_write: 164fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse if (rwsem_atomic_update(-adjustment, sem) & RWSEM_ACTIVE_MASK) 165345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse goto out; 166345af7bf3304410634c21ada4664fda83d4d9a16Michel Lespinasse goto try_again_write; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wait for a lock to be granted 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 172c7af77b584b02d3e321b00203a618a9c93782121Livio Soaresstatic struct rw_semaphore __sched * 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsrwsem_down_failed_common(struct rw_semaphore *sem, 174a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse unsigned int flags, signed long adjustment) 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 176a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse struct rwsem_waiter waiter; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *tsk = current; 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds signed long count; 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_task_state(tsk, TASK_UNINTERRUPTIBLE); 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* set up my own style of waitqueue */ 183ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_lock_irq(&sem->wait_lock); 184a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse waiter.task = tsk; 185a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse waiter.flags = flags; 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds get_task_struct(tsk); 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 188fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse if (list_empty(&sem->wait_list)) 189fd41b33435ada87323cc86b50959fbffe35192c8Michel Lespinasse adjustment += RWSEM_WAITING_BIAS; 190a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse list_add_tail(&waiter.list, &sem->wait_list); 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse /* we're now waiting on the lock, but no longer actively locking */ 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds count = rwsem_atomic_update(adjustment, sem); 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 195424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse /* If there are no active locks, wake the front queued process(es) up. 196424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * 197424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * Alternatively, if we're called from a failed down_write(), there 198424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * were already threads queued before us and there are no active 199424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * writers, the lock must be read owned; so we try to wake any read 200424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse * locks that were queued ahead of us. */ 201424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse if (count == RWSEM_WAITING_BIAS) 20270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse sem = __rwsem_do_wake(sem, RWSEM_WAKE_NO_ACTIVE); 203424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse else if (count > RWSEM_WAITING_BIAS && 204424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse adjustment == -RWSEM_ACTIVE_WRITE_BIAS) 205424acaaeb3a3932d64a9b4bd59df6cf72c22d8f3Michel Lespinasse sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED); 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 207ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_unlock_irq(&sem->wait_lock); 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* wait to be given the lock */ 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (;;) { 211a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse if (!waiter.task) 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds schedule(); 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_task_state(tsk, TASK_UNINTERRUPTIBLE); 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tsk->state = TASK_RUNNING; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sem; 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wait for the read lock to be granted 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 225d123375425d7df4b6081a631fc1203fceafa59b2Thomas Gleixnerstruct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem) 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 227a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ, 228a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse -RWSEM_ACTIVE_READ_BIAS); 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wait for the write lock to be granted 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 234d123375425d7df4b6081a631fc1203fceafa59b2Thomas Gleixnerstruct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem) 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 236a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE, 237a8618a0e8a06f75c6efec2a5477861d704d48b28Michel Lespinasse -RWSEM_ACTIVE_WRITE_BIAS); 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * handle waking up a waiter on the semaphore 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - up_read/up_write has decremented the active part of count if we come here 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 244d123375425d7df4b6081a631fc1203fceafa59b2Thomas Gleixnerstruct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 248ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_lock_irqsave(&sem->wait_lock, flags); 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* do nothing if list empty */ 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!list_empty(&sem->wait_list)) 25270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse sem = __rwsem_do_wake(sem, RWSEM_WAKE_ANY); 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 254ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sem; 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * downgrade a write lock into a read lock 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - caller incremented waiting part of count and discovered it still negative 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - just wake up any readers at the front of the queue 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 264d123375425d7df4b6081a631fc1203fceafa59b2Thomas Gleixnerstruct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 268ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_lock_irqsave(&sem->wait_lock, flags); 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* do nothing if list empty */ 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!list_empty(&sem->wait_list)) 27270bdc6e0644f3535e93bac5c364ca199397e507eMichel Lespinasse sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED); 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 274ddb6c9b58a19edcfac93ac670b066c836ff729f1Thomas Gleixner raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sem; 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(rwsem_down_read_failed); 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(rwsem_down_write_failed); 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(rwsem_wake); 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(rwsem_downgrade_wake); 283