11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Generic waiting primitives. 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 46d49e352ae9aed3f599041b0c0389aa924815f14Nadia Yvette Chambers * (C) 2004 Nadia Yvette Chambers, Oracle 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 79984de1a5a8a96275fcab818f7419af5a3c86e71Paul Gortmaker#include <linux/export.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/wait.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/hash.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 13f07fdec50a13f134ea9608c8fb3f6408c58ef55ePeter Zijlstravoid __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *key) 1421d71f513b6221f482ed6ad45e05f073ae67f319Ingo Molnar{ 1521d71f513b6221f482ed6ad45e05f073ae67f319Ingo Molnar spin_lock_init(&q->lock); 16f07fdec50a13f134ea9608c8fb3f6408c58ef55ePeter Zijlstra lockdep_set_class_and_name(&q->lock, key, name); 1721d71f513b6221f482ed6ad45e05f073ae67f319Ingo Molnar INIT_LIST_HEAD(&q->task_list); 1821d71f513b6221f482ed6ad45e05f073ae67f319Ingo Molnar} 19eb4542b98c81e22e08587b747b21986a45360999Ingo Molnar 202fc391112fb6f3424435a3aa2fda887497b5f807Peter ZijlstraEXPORT_SYMBOL(__init_waitqueue_head); 21eb4542b98c81e22e08587b747b21986a45360999Ingo Molnar 227ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait->flags &= ~WQ_FLAG_EXCLUSIVE; 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __add_wait_queue(q, wait); 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(add_wait_queue); 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 337ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait) 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait->flags |= WQ_FLAG_EXCLUSIVE; 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __add_wait_queue_tail(q, wait); 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(add_wait_queue_exclusive); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 447ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __remove_wait_queue(q, wait); 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(remove_wait_queue); 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 56b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just 57b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve 58b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * number) then we wake all the non-exclusive tasks and one exclusive task. 59b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * 60b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * There are circumstances in which we can try to wake a task which has already 61b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns 62b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * zero in this (rare) case, and we handle it by continuing to scan the queue. 63b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra */ 64b4145872f7049e429718b40b86e1b46659988398Peter Zijlstrastatic void __wake_up_common(wait_queue_head_t *q, unsigned int mode, 65b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra int nr_exclusive, int wake_flags, void *key) 66b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 67b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra wait_queue_t *curr, *next; 68b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 69b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra list_for_each_entry_safe(curr, next, &q->task_list, task_list) { 70b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra unsigned flags = curr->flags; 71b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 72b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra if (curr->func(curr, mode, wake_flags, key) && 73b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) 74b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra break; 75b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra } 76b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 77b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 78b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra/** 79b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * __wake_up - wake up threads blocked on a waitqueue. 80b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @q: the waitqueue 81b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @mode: which threads 82b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @nr_exclusive: how many wake-one or wake-many threads to wake up 83b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @key: is directly passed to the wakeup function 84b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * 85b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * It may be assumed that this function implies a write memory barrier before 86b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * changing the task state if and only if any tasks are woken up. 87b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra */ 88b4145872f7049e429718b40b86e1b46659988398Peter Zijlstravoid __wake_up(wait_queue_head_t *q, unsigned int mode, 89b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra int nr_exclusive, void *key) 90b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 91b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra unsigned long flags; 92b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 93b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra spin_lock_irqsave(&q->lock, flags); 94b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra __wake_up_common(q, mode, nr_exclusive, 0, key); 95b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra spin_unlock_irqrestore(&q->lock, flags); 96b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 97b4145872f7049e429718b40b86e1b46659988398Peter ZijlstraEXPORT_SYMBOL(__wake_up); 98b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 99b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra/* 100b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * Same as __wake_up but called with the spinlock in wait_queue_head_t held. 101b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra */ 102b4145872f7049e429718b40b86e1b46659988398Peter Zijlstravoid __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) 103b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 104b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra __wake_up_common(q, mode, nr, 0, NULL); 105b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 106b4145872f7049e429718b40b86e1b46659988398Peter ZijlstraEXPORT_SYMBOL_GPL(__wake_up_locked); 107b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 108b4145872f7049e429718b40b86e1b46659988398Peter Zijlstravoid __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) 109b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 110b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra __wake_up_common(q, mode, 1, 0, key); 111b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 112b4145872f7049e429718b40b86e1b46659988398Peter ZijlstraEXPORT_SYMBOL_GPL(__wake_up_locked_key); 113b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 114b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra/** 115b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * __wake_up_sync_key - wake up threads blocked on a waitqueue. 116b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @q: the waitqueue 117b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @mode: which threads 118b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @nr_exclusive: how many wake-one or wake-many threads to wake up 119b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * @key: opaque value to be passed to wakeup targets 120b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * 121b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * The sync wakeup differs that the waker knows that it will schedule 122b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * away soon, so while the target thread will be woken up, it will not 123b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * be migrated to another CPU - ie. the two threads are 'synchronized' 124b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * with each other. This can prevent needless bouncing between CPUs. 125b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * 126b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * On UP it can prevent extra preemption. 127b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * 128b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * It may be assumed that this function implies a write memory barrier before 129b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * changing the task state if and only if any tasks are woken up. 130b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra */ 131b4145872f7049e429718b40b86e1b46659988398Peter Zijlstravoid __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, 132b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra int nr_exclusive, void *key) 133b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 134b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra unsigned long flags; 135b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra int wake_flags = 1; /* XXX WF_SYNC */ 136b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 137b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra if (unlikely(!q)) 138b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra return; 139b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 140b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra if (unlikely(nr_exclusive != 1)) 141b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra wake_flags = 0; 142b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 143b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra spin_lock_irqsave(&q->lock, flags); 144b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra __wake_up_common(q, mode, nr_exclusive, wake_flags, key); 145b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra spin_unlock_irqrestore(&q->lock, flags); 146b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 147b4145872f7049e429718b40b86e1b46659988398Peter ZijlstraEXPORT_SYMBOL_GPL(__wake_up_sync_key); 148b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 149b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra/* 150b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra * __wake_up_sync - see __wake_up_sync_key() 151b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra */ 152b4145872f7049e429718b40b86e1b46659988398Peter Zijlstravoid __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) 153b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra{ 154b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra __wake_up_sync_key(q, mode, nr_exclusive, NULL); 155b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra} 156b4145872f7049e429718b40b86e1b46659988398Peter ZijlstraEXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ 157b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra 158b4145872f7049e429718b40b86e1b46659988398Peter Zijlstra/* 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note: we use "set_current_state()" _after_ the wait-queue add, 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * because we need a memory barrier there on SMP, so that any 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wake-function that tests for the wait-queue being active 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * will be guaranteed to see waitqueue addition _or_ subsequent 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * tests in this thread will see the wakeup having taken place. 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * The spin_unlock() itself is semi-permeable and only protects 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * one way (it only protects stuff inside the critical region and 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * stops them from bleeding out - it would still allow subsequent 16859c51591a0ac7568824f541f57de967e88adaa07Michael Opdenacker * loads to move into the critical region). 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1707ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsprepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state) 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (list_empty(&wait->task_list)) 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __add_wait_queue(q, wait); 179a25d644fc0e232f242d1f3baa63c149c42536ff0Tejun Heo set_current_state(state); 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(prepare_to_wait); 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1847ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsprepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state) 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait->flags |= WQ_FLAG_EXCLUSIVE; 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (list_empty(&wait->task_list)) 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __add_wait_queue_tail(q, wait); 193a25d644fc0e232f242d1f3baa63c149c42536ff0Tejun Heo set_current_state(state); 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(prepare_to_wait_exclusive); 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 198c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterovlong prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state) 199c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov{ 200c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov unsigned long flags; 201c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov 202c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov if (signal_pending_state(state, current)) 203c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov return -ERESTARTSYS; 204c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov 205c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov wait->private = current; 206c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov wait->func = autoremove_wake_function; 207c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov 208c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov spin_lock_irqsave(&q->lock, flags); 209c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov if (list_empty(&wait->task_list)) { 210c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov if (wait->flags & WQ_FLAG_EXCLUSIVE) 211c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov __add_wait_queue_tail(q, wait); 212c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov else 213c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov __add_wait_queue(q, wait); 214c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov } 215c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov set_current_state(state); 216c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov spin_unlock_irqrestore(&q->lock, flags); 217c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov 218c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov return 0; 219c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov} 220c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg NesterovEXPORT_SYMBOL(prepare_to_wait_event); 221c2d816443ef305aba8eaf0bf368f4d3d87494f06Oleg Nesterov 222ee2f154a598e96df2ebb01648a7699373bc085c7Randy Dunlap/** 223777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * finish_wait - clean up after waiting in a queue 224777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * @q: waitqueue waited on 225777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * @wait: wait descriptor 226777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * 227777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * Sets current thread back to running state and removes 228777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * the wait descriptor from the given waitqueue if still 229777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * queued. 230777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner */ 2317ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid finish_wait(wait_queue_head_t *q, wait_queue_t *wait) 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long flags; 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __set_current_state(TASK_RUNNING); 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We can check for list emptiness outside the lock 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IFF: 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - we use the "careful" check that verifies both 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the next and prev pointers, so that there cannot 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * be any half-pending updates in progress on other 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * CPU's that we haven't seen yet (and that might 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * still change the stack area. 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - all other users take the lock (ie we can only 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have _one_ other CPU that looks at or modifies 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the list). 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!list_empty_careful(&wait->task_list)) { 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_lock_irqsave(&q->lock, flags); 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_del_init(&wait->task_list); 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds spin_unlock_irqrestore(&q->lock, flags); 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(finish_wait); 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 257ee2f154a598e96df2ebb01648a7699373bc085c7Randy Dunlap/** 258777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * abort_exclusive_wait - abort exclusive waiting in a queue 259777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * @q: waitqueue waited on 260777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * @wait: wait descriptor 261ee2f154a598e96df2ebb01648a7699373bc085c7Randy Dunlap * @mode: runstate of the waiter to be woken 262777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * @key: key to identify a wait bit queue or %NULL 263777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * 264777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * Sets current thread back to running state and removes 265777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * the wait descriptor from the given waitqueue if still 266777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * queued. 267777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * 268777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * Wakes up the next waiter if the caller is concurrently 269777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * woken up through the queue. 270777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * 271777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * This prevents waiter starvation where an exclusive waiter 27225985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * aborts and is woken up concurrently and no one wakes up 273777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner * the next waiter. 274777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner */ 275777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weinervoid abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, 276777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner unsigned int mode, void *key) 277777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner{ 278777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner unsigned long flags; 279777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner 280777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner __set_current_state(TASK_RUNNING); 281777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner spin_lock_irqsave(&q->lock, flags); 282777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner if (!list_empty(&wait->task_list)) 283777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner list_del_init(&wait->task_list); 284777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner else if (waitqueue_active(q)) 28578ddb08feb7d4fbe3c0a9931804c51ee58be4023Johannes Weiner __wake_up_locked_key(q, mode, key); 286777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner spin_unlock_irqrestore(&q->lock, flags); 287777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner} 288777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes WeinerEXPORT_SYMBOL(abort_exclusive_wait); 289777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = default_wake_function(wait, mode, sync, key); 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret) 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_del_init(&wait->task_list); 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(autoremove_wake_function); 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct wait_bit_key *key = arg; 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct wait_bit_queue *wait_bit 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds = container_of(wait, struct wait_bit_queue, wait); 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (wait_bit->key.flags != key->flags || 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_bit->key.bit_nr != key->bit_nr || 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds test_bit(key->bit_nr, key->flags)) 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return autoremove_wake_function(wait, mode, sync, key); 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(wake_bit_function); 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * To allow interruptible waiting and asynchronous (i.e. nonblocking) 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * permitted return codes. Nonzero return codes halt waiting and return. 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3207ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonint __sched 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds__wait_on_bit(wait_queue_head_t *wq, struct wait_bit_queue *q, 322c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown wait_bit_action_f *action, unsigned mode) 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int ret = 0; 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prepare_to_wait(wq, &q->wait, mode); 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (test_bit(q->key.bit_nr, q->key.flags)) 329c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown ret = (*action)(&q->key); 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (test_bit(q->key.bit_nr, q->key.flags) && !ret); 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds finish_wait(wq, &q->wait); 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(__wait_on_bit); 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3367ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonint __sched out_of_line_wait_on_bit(void *word, int bit, 337c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown wait_bit_action_f *action, unsigned mode) 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_queue_head_t *wq = bit_waitqueue(word, bit); 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DEFINE_WAIT_BIT(wait, word, bit); 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return __wait_on_bit(wq, &wait, action, mode); 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(out_of_line_wait_on_bit); 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 346cbbce82209490df8b68da9aec0d642451fe0a668NeilBrownint __sched out_of_line_wait_on_bit_timeout( 347cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown void *word, int bit, wait_bit_action_f *action, 348cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown unsigned mode, unsigned long timeout) 349cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown{ 350cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown wait_queue_head_t *wq = bit_waitqueue(word, bit); 351cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown DEFINE_WAIT_BIT(wait, word, bit); 352cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown 353cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown wait.key.timeout = jiffies + timeout; 354cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return __wait_on_bit(wq, &wait, action, mode); 355cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown} 356cbbce82209490df8b68da9aec0d642451fe0a668NeilBrownEXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout); 357cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown 3587ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonint __sched 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds__wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q, 360c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown wait_bit_action_f *action, unsigned mode) 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 363777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner int ret; 364777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prepare_to_wait_exclusive(wq, &q->wait, mode); 366777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner if (!test_bit(q->key.bit_nr, q->key.flags)) 367777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner continue; 368c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown ret = action(&q->key); 369777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner if (!ret) 370777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner continue; 371777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner abort_exclusive_wait(wq, &q->wait, mode, &q->key); 372777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner return ret; 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (test_and_set_bit(q->key.bit_nr, q->key.flags)); 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds finish_wait(wq, &q->wait); 375777c6c5f1f6e757ae49ecca2ed72d6b1f523c007Johannes Weiner return 0; 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(__wait_on_bit_lock); 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3797ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonint __sched out_of_line_wait_on_bit_lock(void *word, int bit, 380c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown wait_bit_action_f *action, unsigned mode) 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_queue_head_t *wq = bit_waitqueue(word, bit); 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DEFINE_WAIT_BIT(wait, word, bit); 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return __wait_on_bit_lock(wq, &wait, action, mode); 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(out_of_line_wait_on_bit_lock); 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3897ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid __wake_up_bit(wait_queue_head_t *wq, void *word, int bit) 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (waitqueue_active(wq)) 393e64d66c8edf11629aa203328daf898775ee27dd4Matthew Wilcox __wake_up(wq, TASK_NORMAL, 1, &key); 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(__wake_up_bit); 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * wake_up_bit - wake up a waiter on a bit 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @word: the word being waited on, a kernel virtual address 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * @bit: the bit of the word being waited on 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * There is a standard hashed waitqueue table for generic use. This 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * is the part of the hashtable's accessor API that wakes up waiters 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * on a bit. For instance, if one were to have waiters on a bitflag, 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * one would call wake_up_bit() after clearing the bit. 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * In order for this to function properly, as it uses waitqueue_active() 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * internally, some kind of memory barrier must be done prior to calling 4094e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra * this. Typically, this will be smp_mb__after_atomic(), but in some 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * cases where bitflags are manipulated non-atomically under a lock, one 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * may need to use a less regular barrier, such fs/inode.c's smp_mb(), 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * because spin_unlock() does not guarantee a memory barrier. 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4147ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonvoid wake_up_bit(void *word, int bit) 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __wake_up_bit(bit_waitqueue(word, bit), word, bit); 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(wake_up_bit); 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4207ad5b3a505e68cfdc342933d6e0fc0eaa5e0a4f7Harvey Harrisonwait_queue_head_t *bit_waitqueue(void *word, int bit) 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds const int shift = BITS_PER_LONG == 32 ? 5 : 6; 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds const struct zone *zone = page_zone(virt_to_page(word)); 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long val = (unsigned long)word << shift | bit; 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return &zone->wait_table[hash_long(val, zone->wait_table_bits)]; 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(bit_waitqueue); 429cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 430cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells/* 431cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * Manipulate the atomic_t address to produce a better bit waitqueue table hash 432cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * index (we're keying off bit -1, but that would produce a horrible hash 433cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * value). 434cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells */ 435cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howellsstatic inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) 436cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells{ 437cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells if (BITS_PER_LONG == 64) { 438cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells unsigned long q = (unsigned long)p; 439cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return bit_waitqueue((void *)(q & ~1), q & 1); 440cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells } 441cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return bit_waitqueue(p, 0); 442cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells} 443cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 444cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howellsstatic int wake_atomic_t_function(wait_queue_t *wait, unsigned mode, int sync, 445cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells void *arg) 446cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells{ 447cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells struct wait_bit_key *key = arg; 448cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells struct wait_bit_queue *wait_bit 449cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells = container_of(wait, struct wait_bit_queue, wait); 450cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells atomic_t *val = key->flags; 451cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 452cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells if (wait_bit->key.flags != key->flags || 453cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells wait_bit->key.bit_nr != key->bit_nr || 454cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells atomic_read(val) != 0) 455cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return 0; 456cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return autoremove_wake_function(wait, mode, sync, key); 457cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells} 458cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 459cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells/* 460cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, 461cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero 462cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * return codes halt waiting and return. 463cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells */ 464cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howellsstatic __sched 465cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howellsint __wait_on_atomic_t(wait_queue_head_t *wq, struct wait_bit_queue *q, 466cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells int (*action)(atomic_t *), unsigned mode) 467cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells{ 468cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells atomic_t *val; 469cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells int ret = 0; 470cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 471cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells do { 472cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells prepare_to_wait(wq, &q->wait, mode); 473cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells val = q->key.flags; 474cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells if (atomic_read(val) == 0) 47542577ca8c3616baaafdd8f167b2e1fb959026081David Howells break; 47642577ca8c3616baaafdd8f167b2e1fb959026081David Howells ret = (*action)(val); 477cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells } while (!ret && atomic_read(val) != 0); 478cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells finish_wait(wq, &q->wait); 479cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return ret; 480cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells} 481cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 482cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells#define DEFINE_WAIT_ATOMIC_T(name, p) \ 483cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells struct wait_bit_queue name = { \ 484cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ 485cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells .wait = { \ 486cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells .private = current, \ 487cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells .func = wake_atomic_t_function, \ 488cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells .task_list = \ 489cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells LIST_HEAD_INIT((name).wait.task_list), \ 490cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells }, \ 491cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells } 492cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 493cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells__sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *), 494cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells unsigned mode) 495cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells{ 496cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells wait_queue_head_t *wq = atomic_t_waitqueue(p); 497cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells DEFINE_WAIT_ATOMIC_T(wait, p); 498cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 499cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells return __wait_on_atomic_t(wq, &wait, action, mode); 500cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells} 501cb65537ee1134d3cc55c1fa83952bc8eb1212833David HowellsEXPORT_SYMBOL(out_of_line_wait_on_atomic_t); 502cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells 503cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells/** 504cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * wake_up_atomic_t - Wake up a waiter on a atomic_t 5052203547f82b7727e2cd3fee3e56fceae2b2b691cRandy Dunlap * @p: The atomic_t being waited on, a kernel virtual address 506cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * 507cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * Wake up anyone waiting for the atomic_t to go to zero. 508cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * 509cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t 510cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells * check is done by the waiter's wake function, not the by the waker itself). 511cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells */ 512cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howellsvoid wake_up_atomic_t(atomic_t *p) 513cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells{ 514cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR); 515cb65537ee1134d3cc55c1fa83952bc8eb1212833David Howells} 516cb65537ee1134d3cc55c1fa83952bc8eb1212833David HowellsEXPORT_SYMBOL(wake_up_atomic_t); 517743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown 518c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown__sched int bit_wait(struct wait_bit_key *word) 519743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown{ 520743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown if (signal_pending_state(current->state, current)) 521743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown return 1; 522743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown schedule(); 523743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown return 0; 524743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown} 525743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrownEXPORT_SYMBOL(bit_wait); 526743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown 527c1221321b7c25b53204447cff9949a6d5a7ddddcNeilBrown__sched int bit_wait_io(struct wait_bit_key *word) 528743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown{ 529743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown if (signal_pending_state(current->state, current)) 530743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown return 1; 531743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown io_schedule(); 532743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown return 0; 533743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrown} 534743162013d40ca612b4cb53d3a200dff2d9ab26eNeilBrownEXPORT_SYMBOL(bit_wait_io); 535cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown 536cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown__sched int bit_wait_timeout(struct wait_bit_key *word) 537cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown{ 538cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown unsigned long now = ACCESS_ONCE(jiffies); 539cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown if (signal_pending_state(current->state, current)) 540cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return 1; 541cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown if (time_after_eq(now, word->timeout)) 542cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return -EAGAIN; 543cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown schedule_timeout(word->timeout - now); 544cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return 0; 545cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown} 546cbbce82209490df8b68da9aec0d642451fe0a668NeilBrownEXPORT_SYMBOL_GPL(bit_wait_timeout); 547cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown 548cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown__sched int bit_wait_io_timeout(struct wait_bit_key *word) 549cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown{ 550cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown unsigned long now = ACCESS_ONCE(jiffies); 551cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown if (signal_pending_state(current->state, current)) 552cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return 1; 553cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown if (time_after_eq(now, word->timeout)) 554cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return -EAGAIN; 555cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown io_schedule_timeout(word->timeout - now); 556cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown return 0; 557cbbce82209490df8b68da9aec0d642451fe0a668NeilBrown} 558cbbce82209490df8b68da9aec0d642451fe0a668NeilBrownEXPORT_SYMBOL_GPL(bit_wait_io_timeout); 559