1c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifndef _ASM_GENERIC_BITOPS_ATOMIC_H_ 2c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define _ASM_GENERIC_BITOPS_ATOMIC_H_ 3c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 4c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/types.h> 5c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 6c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG)) 7c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG) 8c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 9c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifdef CONFIG_SMP 10c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/spinlock.h> 11c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/cache.h> /* we use L1_CACHE_BYTES */ 12c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 13c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* Use an array of spinlocks for our atomic_ts. 14c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Hash function to index into a different SPINLOCK. 15c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Since "a" is usually an address, use one spinlock per cacheline. 16c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 17c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru# define ATOMIC_HASH_SIZE 4 18c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru# define ATOMIC_HASH(a) (&(__atomic_hash[ (((unsigned long) a)/L1_CACHE_BYTES) & (ATOMIC_HASH_SIZE-1) ])) 19c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 20c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queruextern raw_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned; 21c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 22c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* Can't use raw_spin_lock_irq because of #include problems, so 23c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * this is the substitute */ 24c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define _atomic_spin_lock_irqsave(l,f) do { \ 25c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru raw_spinlock_t *s = ATOMIC_HASH(l); \ 26c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru local_irq_save(f); \ 27c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __raw_spin_lock(s); \ 28c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} while(0) 29c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 30c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define _atomic_spin_unlock_irqrestore(l,f) do { \ 31c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru raw_spinlock_t *s = ATOMIC_HASH(l); \ 32c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __raw_spin_unlock(s); \ 33c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru local_irq_restore(f); \ 34c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} while(0) 35c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 36c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 37c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#else 38c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru# define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0) 39c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru# define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0) 40c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 41c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 42c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* 43c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * NMI events can occur at any time, including when interrupts have been 44c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * disabled by *_irqsave(). So you can get NMI events occurring while a 45c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * *_bit function is holding a spin lock. If the NMI handler also wants 46c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * to do bit manipulation (and they do) then you can get a deadlock 47c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * between the original caller of *_bit() and the NMI handler. 48c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 49c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * by Keith Owens 50c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 51c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 52c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 53c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * set_bit - Atomically set a bit in memory 54c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: the bit to set 55c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: the address to start counting from 56c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 57c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * This function is atomic and may not be reordered. See __set_bit() 58c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * if you do not require the atomic guarantees. 59c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 60c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Note: there are no guarantees that this function will not be reordered 61c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * on non x86 architectures, so if you are writting portable code, 62c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * make sure not to rely on its reordering guarantees. 63c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 64c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Note that @nr may be almost arbitrarily large; this function is not 65c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * restricted to acting on a single-word quantity. 66c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 67c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void set_bit(int nr, volatile unsigned long *addr) 68c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 69c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 70c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 71c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 72c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 73c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 74c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p |= mask; 75c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 76c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 77c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 78c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 79c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * clear_bit - Clears a bit in memory 80c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: Bit to clear 81c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: Address to start counting from 82c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 83c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * clear_bit() is atomic and may not be reordered. However, it does 84c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * not contain a memory barrier, so if it is used for locking purposes, 85c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() 86c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * in order to ensure changes are visible on other processors. 87c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 88c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void clear_bit(int nr, volatile unsigned long *addr) 89c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 90c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 91c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 92c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 93c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 94c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 95c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p &= ~mask; 96c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 97c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 98c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 99c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 100c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * change_bit - Toggle a bit in memory 101c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: Bit to change 102c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: Address to start counting from 103c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 104c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * change_bit() is atomic and may not be reordered. It may be 105c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * reordered on other architectures than x86. 106c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * Note that @nr may be almost arbitrarily large; this function is not 107c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * restricted to acting on a single-word quantity. 108c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 109c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline void change_bit(int nr, volatile unsigned long *addr) 110c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 111c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 112c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 113c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 114c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 115c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 116c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p ^= mask; 117c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 118c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 119c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 120c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 121c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * test_and_set_bit - Set a bit and return its old value 122c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: Bit to set 123c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: Address to count from 124c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 125c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * This operation is atomic and cannot be reordered. 126c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * It may be reordered on other architectures than x86. 127c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * It also implies a memory barrier. 128c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 129c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline int test_and_set_bit(int nr, volatile unsigned long *addr) 130c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 131c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 132c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 133c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long old; 134c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 135c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 136c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 137c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru old = *p; 138c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p = old | mask; 139c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 140c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 141c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return (old & mask) != 0; 142c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 143c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 144c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 145c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * test_and_clear_bit - Clear a bit and return its old value 146c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: Bit to clear 147c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: Address to count from 148c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 149c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * This operation is atomic and cannot be reordered. 150c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * It can be reorderdered on other architectures other than x86. 151c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * It also implies a memory barrier. 152c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 153c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline int test_and_clear_bit(int nr, volatile unsigned long *addr) 154c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 155c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 156c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 157c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long old; 158c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 159c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 160c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 161c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru old = *p; 162c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p = old & ~mask; 163c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 164c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 165c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return (old & mask) != 0; 166c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 167c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 168c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/** 169c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * test_and_change_bit - Change a bit and return its old value 170c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @nr: Bit to change 171c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * @addr: Address to count from 172c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * 173c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * This operation is atomic and cannot be reordered. 174c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * It also implies a memory barrier. 175c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 176c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustatic inline int test_and_change_bit(int nr, volatile unsigned long *addr) 177c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru{ 178c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long mask = BITOP_MASK(nr); 179c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); 180c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long old; 181c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long flags; 182c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 183c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_lock_irqsave(p, flags); 184c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru old = *p; 185c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru *p = old ^ mask; 186c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru _atomic_spin_unlock_irqrestore(p, flags); 187c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 188c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru return (old & mask) != 0; 189c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} 190c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 191c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif /* _ASM_GENERIC_BITOPS_ATOMIC_H */ 192