1#ifndef __ASM_SPINLOCK_H 2#define __ASM_SPINLOCK_H 3 4#include <asm/atomic.h> 5#include <asm/rwlock.h> 6#include <asm/page.h> 7#include <asm/processor.h> 8#include <linux/compiler.h> 9 10#ifdef CONFIG_PARAVIRT 11#include <asm/paravirt.h> 12#else 13#define CLI_STRING "cli" 14#define STI_STRING "sti" 15#define CLI_STI_CLOBBERS 16#define CLI_STI_INPUT_ARGS 17#endif /* CONFIG_PARAVIRT */ 18 19/* 20 * Your basic SMP spinlocks, allowing only a single CPU anywhere 21 * 22 * Simple spin lock operations. There are two variants, one clears IRQ's 23 * on the local processor, one does not. 24 * 25 * We make no fairness assumptions. They have a cost. 26 * 27 * (the type definitions are in asm/spinlock_types.h) 28 */ 29 30static inline int __raw_spin_is_locked(raw_spinlock_t *x) 31{ 32 return *(volatile signed char *)(&(x)->slock) <= 0; 33} 34 35static inline void __raw_spin_lock(raw_spinlock_t *lock) 36{ 37 asm volatile("\n1:\t" 38 LOCK_PREFIX " ; decb %0\n\t" 39 "jns 3f\n" 40 "2:\t" 41 "rep;nop\n\t" 42 "cmpb $0,%0\n\t" 43 "jle 2b\n\t" 44 "jmp 1b\n" 45 "3:\n\t" 46 : "+m" (lock->slock) : : "memory"); 47} 48 49/* 50 * It is easier for the lock validator if interrupts are not re-enabled 51 * in the middle of a lock-acquire. This is a performance feature anyway 52 * so we turn it off: 53 * 54 * NOTE: there's an irqs-on section here, which normally would have to be 55 * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. 56 */ 57#ifndef CONFIG_PROVE_LOCKING 58static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) 59{ 60 asm volatile( 61 "\n1:\t" 62 LOCK_PREFIX " ; decb %[slock]\n\t" 63 "jns 5f\n" 64 "2:\t" 65 "testl $0x200, %[flags]\n\t" 66 "jz 4f\n\t" 67 STI_STRING "\n" 68 "3:\t" 69 "rep;nop\n\t" 70 "cmpb $0, %[slock]\n\t" 71 "jle 3b\n\t" 72 CLI_STRING "\n\t" 73 "jmp 1b\n" 74 "4:\t" 75 "rep;nop\n\t" 76 "cmpb $0, %[slock]\n\t" 77 "jg 1b\n\t" 78 "jmp 4b\n" 79 "5:\n\t" 80 : [slock] "+m" (lock->slock) 81 : [flags] "r" (flags) 82 CLI_STI_INPUT_ARGS 83 : "memory" CLI_STI_CLOBBERS); 84} 85#endif 86 87static inline int __raw_spin_trylock(raw_spinlock_t *lock) 88{ 89 char oldval; 90 asm volatile( 91 "xchgb %b0,%1" 92 :"=q" (oldval), "+m" (lock->slock) 93 :"0" (0) : "memory"); 94 return oldval > 0; 95} 96 97/* 98 * __raw_spin_unlock based on writing $1 to the low byte. 99 * This method works. Despite all the confusion. 100 * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there) 101 * (PPro errata 66, 92) 102 */ 103 104#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) 105 106static inline void __raw_spin_unlock(raw_spinlock_t *lock) 107{ 108 asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); 109} 110 111#else 112 113static inline void __raw_spin_unlock(raw_spinlock_t *lock) 114{ 115 char oldval = 1; 116 117 asm volatile("xchgb %b0, %1" 118 : "=q" (oldval), "+m" (lock->slock) 119 : "0" (oldval) : "memory"); 120} 121 122#endif 123 124static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) 125{ 126 while (__raw_spin_is_locked(lock)) 127 cpu_relax(); 128} 129 130/* 131 * Read-write spinlocks, allowing multiple readers 132 * but only one writer. 133 * 134 * NOTE! it is quite common to have readers in interrupts 135 * but no interrupt writers. For those circumstances we 136 * can "mix" irq-safe locks - any writer needs to get a 137 * irq-safe write-lock, but readers can get non-irqsafe 138 * read-locks. 139 * 140 * On x86, we implement read-write locks as a 32-bit counter 141 * with the high bit (sign) being the "contended" bit. 142 * 143 * The inline assembly is non-obvious. Think about it. 144 * 145 * Changed to use the same technique as rw semaphores. See 146 * semaphore.h for details. -ben 147 * 148 * the helpers are in arch/i386/kernel/semaphore.c 149 */ 150 151/** 152 * read_can_lock - would read_trylock() succeed? 153 * @lock: the rwlock in question. 154 */ 155static inline int __raw_read_can_lock(raw_rwlock_t *x) 156{ 157 return (int)(x)->lock > 0; 158} 159 160/** 161 * write_can_lock - would write_trylock() succeed? 162 * @lock: the rwlock in question. 163 */ 164static inline int __raw_write_can_lock(raw_rwlock_t *x) 165{ 166 return (x)->lock == RW_LOCK_BIAS; 167} 168 169static inline void __raw_read_lock(raw_rwlock_t *rw) 170{ 171 asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" 172 "jns 1f\n" 173 "call __read_lock_failed\n\t" 174 "1:\n" 175 ::"a" (rw) : "memory"); 176} 177 178static inline void __raw_write_lock(raw_rwlock_t *rw) 179{ 180 asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" 181 "jz 1f\n" 182 "call __write_lock_failed\n\t" 183 "1:\n" 184 ::"a" (rw) : "memory"); 185} 186 187static inline int __raw_read_trylock(raw_rwlock_t *lock) 188{ 189 atomic_t *count = (atomic_t *)lock; 190 atomic_dec(count); 191 if (atomic_read(count) >= 0) 192 return 1; 193 atomic_inc(count); 194 return 0; 195} 196 197static inline int __raw_write_trylock(raw_rwlock_t *lock) 198{ 199 atomic_t *count = (atomic_t *)lock; 200 if (atomic_sub_and_test(RW_LOCK_BIAS, count)) 201 return 1; 202 atomic_add(RW_LOCK_BIAS, count); 203 return 0; 204} 205 206static inline void __raw_read_unlock(raw_rwlock_t *rw) 207{ 208 asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory"); 209} 210 211static inline void __raw_write_unlock(raw_rwlock_t *rw) 212{ 213 asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" 214 : "+m" (rw->lock) : : "memory"); 215} 216 217#define _raw_spin_relax(lock) cpu_relax() 218#define _raw_read_relax(lock) cpu_relax() 219#define _raw_write_relax(lock) cpu_relax() 220 221#endif /* __ASM_SPINLOCK_H */ 222