1/* 2 * Copyright 2011 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 */ 14 15#include <linux/spinlock.h> 16#include <linux/module.h> 17#include <asm/processor.h> 18 19#include "spinlock_common.h" 20 21/* 22 * Read the spinlock value without allocating in our cache and without 23 * causing an invalidation to another cpu with a copy of the cacheline. 24 * This is important when we are spinning waiting for the lock. 25 */ 26static inline u32 arch_spin_read_noalloc(void *lock) 27{ 28 return atomic_cmpxchg((atomic_t *)lock, -1, -1); 29} 30 31/* 32 * Wait until the high bits (current) match my ticket. 33 * If we notice the overflow bit set on entry, we clear it. 34 */ 35void arch_spin_lock_slow(arch_spinlock_t *lock, u32 my_ticket) 36{ 37 if (unlikely(my_ticket & __ARCH_SPIN_NEXT_OVERFLOW)) { 38 __insn_fetchand4(&lock->lock, ~__ARCH_SPIN_NEXT_OVERFLOW); 39 my_ticket &= ~__ARCH_SPIN_NEXT_OVERFLOW; 40 } 41 42 for (;;) { 43 u32 val = arch_spin_read_noalloc(lock); 44 u32 delta = my_ticket - arch_spin_current(val); 45 if (delta == 0) 46 return; 47 relax((128 / CYCLES_PER_RELAX_LOOP) * delta); 48 } 49} 50EXPORT_SYMBOL(arch_spin_lock_slow); 51 52/* 53 * Check the lock to see if it is plausible, and try to get it with cmpxchg(). 54 */ 55int arch_spin_trylock(arch_spinlock_t *lock) 56{ 57 u32 val = arch_spin_read_noalloc(lock); 58 if (unlikely(arch_spin_current(val) != arch_spin_next(val))) 59 return 0; 60 return cmpxchg(&lock->lock, val, (val + 1) & ~__ARCH_SPIN_NEXT_OVERFLOW) 61 == val; 62} 63EXPORT_SYMBOL(arch_spin_trylock); 64 65void arch_spin_unlock_wait(arch_spinlock_t *lock) 66{ 67 u32 iterations = 0; 68 while (arch_spin_is_locked(lock)) 69 delay_backoff(iterations++); 70} 71EXPORT_SYMBOL(arch_spin_unlock_wait); 72 73/* 74 * If the read lock fails due to a writer, we retry periodically 75 * until the value is positive and we write our incremented reader count. 76 */ 77void __read_lock_failed(arch_rwlock_t *rw) 78{ 79 u32 val; 80 int iterations = 0; 81 do { 82 delay_backoff(iterations++); 83 val = __insn_fetchaddgez4(&rw->lock, 1); 84 } while (unlikely(arch_write_val_locked(val))); 85} 86EXPORT_SYMBOL(__read_lock_failed); 87 88/* 89 * If we failed because there were readers, clear the "writer" bit 90 * so we don't block additional readers. Otherwise, there was another 91 * writer anyway, so our "fetchor" made no difference. Then wait, 92 * issuing periodic fetchor instructions, till we get the lock. 93 */ 94void __write_lock_failed(arch_rwlock_t *rw, u32 val) 95{ 96 int iterations = 0; 97 do { 98 if (!arch_write_val_locked(val)) 99 val = __insn_fetchand4(&rw->lock, ~__WRITE_LOCK_BIT); 100 delay_backoff(iterations++); 101 val = __insn_fetchor4(&rw->lock, __WRITE_LOCK_BIT); 102 } while (val != 0); 103} 104EXPORT_SYMBOL(__write_lock_failed); 105