118aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 218aecc2b645bbb07851b196452a2af314222069bChris Metcalf * Copyright 2011 Tilera Corporation. All Rights Reserved. 318aecc2b645bbb07851b196452a2af314222069bChris Metcalf * 418aecc2b645bbb07851b196452a2af314222069bChris Metcalf * This program is free software; you can redistribute it and/or 518aecc2b645bbb07851b196452a2af314222069bChris Metcalf * modify it under the terms of the GNU General Public License 618aecc2b645bbb07851b196452a2af314222069bChris Metcalf * as published by the Free Software Foundation, version 2. 718aecc2b645bbb07851b196452a2af314222069bChris Metcalf * 818aecc2b645bbb07851b196452a2af314222069bChris Metcalf * This program is distributed in the hope that it will be useful, but 918aecc2b645bbb07851b196452a2af314222069bChris Metcalf * WITHOUT ANY WARRANTY; without even the implied warranty of 1018aecc2b645bbb07851b196452a2af314222069bChris Metcalf * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 1118aecc2b645bbb07851b196452a2af314222069bChris Metcalf * NON INFRINGEMENT. See the GNU General Public License for 1218aecc2b645bbb07851b196452a2af314222069bChris Metcalf * more details. 1318aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 1418aecc2b645bbb07851b196452a2af314222069bChris Metcalf 1518aecc2b645bbb07851b196452a2af314222069bChris Metcalf#include <linux/spinlock.h> 1618aecc2b645bbb07851b196452a2af314222069bChris Metcalf#include <linux/module.h> 1718aecc2b645bbb07851b196452a2af314222069bChris Metcalf#include <asm/processor.h> 1818aecc2b645bbb07851b196452a2af314222069bChris Metcalf 1918aecc2b645bbb07851b196452a2af314222069bChris Metcalf#include "spinlock_common.h" 2018aecc2b645bbb07851b196452a2af314222069bChris Metcalf 2118aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 2218aecc2b645bbb07851b196452a2af314222069bChris Metcalf * Read the spinlock value without allocating in our cache and without 2318aecc2b645bbb07851b196452a2af314222069bChris Metcalf * causing an invalidation to another cpu with a copy of the cacheline. 2418aecc2b645bbb07851b196452a2af314222069bChris Metcalf * This is important when we are spinning waiting for the lock. 2518aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 2618aecc2b645bbb07851b196452a2af314222069bChris Metcalfstatic inline u32 arch_spin_read_noalloc(void *lock) 2718aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 2818aecc2b645bbb07851b196452a2af314222069bChris Metcalf return atomic_cmpxchg((atomic_t *)lock, -1, -1); 2918aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 3018aecc2b645bbb07851b196452a2af314222069bChris Metcalf 3118aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 3218aecc2b645bbb07851b196452a2af314222069bChris Metcalf * Wait until the high bits (current) match my ticket. 3318aecc2b645bbb07851b196452a2af314222069bChris Metcalf * If we notice the overflow bit set on entry, we clear it. 3418aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 3518aecc2b645bbb07851b196452a2af314222069bChris Metcalfvoid arch_spin_lock_slow(arch_spinlock_t *lock, u32 my_ticket) 3618aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 3718aecc2b645bbb07851b196452a2af314222069bChris Metcalf if (unlikely(my_ticket & __ARCH_SPIN_NEXT_OVERFLOW)) { 3818aecc2b645bbb07851b196452a2af314222069bChris Metcalf __insn_fetchand4(&lock->lock, ~__ARCH_SPIN_NEXT_OVERFLOW); 3918aecc2b645bbb07851b196452a2af314222069bChris Metcalf my_ticket &= ~__ARCH_SPIN_NEXT_OVERFLOW; 4018aecc2b645bbb07851b196452a2af314222069bChris Metcalf } 4118aecc2b645bbb07851b196452a2af314222069bChris Metcalf 4218aecc2b645bbb07851b196452a2af314222069bChris Metcalf for (;;) { 4318aecc2b645bbb07851b196452a2af314222069bChris Metcalf u32 val = arch_spin_read_noalloc(lock); 4418aecc2b645bbb07851b196452a2af314222069bChris Metcalf u32 delta = my_ticket - arch_spin_current(val); 4518aecc2b645bbb07851b196452a2af314222069bChris Metcalf if (delta == 0) 4618aecc2b645bbb07851b196452a2af314222069bChris Metcalf return; 4718aecc2b645bbb07851b196452a2af314222069bChris Metcalf relax((128 / CYCLES_PER_RELAX_LOOP) * delta); 4818aecc2b645bbb07851b196452a2af314222069bChris Metcalf } 4918aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 5018aecc2b645bbb07851b196452a2af314222069bChris MetcalfEXPORT_SYMBOL(arch_spin_lock_slow); 5118aecc2b645bbb07851b196452a2af314222069bChris Metcalf 5218aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 5318aecc2b645bbb07851b196452a2af314222069bChris Metcalf * Check the lock to see if it is plausible, and try to get it with cmpxchg(). 5418aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 5518aecc2b645bbb07851b196452a2af314222069bChris Metcalfint arch_spin_trylock(arch_spinlock_t *lock) 5618aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 5718aecc2b645bbb07851b196452a2af314222069bChris Metcalf u32 val = arch_spin_read_noalloc(lock); 5818aecc2b645bbb07851b196452a2af314222069bChris Metcalf if (unlikely(arch_spin_current(val) != arch_spin_next(val))) 5918aecc2b645bbb07851b196452a2af314222069bChris Metcalf return 0; 6018aecc2b645bbb07851b196452a2af314222069bChris Metcalf return cmpxchg(&lock->lock, val, (val + 1) & ~__ARCH_SPIN_NEXT_OVERFLOW) 6118aecc2b645bbb07851b196452a2af314222069bChris Metcalf == val; 6218aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 6318aecc2b645bbb07851b196452a2af314222069bChris MetcalfEXPORT_SYMBOL(arch_spin_trylock); 6418aecc2b645bbb07851b196452a2af314222069bChris Metcalf 6518aecc2b645bbb07851b196452a2af314222069bChris Metcalfvoid arch_spin_unlock_wait(arch_spinlock_t *lock) 6618aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 6718aecc2b645bbb07851b196452a2af314222069bChris Metcalf u32 iterations = 0; 6818aecc2b645bbb07851b196452a2af314222069bChris Metcalf while (arch_spin_is_locked(lock)) 6918aecc2b645bbb07851b196452a2af314222069bChris Metcalf delay_backoff(iterations++); 7018aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 7118aecc2b645bbb07851b196452a2af314222069bChris MetcalfEXPORT_SYMBOL(arch_spin_unlock_wait); 7218aecc2b645bbb07851b196452a2af314222069bChris Metcalf 7318aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 7418aecc2b645bbb07851b196452a2af314222069bChris Metcalf * If the read lock fails due to a writer, we retry periodically 7518aecc2b645bbb07851b196452a2af314222069bChris Metcalf * until the value is positive and we write our incremented reader count. 7618aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 7718aecc2b645bbb07851b196452a2af314222069bChris Metcalfvoid __read_lock_failed(arch_rwlock_t *rw) 7818aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 7918aecc2b645bbb07851b196452a2af314222069bChris Metcalf u32 val; 8018aecc2b645bbb07851b196452a2af314222069bChris Metcalf int iterations = 0; 8118aecc2b645bbb07851b196452a2af314222069bChris Metcalf do { 8218aecc2b645bbb07851b196452a2af314222069bChris Metcalf delay_backoff(iterations++); 8318aecc2b645bbb07851b196452a2af314222069bChris Metcalf val = __insn_fetchaddgez4(&rw->lock, 1); 8418aecc2b645bbb07851b196452a2af314222069bChris Metcalf } while (unlikely(arch_write_val_locked(val))); 8518aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 8618aecc2b645bbb07851b196452a2af314222069bChris MetcalfEXPORT_SYMBOL(__read_lock_failed); 8718aecc2b645bbb07851b196452a2af314222069bChris Metcalf 8818aecc2b645bbb07851b196452a2af314222069bChris Metcalf/* 8918aecc2b645bbb07851b196452a2af314222069bChris Metcalf * If we failed because there were readers, clear the "writer" bit 9018aecc2b645bbb07851b196452a2af314222069bChris Metcalf * so we don't block additional readers. Otherwise, there was another 9118aecc2b645bbb07851b196452a2af314222069bChris Metcalf * writer anyway, so our "fetchor" made no difference. Then wait, 9218aecc2b645bbb07851b196452a2af314222069bChris Metcalf * issuing periodic fetchor instructions, till we get the lock. 9318aecc2b645bbb07851b196452a2af314222069bChris Metcalf */ 9418aecc2b645bbb07851b196452a2af314222069bChris Metcalfvoid __write_lock_failed(arch_rwlock_t *rw, u32 val) 9518aecc2b645bbb07851b196452a2af314222069bChris Metcalf{ 9618aecc2b645bbb07851b196452a2af314222069bChris Metcalf int iterations = 0; 9718aecc2b645bbb07851b196452a2af314222069bChris Metcalf do { 9818aecc2b645bbb07851b196452a2af314222069bChris Metcalf if (!arch_write_val_locked(val)) 9918aecc2b645bbb07851b196452a2af314222069bChris Metcalf val = __insn_fetchand4(&rw->lock, ~__WRITE_LOCK_BIT); 10018aecc2b645bbb07851b196452a2af314222069bChris Metcalf delay_backoff(iterations++); 10118aecc2b645bbb07851b196452a2af314222069bChris Metcalf val = __insn_fetchor4(&rw->lock, __WRITE_LOCK_BIT); 10218aecc2b645bbb07851b196452a2af314222069bChris Metcalf } while (val != 0); 10318aecc2b645bbb07851b196452a2af314222069bChris Metcalf} 10418aecc2b645bbb07851b196452a2af314222069bChris MetcalfEXPORT_SYMBOL(__write_lock_failed); 105