1b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar/* 2b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Assembly implementation of the mutex fastpath, based on atomic 3b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * decrement/increment. 4b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 5b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * started by Ingo Molnar: 6b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 7b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 8b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar */ 91965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#ifndef _ASM_X86_MUTEX_64_H 101965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#define _ASM_X86_MUTEX_64_H 11b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 12b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar/** 13b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * __mutex_fastpath_lock - decrement and call function if negative 14b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @v: pointer of type atomic_t 15b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @fail_fn: function to call if the result is negative 16b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 17b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Atomically decrements @v and calls <fail_fn> if the result is negative. 18b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar */ 192c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches#define __mutex_fastpath_lock(v, fail_fn) \ 202c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perchesdo { \ 212c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches unsigned long dummy; \ 222c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches \ 232c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches typecheck(atomic_t *, v); \ 242c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches typecheck_fn(void (*)(atomic_t *), fail_fn); \ 252c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches \ 262c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches asm volatile(LOCK_PREFIX " decl (%%rdi)\n" \ 272c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches " jns 1f \n" \ 282c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches " call " #fail_fn "\n" \ 292c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches "1:" \ 302c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "=D" (dummy) \ 312c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "D" (v) \ 322c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "rax", "rsi", "rdx", "rcx", \ 332c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches "r8", "r9", "r10", "r11", "memory"); \ 34b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar} while (0) 35b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 36b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar/** 37b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * __mutex_fastpath_lock_retval - try to take the lock by moving the count 38b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * from 1 to a 0 value 39b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @count: pointer of type atomic_t 40b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @fail_fn: function to call if the original value was not 1 41b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 42b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Change the count from 1 to a value lower than 1, and call <fail_fn> if 43b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * it wasn't 1 originally. This function returns 0 if the fastpath succeeds, 44b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * or anything the slow path function returns 45b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar */ 462c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perchesstatic inline int __mutex_fastpath_lock_retval(atomic_t *count, 472c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches int (*fail_fn)(atomic_t *)) 48b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar{ 49b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar if (unlikely(atomic_dec_return(count) < 0)) 50b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar return fail_fn(count); 51b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar else 52b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar return 0; 53b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar} 54b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 55b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar/** 56b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * __mutex_fastpath_unlock - increment and call function if nonpositive 57b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @v: pointer of type atomic_t 58b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @fail_fn: function to call if the result is nonpositive 59b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 60b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Atomically increments @v and calls <fail_fn> if the result is nonpositive. 61b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar */ 622c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches#define __mutex_fastpath_unlock(v, fail_fn) \ 632c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perchesdo { \ 642c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches unsigned long dummy; \ 652c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches \ 662c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches typecheck(atomic_t *, v); \ 672c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches typecheck_fn(void (*)(atomic_t *), fail_fn); \ 682c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches \ 692c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches asm volatile(LOCK_PREFIX " incl (%%rdi)\n" \ 702c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches " jg 1f\n" \ 712c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches " call " #fail_fn "\n" \ 722c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches "1:" \ 732c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "=D" (dummy) \ 742c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "D" (v) \ 752c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches : "rax", "rsi", "rdx", "rcx", \ 762c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches "r8", "r9", "r10", "r11", "memory"); \ 77b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar} while (0) 78b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 79b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar#define __mutex_slowpath_needs_to_unlock() 1 80b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 81b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar/** 82b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * __mutex_fastpath_trylock - try to acquire the mutex, without waiting 83b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 84b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @count: pointer of type atomic_t 85b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * @fail_fn: fallback function 86b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * 87b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * Change the count from 1 to 0 and return 1 (success), or return 0 (failure) 88b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * if it wasn't 1 originally. [the fallback function is never used on 89b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar * x86_64, because all x86_64 CPUs have a CMPXCHG instruction.] 90b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar */ 912c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perchesstatic inline int __mutex_fastpath_trylock(atomic_t *count, 922c4e8830414de84cc969a1f9685f4c48b91ca6e7Joe Perches int (*fail_fn)(atomic_t *)) 93b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar{ 944cec87361462d570d6a67888feda41e77e0a9562Linus Torvalds if (likely(atomic_cmpxchg(count, 1, 0) == 1)) 95b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar return 1; 96b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar else 97b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar return 0; 98b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar} 99b8aa0361e434f8d9cbe9bb34525af1e8721396d8Ingo Molnar 1001965aae3c98397aad957412413c07e97b1bd4e64H. Peter Anvin#endif /* _ASM_X86_MUTEX_64_H */ 101