110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas/* 210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * Based on arch/arm/include/asm/cmpxchg.h 310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * Copyright (C) 2012 ARM Ltd. 510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * This program is free software; you can redistribute it and/or modify 710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * it under the terms of the GNU General Public License version 2 as 810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * published by the Free Software Foundation. 910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 1010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * This program is distributed in the hope that it will be useful, 1110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * GNU General Public License for more details. 1410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 1510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * You should have received a copy of the GNU General Public License 1610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas */ 1810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#ifndef __ASM_CMPXCHG_H 1910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#define __ASM_CMPXCHG_H 2010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#include <linux/bug.h> 2210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#include <asm/barrier.h> 2410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 2610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 2710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long ret, tmp; 2810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas switch (size) { 3010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 1: 3110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg1\n" 328e86f0b409a44193f1587e87b69c5dcf8f65be67Will Deacon "1: ldxrb %w0, %2\n" 333a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxrb %w1, %w3, %2\n" 3410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 353a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) 363a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 3795c4189689f92fba7ecf9097173404d4928c6e9bWill Deacon : "memory"); 3810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 3910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 2: 4010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg2\n" 418e86f0b409a44193f1587e87b69c5dcf8f65be67Will Deacon "1: ldxrh %w0, %2\n" 423a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxrh %w1, %w3, %2\n" 4310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 443a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr) 453a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 4695c4189689f92fba7ecf9097173404d4928c6e9bWill Deacon : "memory"); 4710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 4810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 4: 4910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg4\n" 508e86f0b409a44193f1587e87b69c5dcf8f65be67Will Deacon "1: ldxr %w0, %2\n" 513a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxr %w1, %w3, %2\n" 5210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 533a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr) 543a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 5595c4189689f92fba7ecf9097173404d4928c6e9bWill Deacon : "memory"); 5610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 5710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 8: 5810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg8\n" 598e86f0b409a44193f1587e87b69c5dcf8f65be67Will Deacon "1: ldxr %0, %2\n" 603a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxr %w1, %3, %2\n" 6110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 623a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr) 633a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 6495c4189689f92fba7ecf9097173404d4928c6e9bWill Deacon : "memory"); 6510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 6610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas default: 6710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas BUILD_BUG(); 6810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } 6910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 708e86f0b409a44193f1587e87b69c5dcf8f65be67Will Deacon smp_mb(); 7110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return ret; 7210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 7310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 7410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#define xchg(ptr,x) \ 75e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon({ \ 76e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon __typeof__(*(ptr)) __ret; \ 77e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon __ret = (__typeof__(*(ptr))) \ 78e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 79e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon __ret; \ 80e1dfda9ced9bea1413a736f0d578f8218a7788ecWill Deacon}) 8110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 8210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 8310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long new, int size) 8410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 8510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long oldval = 0, res; 8610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 8710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas switch (size) { 8810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 1: 8910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 9010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg1\n" 913a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxrb %w1, %2\n" 9210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 9310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 9410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 953a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxrb %w0, %w4, %2\n" 9610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 973a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr) 983a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 9910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 10010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 10110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 10210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 10310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 2: 10410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 10510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg2\n" 1063a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxrh %w1, %2\n" 10710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 10810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 10910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1103a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxrh %w0, %w4, %2\n" 11110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1123a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr) 1133a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 1143a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc"); 11510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 11610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 11710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 11810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 4: 11910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 12010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg4\n" 1213a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxr %w1, %2\n" 12210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 12310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 12410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1253a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxr %w0, %w4, %2\n" 12610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1273a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr) 1283a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 12910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 13010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 13110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 13210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 13310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 8: 13410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 13510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg8\n" 1363a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxr %1, %2\n" 13710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 13810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %1, %3\n" 13910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1403a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxr %w0, %4, %2\n" 14110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1423a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr) 1433a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 14410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 14510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 14610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 14710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 14810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas default: 14910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas BUILD_BUG(); 15010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } 15110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 15210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return oldval; 15310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 15410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 15510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 15610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long new, int size) 15710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 15810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long ret; 15910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 16010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas smp_mb(); 16110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas ret = __cmpxchg(ptr, old, new, size); 16210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas smp_mb(); 16310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 16410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return ret; 16510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 16610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 16760010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton#define cmpxchg(ptr, o, n) \ 16860010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton({ \ 16960010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __typeof__(*(ptr)) __ret; \ 17060010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret = (__typeof__(*(ptr))) \ 17160010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \ 17260010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton sizeof(*(ptr))); \ 17360010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret; \ 17460010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton}) 17560010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton 17660010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton#define cmpxchg_local(ptr, o, n) \ 17760010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton({ \ 17860010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __typeof__(*(ptr)) __ret; \ 17960010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret = (__typeof__(*(ptr))) \ 18060010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __cmpxchg((ptr), (unsigned long)(o), \ 18160010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton (unsigned long)(n), sizeof(*(ptr))); \ 18260010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret; \ 18360010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton}) 18410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 185a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 186a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 187a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang 188cf10b79a7d88edc689479af989b3a88e9adf07ffWill Deacon#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 189cf10b79a7d88edc689479af989b3a88e9adf07ffWill Deacon 19010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#endif /* __ASM_CMPXCHG_H */ 191