cmpxchg.h revision 60010e508111b2fd3d73de56f3b2c2bfc0f9eba1
110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas/* 210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * Based on arch/arm/include/asm/cmpxchg.h 310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * Copyright (C) 2012 ARM Ltd. 510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * This program is free software; you can redistribute it and/or modify 710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * it under the terms of the GNU General Public License version 2 as 810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * published by the Free Software Foundation. 910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 1010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * This program is distributed in the hope that it will be useful, 1110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * GNU General Public License for more details. 1410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * 1510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * You should have received a copy of the GNU General Public License 1610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas */ 1810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#ifndef __ASM_CMPXCHG_H 1910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#define __ASM_CMPXCHG_H 2010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#include <linux/bug.h> 2210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#include <asm/barrier.h> 2410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 2610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 2710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long ret, tmp; 2810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 2910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas switch (size) { 3010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 1: 3110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg1\n" 323a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon "1: ldaxrb %w0, %2\n" 333a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxrb %w1, %w3, %2\n" 3410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 353a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) 363a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 373a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc", "memory"); 3810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 3910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 2: 4010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg2\n" 413a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon "1: ldaxrh %w0, %2\n" 423a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxrh %w1, %w3, %2\n" 4310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 443a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr) 453a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 463a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc", "memory"); 4710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 4810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 4: 4910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg4\n" 503a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon "1: ldaxr %w0, %2\n" 513a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxr %w1, %w3, %2\n" 5210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 533a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr) 543a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 553a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc", "memory"); 5610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 5710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 8: 5810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __xchg8\n" 593a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon "1: ldaxr %0, %2\n" 603a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stlxr %w1, %3, %2\n" 6110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cbnz %w1, 1b\n" 623a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr) 633a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "r" (x) 643a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc", "memory"); 6510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 6610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas default: 6710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas BUILD_BUG(); 6810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } 6910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 7010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return ret; 7110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 7210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 7310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#define xchg(ptr,x) \ 7410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) 7510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 7610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 7710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long new, int size) 7810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 7910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long oldval = 0, res; 8010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 8110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas switch (size) { 8210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 1: 8310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 8410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg1\n" 853a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxrb %w1, %2\n" 8610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 8710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 8810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 893a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxrb %w0, %w4, %2\n" 9010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 913a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr) 923a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 9310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 9410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 9510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 9610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 9710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 2: 9810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 9910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg2\n" 1003a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxrh %w1, %2\n" 10110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 10210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 10310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1043a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxrh %w0, %w4, %2\n" 10510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1063a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr) 1073a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 1083a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "cc"); 10910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 11010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 11110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 11210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 4: 11310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 11410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg4\n" 1153a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxr %w1, %2\n" 11610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 11710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %w1, %w3\n" 11810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1193a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxr %w0, %w4, %2\n" 12010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1213a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr) 1223a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 12310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 12410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 12510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 12610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 12710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas case 8: 12810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas do { 12910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas asm volatile("// __cmpxchg8\n" 1303a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " ldxr %1, %2\n" 13110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " mov %w0, #0\n" 13210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " cmp %1, %3\n" 13310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas " b.ne 1f\n" 1343a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon " stxr %w0, %4, %2\n" 13510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas "1:\n" 1363a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr) 1373a0310eb369aae985d6409d8ff1340146578e5c1Will Deacon : "Ir" (old), "r" (new) 13810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas : "cc"); 13910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } while (res); 14010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas break; 14110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 14210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas default: 14310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas BUILD_BUG(); 14410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas } 14510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 14610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return oldval; 14710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 14810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 14910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinasstatic inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 15010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long new, int size) 15110b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas{ 15210b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas unsigned long ret; 15310b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 15410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas smp_mb(); 15510b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas ret = __cmpxchg(ptr, old, new, size); 15610b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas smp_mb(); 15710b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 15810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas return ret; 15910b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas} 16010b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 16160010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton#define cmpxchg(ptr, o, n) \ 16260010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton({ \ 16360010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __typeof__(*(ptr)) __ret; \ 16460010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret = (__typeof__(*(ptr))) \ 16560010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \ 16660010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton sizeof(*(ptr))); \ 16760010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret; \ 16860010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton}) 16960010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton 17060010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton#define cmpxchg_local(ptr, o, n) \ 17160010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton({ \ 17260010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __typeof__(*(ptr)) __ret; \ 17360010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret = (__typeof__(*(ptr))) \ 17460010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __cmpxchg((ptr), (unsigned long)(o), \ 17560010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton (unsigned long)(n), sizeof(*(ptr))); \ 17660010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton __ret; \ 17760010e508111b2fd3d73de56f3b2c2bfc0f9eba1Mark Hambleton}) 17810b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas 179a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 180a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 181a84b086b874e0c03ca456c1748df7031a8cdf957Chen Gang 182cf10b79a7d88edc689479af989b3a88e9adf07ffWill Deacon#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 183cf10b79a7d88edc689479af989b3a88e9adf07ffWill Deacon 18410b663aef1c24794d32141be86b4dfcc64293bd0Catalin Marinas#endif /* __ASM_CMPXCHG_H */ 185