atomic-x86.h revision d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aa
193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro/* 293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Copyright (C) 2010 The Android Open Source Project 393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Licensed under the Apache License, Version 2.0 (the "License"); 593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * you may not use this file except in compliance with the License. 693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * You may obtain a copy of the License at 793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * http://www.apache.org/licenses/LICENSE-2.0 993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 1093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Unless required by applicable law or agreed to in writing, software 1193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * distributed under the License is distributed on an "AS IS" BASIS, 1293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * See the License for the specific language governing permissions and 1493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * limitations under the License. 1593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro */ 1693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#ifndef ANDROID_CUTILS_ATOMIC_X86_H 1893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#define ANDROID_CUTILS_ATOMIC_X86_H 1993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#include <stdint.h> 2193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_compiler_barrier(void) 2393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 2493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("" : : : "memory"); 2593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 2693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#if ANDROID_SMP == 0 2893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_memory_barrier(void) 2993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 3193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 3293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#else 3393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_memory_barrier(void) 3493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("mfence" : : : "memory"); 3693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 3793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif 3893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 39d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_acquire_load(volatile const int32_t *ptr) 40d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 4193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t value = *ptr; 4293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 4393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return value; 4493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 4593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 46d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_release_load(volatile const int32_t *ptr) 47d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 4893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 4993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return *ptr; 5093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 5293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_atomic_acquire_store(int32_t value, 53d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro volatile int32_t *ptr) 54d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 5593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 5693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 5793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 5993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_atomic_release_store(int32_t value, 60d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro volatile int32_t *ptr) 61d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 6293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 6393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 6493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 6593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 6693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_cas(int32_t old_value, int32_t new_value, 6793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 6893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 6993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev; 7093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; cmpxchgl %1, %2" 7193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "=a" (prev) 7293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "q" (new_value), "m" (*ptr), "0" (old_value) 7393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "memory"); 7493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev != old_value; 7593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 7693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 7793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_acquire_cas(int32_t old_value, 7893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t new_value, 7993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 8093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 8193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Loads are not reordered with other loads. */ 8293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 8393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 8493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 8593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_release_cas(int32_t old_value, 8693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t new_value, 8793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 8893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 8993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Stores are not reordered with other stores. */ 9093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 9193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 9293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 9393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_swap(int32_t new_value, 9493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 9593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 9693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("xchgl %1, %0" 9793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "=r" (new_value) 9893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "m" (*ptr), "0" (new_value) 9993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "memory"); 10093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* new_value now holds the old value of *ptr */ 10193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return new_value; 10293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 10393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 10493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_add(int32_t increment, 10593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 10693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 10793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; xaddl %0, %1" 10893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "+r" (increment), "+m" (*ptr) 10993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : : "memory"); 11093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* increment now holds the old value of *ptr */ 11193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return increment; 11293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 114d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_inc(volatile int32_t *addr) 115d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 11693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(1, addr); 11793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 119d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_dec(volatile int32_t *addr) 120d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 12193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(-1, addr); 12293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 12393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 12493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_and(int32_t value, 12593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 12693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 12793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 12893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 12993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 13093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev & value, ptr); 13193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 13293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 13393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 13493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 13593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) 13693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 13793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 13893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 13993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 14093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev | value, ptr); 14193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 14293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 14393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 14493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 14593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif /* ANDROID_CUTILS_ATOMIC_X86_H */ 146