193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro/* 293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Copyright (C) 2010 The Android Open Source Project 393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Licensed under the Apache License, Version 2.0 (the "License"); 593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * you may not use this file except in compliance with the License. 693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * You may obtain a copy of the License at 793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * http://www.apache.org/licenses/LICENSE-2.0 993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 1093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Unless required by applicable law or agreed to in writing, software 1193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * distributed under the License is distributed on an "AS IS" BASIS, 1293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * See the License for the specific language governing permissions and 1493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * limitations under the License. 1593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro */ 1693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#ifndef ANDROID_CUTILS_ATOMIC_X86_H 1893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#define ANDROID_CUTILS_ATOMIC_X86_H 1993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#include <stdint.h> 2193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 225206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng#ifndef ANDROID_ATOMIC_INLINE 235206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) 245206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng#endif 255206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng 265206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE void android_compiler_barrier(void) 2793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 2893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("" : : : "memory"); 2993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 3093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 3193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#if ANDROID_SMP == 0 325206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) 3393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 3593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 3693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#else 375206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE void android_memory_barrier(void) 3893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("mfence" : : : "memory"); 4093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 4193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif 4293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 435206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 445206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_acquire_load(volatile const int32_t *ptr) 45d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 4693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t value = *ptr; 4793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 4893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return value; 4993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 515206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 525206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_release_load(volatile const int32_t *ptr) 53d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 5493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 5593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return *ptr; 5693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 585206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE void 595206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_acquire_store(int32_t value, volatile int32_t *ptr) 60d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 6193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 6293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 6393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 6493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 655206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE void 665206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_release_store(int32_t value, volatile int32_t *ptr) 67d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 6893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 6993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 7093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 7193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 725206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int 735206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_cas(int32_t old_value, int32_t new_value, volatile int32_t *ptr) 7493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 7593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev; 7693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; cmpxchgl %1, %2" 7793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "=a" (prev) 7893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "q" (new_value), "m" (*ptr), "0" (old_value) 7993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "memory"); 8093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev != old_value; 8193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 8293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 835206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int 845206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_acquire_cas(int32_t old_value, 855206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng int32_t new_value, 865206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng volatile int32_t *ptr) 8793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 8893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Loads are not reordered with other loads. */ 8993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 9093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 9193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 925206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int 935206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_release_cas(int32_t old_value, 945206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng int32_t new_value, 955206d5971208fab8208669dabc98bbb9f7e4a45aBen Cheng volatile int32_t *ptr) 9693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 9793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Stores are not reordered with other stores. */ 9893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 9993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 10093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1015206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 1025206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_add(int32_t increment, volatile int32_t *ptr) 10393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 10493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; xaddl %0, %1" 10593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "+r" (increment), "+m" (*ptr) 10693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : : "memory"); 10793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* increment now holds the old value of *ptr */ 10893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return increment; 10993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1115206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 1125206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_inc(volatile int32_t *addr) 113d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 11493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(1, addr); 11593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1175206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 1185206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_dec(volatile int32_t *addr) 119d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 12093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(-1, addr); 12193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 12293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1235206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 1245206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_and(int32_t value, volatile int32_t *ptr) 12593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 12693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 12793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 12893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 12993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev & value, ptr); 13093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 13193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 13293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 13393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1345206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengextern ANDROID_ATOMIC_INLINE int32_t 1355206d5971208fab8208669dabc98bbb9f7e4a45aBen Chengandroid_atomic_or(int32_t value, volatile int32_t *ptr) 13693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 13793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 13893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 13993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 14093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev | value, ptr); 14193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 14293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 14393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 14493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 14593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif /* ANDROID_CUTILS_ATOMIC_X86_H */ 146