193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro/* 293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Copyright (C) 2010 The Android Open Source Project 393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Licensed under the Apache License, Version 2.0 (the "License"); 593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * you may not use this file except in compliance with the License. 693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * You may obtain a copy of the License at 793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * http://www.apache.org/licenses/LICENSE-2.0 993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * 1093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * Unless required by applicable law or agreed to in writing, software 1193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * distributed under the License is distributed on an "AS IS" BASIS, 1293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * See the License for the specific language governing permissions and 1493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro * limitations under the License. 1593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro */ 1693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 1793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#ifndef ANDROID_CUTILS_ATOMIC_X86_H 1893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#define ANDROID_CUTILS_ATOMIC_X86_H 1993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#include <stdint.h> 2193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_compiler_barrier(void) 2393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 2493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("" : : : "memory"); 2593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 2693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 2793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#if ANDROID_SMP == 0 2893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_memory_barrier(void) 2993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 3193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 32464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstromextern inline void android_memory_store_barrier(void) 33464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom{ 34464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom android_compiler_barrier(); 35464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom} 3693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#else 3793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_memory_barrier(void) 3893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 3993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("mfence" : : : "memory"); 4093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 41464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstromextern inline void android_memory_store_barrier(void) 42464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom{ 43464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom android_compiler_barrier(); 44464431e65fbede57b0d41d230fe6f6dc465c20f8Brian Carlstrom} 4593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif 4693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 47d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_acquire_load(volatile const int32_t *ptr) 48d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 4993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t value = *ptr; 5093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 5193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return value; 5293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 54d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_release_load(volatile const int32_t *ptr) 55d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 5693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 5793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return *ptr; 5893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 5993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 6093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_atomic_acquire_store(int32_t value, 61d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro volatile int32_t *ptr) 62d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 6393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 6493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_memory_barrier(); 6593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 6693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 6793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline void android_atomic_release_store(int32_t value, 68d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro volatile int32_t *ptr) 69d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 7093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro android_compiler_barrier(); 7193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro *ptr = value; 7293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 7393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 7493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_cas(int32_t old_value, int32_t new_value, 7593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 7693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 7793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev; 7893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; cmpxchgl %1, %2" 7993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "=a" (prev) 8093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "q" (new_value), "m" (*ptr), "0" (old_value) 8193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "memory"); 8293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev != old_value; 8393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 8493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 8593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_acquire_cas(int32_t old_value, 8693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t new_value, 8793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 8893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 8993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Loads are not reordered with other loads. */ 9093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 9193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 9293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 9393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int android_atomic_release_cas(int32_t old_value, 9493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t new_value, 9593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 9693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 9793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* Stores are not reordered with other stores. */ 9893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_cas(old_value, new_value, ptr); 9993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 10093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 10193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_add(int32_t increment, 10293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 10393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 10493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro __asm__ __volatile__ ("lock; xaddl %0, %1" 10593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : "+r" (increment), "+m" (*ptr) 10693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro : : "memory"); 10793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro /* increment now holds the old value of *ptr */ 10893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return increment; 10993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 111d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_inc(volatile int32_t *addr) 112d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 11393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(1, addr); 11493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 11593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 116d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiroextern inline int32_t android_atomic_dec(volatile int32_t *addr) 117d55f0adfb5ec4202ad5bd5d188e66c0f6a27b0aaCarl Shapiro{ 11893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return android_atomic_add(-1, addr); 11993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 12093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 12193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_and(int32_t value, 12293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro volatile int32_t *ptr) 12393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 12493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 12593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 12693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 12793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev & value, ptr); 12893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 12993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 13093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 13193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 13293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiroextern inline int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) 13393b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro{ 13493b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro int32_t prev, status; 13593b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro do { 13693b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro prev = *ptr; 13793b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro status = android_atomic_cas(prev, prev | value, ptr); 13893b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro } while (__builtin_expect(status != 0, 0)); 13993b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro return prev; 14093b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro} 14193b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro 14293b0cb40c18cae594c931677be2b9214420610b7Carl Shapiro#endif /* ANDROID_CUTILS_ATOMIC_X86_H */ 143