1c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat/* 2c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * Copyright (C) 2014 The Android Open Source Project 3c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * All rights reserved. 4c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * 5c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * Redistribution and use in source and binary forms, with or without 6c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * modification, are permitted provided that the following conditions 7c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * are met: 8c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * * Redistributions of source code must retain the above copyright 9c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * notice, this list of conditions and the following disclaimer. 10c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * * Redistributions in binary form must reproduce the above copyright 11c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * notice, this list of conditions and the following disclaimer in 12c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * the documentation and/or other materials provided with the 13c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * distribution. 14c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * 15c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat * SUCH DAMAGE. 27c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat */ 28c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 29c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#ifndef ANDROID_CUTILS_ATOMIC_AARCH64_H 30c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#define ANDROID_CUTILS_ATOMIC_AARCH64_H 31c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 32c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#include <stdint.h> 33c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 34c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#ifndef ANDROID_ATOMIC_INLINE 35c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#define ANDROID_ATOMIC_INLINE inline __attribute__((always_inline)) 36c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#endif 37c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 38c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat/* 39c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat TODOAArch64: Revisit the below functions and check for potential 40c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat optimizations using assembly code or otherwise. 41c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat*/ 42c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 43c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 44c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatvoid android_compiler_barrier(void) 45c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 46c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat __asm__ __volatile__ ("" : : : "memory"); 47c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 48c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 49c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 50c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatvoid android_memory_barrier(void) 51c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 52c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat __asm__ __volatile__ ("dmb ish" : : : "memory"); 53c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 54c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 55c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 56c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_acquire_load(volatile const int32_t *ptr) 57c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 58c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat int32_t value = *ptr; 59c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 60c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return value; 61c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 62c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 63c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 64c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_release_load(volatile const int32_t *ptr) 65c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 66c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 67c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return *ptr; 68c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 69c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 70c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 71c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatvoid android_atomic_acquire_store(int32_t value, volatile int32_t *ptr) 72c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 73c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat *ptr = value; 74c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 75c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 76c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 77c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 78c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatvoid android_atomic_release_store(int32_t value, volatile int32_t *ptr) 79c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 80c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 81c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat *ptr = value; 82c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 83c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 84c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 85c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint android_atomic_cas(int32_t old_value, int32_t new_value, 86c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat volatile int32_t *ptr) 87c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 88c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return __sync_val_compare_and_swap(ptr, old_value, new_value) != old_value; 89c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 90c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 91c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 92c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint android_atomic_acquire_cas(int32_t old_value, int32_t new_value, 93c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat volatile int32_t *ptr) 94c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 95c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat int status = android_atomic_cas(old_value, new_value, ptr); 96c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 97c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return status; 98c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 99c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 100c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 101c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint android_atomic_release_cas(int32_t old_value, int32_t new_value, 102c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat volatile int32_t *ptr) 103c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 104c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 105c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return android_atomic_cas(old_value, new_value, ptr); 106c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 107c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 108c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 109c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_add(int32_t increment, volatile int32_t *ptr) 110c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 111c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat int32_t prev, status; 112c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 113c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat do { 114c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat prev = *ptr; 115c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat status = android_atomic_cas(prev, prev + increment, ptr); 116c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat } while (__builtin_expect(status != 0, 0)); 117c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return prev; 118c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 119c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 120c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 121c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_inc(volatile int32_t *addr) 122c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 123c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return android_atomic_add(1, addr); 124c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 125c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 126c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 127c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_dec(volatile int32_t *addr) 128c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 129c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return android_atomic_add(-1, addr); 130c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 131c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 132c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 133c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_and(int32_t value, volatile int32_t *ptr) 134c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 135c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat int32_t prev, status; 136c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 137c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat do { 138c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat prev = *ptr; 139c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat status = android_atomic_cas(prev, prev & value, ptr); 140c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat } while (__builtin_expect(status != 0, 0)); 141c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return prev; 142c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 143c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 144c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatextern ANDROID_ATOMIC_INLINE 145c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhatint32_t android_atomic_or(int32_t value, volatile int32_t *ptr) 146c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat{ 147c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat int32_t prev, status; 148c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat android_memory_barrier(); 149c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat do { 150c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat prev = *ptr; 151c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat status = android_atomic_cas(prev, prev | value, ptr); 152c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat } while (__builtin_expect(status != 0, 0)); 153c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat return prev; 154c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat} 155c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat 156c15d2ce5fcb98bdf2261e365ce8f59122173172aAshok Bhat#endif /* ANDROID_CUTILS_ATOMIC_AARCH64_H */ 157