1/* 2 * Copyright (C) 2006 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <endian.h> 18#include <limits.h> 19#undef _USING_LIBCXX // Prevent using of <atomic>. 20#include <stdatomic.h> 21 22#include <stddef.h> 23 24#include "private/bionic_futex.h" 25 26// This file contains C++ ABI support functions for one time 27// constructors as defined in the "Run-time ABI for the ARM Architecture" 28// section 4.4.2 29// 30// ARM C++ ABI and Itanium/x86 C++ ABI has different definition for 31// one time construction: 32// 33// ARM C++ ABI defines the LSB of guard variable should be tested 34// by compiler-generated code before calling __cxa_guard_acquire et al. 35// 36// The Itanium/x86 C++ ABI defines the low-order _byte_ should be 37// tested instead. 38// 39// Meanwhile, guard variable are 32bit aligned for ARM, and 64bit 40// aligned for x86. 41// 42// Reference documentation: 43// 44// section 3.2.3 of ARM IHI 0041C (for ARM) 45// section 3.3.2 of the Itanium C++ ABI specification v1.83 (for x86). 46// 47// There is no C++ ABI available for other ARCH. But the gcc source 48// shows all other ARCH follow the definition of Itanium/x86 C++ ABI. 49 50#if defined(__arm__) 51// The ARM C++ ABI mandates that guard variables are 32-bit aligned, 32-bit 52// values. The LSB is tested by the compiler-generated code before calling 53// __cxa_guard_acquire. 54union _guard_t { 55 atomic_int state; 56 int32_t aligner; 57}; 58 59#else 60// The Itanium/x86 C++ ABI (used by all other architectures) mandates that 61// guard variables are 64-bit aligned, 64-bit values. The LSB is tested by 62// the compiler-generated code before calling __cxa_guard_acquire. 63union _guard_t { 64 atomic_int state; 65 int64_t aligner; 66}; 67 68#endif 69 70// Set construction state values according to reference documentation. 71// 0 is the initialization value. 72// Arm requires ((*gv & 1) == 1) after __cxa_guard_release, ((*gv & 3) == 0) after __cxa_guard_abort. 73// X86 requires first byte not modified by __cxa_guard_acquire, first byte is non-zero after 74// __cxa_guard_release. 75 76#define CONSTRUCTION_NOT_YET_STARTED 0 77#define CONSTRUCTION_COMPLETE 1 78#define CONSTRUCTION_UNDERWAY_WITHOUT_WAITER 0x100 79#define CONSTRUCTION_UNDERWAY_WITH_WAITER 0x200 80 81extern "C" int __cxa_guard_acquire(_guard_t* gv) { 82 int old_value = atomic_load_explicit(&gv->state, memory_order_relaxed); 83 84 while (true) { 85 if (old_value == CONSTRUCTION_COMPLETE) { 86 // A load_acquire operation is need before exiting with COMPLETE state, as we have to ensure 87 // that all the stores performed by the construction function are observable on this CPU 88 // after we exit. 89 atomic_thread_fence(memory_order_acquire); 90 return 0; 91 } else if (old_value == CONSTRUCTION_NOT_YET_STARTED) { 92 if (!atomic_compare_exchange_weak_explicit(&gv->state, &old_value, 93 CONSTRUCTION_UNDERWAY_WITHOUT_WAITER, 94 memory_order_relaxed, 95 memory_order_relaxed)) { 96 continue; 97 } 98 // The acquire fence may not be needed. But as described in section 3.3.2 of 99 // the Itanium C++ ABI specification, it probably has to behave like the 100 // acquisition of a mutex, which needs an acquire fence. 101 atomic_thread_fence(memory_order_acquire); 102 return 1; 103 } else if (old_value == CONSTRUCTION_UNDERWAY_WITHOUT_WAITER) { 104 if (!atomic_compare_exchange_weak_explicit(&gv->state, &old_value, 105 CONSTRUCTION_UNDERWAY_WITH_WAITER, 106 memory_order_relaxed, 107 memory_order_relaxed)) { 108 continue; 109 } 110 } 111 112 __futex_wait_ex(&gv->state, false, CONSTRUCTION_UNDERWAY_WITH_WAITER, false, nullptr); 113 old_value = atomic_load_explicit(&gv->state, memory_order_relaxed); 114 } 115} 116 117extern "C" void __cxa_guard_release(_guard_t* gv) { 118 // Release fence is used to make all stores performed by the construction function 119 // visible in other threads. 120 int old_value = atomic_exchange_explicit(&gv->state, CONSTRUCTION_COMPLETE, memory_order_release); 121 if (old_value == CONSTRUCTION_UNDERWAY_WITH_WAITER) { 122 __futex_wake_ex(&gv->state, false, INT_MAX); 123 } 124} 125 126extern "C" void __cxa_guard_abort(_guard_t* gv) { 127 // Release fence is used to make all stores performed by the construction function 128 // visible in other threads. 129 int old_value = atomic_exchange_explicit(&gv->state, CONSTRUCTION_NOT_YET_STARTED, memory_order_release); 130 if (old_value == CONSTRUCTION_UNDERWAY_WITH_WAITER) { 131 __futex_wake_ex(&gv->state, false, INT_MAX); 132 } 133} 134