1// Copyright (c) 2008, Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above 11// copyright notice, this list of conditions and the following disclaimer 12// in the documentation and/or other materials provided with the 13// distribution. 14// * Neither the name of Google Inc. nor the names of its 15// contributors may be used to endorse or promote products derived from 16// this software without specific prior written permission. 17// 18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30// --- 31// Author: Paul Pluzhnikov 32// 33// Allow dynamic symbol lookup in the kernel VDSO page. 34// 35// VDSOSupport -- a class representing kernel VDSO (if present). 36// 37 38#include "base/vdso_support.h" 39 40#ifdef HAVE_VDSO_SUPPORT // defined in vdso_support.h 41 42#include <fcntl.h> 43#include <stddef.h> // for ptrdiff_t 44 45#include "base/atomicops.h" // for MemoryBarrier 46#include "base/linux_syscall_support.h" 47#include "base/logging.h" 48#include "base/dynamic_annotations.h" 49#include "base/basictypes.h" // for COMPILE_ASSERT 50 51using base::subtle::MemoryBarrier; 52 53#ifndef AT_SYSINFO_EHDR 54#define AT_SYSINFO_EHDR 33 55#endif 56 57namespace base { 58 59const void *VDSOSupport::vdso_base_ = ElfMemImage::kInvalidBase; 60VDSOSupport::GetCpuFn VDSOSupport::getcpu_fn_ = &InitAndGetCPU; 61VDSOSupport::VDSOSupport() 62 // If vdso_base_ is still set to kInvalidBase, we got here 63 // before VDSOSupport::Init has been called. Call it now. 64 : image_(vdso_base_ == ElfMemImage::kInvalidBase ? Init() : vdso_base_) { 65} 66 67// NOTE: we can't use GoogleOnceInit() below, because we can be 68// called by tcmalloc, and none of the *once* stuff may be functional yet. 69// 70// In addition, we hope that the VDSOSupportHelper constructor 71// causes this code to run before there are any threads, and before 72// InitGoogle() has executed any chroot or setuid calls. 73// 74// Finally, even if there is a race here, it is harmless, because 75// the operation should be idempotent. 76const void *VDSOSupport::Init() { 77 if (vdso_base_ == ElfMemImage::kInvalidBase) { 78 // Valgrind zaps AT_SYSINFO_EHDR and friends from the auxv[] 79 // on stack, and so glibc works as if VDSO was not present. 80 // But going directly to kernel via /proc/self/auxv below bypasses 81 // Valgrind zapping. So we check for Valgrind separately. 82 if (RunningOnValgrind()) { 83 vdso_base_ = NULL; 84 getcpu_fn_ = &GetCPUViaSyscall; 85 return NULL; 86 } 87 int fd = open("/proc/self/auxv", O_RDONLY); 88 if (fd == -1) { 89 // Kernel too old to have a VDSO. 90 vdso_base_ = NULL; 91 getcpu_fn_ = &GetCPUViaSyscall; 92 return NULL; 93 } 94 ElfW(auxv_t) aux; 95 while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) { 96 if (aux.a_type == AT_SYSINFO_EHDR) { 97 COMPILE_ASSERT(sizeof(vdso_base_) == sizeof(aux.a_un.a_val), 98 unexpected_sizeof_pointer_NE_sizeof_a_val); 99 vdso_base_ = reinterpret_cast<void *>(aux.a_un.a_val); 100 break; 101 } 102 } 103 close(fd); 104 if (vdso_base_ == ElfMemImage::kInvalidBase) { 105 // Didn't find AT_SYSINFO_EHDR in auxv[]. 106 vdso_base_ = NULL; 107 } 108 } 109 GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present. 110 if (vdso_base_) { 111 VDSOSupport vdso; 112 SymbolInfo info; 113 if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) { 114 // Casting from an int to a pointer is not legal C++. To emphasize 115 // this, we use a C-style cast rather than a C++-style cast. 116 fn = (GetCpuFn)(info.address); 117 } 118 } 119 // Subtle: this code runs outside of any locks; prevent compiler 120 // from assigning to getcpu_fn_ more than once. 121 base::subtle::MemoryBarrier(); 122 getcpu_fn_ = fn; 123 return vdso_base_; 124} 125 126const void *VDSOSupport::SetBase(const void *base) { 127 CHECK(base != ElfMemImage::kInvalidBase); 128 const void *old_base = vdso_base_; 129 vdso_base_ = base; 130 image_.Init(base); 131 // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO. 132 getcpu_fn_ = &InitAndGetCPU; 133 return old_base; 134} 135 136bool VDSOSupport::LookupSymbol(const char *name, 137 const char *version, 138 int type, 139 SymbolInfo *info) const { 140 return image_.LookupSymbol(name, version, type, info); 141} 142 143bool VDSOSupport::LookupSymbolByAddress(const void *address, 144 SymbolInfo *info_out) const { 145 return image_.LookupSymbolByAddress(address, info_out); 146} 147 148// NOLINT on 'long' because this routine mimics kernel api. 149long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, void *, void *) { // NOLINT 150#if defined(__NR_getcpu) 151 return sys_getcpu(cpu, NULL, NULL); 152#else 153 // x86_64 never implemented sys_getcpu(), except as a VDSO call. 154 errno = ENOSYS; 155 return -1; 156#endif 157} 158 159// Use fast __vdso_getcpu if available. 160long VDSOSupport::InitAndGetCPU(unsigned *cpu, void *x, void *y) { // NOLINT 161 Init(); 162 CHECK_NE(getcpu_fn_, &InitAndGetCPU); // << "Init() did not set getcpu_fn_"; 163 return (*getcpu_fn_)(cpu, x, y); 164} 165 166// This function must be very fast, and may be called from very 167// low level (e.g. tcmalloc). Hence I avoid things like 168// GoogleOnceInit() and ::operator new. 169int GetCPU(void) { 170 unsigned cpu; 171 int ret_code = (*VDSOSupport::getcpu_fn_)(&cpu, NULL, NULL); 172 return ret_code == 0 ? cpu : ret_code; 173} 174 175// We need to make sure VDSOSupport::Init() is called before 176// the main() runs, since it might do something like setuid or 177// chroot. If VDSOSupport 178// is used in any global constructor, this will happen, since 179// VDSOSupport's constructor calls Init. But if not, we need to 180// ensure it here, with a global constructor of our own. This 181// is an allowed exception to the normal rule against non-trivial 182// global constructors. 183static class VDSOInitHelper { 184 public: 185 VDSOInitHelper() { VDSOSupport::Init(); } 186} vdso_init_helper; 187} 188 189#endif // HAVE_VDSO_SUPPORT 190