1// Copyright (c) 2008, Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8//     * Redistributions of source code must retain the above copyright
9// notice, this list of conditions and the following disclaimer.
10//     * Redistributions in binary form must reproduce the above
11// copyright notice, this list of conditions and the following disclaimer
12// in the documentation and/or other materials provided with the
13// distribution.
14//     * Neither the name of Google Inc. nor the names of its
15// contributors may be used to endorse or promote products derived from
16// this software without specific prior written permission.
17//
18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30// ---
31// Author: Paul Pluzhnikov
32//
33// Allow dynamic symbol lookup in the kernel VDSO page.
34//
35// VDSOSupport -- a class representing kernel VDSO (if present).
36//
37
38#include "base/vdso_support.h"
39
40#ifdef HAVE_VDSO_SUPPORT     // defined in vdso_support.h
41
42#include <fcntl.h>
43#include <stddef.h>   // for ptrdiff_t
44
45#include "base/atomicops.h"  // for MemoryBarrier
46#include "base/linux_syscall_support.h"
47#include "base/logging.h"
48#include "base/dynamic_annotations.h"
49#include "base/basictypes.h"  // for COMPILE_ASSERT
50
51using base::subtle::MemoryBarrier;
52
53#ifndef AT_SYSINFO_EHDR
54#define AT_SYSINFO_EHDR 33
55#endif
56
57namespace base {
58
59const void *VDSOSupport::vdso_base_ = ElfMemImage::kInvalidBase;
60VDSOSupport::GetCpuFn VDSOSupport::getcpu_fn_ = &InitAndGetCPU;
61VDSOSupport::VDSOSupport()
62    // If vdso_base_ is still set to kInvalidBase, we got here
63    // before VDSOSupport::Init has been called. Call it now.
64    : image_(vdso_base_ == ElfMemImage::kInvalidBase ? Init() : vdso_base_) {
65}
66
67// NOTE: we can't use GoogleOnceInit() below, because we can be
68// called by tcmalloc, and none of the *once* stuff may be functional yet.
69//
70// In addition, we hope that the VDSOSupportHelper constructor
71// causes this code to run before there are any threads, and before
72// InitGoogle() has executed any chroot or setuid calls.
73//
74// Finally, even if there is a race here, it is harmless, because
75// the operation should be idempotent.
76const void *VDSOSupport::Init() {
77  if (vdso_base_ == ElfMemImage::kInvalidBase) {
78    // Valgrind zaps AT_SYSINFO_EHDR and friends from the auxv[]
79    // on stack, and so glibc works as if VDSO was not present.
80    // But going directly to kernel via /proc/self/auxv below bypasses
81    // Valgrind zapping. So we check for Valgrind separately.
82    if (RunningOnValgrind()) {
83      vdso_base_ = NULL;
84      getcpu_fn_ = &GetCPUViaSyscall;
85      return NULL;
86    }
87    int fd = open("/proc/self/auxv", O_RDONLY);
88    if (fd == -1) {
89      // Kernel too old to have a VDSO.
90      vdso_base_ = NULL;
91      getcpu_fn_ = &GetCPUViaSyscall;
92      return NULL;
93    }
94    ElfW(auxv_t) aux;
95    while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) {
96      if (aux.a_type == AT_SYSINFO_EHDR) {
97        COMPILE_ASSERT(sizeof(vdso_base_) == sizeof(aux.a_un.a_val),
98                       unexpected_sizeof_pointer_NE_sizeof_a_val);
99        vdso_base_ = reinterpret_cast<void *>(aux.a_un.a_val);
100        break;
101      }
102    }
103    close(fd);
104    if (vdso_base_ == ElfMemImage::kInvalidBase) {
105      // Didn't find AT_SYSINFO_EHDR in auxv[].
106      vdso_base_ = NULL;
107    }
108  }
109  GetCpuFn fn = &GetCPUViaSyscall;  // default if VDSO not present.
110  if (vdso_base_) {
111    VDSOSupport vdso;
112    SymbolInfo info;
113    if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) {
114      // Casting from an int to a pointer is not legal C++.  To emphasize
115      // this, we use a C-style cast rather than a C++-style cast.
116      fn = (GetCpuFn)(info.address);
117    }
118  }
119  // Subtle: this code runs outside of any locks; prevent compiler
120  // from assigning to getcpu_fn_ more than once.
121  base::subtle::MemoryBarrier();
122  getcpu_fn_ = fn;
123  return vdso_base_;
124}
125
126const void *VDSOSupport::SetBase(const void *base) {
127  CHECK(base != ElfMemImage::kInvalidBase);
128  const void *old_base = vdso_base_;
129  vdso_base_ = base;
130  image_.Init(base);
131  // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO.
132  getcpu_fn_ = &InitAndGetCPU;
133  return old_base;
134}
135
136bool VDSOSupport::LookupSymbol(const char *name,
137                               const char *version,
138                               int type,
139                               SymbolInfo *info) const {
140  return image_.LookupSymbol(name, version, type, info);
141}
142
143bool VDSOSupport::LookupSymbolByAddress(const void *address,
144                                        SymbolInfo *info_out) const {
145  return image_.LookupSymbolByAddress(address, info_out);
146}
147
148// NOLINT on 'long' because this routine mimics kernel api.
149long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, void *, void *) { // NOLINT
150#if defined(__NR_getcpu)
151  return sys_getcpu(cpu, NULL, NULL);
152#else
153  // x86_64 never implemented sys_getcpu(), except as a VDSO call.
154  errno = ENOSYS;
155  return -1;
156#endif
157}
158
159// Use fast __vdso_getcpu if available.
160long VDSOSupport::InitAndGetCPU(unsigned *cpu, void *x, void *y) { // NOLINT
161  Init();
162  CHECK_NE(getcpu_fn_, &InitAndGetCPU); // << "Init() did not set getcpu_fn_";
163  return (*getcpu_fn_)(cpu, x, y);
164}
165
166// This function must be very fast, and may be called from very
167// low level (e.g. tcmalloc). Hence I avoid things like
168// GoogleOnceInit() and ::operator new.
169int GetCPU(void) {
170  unsigned cpu;
171  int ret_code = (*VDSOSupport::getcpu_fn_)(&cpu, NULL, NULL);
172  return ret_code == 0 ? cpu : ret_code;
173}
174
175// We need to make sure VDSOSupport::Init() is called before
176// the main() runs, since it might do something like setuid or
177// chroot.  If VDSOSupport
178// is used in any global constructor, this will happen, since
179// VDSOSupport's constructor calls Init.  But if not, we need to
180// ensure it here, with a global constructor of our own.  This
181// is an allowed exception to the normal rule against non-trivial
182// global constructors.
183static class VDSOInitHelper {
184 public:
185  VDSOInitHelper() { VDSOSupport::Init(); }
186} vdso_init_helper;
187}
188
189#endif  // HAVE_VDSO_SUPPORT
190