1/* 2 * Copyright 2006 Andi Kleen, SUSE Labs. 3 * Subject to the GNU Public License, v.2 4 * 5 * Fast user context implementation of clock_gettime, gettimeofday, and time. 6 * 7 * The code should have no internal unresolved relocations. 8 * Check with readelf after changing. 9 */ 10 11/* Disable profiling for userspace code: */ 12#define DISABLE_BRANCH_PROFILING 13 14#include <linux/kernel.h> 15#include <linux/posix-timers.h> 16#include <linux/time.h> 17#include <linux/string.h> 18#include <asm/vsyscall.h> 19#include <asm/fixmap.h> 20#include <asm/vgtod.h> 21#include <asm/timex.h> 22#include <asm/hpet.h> 23#include <asm/unistd.h> 24#include <asm/io.h> 25 26#define gtod (&VVAR(vsyscall_gtod_data)) 27 28notrace static cycle_t vread_tsc(void) 29{ 30 cycle_t ret; 31 u64 last; 32 33 /* 34 * Empirically, a fence (of type that depends on the CPU) 35 * before rdtsc is enough to ensure that rdtsc is ordered 36 * with respect to loads. The various CPU manuals are unclear 37 * as to whether rdtsc can be reordered with later loads, 38 * but no one has ever seen it happen. 39 */ 40 rdtsc_barrier(); 41 ret = (cycle_t)vget_cycles(); 42 43 last = VVAR(vsyscall_gtod_data).clock.cycle_last; 44 45 if (likely(ret >= last)) 46 return ret; 47 48 /* 49 * GCC likes to generate cmov here, but this branch is extremely 50 * predictable (it's just a funciton of time and the likely is 51 * very likely) and there's a data dependence, so force GCC 52 * to generate a branch instead. I don't barrier() because 53 * we don't actually need a barrier, and if this function 54 * ever gets inlined it will generate worse code. 55 */ 56 asm volatile (""); 57 return last; 58} 59 60static notrace cycle_t vread_hpet(void) 61{ 62 return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0); 63} 64 65notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) 66{ 67 long ret; 68 asm("syscall" : "=a" (ret) : 69 "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory"); 70 return ret; 71} 72 73notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) 74{ 75 long ret; 76 77 asm("syscall" : "=a" (ret) : 78 "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory"); 79 return ret; 80} 81 82 83notrace static inline long vgetns(void) 84{ 85 long v; 86 cycles_t cycles; 87 if (gtod->clock.vclock_mode == VCLOCK_TSC) 88 cycles = vread_tsc(); 89 else if (gtod->clock.vclock_mode == VCLOCK_HPET) 90 cycles = vread_hpet(); 91 else 92 return 0; 93 v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; 94 return (v * gtod->clock.mult) >> gtod->clock.shift; 95} 96 97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */ 98notrace static int __always_inline do_realtime(struct timespec *ts) 99{ 100 unsigned long seq, ns; 101 int mode; 102 103 do { 104 seq = read_seqcount_begin(>od->seq); 105 mode = gtod->clock.vclock_mode; 106 ts->tv_sec = gtod->wall_time_sec; 107 ts->tv_nsec = gtod->wall_time_nsec; 108 ns = vgetns(); 109 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 110 111 timespec_add_ns(ts, ns); 112 return mode; 113} 114 115notrace static int do_monotonic(struct timespec *ts) 116{ 117 unsigned long seq, ns; 118 int mode; 119 120 do { 121 seq = read_seqcount_begin(>od->seq); 122 mode = gtod->clock.vclock_mode; 123 ts->tv_sec = gtod->monotonic_time_sec; 124 ts->tv_nsec = gtod->monotonic_time_nsec; 125 ns = vgetns(); 126 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 127 timespec_add_ns(ts, ns); 128 129 return mode; 130} 131 132notrace static int do_realtime_coarse(struct timespec *ts) 133{ 134 unsigned long seq; 135 do { 136 seq = read_seqcount_begin(>od->seq); 137 ts->tv_sec = gtod->wall_time_coarse.tv_sec; 138 ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; 139 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 140 return 0; 141} 142 143notrace static int do_monotonic_coarse(struct timespec *ts) 144{ 145 unsigned long seq; 146 do { 147 seq = read_seqcount_begin(>od->seq); 148 ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; 149 ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; 150 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 151 152 return 0; 153} 154 155notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) 156{ 157 int ret = VCLOCK_NONE; 158 159 switch (clock) { 160 case CLOCK_REALTIME: 161 ret = do_realtime(ts); 162 break; 163 case CLOCK_MONOTONIC: 164 ret = do_monotonic(ts); 165 break; 166 case CLOCK_REALTIME_COARSE: 167 return do_realtime_coarse(ts); 168 case CLOCK_MONOTONIC_COARSE: 169 return do_monotonic_coarse(ts); 170 } 171 172 if (ret == VCLOCK_NONE) 173 return vdso_fallback_gettime(clock, ts); 174 return 0; 175} 176int clock_gettime(clockid_t, struct timespec *) 177 __attribute__((weak, alias("__vdso_clock_gettime"))); 178 179notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) 180{ 181 long ret = VCLOCK_NONE; 182 183 if (likely(tv != NULL)) { 184 BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != 185 offsetof(struct timespec, tv_nsec) || 186 sizeof(*tv) != sizeof(struct timespec)); 187 ret = do_realtime((struct timespec *)tv); 188 tv->tv_usec /= 1000; 189 } 190 if (unlikely(tz != NULL)) { 191 /* Avoid memcpy. Some old compilers fail to inline it */ 192 tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest; 193 tz->tz_dsttime = gtod->sys_tz.tz_dsttime; 194 } 195 196 if (ret == VCLOCK_NONE) 197 return vdso_fallback_gtod(tv, tz); 198 return 0; 199} 200int gettimeofday(struct timeval *, struct timezone *) 201 __attribute__((weak, alias("__vdso_gettimeofday"))); 202 203/* 204 * This will break when the xtime seconds get inaccurate, but that is 205 * unlikely 206 */ 207notrace time_t __vdso_time(time_t *t) 208{ 209 /* This is atomic on x86_64 so we don't need any locks. */ 210 time_t result = ACCESS_ONCE(VVAR(vsyscall_gtod_data).wall_time_sec); 211 212 if (t) 213 *t = result; 214 return result; 215} 216int time(time_t *t) 217 __attribute__((weak, alias("__vdso_time"))); 218