1/*
2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
4 *
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
6 *
7 * The code should have no internal unresolved relocations.
8 * Check with readelf after changing.
9 */
10
11/* Disable profiling for userspace code: */
12#define DISABLE_BRANCH_PROFILING
13
14#include <linux/kernel.h>
15#include <linux/posix-timers.h>
16#include <linux/time.h>
17#include <linux/string.h>
18#include <asm/vsyscall.h>
19#include <asm/fixmap.h>
20#include <asm/vgtod.h>
21#include <asm/timex.h>
22#include <asm/hpet.h>
23#include <asm/unistd.h>
24#include <asm/io.h>
25
26#define gtod (&VVAR(vsyscall_gtod_data))
27
28notrace static cycle_t vread_tsc(void)
29{
30	cycle_t ret;
31	u64 last;
32
33	/*
34	 * Empirically, a fence (of type that depends on the CPU)
35	 * before rdtsc is enough to ensure that rdtsc is ordered
36	 * with respect to loads.  The various CPU manuals are unclear
37	 * as to whether rdtsc can be reordered with later loads,
38	 * but no one has ever seen it happen.
39	 */
40	rdtsc_barrier();
41	ret = (cycle_t)vget_cycles();
42
43	last = VVAR(vsyscall_gtod_data).clock.cycle_last;
44
45	if (likely(ret >= last))
46		return ret;
47
48	/*
49	 * GCC likes to generate cmov here, but this branch is extremely
50	 * predictable (it's just a funciton of time and the likely is
51	 * very likely) and there's a data dependence, so force GCC
52	 * to generate a branch instead.  I don't barrier() because
53	 * we don't actually need a barrier, and if this function
54	 * ever gets inlined it will generate worse code.
55	 */
56	asm volatile ("");
57	return last;
58}
59
60static notrace cycle_t vread_hpet(void)
61{
62	return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0);
63}
64
65notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
66{
67	long ret;
68	asm("syscall" : "=a" (ret) :
69	    "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory");
70	return ret;
71}
72
73notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
74{
75	long ret;
76
77	asm("syscall" : "=a" (ret) :
78	    "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
79	return ret;
80}
81
82
83notrace static inline long vgetns(void)
84{
85	long v;
86	cycles_t cycles;
87	if (gtod->clock.vclock_mode == VCLOCK_TSC)
88		cycles = vread_tsc();
89	else if (gtod->clock.vclock_mode == VCLOCK_HPET)
90		cycles = vread_hpet();
91	else
92		return 0;
93	v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask;
94	return (v * gtod->clock.mult) >> gtod->clock.shift;
95}
96
97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
98notrace static int __always_inline do_realtime(struct timespec *ts)
99{
100	unsigned long seq, ns;
101	int mode;
102
103	do {
104		seq = read_seqcount_begin(&gtod->seq);
105		mode = gtod->clock.vclock_mode;
106		ts->tv_sec = gtod->wall_time_sec;
107		ts->tv_nsec = gtod->wall_time_nsec;
108		ns = vgetns();
109	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
110
111	timespec_add_ns(ts, ns);
112	return mode;
113}
114
115notrace static int do_monotonic(struct timespec *ts)
116{
117	unsigned long seq, ns;
118	int mode;
119
120	do {
121		seq = read_seqcount_begin(&gtod->seq);
122		mode = gtod->clock.vclock_mode;
123		ts->tv_sec = gtod->monotonic_time_sec;
124		ts->tv_nsec = gtod->monotonic_time_nsec;
125		ns = vgetns();
126	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
127	timespec_add_ns(ts, ns);
128
129	return mode;
130}
131
132notrace static int do_realtime_coarse(struct timespec *ts)
133{
134	unsigned long seq;
135	do {
136		seq = read_seqcount_begin(&gtod->seq);
137		ts->tv_sec = gtod->wall_time_coarse.tv_sec;
138		ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
139	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
140	return 0;
141}
142
143notrace static int do_monotonic_coarse(struct timespec *ts)
144{
145	unsigned long seq;
146	do {
147		seq = read_seqcount_begin(&gtod->seq);
148		ts->tv_sec = gtod->monotonic_time_coarse.tv_sec;
149		ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec;
150	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
151
152	return 0;
153}
154
155notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
156{
157	int ret = VCLOCK_NONE;
158
159	switch (clock) {
160	case CLOCK_REALTIME:
161		ret = do_realtime(ts);
162		break;
163	case CLOCK_MONOTONIC:
164		ret = do_monotonic(ts);
165		break;
166	case CLOCK_REALTIME_COARSE:
167		return do_realtime_coarse(ts);
168	case CLOCK_MONOTONIC_COARSE:
169		return do_monotonic_coarse(ts);
170	}
171
172	if (ret == VCLOCK_NONE)
173		return vdso_fallback_gettime(clock, ts);
174	return 0;
175}
176int clock_gettime(clockid_t, struct timespec *)
177	__attribute__((weak, alias("__vdso_clock_gettime")));
178
179notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
180{
181	long ret = VCLOCK_NONE;
182
183	if (likely(tv != NULL)) {
184		BUILD_BUG_ON(offsetof(struct timeval, tv_usec) !=
185			     offsetof(struct timespec, tv_nsec) ||
186			     sizeof(*tv) != sizeof(struct timespec));
187		ret = do_realtime((struct timespec *)tv);
188		tv->tv_usec /= 1000;
189	}
190	if (unlikely(tz != NULL)) {
191		/* Avoid memcpy. Some old compilers fail to inline it */
192		tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest;
193		tz->tz_dsttime = gtod->sys_tz.tz_dsttime;
194	}
195
196	if (ret == VCLOCK_NONE)
197		return vdso_fallback_gtod(tv, tz);
198	return 0;
199}
200int gettimeofday(struct timeval *, struct timezone *)
201	__attribute__((weak, alias("__vdso_gettimeofday")));
202
203/*
204 * This will break when the xtime seconds get inaccurate, but that is
205 * unlikely
206 */
207notrace time_t __vdso_time(time_t *t)
208{
209	/* This is atomic on x86_64 so we don't need any locks. */
210	time_t result = ACCESS_ONCE(VVAR(vsyscall_gtod_data).wall_time_sec);
211
212	if (t)
213		*t = result;
214	return result;
215}
216int time(time_t *t)
217	__attribute__((weak, alias("__vdso_time")));
218