1 /*
2  * Copyright 2006 Andi Kleen, SUSE Labs.
3  * Subject to the GNU Public License, v.2
4  *
5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
6  *
7  * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
8  *  sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
9  *
10  * The code should have no internal unresolved relocations.
11  * Check with readelf after changing.
12  */
13 
14 #include <uapi/linux/time.h>
15 #include <asm/vgtod.h>
16 #include <asm/hpet.h>
17 #include <asm/vvar.h>
18 #include <asm/unistd.h>
19 #include <asm/msr.h>
20 #include <linux/math64.h>
21 #include <linux/time.h>
22 
23 #define gtod (&VVAR(vsyscall_gtod_data))
24 
25 extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
26 extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
27 extern time_t __vdso_time(time_t *t);
28 
29 #ifdef CONFIG_HPET_TIMER
30 extern u8 hpet_page
31 	__attribute__((visibility("hidden")));
32 
33 static notrace cycle_t vread_hpet(void)
34 {
35 	return *(const volatile u32 *)(&hpet_page + HPET_COUNTER);
36 }
37 #endif
38 
39 #ifndef BUILD_VDSO32
40 
41 #include <linux/kernel.h>
42 #include <asm/vsyscall.h>
43 #include <asm/fixmap.h>
44 #include <asm/pvclock.h>
45 
46 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
47 {
48 	long ret;
49 	asm("syscall" : "=a" (ret) :
50 	    "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
51 	return ret;
52 }
53 
54 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
55 {
56 	long ret;
57 
58 	asm("syscall" : "=a" (ret) :
59 	    "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
60 	return ret;
61 }
62 
63 #ifdef CONFIG_PARAVIRT_CLOCK
64 
65 static notrace const struct pvclock_vsyscall_time_info *get_pvti(int cpu)
66 {
67 	const struct pvclock_vsyscall_time_info *pvti_base;
68 	int idx = cpu / (PAGE_SIZE/PVTI_SIZE);
69 	int offset = cpu % (PAGE_SIZE/PVTI_SIZE);
70 
71 	BUG_ON(PVCLOCK_FIXMAP_BEGIN + idx > PVCLOCK_FIXMAP_END);
72 
73 	pvti_base = (struct pvclock_vsyscall_time_info *)
74 		    __fix_to_virt(PVCLOCK_FIXMAP_BEGIN+idx);
75 
76 	return &pvti_base[offset];
77 }
78 
79 static notrace cycle_t vread_pvclock(int *mode)
80 {
81 	const struct pvclock_vsyscall_time_info *pvti;
82 	cycle_t ret;
83 	u64 last;
84 	u32 version;
85 	u8 flags;
86 	unsigned cpu, cpu1;
87 
88 
89 	/*
90 	 * Note: hypervisor must guarantee that:
91 	 * 1. cpu ID number maps 1:1 to per-CPU pvclock time info.
92 	 * 2. that per-CPU pvclock time info is updated if the
93 	 *    underlying CPU changes.
94 	 * 3. that version is increased whenever underlying CPU
95 	 *    changes.
96 	 *
97 	 */
98 	do {
99 		cpu = __getcpu() & VGETCPU_CPU_MASK;
100 		/* TODO: We can put vcpu id into higher bits of pvti.version.
101 		 * This will save a couple of cycles by getting rid of
102 		 * __getcpu() calls (Gleb).
103 		 */
104 
105 		pvti = get_pvti(cpu);
106 
107 		version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags);
108 
109 		/*
110 		 * Test we're still on the cpu as well as the version.
111 		 * We could have been migrated just after the first
112 		 * vgetcpu but before fetching the version, so we
113 		 * wouldn't notice a version change.
114 		 */
115 		cpu1 = __getcpu() & VGETCPU_CPU_MASK;
116 	} while (unlikely(cpu != cpu1 ||
117 			  (pvti->pvti.version & 1) ||
118 			  pvti->pvti.version != version));
119 
120 	if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT)))
121 		*mode = VCLOCK_NONE;
122 
123 	/* refer to tsc.c read_tsc() comment for rationale */
124 	last = gtod->cycle_last;
125 
126 	if (likely(ret >= last))
127 		return ret;
128 
129 	return last;
130 }
131 #endif
132 
133 #else
134 
135 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
136 {
137 	long ret;
138 
139 	asm(
140 		"mov %%ebx, %%edx \n"
141 		"mov %2, %%ebx \n"
142 		"call __kernel_vsyscall \n"
143 		"mov %%edx, %%ebx \n"
144 		: "=a" (ret)
145 		: "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
146 		: "memory", "edx");
147 	return ret;
148 }
149 
150 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
151 {
152 	long ret;
153 
154 	asm(
155 		"mov %%ebx, %%edx \n"
156 		"mov %2, %%ebx \n"
157 		"call __kernel_vsyscall \n"
158 		"mov %%edx, %%ebx \n"
159 		: "=a" (ret)
160 		: "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
161 		: "memory", "edx");
162 	return ret;
163 }
164 
165 #ifdef CONFIG_PARAVIRT_CLOCK
166 
167 static notrace cycle_t vread_pvclock(int *mode)
168 {
169 	*mode = VCLOCK_NONE;
170 	return 0;
171 }
172 #endif
173 
174 #endif
175 
176 notrace static cycle_t vread_tsc(void)
177 {
178 	cycle_t ret = (cycle_t)rdtsc_ordered();
179 	u64 last = gtod->cycle_last;
180 
181 	if (likely(ret >= last))
182 		return ret;
183 
184 	/*
185 	 * GCC likes to generate cmov here, but this branch is extremely
186 	 * predictable (it's just a funciton of time and the likely is
187 	 * very likely) and there's a data dependence, so force GCC
188 	 * to generate a branch instead.  I don't barrier() because
189 	 * we don't actually need a barrier, and if this function
190 	 * ever gets inlined it will generate worse code.
191 	 */
192 	asm volatile ("");
193 	return last;
194 }
195 
196 notrace static inline u64 vgetsns(int *mode)
197 {
198 	u64 v;
199 	cycles_t cycles;
200 
201 	if (gtod->vclock_mode == VCLOCK_TSC)
202 		cycles = vread_tsc();
203 #ifdef CONFIG_HPET_TIMER
204 	else if (gtod->vclock_mode == VCLOCK_HPET)
205 		cycles = vread_hpet();
206 #endif
207 #ifdef CONFIG_PARAVIRT_CLOCK
208 	else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
209 		cycles = vread_pvclock(mode);
210 #endif
211 	else
212 		return 0;
213 	v = (cycles - gtod->cycle_last) & gtod->mask;
214 	return v * gtod->mult;
215 }
216 
217 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
218 notrace static int __always_inline do_realtime(struct timespec *ts)
219 {
220 	unsigned long seq;
221 	u64 ns;
222 	int mode;
223 
224 	do {
225 		seq = gtod_read_begin(gtod);
226 		mode = gtod->vclock_mode;
227 		ts->tv_sec = gtod->wall_time_sec;
228 		ns = gtod->wall_time_snsec;
229 		ns += vgetsns(&mode);
230 		ns >>= gtod->shift;
231 	} while (unlikely(gtod_read_retry(gtod, seq)));
232 
233 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
234 	ts->tv_nsec = ns;
235 
236 	return mode;
237 }
238 
239 notrace static int __always_inline do_monotonic(struct timespec *ts)
240 {
241 	unsigned long seq;
242 	u64 ns;
243 	int mode;
244 
245 	do {
246 		seq = gtod_read_begin(gtod);
247 		mode = gtod->vclock_mode;
248 		ts->tv_sec = gtod->monotonic_time_sec;
249 		ns = gtod->monotonic_time_snsec;
250 		ns += vgetsns(&mode);
251 		ns >>= gtod->shift;
252 	} while (unlikely(gtod_read_retry(gtod, seq)));
253 
254 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
255 	ts->tv_nsec = ns;
256 
257 	return mode;
258 }
259 
260 notrace static void do_realtime_coarse(struct timespec *ts)
261 {
262 	unsigned long seq;
263 	do {
264 		seq = gtod_read_begin(gtod);
265 		ts->tv_sec = gtod->wall_time_coarse_sec;
266 		ts->tv_nsec = gtod->wall_time_coarse_nsec;
267 	} while (unlikely(gtod_read_retry(gtod, seq)));
268 }
269 
270 notrace static void do_monotonic_coarse(struct timespec *ts)
271 {
272 	unsigned long seq;
273 	do {
274 		seq = gtod_read_begin(gtod);
275 		ts->tv_sec = gtod->monotonic_time_coarse_sec;
276 		ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
277 	} while (unlikely(gtod_read_retry(gtod, seq)));
278 }
279 
280 notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
281 {
282 	switch (clock) {
283 	case CLOCK_REALTIME:
284 		if (do_realtime(ts) == VCLOCK_NONE)
285 			goto fallback;
286 		break;
287 	case CLOCK_MONOTONIC:
288 		if (do_monotonic(ts) == VCLOCK_NONE)
289 			goto fallback;
290 		break;
291 	case CLOCK_REALTIME_COARSE:
292 		do_realtime_coarse(ts);
293 		break;
294 	case CLOCK_MONOTONIC_COARSE:
295 		do_monotonic_coarse(ts);
296 		break;
297 	default:
298 		goto fallback;
299 	}
300 
301 	return 0;
302 fallback:
303 	return vdso_fallback_gettime(clock, ts);
304 }
305 int clock_gettime(clockid_t, struct timespec *)
306 	__attribute__((weak, alias("__vdso_clock_gettime")));
307 
308 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
309 {
310 	if (likely(tv != NULL)) {
311 		if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
312 			return vdso_fallback_gtod(tv, tz);
313 		tv->tv_usec /= 1000;
314 	}
315 	if (unlikely(tz != NULL)) {
316 		tz->tz_minuteswest = gtod->tz_minuteswest;
317 		tz->tz_dsttime = gtod->tz_dsttime;
318 	}
319 
320 	return 0;
321 }
322 int gettimeofday(struct timeval *, struct timezone *)
323 	__attribute__((weak, alias("__vdso_gettimeofday")));
324 
325 /*
326  * This will break when the xtime seconds get inaccurate, but that is
327  * unlikely
328  */
329 notrace time_t __vdso_time(time_t *t)
330 {
331 	/* This is atomic on x86 so we don't need any locks. */
332 	time_t result = ACCESS_ONCE(gtod->wall_time_sec);
333 
334 	if (t)
335 		*t = result;
336 	return result;
337 }
338 int time(time_t *t)
339 	__attribute__((weak, alias("__vdso_time")));
340