19a08862aSNagarathnam Muthusamy /* 29a08862aSNagarathnam Muthusamy * Copyright 2006 Andi Kleen, SUSE Labs. 39a08862aSNagarathnam Muthusamy * Subject to the GNU Public License, v.2 49a08862aSNagarathnam Muthusamy * 59a08862aSNagarathnam Muthusamy * Fast user context implementation of clock_gettime, gettimeofday, and time. 69a08862aSNagarathnam Muthusamy * 79a08862aSNagarathnam Muthusamy * The code should have no internal unresolved relocations. 89a08862aSNagarathnam Muthusamy * Check with readelf after changing. 99a08862aSNagarathnam Muthusamy * Also alternative() doesn't work. 109a08862aSNagarathnam Muthusamy */ 119a08862aSNagarathnam Muthusamy /* 129a08862aSNagarathnam Muthusamy * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. 139a08862aSNagarathnam Muthusamy */ 149a08862aSNagarathnam Muthusamy 159a08862aSNagarathnam Muthusamy /* Disable profiling for userspace code: */ 169a08862aSNagarathnam Muthusamy #ifndef DISABLE_BRANCH_PROFILING 179a08862aSNagarathnam Muthusamy #define DISABLE_BRANCH_PROFILING 189a08862aSNagarathnam Muthusamy #endif 199a08862aSNagarathnam Muthusamy 209a08862aSNagarathnam Muthusamy #include <linux/kernel.h> 219a08862aSNagarathnam Muthusamy #include <linux/time.h> 229a08862aSNagarathnam Muthusamy #include <linux/string.h> 239a08862aSNagarathnam Muthusamy #include <asm/io.h> 249a08862aSNagarathnam Muthusamy #include <asm/unistd.h> 259a08862aSNagarathnam Muthusamy #include <asm/timex.h> 269a08862aSNagarathnam Muthusamy #include <asm/clocksource.h> 279a08862aSNagarathnam Muthusamy #include <asm/vvar.h> 289a08862aSNagarathnam Muthusamy 299a08862aSNagarathnam Muthusamy #undef TICK_PRIV_BIT 309a08862aSNagarathnam Muthusamy #ifdef CONFIG_SPARC64 319a08862aSNagarathnam Muthusamy #define TICK_PRIV_BIT (1UL << 63) 329a08862aSNagarathnam Muthusamy #else 339a08862aSNagarathnam Muthusamy #define TICK_PRIV_BIT (1ULL << 63) 349a08862aSNagarathnam Muthusamy #endif 359a08862aSNagarathnam Muthusamy 36776ca154SDavid S. Miller #ifdef CONFIG_SPARC64 379a08862aSNagarathnam Muthusamy #define SYSCALL_STRING \ 389a08862aSNagarathnam Muthusamy "ta 0x6d;" \ 39776ca154SDavid S. Miller "bcs,a 1f;" \ 409a08862aSNagarathnam Muthusamy " sub %%g0, %%o0, %%o0;" \ 41776ca154SDavid S. Miller "1:" 42776ca154SDavid S. Miller #else 43776ca154SDavid S. Miller #define SYSCALL_STRING \ 44776ca154SDavid S. Miller "ta 0x10;" \ 45776ca154SDavid S. Miller "bcs,a 1f;" \ 46776ca154SDavid S. Miller " sub %%g0, %%o0, %%o0;" \ 47776ca154SDavid S. Miller "1:" 48776ca154SDavid S. Miller #endif 499a08862aSNagarathnam Muthusamy 509a08862aSNagarathnam Muthusamy #define SYSCALL_CLOBBERS \ 519a08862aSNagarathnam Muthusamy "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ 529a08862aSNagarathnam Muthusamy "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ 539a08862aSNagarathnam Muthusamy "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ 549a08862aSNagarathnam Muthusamy "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ 559a08862aSNagarathnam Muthusamy "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ 569a08862aSNagarathnam Muthusamy "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ 579a08862aSNagarathnam Muthusamy "cc", "memory" 589a08862aSNagarathnam Muthusamy 599a08862aSNagarathnam Muthusamy /* 609a08862aSNagarathnam Muthusamy * Compute the vvar page's address in the process address space, and return it 619a08862aSNagarathnam Muthusamy * as a pointer to the vvar_data. 629a08862aSNagarathnam Muthusamy */ 639a08862aSNagarathnam Muthusamy static notrace noinline struct vvar_data * 649a08862aSNagarathnam Muthusamy get_vvar_data(void) 659a08862aSNagarathnam Muthusamy { 669a08862aSNagarathnam Muthusamy unsigned long ret; 679a08862aSNagarathnam Muthusamy 689a08862aSNagarathnam Muthusamy /* 699a08862aSNagarathnam Muthusamy * vdso data page is the first vDSO page so grab the return address 709a08862aSNagarathnam Muthusamy * and move up a page to get to the data page. 719a08862aSNagarathnam Muthusamy */ 729a08862aSNagarathnam Muthusamy ret = (unsigned long)__builtin_return_address(0); 739a08862aSNagarathnam Muthusamy ret &= ~(8192 - 1); 749a08862aSNagarathnam Muthusamy ret -= 8192; 759a08862aSNagarathnam Muthusamy 769a08862aSNagarathnam Muthusamy return (struct vvar_data *) ret; 779a08862aSNagarathnam Muthusamy } 789a08862aSNagarathnam Muthusamy 799a08862aSNagarathnam Muthusamy static notrace long 809a08862aSNagarathnam Muthusamy vdso_fallback_gettime(long clock, struct timespec *ts) 819a08862aSNagarathnam Muthusamy { 829a08862aSNagarathnam Muthusamy register long num __asm__("g1") = __NR_clock_gettime; 839a08862aSNagarathnam Muthusamy register long o0 __asm__("o0") = clock; 849a08862aSNagarathnam Muthusamy register long o1 __asm__("o1") = (long) ts; 859a08862aSNagarathnam Muthusamy 869a08862aSNagarathnam Muthusamy __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), 879a08862aSNagarathnam Muthusamy "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); 889a08862aSNagarathnam Muthusamy return o0; 899a08862aSNagarathnam Muthusamy } 909a08862aSNagarathnam Muthusamy 919a08862aSNagarathnam Muthusamy static notrace __always_inline long 929a08862aSNagarathnam Muthusamy vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz) 939a08862aSNagarathnam Muthusamy { 949a08862aSNagarathnam Muthusamy register long num __asm__("g1") = __NR_gettimeofday; 959a08862aSNagarathnam Muthusamy register long o0 __asm__("o0") = (long) tv; 969a08862aSNagarathnam Muthusamy register long o1 __asm__("o1") = (long) tz; 979a08862aSNagarathnam Muthusamy 989a08862aSNagarathnam Muthusamy __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), 999a08862aSNagarathnam Muthusamy "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); 1009a08862aSNagarathnam Muthusamy return o0; 1019a08862aSNagarathnam Muthusamy } 1029a08862aSNagarathnam Muthusamy 1039a08862aSNagarathnam Muthusamy #ifdef CONFIG_SPARC64 1049a08862aSNagarathnam Muthusamy static notrace noinline u64 1059a08862aSNagarathnam Muthusamy vread_tick(void) { 1069a08862aSNagarathnam Muthusamy u64 ret; 1079a08862aSNagarathnam Muthusamy 1082f6c9bf3SDavid S. Miller __asm__ __volatile__("1:\n\t" 1092f6c9bf3SDavid S. Miller "rd %%tick, %0\n\t" 1102f6c9bf3SDavid S. Miller ".pushsection .tick_patch, \"a\"\n\t" 1112f6c9bf3SDavid S. Miller ".word 1b - ., 1f - .\n\t" 1122f6c9bf3SDavid S. Miller ".popsection\n\t" 1132f6c9bf3SDavid S. Miller ".pushsection .tick_patch_replacement, \"ax\"\n\t" 1142f6c9bf3SDavid S. Miller "1:\n\t" 1152f6c9bf3SDavid S. Miller "rd %%asr24, %0\n\t" 1162f6c9bf3SDavid S. Miller ".popsection\n" 1172f6c9bf3SDavid S. Miller : "=r" (ret)); 1189a08862aSNagarathnam Muthusamy return ret & ~TICK_PRIV_BIT; 1199a08862aSNagarathnam Muthusamy } 1209a08862aSNagarathnam Muthusamy #else 1219a08862aSNagarathnam Muthusamy static notrace noinline u64 1229a08862aSNagarathnam Muthusamy vread_tick(void) 1239a08862aSNagarathnam Muthusamy { 1242f6c9bf3SDavid S. Miller register unsigned long long ret asm("o4"); 1259a08862aSNagarathnam Muthusamy 1262f6c9bf3SDavid S. Miller __asm__ __volatile__("1:\n\t" 1272f6c9bf3SDavid S. Miller "rd %%tick, %L0\n\t" 1282f6c9bf3SDavid S. Miller "srlx %L0, 32, %H0\n\t" 1292f6c9bf3SDavid S. Miller ".pushsection .tick_patch, \"a\"\n\t" 1302f6c9bf3SDavid S. Miller ".word 1b - ., 1f - .\n\t" 1312f6c9bf3SDavid S. Miller ".popsection\n\t" 1322f6c9bf3SDavid S. Miller ".pushsection .tick_patch_replacement, \"ax\"\n\t" 1332f6c9bf3SDavid S. Miller "1:\n\t" 1342f6c9bf3SDavid S. Miller "rd %%asr24, %L0\n\t" 1352f6c9bf3SDavid S. Miller ".popsection\n" 1362f6c9bf3SDavid S. Miller : "=r" (ret)); 1372f6c9bf3SDavid S. Miller return ret; 1389a08862aSNagarathnam Muthusamy } 1399a08862aSNagarathnam Muthusamy #endif 1409a08862aSNagarathnam Muthusamy 1419a08862aSNagarathnam Muthusamy static notrace inline u64 1429a08862aSNagarathnam Muthusamy vgetsns(struct vvar_data *vvar) 1439a08862aSNagarathnam Muthusamy { 1449a08862aSNagarathnam Muthusamy u64 v; 1459a08862aSNagarathnam Muthusamy u64 cycles; 1469a08862aSNagarathnam Muthusamy 1479a08862aSNagarathnam Muthusamy cycles = vread_tick(); 1489a08862aSNagarathnam Muthusamy v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; 1499a08862aSNagarathnam Muthusamy return v * vvar->clock.mult; 1509a08862aSNagarathnam Muthusamy } 1519a08862aSNagarathnam Muthusamy 1529a08862aSNagarathnam Muthusamy static notrace noinline int 1539a08862aSNagarathnam Muthusamy do_realtime(struct vvar_data *vvar, struct timespec *ts) 1549a08862aSNagarathnam Muthusamy { 1559a08862aSNagarathnam Muthusamy unsigned long seq; 1569a08862aSNagarathnam Muthusamy u64 ns; 1579a08862aSNagarathnam Muthusamy 1589a08862aSNagarathnam Muthusamy ts->tv_nsec = 0; 1599a08862aSNagarathnam Muthusamy do { 1609a08862aSNagarathnam Muthusamy seq = vvar_read_begin(vvar); 1619a08862aSNagarathnam Muthusamy ts->tv_sec = vvar->wall_time_sec; 1629a08862aSNagarathnam Muthusamy ns = vvar->wall_time_snsec; 1639a08862aSNagarathnam Muthusamy ns += vgetsns(vvar); 1649a08862aSNagarathnam Muthusamy ns >>= vvar->clock.shift; 1659a08862aSNagarathnam Muthusamy } while (unlikely(vvar_read_retry(vvar, seq))); 1669a08862aSNagarathnam Muthusamy 1679a08862aSNagarathnam Muthusamy timespec_add_ns(ts, ns); 1689a08862aSNagarathnam Muthusamy 1699a08862aSNagarathnam Muthusamy return 0; 1709a08862aSNagarathnam Muthusamy } 1719a08862aSNagarathnam Muthusamy 1729a08862aSNagarathnam Muthusamy static notrace noinline int 1739a08862aSNagarathnam Muthusamy do_monotonic(struct vvar_data *vvar, struct timespec *ts) 1749a08862aSNagarathnam Muthusamy { 1759a08862aSNagarathnam Muthusamy unsigned long seq; 1769a08862aSNagarathnam Muthusamy u64 ns; 1779a08862aSNagarathnam Muthusamy 1789a08862aSNagarathnam Muthusamy ts->tv_nsec = 0; 1799a08862aSNagarathnam Muthusamy do { 1809a08862aSNagarathnam Muthusamy seq = vvar_read_begin(vvar); 1819a08862aSNagarathnam Muthusamy ts->tv_sec = vvar->monotonic_time_sec; 1829a08862aSNagarathnam Muthusamy ns = vvar->monotonic_time_snsec; 1839a08862aSNagarathnam Muthusamy ns += vgetsns(vvar); 1849a08862aSNagarathnam Muthusamy ns >>= vvar->clock.shift; 1859a08862aSNagarathnam Muthusamy } while (unlikely(vvar_read_retry(vvar, seq))); 1869a08862aSNagarathnam Muthusamy 1879a08862aSNagarathnam Muthusamy timespec_add_ns(ts, ns); 1889a08862aSNagarathnam Muthusamy 1899a08862aSNagarathnam Muthusamy return 0; 1909a08862aSNagarathnam Muthusamy } 1919a08862aSNagarathnam Muthusamy 1929a08862aSNagarathnam Muthusamy static notrace noinline int 1939a08862aSNagarathnam Muthusamy do_realtime_coarse(struct vvar_data *vvar, struct timespec *ts) 1949a08862aSNagarathnam Muthusamy { 1959a08862aSNagarathnam Muthusamy unsigned long seq; 1969a08862aSNagarathnam Muthusamy 1979a08862aSNagarathnam Muthusamy do { 1989a08862aSNagarathnam Muthusamy seq = vvar_read_begin(vvar); 1999a08862aSNagarathnam Muthusamy ts->tv_sec = vvar->wall_time_coarse_sec; 2009a08862aSNagarathnam Muthusamy ts->tv_nsec = vvar->wall_time_coarse_nsec; 2019a08862aSNagarathnam Muthusamy } while (unlikely(vvar_read_retry(vvar, seq))); 2029a08862aSNagarathnam Muthusamy return 0; 2039a08862aSNagarathnam Muthusamy } 2049a08862aSNagarathnam Muthusamy 2059a08862aSNagarathnam Muthusamy static notrace noinline int 2069a08862aSNagarathnam Muthusamy do_monotonic_coarse(struct vvar_data *vvar, struct timespec *ts) 2079a08862aSNagarathnam Muthusamy { 2089a08862aSNagarathnam Muthusamy unsigned long seq; 2099a08862aSNagarathnam Muthusamy 2109a08862aSNagarathnam Muthusamy do { 2119a08862aSNagarathnam Muthusamy seq = vvar_read_begin(vvar); 2129a08862aSNagarathnam Muthusamy ts->tv_sec = vvar->monotonic_time_coarse_sec; 2139a08862aSNagarathnam Muthusamy ts->tv_nsec = vvar->monotonic_time_coarse_nsec; 2149a08862aSNagarathnam Muthusamy } while (unlikely(vvar_read_retry(vvar, seq))); 2159a08862aSNagarathnam Muthusamy 2169a08862aSNagarathnam Muthusamy return 0; 2179a08862aSNagarathnam Muthusamy } 2189a08862aSNagarathnam Muthusamy 2199a08862aSNagarathnam Muthusamy notrace int 2209a08862aSNagarathnam Muthusamy __vdso_clock_gettime(clockid_t clock, struct timespec *ts) 2219a08862aSNagarathnam Muthusamy { 2229a08862aSNagarathnam Muthusamy struct vvar_data *vvd = get_vvar_data(); 2239a08862aSNagarathnam Muthusamy 2249a08862aSNagarathnam Muthusamy switch (clock) { 2259a08862aSNagarathnam Muthusamy case CLOCK_REALTIME: 2269a08862aSNagarathnam Muthusamy if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) 2279a08862aSNagarathnam Muthusamy break; 2289a08862aSNagarathnam Muthusamy return do_realtime(vvd, ts); 2299a08862aSNagarathnam Muthusamy case CLOCK_MONOTONIC: 2309a08862aSNagarathnam Muthusamy if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) 2319a08862aSNagarathnam Muthusamy break; 2329a08862aSNagarathnam Muthusamy return do_monotonic(vvd, ts); 2339a08862aSNagarathnam Muthusamy case CLOCK_REALTIME_COARSE: 2349a08862aSNagarathnam Muthusamy return do_realtime_coarse(vvd, ts); 2359a08862aSNagarathnam Muthusamy case CLOCK_MONOTONIC_COARSE: 2369a08862aSNagarathnam Muthusamy return do_monotonic_coarse(vvd, ts); 2379a08862aSNagarathnam Muthusamy } 2389a08862aSNagarathnam Muthusamy /* 2399a08862aSNagarathnam Muthusamy * Unknown clock ID ? Fall back to the syscall. 2409a08862aSNagarathnam Muthusamy */ 2419a08862aSNagarathnam Muthusamy return vdso_fallback_gettime(clock, ts); 2429a08862aSNagarathnam Muthusamy } 2439a08862aSNagarathnam Muthusamy int 2449a08862aSNagarathnam Muthusamy clock_gettime(clockid_t, struct timespec *) 2459a08862aSNagarathnam Muthusamy __attribute__((weak, alias("__vdso_clock_gettime"))); 2469a08862aSNagarathnam Muthusamy 2479a08862aSNagarathnam Muthusamy notrace int 2489a08862aSNagarathnam Muthusamy __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) 2499a08862aSNagarathnam Muthusamy { 2509a08862aSNagarathnam Muthusamy struct vvar_data *vvd = get_vvar_data(); 2519a08862aSNagarathnam Muthusamy 2529a08862aSNagarathnam Muthusamy if (likely(vvd->vclock_mode != VCLOCK_NONE)) { 2539a08862aSNagarathnam Muthusamy if (likely(tv != NULL)) { 2549a08862aSNagarathnam Muthusamy union tstv_t { 2559a08862aSNagarathnam Muthusamy struct timespec ts; 2569a08862aSNagarathnam Muthusamy struct timeval tv; 2579a08862aSNagarathnam Muthusamy } *tstv = (union tstv_t *) tv; 2589a08862aSNagarathnam Muthusamy do_realtime(vvd, &tstv->ts); 2599a08862aSNagarathnam Muthusamy /* 2609a08862aSNagarathnam Muthusamy * Assign before dividing to ensure that the division is 2619a08862aSNagarathnam Muthusamy * done in the type of tv_usec, not tv_nsec. 2629a08862aSNagarathnam Muthusamy * 2639a08862aSNagarathnam Muthusamy * There cannot be > 1 billion usec in a second: 2649a08862aSNagarathnam Muthusamy * do_realtime() has already distributed such overflow 2659a08862aSNagarathnam Muthusamy * into tv_sec. So we can assign it to an int safely. 2669a08862aSNagarathnam Muthusamy */ 2679a08862aSNagarathnam Muthusamy tstv->tv.tv_usec = tstv->ts.tv_nsec; 2689a08862aSNagarathnam Muthusamy tstv->tv.tv_usec /= 1000; 2699a08862aSNagarathnam Muthusamy } 2709a08862aSNagarathnam Muthusamy if (unlikely(tz != NULL)) { 2719a08862aSNagarathnam Muthusamy /* Avoid memcpy. Some old compilers fail to inline it */ 2729a08862aSNagarathnam Muthusamy tz->tz_minuteswest = vvd->tz_minuteswest; 2739a08862aSNagarathnam Muthusamy tz->tz_dsttime = vvd->tz_dsttime; 2749a08862aSNagarathnam Muthusamy } 2759a08862aSNagarathnam Muthusamy return 0; 2769a08862aSNagarathnam Muthusamy } 2779a08862aSNagarathnam Muthusamy return vdso_fallback_gettimeofday(tv, tz); 2789a08862aSNagarathnam Muthusamy } 2799a08862aSNagarathnam Muthusamy int 2809a08862aSNagarathnam Muthusamy gettimeofday(struct timeval *, struct timezone *) 2819a08862aSNagarathnam Muthusamy __attribute__((weak, alias("__vdso_gettimeofday"))); 282