1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <vdso/datapage.h> 11 #include <vdso/helpers.h> 12 13 /* 14 * The generic vDSO implementation requires that gettimeofday.h 15 * provides: 16 * - __arch_get_vdso_data(): to get the vdso datapage. 17 * - __arch_get_hw_counter(): to get the hw counter based on the 18 * clock_mode. 19 * - gettimeofday_fallback(): fallback for gettimeofday. 20 * - clock_gettime_fallback(): fallback for clock_gettime. 21 * - clock_getres_fallback(): fallback for clock_getres. 22 */ 23 #ifdef ENABLE_COMPAT_VDSO 24 #include <asm/vdso/compat_gettimeofday.h> 25 #else 26 #include <asm/vdso/gettimeofday.h> 27 #endif /* ENABLE_COMPAT_VDSO */ 28 29 #ifndef vdso_calc_delta 30 /* 31 * Default implementation which works for all sane clocksources. That 32 * obviously excludes x86/TSC. 33 */ 34 static __always_inline 35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 36 { 37 return ((cycles - last) & mask) * mult; 38 } 39 #endif 40 41 static int do_hres(const struct vdso_data *vd, clockid_t clk, 42 struct __kernel_timespec *ts) 43 { 44 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 45 u64 cycles, last, sec, ns; 46 u32 seq; 47 48 do { 49 seq = vdso_read_begin(vd); 50 cycles = __arch_get_hw_counter(vd->clock_mode); 51 ns = vdso_ts->nsec; 52 last = vd->cycle_last; 53 if (unlikely((s64)cycles < 0)) 54 return -1; 55 56 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 57 ns >>= vd->shift; 58 sec = vdso_ts->sec; 59 } while (unlikely(vdso_read_retry(vd, seq))); 60 61 /* 62 * Do this outside the loop: a race inside the loop could result 63 * in __iter_div_u64_rem() being extremely slow. 64 */ 65 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 66 ts->tv_nsec = ns; 67 68 return 0; 69 } 70 71 static void do_coarse(const struct vdso_data *vd, clockid_t clk, 72 struct __kernel_timespec *ts) 73 { 74 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 75 u32 seq; 76 77 do { 78 seq = vdso_read_begin(vd); 79 ts->tv_sec = vdso_ts->sec; 80 ts->tv_nsec = vdso_ts->nsec; 81 } while (unlikely(vdso_read_retry(vd, seq))); 82 } 83 84 static __maybe_unused int 85 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) 86 { 87 const struct vdso_data *vd = __arch_get_vdso_data(); 88 u32 msk; 89 90 /* Check for negative values or invalid clocks */ 91 if (unlikely((u32) clock >= MAX_CLOCKS)) 92 return -1; 93 94 /* 95 * Convert the clockid to a bitmask and use it to check which 96 * clocks are handled in the VDSO directly. 97 */ 98 msk = 1U << clock; 99 if (likely(msk & VDSO_HRES)) { 100 return do_hres(&vd[CS_HRES_COARSE], clock, ts); 101 } else if (msk & VDSO_COARSE) { 102 do_coarse(&vd[CS_HRES_COARSE], clock, ts); 103 return 0; 104 } else if (msk & VDSO_RAW) { 105 return do_hres(&vd[CS_RAW], clock, ts); 106 } 107 return -1; 108 } 109 110 static __maybe_unused int 111 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 112 { 113 int ret = __cvdso_clock_gettime_common(clock, ts); 114 115 if (unlikely(ret)) 116 return clock_gettime_fallback(clock, ts); 117 return 0; 118 } 119 120 static __maybe_unused int 121 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 122 { 123 struct __kernel_timespec ts; 124 int ret; 125 126 ret = __cvdso_clock_gettime_common(clock, &ts); 127 128 #ifdef VDSO_HAS_32BIT_FALLBACK 129 if (unlikely(ret)) 130 return clock_gettime32_fallback(clock, res); 131 #else 132 if (unlikely(ret)) 133 ret = clock_gettime_fallback(clock, &ts); 134 #endif 135 136 if (likely(!ret)) { 137 res->tv_sec = ts.tv_sec; 138 res->tv_nsec = ts.tv_nsec; 139 } 140 return ret; 141 } 142 143 static __maybe_unused int 144 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 145 { 146 const struct vdso_data *vd = __arch_get_vdso_data(); 147 148 if (likely(tv != NULL)) { 149 struct __kernel_timespec ts; 150 151 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 152 return gettimeofday_fallback(tv, tz); 153 154 tv->tv_sec = ts.tv_sec; 155 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 156 } 157 158 if (unlikely(tz != NULL)) { 159 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 160 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 161 } 162 163 return 0; 164 } 165 166 #ifdef VDSO_HAS_TIME 167 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 168 { 169 const struct vdso_data *vd = __arch_get_vdso_data(); 170 __kernel_old_time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 171 172 if (time) 173 *time = t; 174 175 return t; 176 } 177 #endif /* VDSO_HAS_TIME */ 178 179 #ifdef VDSO_HAS_CLOCK_GETRES 180 static __maybe_unused 181 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) 182 { 183 const struct vdso_data *vd = __arch_get_vdso_data(); 184 u64 hrtimer_res; 185 u32 msk; 186 u64 ns; 187 188 /* Check for negative values or invalid clocks */ 189 if (unlikely((u32) clock >= MAX_CLOCKS)) 190 return -1; 191 192 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 193 /* 194 * Convert the clockid to a bitmask and use it to check which 195 * clocks are handled in the VDSO directly. 196 */ 197 msk = 1U << clock; 198 if (msk & VDSO_HRES) { 199 /* 200 * Preserves the behaviour of posix_get_hrtimer_res(). 201 */ 202 ns = hrtimer_res; 203 } else if (msk & VDSO_COARSE) { 204 /* 205 * Preserves the behaviour of posix_get_coarse_res(). 206 */ 207 ns = LOW_RES_NSEC; 208 } else if (msk & VDSO_RAW) { 209 /* 210 * Preserves the behaviour of posix_get_hrtimer_res(). 211 */ 212 ns = hrtimer_res; 213 } else { 214 return -1; 215 } 216 217 if (likely(res)) { 218 res->tv_sec = 0; 219 res->tv_nsec = ns; 220 } 221 return 0; 222 } 223 224 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 225 { 226 int ret = __cvdso_clock_getres_common(clock, res); 227 228 if (unlikely(ret)) 229 return clock_getres_fallback(clock, res); 230 return 0; 231 } 232 233 static __maybe_unused int 234 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 235 { 236 struct __kernel_timespec ts; 237 int ret; 238 239 ret = __cvdso_clock_getres_common(clock, &ts); 240 241 #ifdef VDSO_HAS_32BIT_FALLBACK 242 if (unlikely(ret)) 243 return clock_getres32_fallback(clock, res); 244 #else 245 if (unlikely(ret)) 246 ret = clock_getres_fallback(clock, &ts); 247 #endif 248 249 if (likely(!ret && res)) { 250 res->tv_sec = ts.tv_sec; 251 res->tv_nsec = ts.tv_nsec; 252 } 253 return ret; 254 } 255 #endif /* VDSO_HAS_CLOCK_GETRES */ 256