1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <vdso/datapage.h> 11 #include <vdso/helpers.h> 12 13 /* 14 * The generic vDSO implementation requires that gettimeofday.h 15 * provides: 16 * - __arch_get_vdso_data(): to get the vdso datapage. 17 * - __arch_get_hw_counter(): to get the hw counter based on the 18 * clock_mode. 19 * - gettimeofday_fallback(): fallback for gettimeofday. 20 * - clock_gettime_fallback(): fallback for clock_gettime. 21 * - clock_getres_fallback(): fallback for clock_getres. 22 */ 23 #ifdef ENABLE_COMPAT_VDSO 24 #include <asm/vdso/compat_gettimeofday.h> 25 #else 26 #include <asm/vdso/gettimeofday.h> 27 #endif /* ENABLE_COMPAT_VDSO */ 28 29 #ifndef vdso_calc_delta 30 /* 31 * Default implementation which works for all sane clocksources. That 32 * obviously excludes x86/TSC. 33 */ 34 static __always_inline 35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 36 { 37 return ((cycles - last) & mask) * mult; 38 } 39 #endif 40 41 #ifndef __arch_vdso_hres_capable 42 static inline bool __arch_vdso_hres_capable(void) 43 { 44 return true; 45 } 46 #endif 47 48 #ifdef CONFIG_TIME_NS 49 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 50 struct __kernel_timespec *ts) 51 { 52 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 53 const struct timens_offset *offs = &vdns->offset[clk]; 54 const struct vdso_timestamp *vdso_ts; 55 u64 cycles, last, ns; 56 u32 seq; 57 s64 sec; 58 59 if (clk != CLOCK_MONOTONIC_RAW) 60 vd = &vd[CS_HRES_COARSE]; 61 else 62 vd = &vd[CS_RAW]; 63 vdso_ts = &vd->basetime[clk]; 64 65 do { 66 seq = vdso_read_begin(vd); 67 cycles = __arch_get_hw_counter(vd->clock_mode); 68 ns = vdso_ts->nsec; 69 last = vd->cycle_last; 70 if (unlikely((s64)cycles < 0)) 71 return -1; 72 73 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 74 ns >>= vd->shift; 75 sec = vdso_ts->sec; 76 } while (unlikely(vdso_read_retry(vd, seq))); 77 78 /* Add the namespace offset */ 79 sec += offs->sec; 80 ns += offs->nsec; 81 82 /* 83 * Do this outside the loop: a race inside the loop could result 84 * in __iter_div_u64_rem() being extremely slow. 85 */ 86 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 87 ts->tv_nsec = ns; 88 89 return 0; 90 } 91 #else 92 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 93 { 94 return NULL; 95 } 96 97 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 98 struct __kernel_timespec *ts) 99 { 100 return -EINVAL; 101 } 102 #endif 103 104 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 105 struct __kernel_timespec *ts) 106 { 107 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 108 u64 cycles, last, sec, ns; 109 u32 seq; 110 111 /* Allows to compile the high resolution parts out */ 112 if (!__arch_vdso_hres_capable()) 113 return -1; 114 115 do { 116 /* 117 * Open coded to handle VCLOCK_TIMENS. Time namespace 118 * enabled tasks have a special VVAR page installed which 119 * has vd->seq set to 1 and vd->clock_mode set to 120 * VCLOCK_TIMENS. For non time namespace affected tasks 121 * this does not affect performance because if vd->seq is 122 * odd, i.e. a concurrent update is in progress the extra 123 * check for vd->clock_mode is just a few extra 124 * instructions while spin waiting for vd->seq to become 125 * even again. 126 */ 127 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 128 if (IS_ENABLED(CONFIG_TIME_NS) && 129 vd->clock_mode == VCLOCK_TIMENS) 130 return do_hres_timens(vd, clk, ts); 131 cpu_relax(); 132 } 133 smp_rmb(); 134 135 cycles = __arch_get_hw_counter(vd->clock_mode); 136 ns = vdso_ts->nsec; 137 last = vd->cycle_last; 138 if (unlikely((s64)cycles < 0)) 139 return -1; 140 141 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 142 ns >>= vd->shift; 143 sec = vdso_ts->sec; 144 } while (unlikely(vdso_read_retry(vd, seq))); 145 146 /* 147 * Do this outside the loop: a race inside the loop could result 148 * in __iter_div_u64_rem() being extremely slow. 149 */ 150 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 151 ts->tv_nsec = ns; 152 153 return 0; 154 } 155 156 #ifdef CONFIG_TIME_NS 157 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 158 struct __kernel_timespec *ts) 159 { 160 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 161 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 162 const struct timens_offset *offs = &vdns->offset[clk]; 163 u64 nsec; 164 s64 sec; 165 s32 seq; 166 167 do { 168 seq = vdso_read_begin(vd); 169 sec = vdso_ts->sec; 170 nsec = vdso_ts->nsec; 171 } while (unlikely(vdso_read_retry(vd, seq))); 172 173 /* Add the namespace offset */ 174 sec += offs->sec; 175 nsec += offs->nsec; 176 177 /* 178 * Do this outside the loop: a race inside the loop could result 179 * in __iter_div_u64_rem() being extremely slow. 180 */ 181 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 182 ts->tv_nsec = nsec; 183 return 0; 184 } 185 #else 186 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 187 struct __kernel_timespec *ts) 188 { 189 return -1; 190 } 191 #endif 192 193 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 194 struct __kernel_timespec *ts) 195 { 196 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 197 u32 seq; 198 199 do { 200 /* 201 * Open coded to handle VCLOCK_TIMENS. See comment in 202 * do_hres(). 203 */ 204 while ((seq = READ_ONCE(vd->seq)) & 1) { 205 if (IS_ENABLED(CONFIG_TIME_NS) && 206 vd->clock_mode == VCLOCK_TIMENS) 207 return do_coarse_timens(vd, clk, ts); 208 cpu_relax(); 209 } 210 smp_rmb(); 211 212 ts->tv_sec = vdso_ts->sec; 213 ts->tv_nsec = vdso_ts->nsec; 214 } while (unlikely(vdso_read_retry(vd, seq))); 215 216 return 0; 217 } 218 219 static __maybe_unused int 220 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) 221 { 222 const struct vdso_data *vd = __arch_get_vdso_data(); 223 u32 msk; 224 225 /* Check for negative values or invalid clocks */ 226 if (unlikely((u32) clock >= MAX_CLOCKS)) 227 return -1; 228 229 /* 230 * Convert the clockid to a bitmask and use it to check which 231 * clocks are handled in the VDSO directly. 232 */ 233 msk = 1U << clock; 234 if (likely(msk & VDSO_HRES)) 235 vd = &vd[CS_HRES_COARSE]; 236 else if (msk & VDSO_COARSE) 237 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 238 else if (msk & VDSO_RAW) 239 vd = &vd[CS_RAW]; 240 else 241 return -1; 242 243 return do_hres(vd, clock, ts); 244 } 245 246 static __maybe_unused int 247 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 248 { 249 int ret = __cvdso_clock_gettime_common(clock, ts); 250 251 if (unlikely(ret)) 252 return clock_gettime_fallback(clock, ts); 253 return 0; 254 } 255 256 #ifdef BUILD_VDSO32 257 static __maybe_unused int 258 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 259 { 260 struct __kernel_timespec ts; 261 int ret; 262 263 ret = __cvdso_clock_gettime_common(clock, &ts); 264 265 if (unlikely(ret)) 266 return clock_gettime32_fallback(clock, res); 267 268 /* For ret == 0 */ 269 res->tv_sec = ts.tv_sec; 270 res->tv_nsec = ts.tv_nsec; 271 272 return ret; 273 } 274 #endif /* BUILD_VDSO32 */ 275 276 static __maybe_unused int 277 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 278 { 279 const struct vdso_data *vd = __arch_get_vdso_data(); 280 281 if (likely(tv != NULL)) { 282 struct __kernel_timespec ts; 283 284 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 285 return gettimeofday_fallback(tv, tz); 286 287 tv->tv_sec = ts.tv_sec; 288 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 289 } 290 291 if (unlikely(tz != NULL)) { 292 if (IS_ENABLED(CONFIG_TIME_NS) && 293 vd->clock_mode == VCLOCK_TIMENS) 294 vd = __arch_get_timens_vdso_data(); 295 296 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 297 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 298 } 299 300 return 0; 301 } 302 303 #ifdef VDSO_HAS_TIME 304 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 305 { 306 const struct vdso_data *vd = __arch_get_vdso_data(); 307 __kernel_old_time_t t; 308 309 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 310 vd = __arch_get_timens_vdso_data(); 311 312 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 313 314 if (time) 315 *time = t; 316 317 return t; 318 } 319 #endif /* VDSO_HAS_TIME */ 320 321 #ifdef VDSO_HAS_CLOCK_GETRES 322 static __maybe_unused 323 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) 324 { 325 const struct vdso_data *vd = __arch_get_vdso_data(); 326 u32 msk; 327 u64 ns; 328 329 /* Check for negative values or invalid clocks */ 330 if (unlikely((u32) clock >= MAX_CLOCKS)) 331 return -1; 332 333 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 334 vd = __arch_get_timens_vdso_data(); 335 336 /* 337 * Convert the clockid to a bitmask and use it to check which 338 * clocks are handled in the VDSO directly. 339 */ 340 msk = 1U << clock; 341 if (msk & (VDSO_HRES | VDSO_RAW)) { 342 /* 343 * Preserves the behaviour of posix_get_hrtimer_res(). 344 */ 345 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 346 } else if (msk & VDSO_COARSE) { 347 /* 348 * Preserves the behaviour of posix_get_coarse_res(). 349 */ 350 ns = LOW_RES_NSEC; 351 } else { 352 return -1; 353 } 354 355 if (likely(res)) { 356 res->tv_sec = 0; 357 res->tv_nsec = ns; 358 } 359 return 0; 360 } 361 362 static __maybe_unused 363 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 364 { 365 int ret = __cvdso_clock_getres_common(clock, res); 366 367 if (unlikely(ret)) 368 return clock_getres_fallback(clock, res); 369 return 0; 370 } 371 372 #ifdef BUILD_VDSO32 373 static __maybe_unused int 374 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 375 { 376 struct __kernel_timespec ts; 377 int ret; 378 379 ret = __cvdso_clock_getres_common(clock, &ts); 380 381 if (unlikely(ret)) 382 return clock_getres32_fallback(clock, res); 383 384 if (likely(res)) { 385 res->tv_sec = ts.tv_sec; 386 res->tv_nsec = ts.tv_nsec; 387 } 388 return ret; 389 } 390 #endif /* BUILD_VDSO32 */ 391 #endif /* VDSO_HAS_CLOCK_GETRES */ 392