1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <linux/clocksource.h> 11 #include <vdso/datapage.h> 12 #include <vdso/helpers.h> 13 14 /* 15 * The generic vDSO implementation requires that gettimeofday.h 16 * provides: 17 * - __arch_get_vdso_data(): to get the vdso datapage. 18 * - __arch_get_hw_counter(): to get the hw counter based on the 19 * clock_mode. 20 * - gettimeofday_fallback(): fallback for gettimeofday. 21 * - clock_gettime_fallback(): fallback for clock_gettime. 22 * - clock_getres_fallback(): fallback for clock_getres. 23 */ 24 #ifdef ENABLE_COMPAT_VDSO 25 #include <asm/vdso/compat_gettimeofday.h> 26 #else 27 #include <asm/vdso/gettimeofday.h> 28 #endif /* ENABLE_COMPAT_VDSO */ 29 30 #ifndef vdso_calc_delta 31 /* 32 * Default implementation which works for all sane clocksources. That 33 * obviously excludes x86/TSC. 34 */ 35 static __always_inline 36 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 37 { 38 return ((cycles - last) & mask) * mult; 39 } 40 #endif 41 42 #ifndef vdso_shift_ns 43 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 44 { 45 return ns >> shift; 46 } 47 #endif 48 49 #ifndef __arch_vdso_hres_capable 50 static inline bool __arch_vdso_hres_capable(void) 51 { 52 return true; 53 } 54 #endif 55 56 #ifndef vdso_clocksource_ok 57 static inline bool vdso_clocksource_ok(const struct vdso_data *vd) 58 { 59 return vd->clock_mode != VDSO_CLOCKMODE_NONE; 60 } 61 #endif 62 63 #ifdef CONFIG_TIME_NS 64 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 65 struct __kernel_timespec *ts) 66 { 67 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 68 const struct timens_offset *offs = &vdns->offset[clk]; 69 const struct vdso_timestamp *vdso_ts; 70 u64 cycles, last, ns; 71 u32 seq; 72 s64 sec; 73 74 if (clk != CLOCK_MONOTONIC_RAW) 75 vd = &vd[CS_HRES_COARSE]; 76 else 77 vd = &vd[CS_RAW]; 78 vdso_ts = &vd->basetime[clk]; 79 80 do { 81 seq = vdso_read_begin(vd); 82 83 if (unlikely(!vdso_clocksource_ok(vd))) 84 return -1; 85 86 cycles = __arch_get_hw_counter(vd->clock_mode); 87 ns = vdso_ts->nsec; 88 last = vd->cycle_last; 89 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 90 ns = vdso_shift_ns(ns, vd->shift); 91 sec = vdso_ts->sec; 92 } while (unlikely(vdso_read_retry(vd, seq))); 93 94 /* Add the namespace offset */ 95 sec += offs->sec; 96 ns += offs->nsec; 97 98 /* 99 * Do this outside the loop: a race inside the loop could result 100 * in __iter_div_u64_rem() being extremely slow. 101 */ 102 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 103 ts->tv_nsec = ns; 104 105 return 0; 106 } 107 #else 108 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 109 { 110 return NULL; 111 } 112 113 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 114 struct __kernel_timespec *ts) 115 { 116 return -EINVAL; 117 } 118 #endif 119 120 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 121 struct __kernel_timespec *ts) 122 { 123 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 124 u64 cycles, last, sec, ns; 125 u32 seq; 126 127 /* Allows to compile the high resolution parts out */ 128 if (!__arch_vdso_hres_capable()) 129 return -1; 130 131 do { 132 /* 133 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace 134 * enabled tasks have a special VVAR page installed which 135 * has vd->seq set to 1 and vd->clock_mode set to 136 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks 137 * this does not affect performance because if vd->seq is 138 * odd, i.e. a concurrent update is in progress the extra 139 * check for vd->clock_mode is just a few extra 140 * instructions while spin waiting for vd->seq to become 141 * even again. 142 */ 143 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 144 if (IS_ENABLED(CONFIG_TIME_NS) && 145 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 146 return do_hres_timens(vd, clk, ts); 147 cpu_relax(); 148 } 149 smp_rmb(); 150 151 if (unlikely(!vdso_clocksource_ok(vd))) 152 return -1; 153 154 cycles = __arch_get_hw_counter(vd->clock_mode); 155 ns = vdso_ts->nsec; 156 last = vd->cycle_last; 157 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 158 ns = vdso_shift_ns(ns, vd->shift); 159 sec = vdso_ts->sec; 160 } while (unlikely(vdso_read_retry(vd, seq))); 161 162 /* 163 * Do this outside the loop: a race inside the loop could result 164 * in __iter_div_u64_rem() being extremely slow. 165 */ 166 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 167 ts->tv_nsec = ns; 168 169 return 0; 170 } 171 172 #ifdef CONFIG_TIME_NS 173 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 174 struct __kernel_timespec *ts) 175 { 176 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 177 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 178 const struct timens_offset *offs = &vdns->offset[clk]; 179 u64 nsec; 180 s64 sec; 181 s32 seq; 182 183 do { 184 seq = vdso_read_begin(vd); 185 sec = vdso_ts->sec; 186 nsec = vdso_ts->nsec; 187 } while (unlikely(vdso_read_retry(vd, seq))); 188 189 /* Add the namespace offset */ 190 sec += offs->sec; 191 nsec += offs->nsec; 192 193 /* 194 * Do this outside the loop: a race inside the loop could result 195 * in __iter_div_u64_rem() being extremely slow. 196 */ 197 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 198 ts->tv_nsec = nsec; 199 return 0; 200 } 201 #else 202 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 203 struct __kernel_timespec *ts) 204 { 205 return -1; 206 } 207 #endif 208 209 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 210 struct __kernel_timespec *ts) 211 { 212 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 213 u32 seq; 214 215 do { 216 /* 217 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in 218 * do_hres(). 219 */ 220 while ((seq = READ_ONCE(vd->seq)) & 1) { 221 if (IS_ENABLED(CONFIG_TIME_NS) && 222 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 223 return do_coarse_timens(vd, clk, ts); 224 cpu_relax(); 225 } 226 smp_rmb(); 227 228 ts->tv_sec = vdso_ts->sec; 229 ts->tv_nsec = vdso_ts->nsec; 230 } while (unlikely(vdso_read_retry(vd, seq))); 231 232 return 0; 233 } 234 235 static __maybe_unused int 236 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, 237 struct __kernel_timespec *ts) 238 { 239 u32 msk; 240 241 /* Check for negative values or invalid clocks */ 242 if (unlikely((u32) clock >= MAX_CLOCKS)) 243 return -1; 244 245 /* 246 * Convert the clockid to a bitmask and use it to check which 247 * clocks are handled in the VDSO directly. 248 */ 249 msk = 1U << clock; 250 if (likely(msk & VDSO_HRES)) 251 vd = &vd[CS_HRES_COARSE]; 252 else if (msk & VDSO_COARSE) 253 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 254 else if (msk & VDSO_RAW) 255 vd = &vd[CS_RAW]; 256 else 257 return -1; 258 259 return do_hres(vd, clock, ts); 260 } 261 262 static __maybe_unused int 263 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, 264 struct __kernel_timespec *ts) 265 { 266 int ret = __cvdso_clock_gettime_common(vd, clock, ts); 267 268 if (unlikely(ret)) 269 return clock_gettime_fallback(clock, ts); 270 return 0; 271 } 272 273 static __maybe_unused int 274 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 275 { 276 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); 277 } 278 279 #ifdef BUILD_VDSO32 280 static __maybe_unused int 281 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, 282 struct old_timespec32 *res) 283 { 284 struct __kernel_timespec ts; 285 int ret; 286 287 ret = __cvdso_clock_gettime_common(vd, clock, &ts); 288 289 if (unlikely(ret)) 290 return clock_gettime32_fallback(clock, res); 291 292 /* For ret == 0 */ 293 res->tv_sec = ts.tv_sec; 294 res->tv_nsec = ts.tv_nsec; 295 296 return ret; 297 } 298 299 static __maybe_unused int 300 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 301 { 302 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); 303 } 304 #endif /* BUILD_VDSO32 */ 305 306 static __maybe_unused int 307 __cvdso_gettimeofday_data(const struct vdso_data *vd, 308 struct __kernel_old_timeval *tv, struct timezone *tz) 309 { 310 311 if (likely(tv != NULL)) { 312 struct __kernel_timespec ts; 313 314 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 315 return gettimeofday_fallback(tv, tz); 316 317 tv->tv_sec = ts.tv_sec; 318 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 319 } 320 321 if (unlikely(tz != NULL)) { 322 if (IS_ENABLED(CONFIG_TIME_NS) && 323 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 324 vd = __arch_get_timens_vdso_data(); 325 326 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 327 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 328 } 329 330 return 0; 331 } 332 333 static __maybe_unused int 334 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 335 { 336 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); 337 } 338 339 #ifdef VDSO_HAS_TIME 340 static __maybe_unused __kernel_old_time_t 341 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) 342 { 343 __kernel_old_time_t t; 344 345 if (IS_ENABLED(CONFIG_TIME_NS) && 346 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 347 vd = __arch_get_timens_vdso_data(); 348 349 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 350 351 if (time) 352 *time = t; 353 354 return t; 355 } 356 357 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 358 { 359 return __cvdso_time_data(__arch_get_vdso_data(), time); 360 } 361 #endif /* VDSO_HAS_TIME */ 362 363 #ifdef VDSO_HAS_CLOCK_GETRES 364 static __maybe_unused 365 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, 366 struct __kernel_timespec *res) 367 { 368 u32 msk; 369 u64 ns; 370 371 /* Check for negative values or invalid clocks */ 372 if (unlikely((u32) clock >= MAX_CLOCKS)) 373 return -1; 374 375 if (IS_ENABLED(CONFIG_TIME_NS) && 376 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 377 vd = __arch_get_timens_vdso_data(); 378 379 /* 380 * Convert the clockid to a bitmask and use it to check which 381 * clocks are handled in the VDSO directly. 382 */ 383 msk = 1U << clock; 384 if (msk & (VDSO_HRES | VDSO_RAW)) { 385 /* 386 * Preserves the behaviour of posix_get_hrtimer_res(). 387 */ 388 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 389 } else if (msk & VDSO_COARSE) { 390 /* 391 * Preserves the behaviour of posix_get_coarse_res(). 392 */ 393 ns = LOW_RES_NSEC; 394 } else { 395 return -1; 396 } 397 398 if (likely(res)) { 399 res->tv_sec = 0; 400 res->tv_nsec = ns; 401 } 402 return 0; 403 } 404 405 static __maybe_unused 406 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, 407 struct __kernel_timespec *res) 408 { 409 int ret = __cvdso_clock_getres_common(vd, clock, res); 410 411 if (unlikely(ret)) 412 return clock_getres_fallback(clock, res); 413 return 0; 414 } 415 416 static __maybe_unused 417 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 418 { 419 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); 420 } 421 422 #ifdef BUILD_VDSO32 423 static __maybe_unused int 424 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, 425 struct old_timespec32 *res) 426 { 427 struct __kernel_timespec ts; 428 int ret; 429 430 ret = __cvdso_clock_getres_common(vd, clock, &ts); 431 432 if (unlikely(ret)) 433 return clock_getres32_fallback(clock, res); 434 435 if (likely(res)) { 436 res->tv_sec = ts.tv_sec; 437 res->tv_nsec = ts.tv_nsec; 438 } 439 return ret; 440 } 441 442 static __maybe_unused int 443 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 444 { 445 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), 446 clock, res); 447 } 448 #endif /* BUILD_VDSO32 */ 449 #endif /* VDSO_HAS_CLOCK_GETRES */ 450