1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 #ifndef vdso_calc_delta 9 /* 10 * Default implementation which works for all sane clocksources. That 11 * obviously excludes x86/TSC. 12 */ 13 static __always_inline 14 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 15 { 16 return ((cycles - last) & mask) * mult; 17 } 18 #endif 19 20 #ifndef vdso_shift_ns 21 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 22 { 23 return ns >> shift; 24 } 25 #endif 26 27 #ifndef __arch_vdso_hres_capable 28 static inline bool __arch_vdso_hres_capable(void) 29 { 30 return true; 31 } 32 #endif 33 34 #ifndef vdso_clocksource_ok 35 static inline bool vdso_clocksource_ok(const struct vdso_data *vd) 36 { 37 return vd->clock_mode != VDSO_CLOCKMODE_NONE; 38 } 39 #endif 40 41 #ifdef CONFIG_TIME_NS 42 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 43 struct __kernel_timespec *ts) 44 { 45 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 46 const struct timens_offset *offs = &vdns->offset[clk]; 47 const struct vdso_timestamp *vdso_ts; 48 u64 cycles, last, ns; 49 u32 seq; 50 s64 sec; 51 52 if (clk != CLOCK_MONOTONIC_RAW) 53 vd = &vd[CS_HRES_COARSE]; 54 else 55 vd = &vd[CS_RAW]; 56 vdso_ts = &vd->basetime[clk]; 57 58 do { 59 seq = vdso_read_begin(vd); 60 61 if (unlikely(!vdso_clocksource_ok(vd))) 62 return -1; 63 64 cycles = __arch_get_hw_counter(vd->clock_mode); 65 ns = vdso_ts->nsec; 66 last = vd->cycle_last; 67 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 68 ns = vdso_shift_ns(ns, vd->shift); 69 sec = vdso_ts->sec; 70 } while (unlikely(vdso_read_retry(vd, seq))); 71 72 /* Add the namespace offset */ 73 sec += offs->sec; 74 ns += offs->nsec; 75 76 /* 77 * Do this outside the loop: a race inside the loop could result 78 * in __iter_div_u64_rem() being extremely slow. 79 */ 80 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 81 ts->tv_nsec = ns; 82 83 return 0; 84 } 85 #else 86 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 87 { 88 return NULL; 89 } 90 91 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 92 struct __kernel_timespec *ts) 93 { 94 return -EINVAL; 95 } 96 #endif 97 98 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 99 struct __kernel_timespec *ts) 100 { 101 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 102 u64 cycles, last, sec, ns; 103 u32 seq; 104 105 /* Allows to compile the high resolution parts out */ 106 if (!__arch_vdso_hres_capable()) 107 return -1; 108 109 do { 110 /* 111 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace 112 * enabled tasks have a special VVAR page installed which 113 * has vd->seq set to 1 and vd->clock_mode set to 114 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks 115 * this does not affect performance because if vd->seq is 116 * odd, i.e. a concurrent update is in progress the extra 117 * check for vd->clock_mode is just a few extra 118 * instructions while spin waiting for vd->seq to become 119 * even again. 120 */ 121 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 122 if (IS_ENABLED(CONFIG_TIME_NS) && 123 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 124 return do_hres_timens(vd, clk, ts); 125 cpu_relax(); 126 } 127 smp_rmb(); 128 129 if (unlikely(!vdso_clocksource_ok(vd))) 130 return -1; 131 132 cycles = __arch_get_hw_counter(vd->clock_mode); 133 ns = vdso_ts->nsec; 134 last = vd->cycle_last; 135 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 136 ns = vdso_shift_ns(ns, vd->shift); 137 sec = vdso_ts->sec; 138 } while (unlikely(vdso_read_retry(vd, seq))); 139 140 /* 141 * Do this outside the loop: a race inside the loop could result 142 * in __iter_div_u64_rem() being extremely slow. 143 */ 144 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 145 ts->tv_nsec = ns; 146 147 return 0; 148 } 149 150 #ifdef CONFIG_TIME_NS 151 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 152 struct __kernel_timespec *ts) 153 { 154 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 155 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 156 const struct timens_offset *offs = &vdns->offset[clk]; 157 u64 nsec; 158 s64 sec; 159 s32 seq; 160 161 do { 162 seq = vdso_read_begin(vd); 163 sec = vdso_ts->sec; 164 nsec = vdso_ts->nsec; 165 } while (unlikely(vdso_read_retry(vd, seq))); 166 167 /* Add the namespace offset */ 168 sec += offs->sec; 169 nsec += offs->nsec; 170 171 /* 172 * Do this outside the loop: a race inside the loop could result 173 * in __iter_div_u64_rem() being extremely slow. 174 */ 175 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 176 ts->tv_nsec = nsec; 177 return 0; 178 } 179 #else 180 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 181 struct __kernel_timespec *ts) 182 { 183 return -1; 184 } 185 #endif 186 187 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 188 struct __kernel_timespec *ts) 189 { 190 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 191 u32 seq; 192 193 do { 194 /* 195 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in 196 * do_hres(). 197 */ 198 while ((seq = READ_ONCE(vd->seq)) & 1) { 199 if (IS_ENABLED(CONFIG_TIME_NS) && 200 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 201 return do_coarse_timens(vd, clk, ts); 202 cpu_relax(); 203 } 204 smp_rmb(); 205 206 ts->tv_sec = vdso_ts->sec; 207 ts->tv_nsec = vdso_ts->nsec; 208 } while (unlikely(vdso_read_retry(vd, seq))); 209 210 return 0; 211 } 212 213 static __maybe_unused int 214 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, 215 struct __kernel_timespec *ts) 216 { 217 u32 msk; 218 219 /* Check for negative values or invalid clocks */ 220 if (unlikely((u32) clock >= MAX_CLOCKS)) 221 return -1; 222 223 /* 224 * Convert the clockid to a bitmask and use it to check which 225 * clocks are handled in the VDSO directly. 226 */ 227 msk = 1U << clock; 228 if (likely(msk & VDSO_HRES)) 229 vd = &vd[CS_HRES_COARSE]; 230 else if (msk & VDSO_COARSE) 231 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 232 else if (msk & VDSO_RAW) 233 vd = &vd[CS_RAW]; 234 else 235 return -1; 236 237 return do_hres(vd, clock, ts); 238 } 239 240 static __maybe_unused int 241 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, 242 struct __kernel_timespec *ts) 243 { 244 int ret = __cvdso_clock_gettime_common(vd, clock, ts); 245 246 if (unlikely(ret)) 247 return clock_gettime_fallback(clock, ts); 248 return 0; 249 } 250 251 static __maybe_unused int 252 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 253 { 254 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); 255 } 256 257 #ifdef BUILD_VDSO32 258 static __maybe_unused int 259 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, 260 struct old_timespec32 *res) 261 { 262 struct __kernel_timespec ts; 263 int ret; 264 265 ret = __cvdso_clock_gettime_common(vd, clock, &ts); 266 267 if (unlikely(ret)) 268 return clock_gettime32_fallback(clock, res); 269 270 /* For ret == 0 */ 271 res->tv_sec = ts.tv_sec; 272 res->tv_nsec = ts.tv_nsec; 273 274 return ret; 275 } 276 277 static __maybe_unused int 278 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 279 { 280 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); 281 } 282 #endif /* BUILD_VDSO32 */ 283 284 static __maybe_unused int 285 __cvdso_gettimeofday_data(const struct vdso_data *vd, 286 struct __kernel_old_timeval *tv, struct timezone *tz) 287 { 288 289 if (likely(tv != NULL)) { 290 struct __kernel_timespec ts; 291 292 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 293 return gettimeofday_fallback(tv, tz); 294 295 tv->tv_sec = ts.tv_sec; 296 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 297 } 298 299 if (unlikely(tz != NULL)) { 300 if (IS_ENABLED(CONFIG_TIME_NS) && 301 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 302 vd = __arch_get_timens_vdso_data(); 303 304 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 305 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 306 } 307 308 return 0; 309 } 310 311 static __maybe_unused int 312 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 313 { 314 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); 315 } 316 317 #ifdef VDSO_HAS_TIME 318 static __maybe_unused __kernel_old_time_t 319 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) 320 { 321 __kernel_old_time_t t; 322 323 if (IS_ENABLED(CONFIG_TIME_NS) && 324 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 325 vd = __arch_get_timens_vdso_data(); 326 327 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 328 329 if (time) 330 *time = t; 331 332 return t; 333 } 334 335 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 336 { 337 return __cvdso_time_data(__arch_get_vdso_data(), time); 338 } 339 #endif /* VDSO_HAS_TIME */ 340 341 #ifdef VDSO_HAS_CLOCK_GETRES 342 static __maybe_unused 343 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, 344 struct __kernel_timespec *res) 345 { 346 u32 msk; 347 u64 ns; 348 349 /* Check for negative values or invalid clocks */ 350 if (unlikely((u32) clock >= MAX_CLOCKS)) 351 return -1; 352 353 if (IS_ENABLED(CONFIG_TIME_NS) && 354 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 355 vd = __arch_get_timens_vdso_data(); 356 357 /* 358 * Convert the clockid to a bitmask and use it to check which 359 * clocks are handled in the VDSO directly. 360 */ 361 msk = 1U << clock; 362 if (msk & (VDSO_HRES | VDSO_RAW)) { 363 /* 364 * Preserves the behaviour of posix_get_hrtimer_res(). 365 */ 366 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 367 } else if (msk & VDSO_COARSE) { 368 /* 369 * Preserves the behaviour of posix_get_coarse_res(). 370 */ 371 ns = LOW_RES_NSEC; 372 } else { 373 return -1; 374 } 375 376 if (likely(res)) { 377 res->tv_sec = 0; 378 res->tv_nsec = ns; 379 } 380 return 0; 381 } 382 383 static __maybe_unused 384 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, 385 struct __kernel_timespec *res) 386 { 387 int ret = __cvdso_clock_getres_common(vd, clock, res); 388 389 if (unlikely(ret)) 390 return clock_getres_fallback(clock, res); 391 return 0; 392 } 393 394 static __maybe_unused 395 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 396 { 397 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); 398 } 399 400 #ifdef BUILD_VDSO32 401 static __maybe_unused int 402 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, 403 struct old_timespec32 *res) 404 { 405 struct __kernel_timespec ts; 406 int ret; 407 408 ret = __cvdso_clock_getres_common(vd, clock, &ts); 409 410 if (unlikely(ret)) 411 return clock_getres32_fallback(clock, res); 412 413 if (likely(res)) { 414 res->tv_sec = ts.tv_sec; 415 res->tv_nsec = ts.tv_nsec; 416 } 417 return ret; 418 } 419 420 static __maybe_unused int 421 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 422 { 423 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), 424 clock, res); 425 } 426 #endif /* BUILD_VDSO32 */ 427 #endif /* VDSO_HAS_CLOCK_GETRES */ 428