1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <linux/clocksource.h> 11 #include <vdso/datapage.h> 12 #include <vdso/helpers.h> 13 14 /* 15 * The generic vDSO implementation requires that gettimeofday.h 16 * provides: 17 * - __arch_get_vdso_data(): to get the vdso datapage. 18 * - __arch_get_hw_counter(): to get the hw counter based on the 19 * clock_mode. 20 * - gettimeofday_fallback(): fallback for gettimeofday. 21 * - clock_gettime_fallback(): fallback for clock_gettime. 22 * - clock_getres_fallback(): fallback for clock_getres. 23 */ 24 #ifdef ENABLE_COMPAT_VDSO 25 #include <asm/vdso/compat_gettimeofday.h> 26 #else 27 #include <asm/vdso/gettimeofday.h> 28 #endif /* ENABLE_COMPAT_VDSO */ 29 30 #ifndef vdso_calc_delta 31 /* 32 * Default implementation which works for all sane clocksources. That 33 * obviously excludes x86/TSC. 34 */ 35 static __always_inline 36 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 37 { 38 return ((cycles - last) & mask) * mult; 39 } 40 #endif 41 42 #ifndef __arch_vdso_hres_capable 43 static inline bool __arch_vdso_hres_capable(void) 44 { 45 return true; 46 } 47 #endif 48 49 #ifdef CONFIG_TIME_NS 50 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 51 struct __kernel_timespec *ts) 52 { 53 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 54 const struct timens_offset *offs = &vdns->offset[clk]; 55 const struct vdso_timestamp *vdso_ts; 56 u64 cycles, last, ns; 57 u32 seq; 58 s64 sec; 59 60 if (clk != CLOCK_MONOTONIC_RAW) 61 vd = &vd[CS_HRES_COARSE]; 62 else 63 vd = &vd[CS_RAW]; 64 vdso_ts = &vd->basetime[clk]; 65 66 do { 67 seq = vdso_read_begin(vd); 68 69 if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) 70 return -1; 71 72 cycles = __arch_get_hw_counter(vd->clock_mode); 73 ns = vdso_ts->nsec; 74 last = vd->cycle_last; 75 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 76 ns >>= vd->shift; 77 sec = vdso_ts->sec; 78 } while (unlikely(vdso_read_retry(vd, seq))); 79 80 /* Add the namespace offset */ 81 sec += offs->sec; 82 ns += offs->nsec; 83 84 /* 85 * Do this outside the loop: a race inside the loop could result 86 * in __iter_div_u64_rem() being extremely slow. 87 */ 88 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 89 ts->tv_nsec = ns; 90 91 return 0; 92 } 93 #else 94 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 95 { 96 return NULL; 97 } 98 99 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 100 struct __kernel_timespec *ts) 101 { 102 return -EINVAL; 103 } 104 #endif 105 106 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 107 struct __kernel_timespec *ts) 108 { 109 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 110 u64 cycles, last, sec, ns; 111 u32 seq; 112 113 /* Allows to compile the high resolution parts out */ 114 if (!__arch_vdso_hres_capable()) 115 return -1; 116 117 do { 118 /* 119 * Open coded to handle VCLOCK_TIMENS. Time namespace 120 * enabled tasks have a special VVAR page installed which 121 * has vd->seq set to 1 and vd->clock_mode set to 122 * VCLOCK_TIMENS. For non time namespace affected tasks 123 * this does not affect performance because if vd->seq is 124 * odd, i.e. a concurrent update is in progress the extra 125 * check for vd->clock_mode is just a few extra 126 * instructions while spin waiting for vd->seq to become 127 * even again. 128 */ 129 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 130 if (IS_ENABLED(CONFIG_TIME_NS) && 131 vd->clock_mode == VCLOCK_TIMENS) 132 return do_hres_timens(vd, clk, ts); 133 cpu_relax(); 134 } 135 smp_rmb(); 136 137 if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) 138 return -1; 139 140 cycles = __arch_get_hw_counter(vd->clock_mode); 141 ns = vdso_ts->nsec; 142 last = vd->cycle_last; 143 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 144 ns >>= vd->shift; 145 sec = vdso_ts->sec; 146 } while (unlikely(vdso_read_retry(vd, seq))); 147 148 /* 149 * Do this outside the loop: a race inside the loop could result 150 * in __iter_div_u64_rem() being extremely slow. 151 */ 152 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 153 ts->tv_nsec = ns; 154 155 return 0; 156 } 157 158 #ifdef CONFIG_TIME_NS 159 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 160 struct __kernel_timespec *ts) 161 { 162 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 163 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 164 const struct timens_offset *offs = &vdns->offset[clk]; 165 u64 nsec; 166 s64 sec; 167 s32 seq; 168 169 do { 170 seq = vdso_read_begin(vd); 171 sec = vdso_ts->sec; 172 nsec = vdso_ts->nsec; 173 } while (unlikely(vdso_read_retry(vd, seq))); 174 175 /* Add the namespace offset */ 176 sec += offs->sec; 177 nsec += offs->nsec; 178 179 /* 180 * Do this outside the loop: a race inside the loop could result 181 * in __iter_div_u64_rem() being extremely slow. 182 */ 183 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 184 ts->tv_nsec = nsec; 185 return 0; 186 } 187 #else 188 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 189 struct __kernel_timespec *ts) 190 { 191 return -1; 192 } 193 #endif 194 195 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 196 struct __kernel_timespec *ts) 197 { 198 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 199 u32 seq; 200 201 do { 202 /* 203 * Open coded to handle VCLOCK_TIMENS. See comment in 204 * do_hres(). 205 */ 206 while ((seq = READ_ONCE(vd->seq)) & 1) { 207 if (IS_ENABLED(CONFIG_TIME_NS) && 208 vd->clock_mode == VCLOCK_TIMENS) 209 return do_coarse_timens(vd, clk, ts); 210 cpu_relax(); 211 } 212 smp_rmb(); 213 214 ts->tv_sec = vdso_ts->sec; 215 ts->tv_nsec = vdso_ts->nsec; 216 } while (unlikely(vdso_read_retry(vd, seq))); 217 218 return 0; 219 } 220 221 static __maybe_unused int 222 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) 223 { 224 const struct vdso_data *vd = __arch_get_vdso_data(); 225 u32 msk; 226 227 /* Check for negative values or invalid clocks */ 228 if (unlikely((u32) clock >= MAX_CLOCKS)) 229 return -1; 230 231 /* 232 * Convert the clockid to a bitmask and use it to check which 233 * clocks are handled in the VDSO directly. 234 */ 235 msk = 1U << clock; 236 if (likely(msk & VDSO_HRES)) 237 vd = &vd[CS_HRES_COARSE]; 238 else if (msk & VDSO_COARSE) 239 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 240 else if (msk & VDSO_RAW) 241 vd = &vd[CS_RAW]; 242 else 243 return -1; 244 245 return do_hres(vd, clock, ts); 246 } 247 248 static __maybe_unused int 249 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 250 { 251 int ret = __cvdso_clock_gettime_common(clock, ts); 252 253 if (unlikely(ret)) 254 return clock_gettime_fallback(clock, ts); 255 return 0; 256 } 257 258 #ifdef BUILD_VDSO32 259 static __maybe_unused int 260 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 261 { 262 struct __kernel_timespec ts; 263 int ret; 264 265 ret = __cvdso_clock_gettime_common(clock, &ts); 266 267 if (unlikely(ret)) 268 return clock_gettime32_fallback(clock, res); 269 270 /* For ret == 0 */ 271 res->tv_sec = ts.tv_sec; 272 res->tv_nsec = ts.tv_nsec; 273 274 return ret; 275 } 276 #endif /* BUILD_VDSO32 */ 277 278 static __maybe_unused int 279 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 280 { 281 const struct vdso_data *vd = __arch_get_vdso_data(); 282 283 if (likely(tv != NULL)) { 284 struct __kernel_timespec ts; 285 286 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 287 return gettimeofday_fallback(tv, tz); 288 289 tv->tv_sec = ts.tv_sec; 290 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 291 } 292 293 if (unlikely(tz != NULL)) { 294 if (IS_ENABLED(CONFIG_TIME_NS) && 295 vd->clock_mode == VCLOCK_TIMENS) 296 vd = __arch_get_timens_vdso_data(); 297 298 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 299 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 300 } 301 302 return 0; 303 } 304 305 #ifdef VDSO_HAS_TIME 306 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 307 { 308 const struct vdso_data *vd = __arch_get_vdso_data(); 309 __kernel_old_time_t t; 310 311 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 312 vd = __arch_get_timens_vdso_data(); 313 314 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 315 316 if (time) 317 *time = t; 318 319 return t; 320 } 321 #endif /* VDSO_HAS_TIME */ 322 323 #ifdef VDSO_HAS_CLOCK_GETRES 324 static __maybe_unused 325 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) 326 { 327 const struct vdso_data *vd = __arch_get_vdso_data(); 328 u32 msk; 329 u64 ns; 330 331 /* Check for negative values or invalid clocks */ 332 if (unlikely((u32) clock >= MAX_CLOCKS)) 333 return -1; 334 335 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 336 vd = __arch_get_timens_vdso_data(); 337 338 /* 339 * Convert the clockid to a bitmask and use it to check which 340 * clocks are handled in the VDSO directly. 341 */ 342 msk = 1U << clock; 343 if (msk & (VDSO_HRES | VDSO_RAW)) { 344 /* 345 * Preserves the behaviour of posix_get_hrtimer_res(). 346 */ 347 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 348 } else if (msk & VDSO_COARSE) { 349 /* 350 * Preserves the behaviour of posix_get_coarse_res(). 351 */ 352 ns = LOW_RES_NSEC; 353 } else { 354 return -1; 355 } 356 357 if (likely(res)) { 358 res->tv_sec = 0; 359 res->tv_nsec = ns; 360 } 361 return 0; 362 } 363 364 static __maybe_unused 365 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 366 { 367 int ret = __cvdso_clock_getres_common(clock, res); 368 369 if (unlikely(ret)) 370 return clock_getres_fallback(clock, res); 371 return 0; 372 } 373 374 #ifdef BUILD_VDSO32 375 static __maybe_unused int 376 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 377 { 378 struct __kernel_timespec ts; 379 int ret; 380 381 ret = __cvdso_clock_getres_common(clock, &ts); 382 383 if (unlikely(ret)) 384 return clock_getres32_fallback(clock, res); 385 386 if (likely(res)) { 387 res->tv_sec = ts.tv_sec; 388 res->tv_nsec = ts.tv_nsec; 389 } 390 return ret; 391 } 392 #endif /* BUILD_VDSO32 */ 393 #endif /* VDSO_HAS_CLOCK_GETRES */ 394