1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <vdso/datapage.h> 11 #include <vdso/helpers.h> 12 13 /* 14 * The generic vDSO implementation requires that gettimeofday.h 15 * provides: 16 * - __arch_get_vdso_data(): to get the vdso datapage. 17 * - __arch_get_hw_counter(): to get the hw counter based on the 18 * clock_mode. 19 * - gettimeofday_fallback(): fallback for gettimeofday. 20 * - clock_gettime_fallback(): fallback for clock_gettime. 21 * - clock_getres_fallback(): fallback for clock_getres. 22 */ 23 #ifdef ENABLE_COMPAT_VDSO 24 #include <asm/vdso/compat_gettimeofday.h> 25 #else 26 #include <asm/vdso/gettimeofday.h> 27 #endif /* ENABLE_COMPAT_VDSO */ 28 29 #ifndef vdso_calc_delta 30 /* 31 * Default implementation which works for all sane clocksources. That 32 * obviously excludes x86/TSC. 33 */ 34 static __always_inline 35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) 36 { 37 return ((cycles - last) & mask) * mult; 38 } 39 #endif 40 41 #ifdef CONFIG_TIME_NS 42 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 43 struct __kernel_timespec *ts) 44 { 45 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 46 const struct timens_offset *offs = &vdns->offset[clk]; 47 const struct vdso_timestamp *vdso_ts; 48 u64 cycles, last, ns; 49 u32 seq; 50 s64 sec; 51 52 if (clk != CLOCK_MONOTONIC_RAW) 53 vd = &vd[CS_HRES_COARSE]; 54 else 55 vd = &vd[CS_RAW]; 56 vdso_ts = &vd->basetime[clk]; 57 58 do { 59 seq = vdso_read_begin(vd); 60 cycles = __arch_get_hw_counter(vd->clock_mode); 61 ns = vdso_ts->nsec; 62 last = vd->cycle_last; 63 if (unlikely((s64)cycles < 0)) 64 return -1; 65 66 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 67 ns >>= vd->shift; 68 sec = vdso_ts->sec; 69 } while (unlikely(vdso_read_retry(vd, seq))); 70 71 /* Add the namespace offset */ 72 sec += offs->sec; 73 ns += offs->nsec; 74 75 /* 76 * Do this outside the loop: a race inside the loop could result 77 * in __iter_div_u64_rem() being extremely slow. 78 */ 79 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 80 ts->tv_nsec = ns; 81 82 return 0; 83 } 84 #else 85 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 86 { 87 return NULL; 88 } 89 90 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 91 struct __kernel_timespec *ts) 92 { 93 return -EINVAL; 94 } 95 #endif 96 97 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 98 struct __kernel_timespec *ts) 99 { 100 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 101 u64 cycles, last, sec, ns; 102 u32 seq; 103 104 do { 105 /* 106 * Open coded to handle VCLOCK_TIMENS. Time namespace 107 * enabled tasks have a special VVAR page installed which 108 * has vd->seq set to 1 and vd->clock_mode set to 109 * VCLOCK_TIMENS. For non time namespace affected tasks 110 * this does not affect performance because if vd->seq is 111 * odd, i.e. a concurrent update is in progress the extra 112 * check for vd->clock_mode is just a few extra 113 * instructions while spin waiting for vd->seq to become 114 * even again. 115 */ 116 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 117 if (IS_ENABLED(CONFIG_TIME_NS) && 118 vd->clock_mode == VCLOCK_TIMENS) 119 return do_hres_timens(vd, clk, ts); 120 cpu_relax(); 121 } 122 smp_rmb(); 123 124 cycles = __arch_get_hw_counter(vd->clock_mode); 125 ns = vdso_ts->nsec; 126 last = vd->cycle_last; 127 if (unlikely((s64)cycles < 0)) 128 return -1; 129 130 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); 131 ns >>= vd->shift; 132 sec = vdso_ts->sec; 133 } while (unlikely(vdso_read_retry(vd, seq))); 134 135 /* 136 * Do this outside the loop: a race inside the loop could result 137 * in __iter_div_u64_rem() being extremely slow. 138 */ 139 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 140 ts->tv_nsec = ns; 141 142 return 0; 143 } 144 145 #ifdef CONFIG_TIME_NS 146 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 147 struct __kernel_timespec *ts) 148 { 149 const struct vdso_data *vd = __arch_get_timens_vdso_data(); 150 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 151 const struct timens_offset *offs = &vdns->offset[clk]; 152 u64 nsec; 153 s64 sec; 154 s32 seq; 155 156 do { 157 seq = vdso_read_begin(vd); 158 sec = vdso_ts->sec; 159 nsec = vdso_ts->nsec; 160 } while (unlikely(vdso_read_retry(vd, seq))); 161 162 /* Add the namespace offset */ 163 sec += offs->sec; 164 nsec += offs->nsec; 165 166 /* 167 * Do this outside the loop: a race inside the loop could result 168 * in __iter_div_u64_rem() being extremely slow. 169 */ 170 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 171 ts->tv_nsec = nsec; 172 return 0; 173 } 174 #else 175 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 176 struct __kernel_timespec *ts) 177 { 178 return -1; 179 } 180 #endif 181 182 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 183 struct __kernel_timespec *ts) 184 { 185 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 186 u32 seq; 187 188 do { 189 /* 190 * Open coded to handle VCLOCK_TIMENS. See comment in 191 * do_hres(). 192 */ 193 while ((seq = READ_ONCE(vd->seq)) & 1) { 194 if (IS_ENABLED(CONFIG_TIME_NS) && 195 vd->clock_mode == VCLOCK_TIMENS) 196 return do_coarse_timens(vd, clk, ts); 197 cpu_relax(); 198 } 199 smp_rmb(); 200 201 ts->tv_sec = vdso_ts->sec; 202 ts->tv_nsec = vdso_ts->nsec; 203 } while (unlikely(vdso_read_retry(vd, seq))); 204 205 return 0; 206 } 207 208 static __maybe_unused int 209 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) 210 { 211 const struct vdso_data *vd = __arch_get_vdso_data(); 212 u32 msk; 213 214 /* Check for negative values or invalid clocks */ 215 if (unlikely((u32) clock >= MAX_CLOCKS)) 216 return -1; 217 218 /* 219 * Convert the clockid to a bitmask and use it to check which 220 * clocks are handled in the VDSO directly. 221 */ 222 msk = 1U << clock; 223 if (likely(msk & VDSO_HRES)) 224 vd = &vd[CS_HRES_COARSE]; 225 else if (msk & VDSO_COARSE) 226 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 227 else if (msk & VDSO_RAW) 228 vd = &vd[CS_RAW]; 229 else 230 return -1; 231 232 return do_hres(vd, clock, ts); 233 } 234 235 static __maybe_unused int 236 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 237 { 238 int ret = __cvdso_clock_gettime_common(clock, ts); 239 240 if (unlikely(ret)) 241 return clock_gettime_fallback(clock, ts); 242 return 0; 243 } 244 245 #ifdef BUILD_VDSO32 246 static __maybe_unused int 247 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 248 { 249 struct __kernel_timespec ts; 250 int ret; 251 252 ret = __cvdso_clock_gettime_common(clock, &ts); 253 254 if (unlikely(ret)) 255 return clock_gettime32_fallback(clock, res); 256 257 /* For ret == 0 */ 258 res->tv_sec = ts.tv_sec; 259 res->tv_nsec = ts.tv_nsec; 260 261 return ret; 262 } 263 #endif /* BUILD_VDSO32 */ 264 265 static __maybe_unused int 266 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 267 { 268 const struct vdso_data *vd = __arch_get_vdso_data(); 269 270 if (likely(tv != NULL)) { 271 struct __kernel_timespec ts; 272 273 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 274 return gettimeofday_fallback(tv, tz); 275 276 tv->tv_sec = ts.tv_sec; 277 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 278 } 279 280 if (unlikely(tz != NULL)) { 281 if (IS_ENABLED(CONFIG_TIME_NS) && 282 vd->clock_mode == VCLOCK_TIMENS) 283 vd = __arch_get_timens_vdso_data(); 284 285 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 286 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 287 } 288 289 return 0; 290 } 291 292 #ifdef VDSO_HAS_TIME 293 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 294 { 295 const struct vdso_data *vd = __arch_get_vdso_data(); 296 __kernel_old_time_t t; 297 298 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 299 vd = __arch_get_timens_vdso_data(); 300 301 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 302 303 if (time) 304 *time = t; 305 306 return t; 307 } 308 #endif /* VDSO_HAS_TIME */ 309 310 #ifdef VDSO_HAS_CLOCK_GETRES 311 static __maybe_unused 312 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) 313 { 314 const struct vdso_data *vd = __arch_get_vdso_data(); 315 u32 msk; 316 u64 ns; 317 318 /* Check for negative values or invalid clocks */ 319 if (unlikely((u32) clock >= MAX_CLOCKS)) 320 return -1; 321 322 if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) 323 vd = __arch_get_timens_vdso_data(); 324 325 /* 326 * Convert the clockid to a bitmask and use it to check which 327 * clocks are handled in the VDSO directly. 328 */ 329 msk = 1U << clock; 330 if (msk & (VDSO_HRES | VDSO_RAW)) { 331 /* 332 * Preserves the behaviour of posix_get_hrtimer_res(). 333 */ 334 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 335 } else if (msk & VDSO_COARSE) { 336 /* 337 * Preserves the behaviour of posix_get_coarse_res(). 338 */ 339 ns = LOW_RES_NSEC; 340 } else { 341 return -1; 342 } 343 344 if (likely(res)) { 345 res->tv_sec = 0; 346 res->tv_nsec = ns; 347 } 348 return 0; 349 } 350 351 static __maybe_unused 352 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 353 { 354 int ret = __cvdso_clock_getres_common(clock, res); 355 356 if (unlikely(ret)) 357 return clock_getres_fallback(clock, res); 358 return 0; 359 } 360 361 #ifdef BUILD_VDSO32 362 static __maybe_unused int 363 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 364 { 365 struct __kernel_timespec ts; 366 int ret; 367 368 ret = __cvdso_clock_getres_common(clock, &ts); 369 370 if (unlikely(ret)) 371 return clock_getres32_fallback(clock, res); 372 373 if (likely(res)) { 374 res->tv_sec = ts.tv_sec; 375 res->tv_nsec = ts.tv_nsec; 376 } 377 return ret; 378 } 379 #endif /* BUILD_VDSO32 */ 380 #endif /* VDSO_HAS_CLOCK_GETRES */ 381