1 /* 2 * linux/kernel/time/timekeeping.c 3 * 4 * Kernel timekeeping code and accessor functions 5 * 6 * This code was moved from linux/kernel/timer.c. 7 * Please see that file for copyright and history logs. 8 * 9 */ 10 11 #include <linux/timekeeper_internal.h> 12 #include <linux/module.h> 13 #include <linux/interrupt.h> 14 #include <linux/percpu.h> 15 #include <linux/init.h> 16 #include <linux/mm.h> 17 #include <linux/nmi.h> 18 #include <linux/sched.h> 19 #include <linux/sched/loadavg.h> 20 #include <linux/syscore_ops.h> 21 #include <linux/clocksource.h> 22 #include <linux/jiffies.h> 23 #include <linux/time.h> 24 #include <linux/tick.h> 25 #include <linux/stop_machine.h> 26 #include <linux/pvclock_gtod.h> 27 #include <linux/compiler.h> 28 29 #include "tick-internal.h" 30 #include "ntp_internal.h" 31 #include "timekeeping_internal.h" 32 33 #define TK_CLEAR_NTP (1 << 0) 34 #define TK_MIRROR (1 << 1) 35 #define TK_CLOCK_WAS_SET (1 << 2) 36 37 /* 38 * The most important data for readout fits into a single 64 byte 39 * cache line. 40 */ 41 static struct { 42 seqcount_t seq; 43 struct timekeeper timekeeper; 44 } tk_core ____cacheline_aligned; 45 46 static DEFINE_RAW_SPINLOCK(timekeeper_lock); 47 static struct timekeeper shadow_timekeeper; 48 49 /** 50 * struct tk_fast - NMI safe timekeeper 51 * @seq: Sequence counter for protecting updates. The lowest bit 52 * is the index for the tk_read_base array 53 * @base: tk_read_base array. Access is indexed by the lowest bit of 54 * @seq. 55 * 56 * See @update_fast_timekeeper() below. 57 */ 58 struct tk_fast { 59 seqcount_t seq; 60 struct tk_read_base base[2]; 61 }; 62 63 static struct tk_fast tk_fast_mono ____cacheline_aligned; 64 static struct tk_fast tk_fast_raw ____cacheline_aligned; 65 66 /* flag for if timekeeping is suspended */ 67 int __read_mostly timekeeping_suspended; 68 69 static inline void tk_normalize_xtime(struct timekeeper *tk) 70 { 71 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { 72 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 73 tk->xtime_sec++; 74 } 75 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { 76 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 77 tk->raw_sec++; 78 } 79 } 80 81 static inline struct timespec64 tk_xtime(struct timekeeper *tk) 82 { 83 struct timespec64 ts; 84 85 ts.tv_sec = tk->xtime_sec; 86 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 87 return ts; 88 } 89 90 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) 91 { 92 tk->xtime_sec = ts->tv_sec; 93 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; 94 } 95 96 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) 97 { 98 tk->xtime_sec += ts->tv_sec; 99 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; 100 tk_normalize_xtime(tk); 101 } 102 103 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) 104 { 105 struct timespec64 tmp; 106 107 /* 108 * Verify consistency of: offset_real = -wall_to_monotonic 109 * before modifying anything 110 */ 111 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, 112 -tk->wall_to_monotonic.tv_nsec); 113 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); 114 tk->wall_to_monotonic = wtm; 115 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); 116 tk->offs_real = timespec64_to_ktime(tmp); 117 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); 118 } 119 120 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) 121 { 122 tk->offs_boot = ktime_add(tk->offs_boot, delta); 123 } 124 125 /* 126 * tk_clock_read - atomic clocksource read() helper 127 * 128 * This helper is necessary to use in the read paths because, while the 129 * seqlock ensures we don't return a bad value while structures are updated, 130 * it doesn't protect from potential crashes. There is the possibility that 131 * the tkr's clocksource may change between the read reference, and the 132 * clock reference passed to the read function. This can cause crashes if 133 * the wrong clocksource is passed to the wrong read function. 134 * This isn't necessary to use when holding the timekeeper_lock or doing 135 * a read of the fast-timekeeper tkrs (which is protected by its own locking 136 * and update logic). 137 */ 138 static inline u64 tk_clock_read(struct tk_read_base *tkr) 139 { 140 struct clocksource *clock = READ_ONCE(tkr->clock); 141 142 return clock->read(clock); 143 } 144 145 #ifdef CONFIG_DEBUG_TIMEKEEPING 146 #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */ 147 148 static void timekeeping_check_update(struct timekeeper *tk, u64 offset) 149 { 150 151 u64 max_cycles = tk->tkr_mono.clock->max_cycles; 152 const char *name = tk->tkr_mono.clock->name; 153 154 if (offset > max_cycles) { 155 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n", 156 offset, name, max_cycles); 157 printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n"); 158 } else { 159 if (offset > (max_cycles >> 1)) { 160 printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n", 161 offset, name, max_cycles >> 1); 162 printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n"); 163 } 164 } 165 166 if (tk->underflow_seen) { 167 if (jiffies - tk->last_warning > WARNING_FREQ) { 168 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name); 169 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n"); 170 printk_deferred(" Your kernel is probably still fine.\n"); 171 tk->last_warning = jiffies; 172 } 173 tk->underflow_seen = 0; 174 } 175 176 if (tk->overflow_seen) { 177 if (jiffies - tk->last_warning > WARNING_FREQ) { 178 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name); 179 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n"); 180 printk_deferred(" Your kernel is probably still fine.\n"); 181 tk->last_warning = jiffies; 182 } 183 tk->overflow_seen = 0; 184 } 185 } 186 187 static inline u64 timekeeping_get_delta(struct tk_read_base *tkr) 188 { 189 struct timekeeper *tk = &tk_core.timekeeper; 190 u64 now, last, mask, max, delta; 191 unsigned int seq; 192 193 /* 194 * Since we're called holding a seqlock, the data may shift 195 * under us while we're doing the calculation. This can cause 196 * false positives, since we'd note a problem but throw the 197 * results away. So nest another seqlock here to atomically 198 * grab the points we are checking with. 199 */ 200 do { 201 seq = read_seqcount_begin(&tk_core.seq); 202 now = tk_clock_read(tkr); 203 last = tkr->cycle_last; 204 mask = tkr->mask; 205 max = tkr->clock->max_cycles; 206 } while (read_seqcount_retry(&tk_core.seq, seq)); 207 208 delta = clocksource_delta(now, last, mask); 209 210 /* 211 * Try to catch underflows by checking if we are seeing small 212 * mask-relative negative values. 213 */ 214 if (unlikely((~delta & mask) < (mask >> 3))) { 215 tk->underflow_seen = 1; 216 delta = 0; 217 } 218 219 /* Cap delta value to the max_cycles values to avoid mult overflows */ 220 if (unlikely(delta > max)) { 221 tk->overflow_seen = 1; 222 delta = tkr->clock->max_cycles; 223 } 224 225 return delta; 226 } 227 #else 228 static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset) 229 { 230 } 231 static inline u64 timekeeping_get_delta(struct tk_read_base *tkr) 232 { 233 u64 cycle_now, delta; 234 235 /* read clocksource */ 236 cycle_now = tk_clock_read(tkr); 237 238 /* calculate the delta since the last update_wall_time */ 239 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); 240 241 return delta; 242 } 243 #endif 244 245 /** 246 * tk_setup_internals - Set up internals to use clocksource clock. 247 * 248 * @tk: The target timekeeper to setup. 249 * @clock: Pointer to clocksource. 250 * 251 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment 252 * pair and interval request. 253 * 254 * Unless you're the timekeeping code, you should not be using this! 255 */ 256 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) 257 { 258 u64 interval; 259 u64 tmp, ntpinterval; 260 struct clocksource *old_clock; 261 262 ++tk->cs_was_changed_seq; 263 old_clock = tk->tkr_mono.clock; 264 tk->tkr_mono.clock = clock; 265 tk->tkr_mono.mask = clock->mask; 266 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); 267 268 tk->tkr_raw.clock = clock; 269 tk->tkr_raw.mask = clock->mask; 270 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 271 272 /* Do the ns -> cycle conversion first, using original mult */ 273 tmp = NTP_INTERVAL_LENGTH; 274 tmp <<= clock->shift; 275 ntpinterval = tmp; 276 tmp += clock->mult/2; 277 do_div(tmp, clock->mult); 278 if (tmp == 0) 279 tmp = 1; 280 281 interval = (u64) tmp; 282 tk->cycle_interval = interval; 283 284 /* Go back from cycles -> shifted ns */ 285 tk->xtime_interval = interval * clock->mult; 286 tk->xtime_remainder = ntpinterval - tk->xtime_interval; 287 tk->raw_interval = interval * clock->mult; 288 289 /* if changing clocks, convert xtime_nsec shift units */ 290 if (old_clock) { 291 int shift_change = clock->shift - old_clock->shift; 292 if (shift_change < 0) { 293 tk->tkr_mono.xtime_nsec >>= -shift_change; 294 tk->tkr_raw.xtime_nsec >>= -shift_change; 295 } else { 296 tk->tkr_mono.xtime_nsec <<= shift_change; 297 tk->tkr_raw.xtime_nsec <<= shift_change; 298 } 299 } 300 301 tk->tkr_mono.shift = clock->shift; 302 tk->tkr_raw.shift = clock->shift; 303 304 tk->ntp_error = 0; 305 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 306 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; 307 308 /* 309 * The timekeeper keeps its own mult values for the currently 310 * active clocksource. These value will be adjusted via NTP 311 * to counteract clock drifting. 312 */ 313 tk->tkr_mono.mult = clock->mult; 314 tk->tkr_raw.mult = clock->mult; 315 tk->ntp_err_mult = 0; 316 } 317 318 /* Timekeeper helper functions. */ 319 320 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET 321 static u32 default_arch_gettimeoffset(void) { return 0; } 322 u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; 323 #else 324 static inline u32 arch_gettimeoffset(void) { return 0; } 325 #endif 326 327 static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, u64 delta) 328 { 329 u64 nsec; 330 331 nsec = delta * tkr->mult + tkr->xtime_nsec; 332 nsec >>= tkr->shift; 333 334 /* If arch requires, add in get_arch_timeoffset() */ 335 return nsec + arch_gettimeoffset(); 336 } 337 338 static inline u64 timekeeping_get_ns(struct tk_read_base *tkr) 339 { 340 u64 delta; 341 342 delta = timekeeping_get_delta(tkr); 343 return timekeeping_delta_to_ns(tkr, delta); 344 } 345 346 static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr, u64 cycles) 347 { 348 u64 delta; 349 350 /* calculate the delta since the last update_wall_time */ 351 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask); 352 return timekeeping_delta_to_ns(tkr, delta); 353 } 354 355 /** 356 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. 357 * @tkr: Timekeeping readout base from which we take the update 358 * 359 * We want to use this from any context including NMI and tracing / 360 * instrumenting the timekeeping code itself. 361 * 362 * Employ the latch technique; see @raw_write_seqcount_latch. 363 * 364 * So if a NMI hits the update of base[0] then it will use base[1] 365 * which is still consistent. In the worst case this can result is a 366 * slightly wrong timestamp (a few nanoseconds). See 367 * @ktime_get_mono_fast_ns. 368 */ 369 static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf) 370 { 371 struct tk_read_base *base = tkf->base; 372 373 /* Force readers off to base[1] */ 374 raw_write_seqcount_latch(&tkf->seq); 375 376 /* Update base[0] */ 377 memcpy(base, tkr, sizeof(*base)); 378 379 /* Force readers back to base[0] */ 380 raw_write_seqcount_latch(&tkf->seq); 381 382 /* Update base[1] */ 383 memcpy(base + 1, base, sizeof(*base)); 384 } 385 386 /** 387 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic 388 * 389 * This timestamp is not guaranteed to be monotonic across an update. 390 * The timestamp is calculated by: 391 * 392 * now = base_mono + clock_delta * slope 393 * 394 * So if the update lowers the slope, readers who are forced to the 395 * not yet updated second array are still using the old steeper slope. 396 * 397 * tmono 398 * ^ 399 * | o n 400 * | o n 401 * | u 402 * | o 403 * |o 404 * |12345678---> reader order 405 * 406 * o = old slope 407 * u = update 408 * n = new slope 409 * 410 * So reader 6 will observe time going backwards versus reader 5. 411 * 412 * While other CPUs are likely to be able observe that, the only way 413 * for a CPU local observation is when an NMI hits in the middle of 414 * the update. Timestamps taken from that NMI context might be ahead 415 * of the following timestamps. Callers need to be aware of that and 416 * deal with it. 417 */ 418 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) 419 { 420 struct tk_read_base *tkr; 421 unsigned int seq; 422 u64 now; 423 424 do { 425 seq = raw_read_seqcount_latch(&tkf->seq); 426 tkr = tkf->base + (seq & 0x01); 427 now = ktime_to_ns(tkr->base); 428 429 now += timekeeping_delta_to_ns(tkr, 430 clocksource_delta( 431 tk_clock_read(tkr), 432 tkr->cycle_last, 433 tkr->mask)); 434 } while (read_seqcount_retry(&tkf->seq, seq)); 435 436 return now; 437 } 438 439 u64 ktime_get_mono_fast_ns(void) 440 { 441 return __ktime_get_fast_ns(&tk_fast_mono); 442 } 443 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); 444 445 u64 ktime_get_raw_fast_ns(void) 446 { 447 return __ktime_get_fast_ns(&tk_fast_raw); 448 } 449 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); 450 451 /** 452 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock. 453 * 454 * To keep it NMI safe since we're accessing from tracing, we're not using a 455 * separate timekeeper with updates to monotonic clock and boot offset 456 * protected with seqlocks. This has the following minor side effects: 457 * 458 * (1) Its possible that a timestamp be taken after the boot offset is updated 459 * but before the timekeeper is updated. If this happens, the new boot offset 460 * is added to the old timekeeping making the clock appear to update slightly 461 * earlier: 462 * CPU 0 CPU 1 463 * timekeeping_inject_sleeptime64() 464 * __timekeeping_inject_sleeptime(tk, delta); 465 * timestamp(); 466 * timekeeping_update(tk, TK_CLEAR_NTP...); 467 * 468 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be 469 * partially updated. Since the tk->offs_boot update is a rare event, this 470 * should be a rare occurrence which postprocessing should be able to handle. 471 */ 472 u64 notrace ktime_get_boot_fast_ns(void) 473 { 474 struct timekeeper *tk = &tk_core.timekeeper; 475 476 return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot)); 477 } 478 EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); 479 480 /* Suspend-time cycles value for halted fast timekeeper. */ 481 static u64 cycles_at_suspend; 482 483 static u64 dummy_clock_read(struct clocksource *cs) 484 { 485 return cycles_at_suspend; 486 } 487 488 static struct clocksource dummy_clock = { 489 .read = dummy_clock_read, 490 }; 491 492 /** 493 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource. 494 * @tk: Timekeeper to snapshot. 495 * 496 * It generally is unsafe to access the clocksource after timekeeping has been 497 * suspended, so take a snapshot of the readout base of @tk and use it as the 498 * fast timekeeper's readout base while suspended. It will return the same 499 * number of cycles every time until timekeeping is resumed at which time the 500 * proper readout base for the fast timekeeper will be restored automatically. 501 */ 502 static void halt_fast_timekeeper(struct timekeeper *tk) 503 { 504 static struct tk_read_base tkr_dummy; 505 struct tk_read_base *tkr = &tk->tkr_mono; 506 507 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 508 cycles_at_suspend = tk_clock_read(tkr); 509 tkr_dummy.clock = &dummy_clock; 510 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono); 511 512 tkr = &tk->tkr_raw; 513 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 514 tkr_dummy.clock = &dummy_clock; 515 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw); 516 } 517 518 #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD 519 #warning Please contact your maintainers, as GENERIC_TIME_VSYSCALL_OLD compatibity will disappear soon. 520 521 static inline void update_vsyscall(struct timekeeper *tk) 522 { 523 struct timespec xt, wm; 524 525 xt = timespec64_to_timespec(tk_xtime(tk)); 526 wm = timespec64_to_timespec(tk->wall_to_monotonic); 527 update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult, 528 tk->tkr_mono.cycle_last); 529 } 530 531 static inline void old_vsyscall_fixup(struct timekeeper *tk) 532 { 533 s64 remainder; 534 535 /* 536 * Store only full nanoseconds into xtime_nsec after rounding 537 * it up and add the remainder to the error difference. 538 * XXX - This is necessary to avoid small 1ns inconsistnecies caused 539 * by truncating the remainder in vsyscalls. However, it causes 540 * additional work to be done in timekeeping_adjust(). Once 541 * the vsyscall implementations are converted to use xtime_nsec 542 * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD 543 * users are removed, this can be killed. 544 */ 545 remainder = tk->tkr_mono.xtime_nsec & ((1ULL << tk->tkr_mono.shift) - 1); 546 if (remainder != 0) { 547 tk->tkr_mono.xtime_nsec -= remainder; 548 tk->tkr_mono.xtime_nsec += 1ULL << tk->tkr_mono.shift; 549 tk->ntp_error += remainder << tk->ntp_error_shift; 550 tk->ntp_error -= (1ULL << tk->tkr_mono.shift) << tk->ntp_error_shift; 551 } 552 } 553 #else 554 #define old_vsyscall_fixup(tk) 555 #endif 556 557 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); 558 559 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) 560 { 561 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); 562 } 563 564 /** 565 * pvclock_gtod_register_notifier - register a pvclock timedata update listener 566 */ 567 int pvclock_gtod_register_notifier(struct notifier_block *nb) 568 { 569 struct timekeeper *tk = &tk_core.timekeeper; 570 unsigned long flags; 571 int ret; 572 573 raw_spin_lock_irqsave(&timekeeper_lock, flags); 574 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); 575 update_pvclock_gtod(tk, true); 576 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 577 578 return ret; 579 } 580 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier); 581 582 /** 583 * pvclock_gtod_unregister_notifier - unregister a pvclock 584 * timedata update listener 585 */ 586 int pvclock_gtod_unregister_notifier(struct notifier_block *nb) 587 { 588 unsigned long flags; 589 int ret; 590 591 raw_spin_lock_irqsave(&timekeeper_lock, flags); 592 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); 593 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 594 595 return ret; 596 } 597 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); 598 599 /* 600 * tk_update_leap_state - helper to update the next_leap_ktime 601 */ 602 static inline void tk_update_leap_state(struct timekeeper *tk) 603 { 604 tk->next_leap_ktime = ntp_get_next_leap(); 605 if (tk->next_leap_ktime != KTIME_MAX) 606 /* Convert to monotonic time */ 607 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); 608 } 609 610 /* 611 * Update the ktime_t based scalar nsec members of the timekeeper 612 */ 613 static inline void tk_update_ktime_data(struct timekeeper *tk) 614 { 615 u64 seconds; 616 u32 nsec; 617 618 /* 619 * The xtime based monotonic readout is: 620 * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); 621 * The ktime based monotonic readout is: 622 * nsec = base_mono + now(); 623 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec 624 */ 625 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); 626 nsec = (u32) tk->wall_to_monotonic.tv_nsec; 627 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 628 629 /* 630 * The sum of the nanoseconds portions of xtime and 631 * wall_to_monotonic can be greater/equal one second. Take 632 * this into account before updating tk->ktime_sec. 633 */ 634 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 635 if (nsec >= NSEC_PER_SEC) 636 seconds++; 637 tk->ktime_sec = seconds; 638 639 /* Update the monotonic raw base */ 640 seconds = tk->raw_sec; 641 nsec = (u32)(tk->tkr_raw.xtime_nsec >> tk->tkr_raw.shift); 642 tk->tkr_raw.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 643 } 644 645 /* must hold timekeeper_lock */ 646 static void timekeeping_update(struct timekeeper *tk, unsigned int action) 647 { 648 if (action & TK_CLEAR_NTP) { 649 tk->ntp_error = 0; 650 ntp_clear(); 651 } 652 653 tk_update_leap_state(tk); 654 tk_update_ktime_data(tk); 655 656 update_vsyscall(tk); 657 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); 658 659 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 660 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); 661 662 if (action & TK_CLOCK_WAS_SET) 663 tk->clock_was_set_seq++; 664 /* 665 * The mirroring of the data to the shadow-timekeeper needs 666 * to happen last here to ensure we don't over-write the 667 * timekeeper structure on the next update with stale data 668 */ 669 if (action & TK_MIRROR) 670 memcpy(&shadow_timekeeper, &tk_core.timekeeper, 671 sizeof(tk_core.timekeeper)); 672 } 673 674 /** 675 * timekeeping_forward_now - update clock to the current time 676 * 677 * Forward the current clock to update its state since the last call to 678 * update_wall_time(). This is useful before significant clock changes, 679 * as it avoids having to deal with this time offset explicitly. 680 */ 681 static void timekeeping_forward_now(struct timekeeper *tk) 682 { 683 u64 cycle_now, delta; 684 685 cycle_now = tk_clock_read(&tk->tkr_mono); 686 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask); 687 tk->tkr_mono.cycle_last = cycle_now; 688 tk->tkr_raw.cycle_last = cycle_now; 689 690 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult; 691 692 /* If arch requires, add in get_arch_timeoffset() */ 693 tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift; 694 695 696 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult; 697 698 /* If arch requires, add in get_arch_timeoffset() */ 699 tk->tkr_raw.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_raw.shift; 700 701 tk_normalize_xtime(tk); 702 } 703 704 /** 705 * __getnstimeofday64 - Returns the time of day in a timespec64. 706 * @ts: pointer to the timespec to be set 707 * 708 * Updates the time of day in the timespec. 709 * Returns 0 on success, or -ve when suspended (timespec will be undefined). 710 */ 711 int __getnstimeofday64(struct timespec64 *ts) 712 { 713 struct timekeeper *tk = &tk_core.timekeeper; 714 unsigned long seq; 715 u64 nsecs; 716 717 do { 718 seq = read_seqcount_begin(&tk_core.seq); 719 720 ts->tv_sec = tk->xtime_sec; 721 nsecs = timekeeping_get_ns(&tk->tkr_mono); 722 723 } while (read_seqcount_retry(&tk_core.seq, seq)); 724 725 ts->tv_nsec = 0; 726 timespec64_add_ns(ts, nsecs); 727 728 /* 729 * Do not bail out early, in case there were callers still using 730 * the value, even in the face of the WARN_ON. 731 */ 732 if (unlikely(timekeeping_suspended)) 733 return -EAGAIN; 734 return 0; 735 } 736 EXPORT_SYMBOL(__getnstimeofday64); 737 738 /** 739 * getnstimeofday64 - Returns the time of day in a timespec64. 740 * @ts: pointer to the timespec64 to be set 741 * 742 * Returns the time of day in a timespec64 (WARN if suspended). 743 */ 744 void getnstimeofday64(struct timespec64 *ts) 745 { 746 WARN_ON(__getnstimeofday64(ts)); 747 } 748 EXPORT_SYMBOL(getnstimeofday64); 749 750 ktime_t ktime_get(void) 751 { 752 struct timekeeper *tk = &tk_core.timekeeper; 753 unsigned int seq; 754 ktime_t base; 755 u64 nsecs; 756 757 WARN_ON(timekeeping_suspended); 758 759 do { 760 seq = read_seqcount_begin(&tk_core.seq); 761 base = tk->tkr_mono.base; 762 nsecs = timekeeping_get_ns(&tk->tkr_mono); 763 764 } while (read_seqcount_retry(&tk_core.seq, seq)); 765 766 return ktime_add_ns(base, nsecs); 767 } 768 EXPORT_SYMBOL_GPL(ktime_get); 769 770 u32 ktime_get_resolution_ns(void) 771 { 772 struct timekeeper *tk = &tk_core.timekeeper; 773 unsigned int seq; 774 u32 nsecs; 775 776 WARN_ON(timekeeping_suspended); 777 778 do { 779 seq = read_seqcount_begin(&tk_core.seq); 780 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; 781 } while (read_seqcount_retry(&tk_core.seq, seq)); 782 783 return nsecs; 784 } 785 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns); 786 787 static ktime_t *offsets[TK_OFFS_MAX] = { 788 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real, 789 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot, 790 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai, 791 }; 792 793 ktime_t ktime_get_with_offset(enum tk_offsets offs) 794 { 795 struct timekeeper *tk = &tk_core.timekeeper; 796 unsigned int seq; 797 ktime_t base, *offset = offsets[offs]; 798 u64 nsecs; 799 800 WARN_ON(timekeeping_suspended); 801 802 do { 803 seq = read_seqcount_begin(&tk_core.seq); 804 base = ktime_add(tk->tkr_mono.base, *offset); 805 nsecs = timekeeping_get_ns(&tk->tkr_mono); 806 807 } while (read_seqcount_retry(&tk_core.seq, seq)); 808 809 return ktime_add_ns(base, nsecs); 810 811 } 812 EXPORT_SYMBOL_GPL(ktime_get_with_offset); 813 814 /** 815 * ktime_mono_to_any() - convert mononotic time to any other time 816 * @tmono: time to convert. 817 * @offs: which offset to use 818 */ 819 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) 820 { 821 ktime_t *offset = offsets[offs]; 822 unsigned long seq; 823 ktime_t tconv; 824 825 do { 826 seq = read_seqcount_begin(&tk_core.seq); 827 tconv = ktime_add(tmono, *offset); 828 } while (read_seqcount_retry(&tk_core.seq, seq)); 829 830 return tconv; 831 } 832 EXPORT_SYMBOL_GPL(ktime_mono_to_any); 833 834 /** 835 * ktime_get_raw - Returns the raw monotonic time in ktime_t format 836 */ 837 ktime_t ktime_get_raw(void) 838 { 839 struct timekeeper *tk = &tk_core.timekeeper; 840 unsigned int seq; 841 ktime_t base; 842 u64 nsecs; 843 844 do { 845 seq = read_seqcount_begin(&tk_core.seq); 846 base = tk->tkr_raw.base; 847 nsecs = timekeeping_get_ns(&tk->tkr_raw); 848 849 } while (read_seqcount_retry(&tk_core.seq, seq)); 850 851 return ktime_add_ns(base, nsecs); 852 } 853 EXPORT_SYMBOL_GPL(ktime_get_raw); 854 855 /** 856 * ktime_get_ts64 - get the monotonic clock in timespec64 format 857 * @ts: pointer to timespec variable 858 * 859 * The function calculates the monotonic clock from the realtime 860 * clock and the wall_to_monotonic offset and stores the result 861 * in normalized timespec64 format in the variable pointed to by @ts. 862 */ 863 void ktime_get_ts64(struct timespec64 *ts) 864 { 865 struct timekeeper *tk = &tk_core.timekeeper; 866 struct timespec64 tomono; 867 unsigned int seq; 868 u64 nsec; 869 870 WARN_ON(timekeeping_suspended); 871 872 do { 873 seq = read_seqcount_begin(&tk_core.seq); 874 ts->tv_sec = tk->xtime_sec; 875 nsec = timekeeping_get_ns(&tk->tkr_mono); 876 tomono = tk->wall_to_monotonic; 877 878 } while (read_seqcount_retry(&tk_core.seq, seq)); 879 880 ts->tv_sec += tomono.tv_sec; 881 ts->tv_nsec = 0; 882 timespec64_add_ns(ts, nsec + tomono.tv_nsec); 883 } 884 EXPORT_SYMBOL_GPL(ktime_get_ts64); 885 886 /** 887 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC 888 * 889 * Returns the seconds portion of CLOCK_MONOTONIC with a single non 890 * serialized read. tk->ktime_sec is of type 'unsigned long' so this 891 * works on both 32 and 64 bit systems. On 32 bit systems the readout 892 * covers ~136 years of uptime which should be enough to prevent 893 * premature wrap arounds. 894 */ 895 time64_t ktime_get_seconds(void) 896 { 897 struct timekeeper *tk = &tk_core.timekeeper; 898 899 WARN_ON(timekeeping_suspended); 900 return tk->ktime_sec; 901 } 902 EXPORT_SYMBOL_GPL(ktime_get_seconds); 903 904 /** 905 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME 906 * 907 * Returns the wall clock seconds since 1970. This replaces the 908 * get_seconds() interface which is not y2038 safe on 32bit systems. 909 * 910 * For 64bit systems the fast access to tk->xtime_sec is preserved. On 911 * 32bit systems the access must be protected with the sequence 912 * counter to provide "atomic" access to the 64bit tk->xtime_sec 913 * value. 914 */ 915 time64_t ktime_get_real_seconds(void) 916 { 917 struct timekeeper *tk = &tk_core.timekeeper; 918 time64_t seconds; 919 unsigned int seq; 920 921 if (IS_ENABLED(CONFIG_64BIT)) 922 return tk->xtime_sec; 923 924 do { 925 seq = read_seqcount_begin(&tk_core.seq); 926 seconds = tk->xtime_sec; 927 928 } while (read_seqcount_retry(&tk_core.seq, seq)); 929 930 return seconds; 931 } 932 EXPORT_SYMBOL_GPL(ktime_get_real_seconds); 933 934 /** 935 * __ktime_get_real_seconds - The same as ktime_get_real_seconds 936 * but without the sequence counter protect. This internal function 937 * is called just when timekeeping lock is already held. 938 */ 939 time64_t __ktime_get_real_seconds(void) 940 { 941 struct timekeeper *tk = &tk_core.timekeeper; 942 943 return tk->xtime_sec; 944 } 945 946 /** 947 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter 948 * @systime_snapshot: pointer to struct receiving the system time snapshot 949 */ 950 void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot) 951 { 952 struct timekeeper *tk = &tk_core.timekeeper; 953 unsigned long seq; 954 ktime_t base_raw; 955 ktime_t base_real; 956 u64 nsec_raw; 957 u64 nsec_real; 958 u64 now; 959 960 WARN_ON_ONCE(timekeeping_suspended); 961 962 do { 963 seq = read_seqcount_begin(&tk_core.seq); 964 now = tk_clock_read(&tk->tkr_mono); 965 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; 966 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; 967 base_real = ktime_add(tk->tkr_mono.base, 968 tk_core.timekeeper.offs_real); 969 base_raw = tk->tkr_raw.base; 970 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); 971 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); 972 } while (read_seqcount_retry(&tk_core.seq, seq)); 973 974 systime_snapshot->cycles = now; 975 systime_snapshot->real = ktime_add_ns(base_real, nsec_real); 976 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw); 977 } 978 EXPORT_SYMBOL_GPL(ktime_get_snapshot); 979 980 /* Scale base by mult/div checking for overflow */ 981 static int scale64_check_overflow(u64 mult, u64 div, u64 *base) 982 { 983 u64 tmp, rem; 984 985 tmp = div64_u64_rem(*base, div, &rem); 986 987 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) || 988 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem))) 989 return -EOVERFLOW; 990 tmp *= mult; 991 rem *= mult; 992 993 do_div(rem, div); 994 *base = tmp + rem; 995 return 0; 996 } 997 998 /** 999 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval 1000 * @history: Snapshot representing start of history 1001 * @partial_history_cycles: Cycle offset into history (fractional part) 1002 * @total_history_cycles: Total history length in cycles 1003 * @discontinuity: True indicates clock was set on history period 1004 * @ts: Cross timestamp that should be adjusted using 1005 * partial/total ratio 1006 * 1007 * Helper function used by get_device_system_crosststamp() to correct the 1008 * crosstimestamp corresponding to the start of the current interval to the 1009 * system counter value (timestamp point) provided by the driver. The 1010 * total_history_* quantities are the total history starting at the provided 1011 * reference point and ending at the start of the current interval. The cycle 1012 * count between the driver timestamp point and the start of the current 1013 * interval is partial_history_cycles. 1014 */ 1015 static int adjust_historical_crosststamp(struct system_time_snapshot *history, 1016 u64 partial_history_cycles, 1017 u64 total_history_cycles, 1018 bool discontinuity, 1019 struct system_device_crosststamp *ts) 1020 { 1021 struct timekeeper *tk = &tk_core.timekeeper; 1022 u64 corr_raw, corr_real; 1023 bool interp_forward; 1024 int ret; 1025 1026 if (total_history_cycles == 0 || partial_history_cycles == 0) 1027 return 0; 1028 1029 /* Interpolate shortest distance from beginning or end of history */ 1030 interp_forward = partial_history_cycles > total_history_cycles / 2; 1031 partial_history_cycles = interp_forward ? 1032 total_history_cycles - partial_history_cycles : 1033 partial_history_cycles; 1034 1035 /* 1036 * Scale the monotonic raw time delta by: 1037 * partial_history_cycles / total_history_cycles 1038 */ 1039 corr_raw = (u64)ktime_to_ns( 1040 ktime_sub(ts->sys_monoraw, history->raw)); 1041 ret = scale64_check_overflow(partial_history_cycles, 1042 total_history_cycles, &corr_raw); 1043 if (ret) 1044 return ret; 1045 1046 /* 1047 * If there is a discontinuity in the history, scale monotonic raw 1048 * correction by: 1049 * mult(real)/mult(raw) yielding the realtime correction 1050 * Otherwise, calculate the realtime correction similar to monotonic 1051 * raw calculation 1052 */ 1053 if (discontinuity) { 1054 corr_real = mul_u64_u32_div 1055 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); 1056 } else { 1057 corr_real = (u64)ktime_to_ns( 1058 ktime_sub(ts->sys_realtime, history->real)); 1059 ret = scale64_check_overflow(partial_history_cycles, 1060 total_history_cycles, &corr_real); 1061 if (ret) 1062 return ret; 1063 } 1064 1065 /* Fixup monotonic raw and real time time values */ 1066 if (interp_forward) { 1067 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw); 1068 ts->sys_realtime = ktime_add_ns(history->real, corr_real); 1069 } else { 1070 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); 1071 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); 1072 } 1073 1074 return 0; 1075 } 1076 1077 /* 1078 * cycle_between - true if test occurs chronologically between before and after 1079 */ 1080 static bool cycle_between(u64 before, u64 test, u64 after) 1081 { 1082 if (test > before && test < after) 1083 return true; 1084 if (test < before && before > after) 1085 return true; 1086 return false; 1087 } 1088 1089 /** 1090 * get_device_system_crosststamp - Synchronously capture system/device timestamp 1091 * @get_time_fn: Callback to get simultaneous device time and 1092 * system counter from the device driver 1093 * @ctx: Context passed to get_time_fn() 1094 * @history_begin: Historical reference point used to interpolate system 1095 * time when counter provided by the driver is before the current interval 1096 * @xtstamp: Receives simultaneously captured system and device time 1097 * 1098 * Reads a timestamp from a device and correlates it to system time 1099 */ 1100 int get_device_system_crosststamp(int (*get_time_fn) 1101 (ktime_t *device_time, 1102 struct system_counterval_t *sys_counterval, 1103 void *ctx), 1104 void *ctx, 1105 struct system_time_snapshot *history_begin, 1106 struct system_device_crosststamp *xtstamp) 1107 { 1108 struct system_counterval_t system_counterval; 1109 struct timekeeper *tk = &tk_core.timekeeper; 1110 u64 cycles, now, interval_start; 1111 unsigned int clock_was_set_seq = 0; 1112 ktime_t base_real, base_raw; 1113 u64 nsec_real, nsec_raw; 1114 u8 cs_was_changed_seq; 1115 unsigned long seq; 1116 bool do_interp; 1117 int ret; 1118 1119 do { 1120 seq = read_seqcount_begin(&tk_core.seq); 1121 /* 1122 * Try to synchronously capture device time and a system 1123 * counter value calling back into the device driver 1124 */ 1125 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx); 1126 if (ret) 1127 return ret; 1128 1129 /* 1130 * Verify that the clocksource associated with the captured 1131 * system counter value is the same as the currently installed 1132 * timekeeper clocksource 1133 */ 1134 if (tk->tkr_mono.clock != system_counterval.cs) 1135 return -ENODEV; 1136 cycles = system_counterval.cycles; 1137 1138 /* 1139 * Check whether the system counter value provided by the 1140 * device driver is on the current timekeeping interval. 1141 */ 1142 now = tk_clock_read(&tk->tkr_mono); 1143 interval_start = tk->tkr_mono.cycle_last; 1144 if (!cycle_between(interval_start, cycles, now)) { 1145 clock_was_set_seq = tk->clock_was_set_seq; 1146 cs_was_changed_seq = tk->cs_was_changed_seq; 1147 cycles = interval_start; 1148 do_interp = true; 1149 } else { 1150 do_interp = false; 1151 } 1152 1153 base_real = ktime_add(tk->tkr_mono.base, 1154 tk_core.timekeeper.offs_real); 1155 base_raw = tk->tkr_raw.base; 1156 1157 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, 1158 system_counterval.cycles); 1159 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, 1160 system_counterval.cycles); 1161 } while (read_seqcount_retry(&tk_core.seq, seq)); 1162 1163 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real); 1164 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); 1165 1166 /* 1167 * Interpolate if necessary, adjusting back from the start of the 1168 * current interval 1169 */ 1170 if (do_interp) { 1171 u64 partial_history_cycles, total_history_cycles; 1172 bool discontinuity; 1173 1174 /* 1175 * Check that the counter value occurs after the provided 1176 * history reference and that the history doesn't cross a 1177 * clocksource change 1178 */ 1179 if (!history_begin || 1180 !cycle_between(history_begin->cycles, 1181 system_counterval.cycles, cycles) || 1182 history_begin->cs_was_changed_seq != cs_was_changed_seq) 1183 return -EINVAL; 1184 partial_history_cycles = cycles - system_counterval.cycles; 1185 total_history_cycles = cycles - history_begin->cycles; 1186 discontinuity = 1187 history_begin->clock_was_set_seq != clock_was_set_seq; 1188 1189 ret = adjust_historical_crosststamp(history_begin, 1190 partial_history_cycles, 1191 total_history_cycles, 1192 discontinuity, xtstamp); 1193 if (ret) 1194 return ret; 1195 } 1196 1197 return 0; 1198 } 1199 EXPORT_SYMBOL_GPL(get_device_system_crosststamp); 1200 1201 /** 1202 * do_gettimeofday - Returns the time of day in a timeval 1203 * @tv: pointer to the timeval to be set 1204 * 1205 * NOTE: Users should be converted to using getnstimeofday() 1206 */ 1207 void do_gettimeofday(struct timeval *tv) 1208 { 1209 struct timespec64 now; 1210 1211 getnstimeofday64(&now); 1212 tv->tv_sec = now.tv_sec; 1213 tv->tv_usec = now.tv_nsec/1000; 1214 } 1215 EXPORT_SYMBOL(do_gettimeofday); 1216 1217 /** 1218 * do_settimeofday64 - Sets the time of day. 1219 * @ts: pointer to the timespec64 variable containing the new time 1220 * 1221 * Sets the time of day to the new time and update NTP and notify hrtimers 1222 */ 1223 int do_settimeofday64(const struct timespec64 *ts) 1224 { 1225 struct timekeeper *tk = &tk_core.timekeeper; 1226 struct timespec64 ts_delta, xt; 1227 unsigned long flags; 1228 int ret = 0; 1229 1230 if (!timespec64_valid_strict(ts)) 1231 return -EINVAL; 1232 1233 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1234 write_seqcount_begin(&tk_core.seq); 1235 1236 timekeeping_forward_now(tk); 1237 1238 xt = tk_xtime(tk); 1239 ts_delta.tv_sec = ts->tv_sec - xt.tv_sec; 1240 ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec; 1241 1242 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) { 1243 ret = -EINVAL; 1244 goto out; 1245 } 1246 1247 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta)); 1248 1249 tk_set_xtime(tk, ts); 1250 out: 1251 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 1252 1253 write_seqcount_end(&tk_core.seq); 1254 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1255 1256 /* signal hrtimers about time change */ 1257 clock_was_set(); 1258 1259 return ret; 1260 } 1261 EXPORT_SYMBOL(do_settimeofday64); 1262 1263 /** 1264 * timekeeping_inject_offset - Adds or subtracts from the current time. 1265 * @tv: pointer to the timespec variable containing the offset 1266 * 1267 * Adds or subtracts an offset value from the current time. 1268 */ 1269 int timekeeping_inject_offset(struct timespec *ts) 1270 { 1271 struct timekeeper *tk = &tk_core.timekeeper; 1272 unsigned long flags; 1273 struct timespec64 ts64, tmp; 1274 int ret = 0; 1275 1276 if (!timespec_inject_offset_valid(ts)) 1277 return -EINVAL; 1278 1279 ts64 = timespec_to_timespec64(*ts); 1280 1281 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1282 write_seqcount_begin(&tk_core.seq); 1283 1284 timekeeping_forward_now(tk); 1285 1286 /* Make sure the proposed value is valid */ 1287 tmp = timespec64_add(tk_xtime(tk), ts64); 1288 if (timespec64_compare(&tk->wall_to_monotonic, &ts64) > 0 || 1289 !timespec64_valid_strict(&tmp)) { 1290 ret = -EINVAL; 1291 goto error; 1292 } 1293 1294 tk_xtime_add(tk, &ts64); 1295 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64)); 1296 1297 error: /* even if we error out, we forwarded the time, so call update */ 1298 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 1299 1300 write_seqcount_end(&tk_core.seq); 1301 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1302 1303 /* signal hrtimers about time change */ 1304 clock_was_set(); 1305 1306 return ret; 1307 } 1308 EXPORT_SYMBOL(timekeeping_inject_offset); 1309 1310 /** 1311 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic 1312 * 1313 */ 1314 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) 1315 { 1316 tk->tai_offset = tai_offset; 1317 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 1318 } 1319 1320 /** 1321 * change_clocksource - Swaps clocksources if a new one is available 1322 * 1323 * Accumulates current time interval and initializes new clocksource 1324 */ 1325 static int change_clocksource(void *data) 1326 { 1327 struct timekeeper *tk = &tk_core.timekeeper; 1328 struct clocksource *new, *old; 1329 unsigned long flags; 1330 1331 new = (struct clocksource *) data; 1332 1333 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1334 write_seqcount_begin(&tk_core.seq); 1335 1336 timekeeping_forward_now(tk); 1337 /* 1338 * If the cs is in module, get a module reference. Succeeds 1339 * for built-in code (owner == NULL) as well. 1340 */ 1341 if (try_module_get(new->owner)) { 1342 if (!new->enable || new->enable(new) == 0) { 1343 old = tk->tkr_mono.clock; 1344 tk_setup_internals(tk, new); 1345 if (old->disable) 1346 old->disable(old); 1347 module_put(old->owner); 1348 } else { 1349 module_put(new->owner); 1350 } 1351 } 1352 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 1353 1354 write_seqcount_end(&tk_core.seq); 1355 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1356 1357 return 0; 1358 } 1359 1360 /** 1361 * timekeeping_notify - Install a new clock source 1362 * @clock: pointer to the clock source 1363 * 1364 * This function is called from clocksource.c after a new, better clock 1365 * source has been registered. The caller holds the clocksource_mutex. 1366 */ 1367 int timekeeping_notify(struct clocksource *clock) 1368 { 1369 struct timekeeper *tk = &tk_core.timekeeper; 1370 1371 if (tk->tkr_mono.clock == clock) 1372 return 0; 1373 stop_machine(change_clocksource, clock, NULL); 1374 tick_clock_notify(); 1375 return tk->tkr_mono.clock == clock ? 0 : -1; 1376 } 1377 1378 /** 1379 * getrawmonotonic64 - Returns the raw monotonic time in a timespec 1380 * @ts: pointer to the timespec64 to be set 1381 * 1382 * Returns the raw monotonic time (completely un-modified by ntp) 1383 */ 1384 void getrawmonotonic64(struct timespec64 *ts) 1385 { 1386 struct timekeeper *tk = &tk_core.timekeeper; 1387 unsigned long seq; 1388 u64 nsecs; 1389 1390 do { 1391 seq = read_seqcount_begin(&tk_core.seq); 1392 ts->tv_sec = tk->raw_sec; 1393 nsecs = timekeeping_get_ns(&tk->tkr_raw); 1394 1395 } while (read_seqcount_retry(&tk_core.seq, seq)); 1396 1397 ts->tv_nsec = 0; 1398 timespec64_add_ns(ts, nsecs); 1399 } 1400 EXPORT_SYMBOL(getrawmonotonic64); 1401 1402 1403 /** 1404 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 1405 */ 1406 int timekeeping_valid_for_hres(void) 1407 { 1408 struct timekeeper *tk = &tk_core.timekeeper; 1409 unsigned long seq; 1410 int ret; 1411 1412 do { 1413 seq = read_seqcount_begin(&tk_core.seq); 1414 1415 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 1416 1417 } while (read_seqcount_retry(&tk_core.seq, seq)); 1418 1419 return ret; 1420 } 1421 1422 /** 1423 * timekeeping_max_deferment - Returns max time the clocksource can be deferred 1424 */ 1425 u64 timekeeping_max_deferment(void) 1426 { 1427 struct timekeeper *tk = &tk_core.timekeeper; 1428 unsigned long seq; 1429 u64 ret; 1430 1431 do { 1432 seq = read_seqcount_begin(&tk_core.seq); 1433 1434 ret = tk->tkr_mono.clock->max_idle_ns; 1435 1436 } while (read_seqcount_retry(&tk_core.seq, seq)); 1437 1438 return ret; 1439 } 1440 1441 /** 1442 * read_persistent_clock - Return time from the persistent clock. 1443 * 1444 * Weak dummy function for arches that do not yet support it. 1445 * Reads the time from the battery backed persistent clock. 1446 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported. 1447 * 1448 * XXX - Do be sure to remove it once all arches implement it. 1449 */ 1450 void __weak read_persistent_clock(struct timespec *ts) 1451 { 1452 ts->tv_sec = 0; 1453 ts->tv_nsec = 0; 1454 } 1455 1456 void __weak read_persistent_clock64(struct timespec64 *ts64) 1457 { 1458 struct timespec ts; 1459 1460 read_persistent_clock(&ts); 1461 *ts64 = timespec_to_timespec64(ts); 1462 } 1463 1464 /** 1465 * read_boot_clock64 - Return time of the system start. 1466 * 1467 * Weak dummy function for arches that do not yet support it. 1468 * Function to read the exact time the system has been started. 1469 * Returns a timespec64 with tv_sec=0 and tv_nsec=0 if unsupported. 1470 * 1471 * XXX - Do be sure to remove it once all arches implement it. 1472 */ 1473 void __weak read_boot_clock64(struct timespec64 *ts) 1474 { 1475 ts->tv_sec = 0; 1476 ts->tv_nsec = 0; 1477 } 1478 1479 /* Flag for if timekeeping_resume() has injected sleeptime */ 1480 static bool sleeptime_injected; 1481 1482 /* Flag for if there is a persistent clock on this platform */ 1483 static bool persistent_clock_exists; 1484 1485 /* 1486 * timekeeping_init - Initializes the clocksource and common timekeeping values 1487 */ 1488 void __init timekeeping_init(void) 1489 { 1490 struct timekeeper *tk = &tk_core.timekeeper; 1491 struct clocksource *clock; 1492 unsigned long flags; 1493 struct timespec64 now, boot, tmp; 1494 1495 read_persistent_clock64(&now); 1496 if (!timespec64_valid_strict(&now)) { 1497 pr_warn("WARNING: Persistent clock returned invalid value!\n" 1498 " Check your CMOS/BIOS settings.\n"); 1499 now.tv_sec = 0; 1500 now.tv_nsec = 0; 1501 } else if (now.tv_sec || now.tv_nsec) 1502 persistent_clock_exists = true; 1503 1504 read_boot_clock64(&boot); 1505 if (!timespec64_valid_strict(&boot)) { 1506 pr_warn("WARNING: Boot clock returned invalid value!\n" 1507 " Check your CMOS/BIOS settings.\n"); 1508 boot.tv_sec = 0; 1509 boot.tv_nsec = 0; 1510 } 1511 1512 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1513 write_seqcount_begin(&tk_core.seq); 1514 ntp_init(); 1515 1516 clock = clocksource_default_clock(); 1517 if (clock->enable) 1518 clock->enable(clock); 1519 tk_setup_internals(tk, clock); 1520 1521 tk_set_xtime(tk, &now); 1522 tk->raw_sec = 0; 1523 if (boot.tv_sec == 0 && boot.tv_nsec == 0) 1524 boot = tk_xtime(tk); 1525 1526 set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec); 1527 tk_set_wall_to_mono(tk, tmp); 1528 1529 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 1530 1531 write_seqcount_end(&tk_core.seq); 1532 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1533 } 1534 1535 /* time in seconds when suspend began for persistent clock */ 1536 static struct timespec64 timekeeping_suspend_time; 1537 1538 /** 1539 * __timekeeping_inject_sleeptime - Internal function to add sleep interval 1540 * @delta: pointer to a timespec delta value 1541 * 1542 * Takes a timespec offset measuring a suspend interval and properly 1543 * adds the sleep offset to the timekeeping variables. 1544 */ 1545 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, 1546 struct timespec64 *delta) 1547 { 1548 if (!timespec64_valid_strict(delta)) { 1549 printk_deferred(KERN_WARNING 1550 "__timekeeping_inject_sleeptime: Invalid " 1551 "sleep delta value!\n"); 1552 return; 1553 } 1554 tk_xtime_add(tk, delta); 1555 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); 1556 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); 1557 tk_debug_account_sleep_time(delta); 1558 } 1559 1560 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE) 1561 /** 1562 * We have three kinds of time sources to use for sleep time 1563 * injection, the preference order is: 1564 * 1) non-stop clocksource 1565 * 2) persistent clock (ie: RTC accessible when irqs are off) 1566 * 3) RTC 1567 * 1568 * 1) and 2) are used by timekeeping, 3) by RTC subsystem. 1569 * If system has neither 1) nor 2), 3) will be used finally. 1570 * 1571 * 1572 * If timekeeping has injected sleeptime via either 1) or 2), 1573 * 3) becomes needless, so in this case we don't need to call 1574 * rtc_resume(), and this is what timekeeping_rtc_skipresume() 1575 * means. 1576 */ 1577 bool timekeeping_rtc_skipresume(void) 1578 { 1579 return sleeptime_injected; 1580 } 1581 1582 /** 1583 * 1) can be determined whether to use or not only when doing 1584 * timekeeping_resume() which is invoked after rtc_suspend(), 1585 * so we can't skip rtc_suspend() surely if system has 1). 1586 * 1587 * But if system has 2), 2) will definitely be used, so in this 1588 * case we don't need to call rtc_suspend(), and this is what 1589 * timekeeping_rtc_skipsuspend() means. 1590 */ 1591 bool timekeeping_rtc_skipsuspend(void) 1592 { 1593 return persistent_clock_exists; 1594 } 1595 1596 /** 1597 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values 1598 * @delta: pointer to a timespec64 delta value 1599 * 1600 * This hook is for architectures that cannot support read_persistent_clock64 1601 * because their RTC/persistent clock is only accessible when irqs are enabled. 1602 * and also don't have an effective nonstop clocksource. 1603 * 1604 * This function should only be called by rtc_resume(), and allows 1605 * a suspend offset to be injected into the timekeeping values. 1606 */ 1607 void timekeeping_inject_sleeptime64(struct timespec64 *delta) 1608 { 1609 struct timekeeper *tk = &tk_core.timekeeper; 1610 unsigned long flags; 1611 1612 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1613 write_seqcount_begin(&tk_core.seq); 1614 1615 timekeeping_forward_now(tk); 1616 1617 __timekeeping_inject_sleeptime(tk, delta); 1618 1619 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); 1620 1621 write_seqcount_end(&tk_core.seq); 1622 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1623 1624 /* signal hrtimers about time change */ 1625 clock_was_set(); 1626 } 1627 #endif 1628 1629 /** 1630 * timekeeping_resume - Resumes the generic timekeeping subsystem. 1631 */ 1632 void timekeeping_resume(void) 1633 { 1634 struct timekeeper *tk = &tk_core.timekeeper; 1635 struct clocksource *clock = tk->tkr_mono.clock; 1636 unsigned long flags; 1637 struct timespec64 ts_new, ts_delta; 1638 u64 cycle_now; 1639 1640 sleeptime_injected = false; 1641 read_persistent_clock64(&ts_new); 1642 1643 clockevents_resume(); 1644 clocksource_resume(); 1645 1646 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1647 write_seqcount_begin(&tk_core.seq); 1648 1649 /* 1650 * After system resumes, we need to calculate the suspended time and 1651 * compensate it for the OS time. There are 3 sources that could be 1652 * used: Nonstop clocksource during suspend, persistent clock and rtc 1653 * device. 1654 * 1655 * One specific platform may have 1 or 2 or all of them, and the 1656 * preference will be: 1657 * suspend-nonstop clocksource -> persistent clock -> rtc 1658 * The less preferred source will only be tried if there is no better 1659 * usable source. The rtc part is handled separately in rtc core code. 1660 */ 1661 cycle_now = tk_clock_read(&tk->tkr_mono); 1662 if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) && 1663 cycle_now > tk->tkr_mono.cycle_last) { 1664 u64 nsec, cyc_delta; 1665 1666 cyc_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, 1667 tk->tkr_mono.mask); 1668 nsec = mul_u64_u32_shr(cyc_delta, clock->mult, clock->shift); 1669 ts_delta = ns_to_timespec64(nsec); 1670 sleeptime_injected = true; 1671 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) { 1672 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time); 1673 sleeptime_injected = true; 1674 } 1675 1676 if (sleeptime_injected) 1677 __timekeeping_inject_sleeptime(tk, &ts_delta); 1678 1679 /* Re-base the last cycle value */ 1680 tk->tkr_mono.cycle_last = cycle_now; 1681 tk->tkr_raw.cycle_last = cycle_now; 1682 1683 tk->ntp_error = 0; 1684 timekeeping_suspended = 0; 1685 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 1686 write_seqcount_end(&tk_core.seq); 1687 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1688 1689 touch_softlockup_watchdog(); 1690 1691 tick_resume(); 1692 hrtimers_resume(); 1693 } 1694 1695 int timekeeping_suspend(void) 1696 { 1697 struct timekeeper *tk = &tk_core.timekeeper; 1698 unsigned long flags; 1699 struct timespec64 delta, delta_delta; 1700 static struct timespec64 old_delta; 1701 1702 read_persistent_clock64(&timekeeping_suspend_time); 1703 1704 /* 1705 * On some systems the persistent_clock can not be detected at 1706 * timekeeping_init by its return value, so if we see a valid 1707 * value returned, update the persistent_clock_exists flag. 1708 */ 1709 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec) 1710 persistent_clock_exists = true; 1711 1712 raw_spin_lock_irqsave(&timekeeper_lock, flags); 1713 write_seqcount_begin(&tk_core.seq); 1714 timekeeping_forward_now(tk); 1715 timekeeping_suspended = 1; 1716 1717 if (persistent_clock_exists) { 1718 /* 1719 * To avoid drift caused by repeated suspend/resumes, 1720 * which each can add ~1 second drift error, 1721 * try to compensate so the difference in system time 1722 * and persistent_clock time stays close to constant. 1723 */ 1724 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time); 1725 delta_delta = timespec64_sub(delta, old_delta); 1726 if (abs(delta_delta.tv_sec) >= 2) { 1727 /* 1728 * if delta_delta is too large, assume time correction 1729 * has occurred and set old_delta to the current delta. 1730 */ 1731 old_delta = delta; 1732 } else { 1733 /* Otherwise try to adjust old_system to compensate */ 1734 timekeeping_suspend_time = 1735 timespec64_add(timekeeping_suspend_time, delta_delta); 1736 } 1737 } 1738 1739 timekeeping_update(tk, TK_MIRROR); 1740 halt_fast_timekeeper(tk); 1741 write_seqcount_end(&tk_core.seq); 1742 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 1743 1744 tick_suspend(); 1745 clocksource_suspend(); 1746 clockevents_suspend(); 1747 1748 return 0; 1749 } 1750 1751 /* sysfs resume/suspend bits for timekeeping */ 1752 static struct syscore_ops timekeeping_syscore_ops = { 1753 .resume = timekeeping_resume, 1754 .suspend = timekeeping_suspend, 1755 }; 1756 1757 static int __init timekeeping_init_ops(void) 1758 { 1759 register_syscore_ops(&timekeeping_syscore_ops); 1760 return 0; 1761 } 1762 device_initcall(timekeeping_init_ops); 1763 1764 /* 1765 * Apply a multiplier adjustment to the timekeeper 1766 */ 1767 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, 1768 s64 offset, 1769 bool negative, 1770 int adj_scale) 1771 { 1772 s64 interval = tk->cycle_interval; 1773 s32 mult_adj = 1; 1774 1775 if (negative) { 1776 mult_adj = -mult_adj; 1777 interval = -interval; 1778 offset = -offset; 1779 } 1780 mult_adj <<= adj_scale; 1781 interval <<= adj_scale; 1782 offset <<= adj_scale; 1783 1784 /* 1785 * So the following can be confusing. 1786 * 1787 * To keep things simple, lets assume mult_adj == 1 for now. 1788 * 1789 * When mult_adj != 1, remember that the interval and offset values 1790 * have been appropriately scaled so the math is the same. 1791 * 1792 * The basic idea here is that we're increasing the multiplier 1793 * by one, this causes the xtime_interval to be incremented by 1794 * one cycle_interval. This is because: 1795 * xtime_interval = cycle_interval * mult 1796 * So if mult is being incremented by one: 1797 * xtime_interval = cycle_interval * (mult + 1) 1798 * Its the same as: 1799 * xtime_interval = (cycle_interval * mult) + cycle_interval 1800 * Which can be shortened to: 1801 * xtime_interval += cycle_interval 1802 * 1803 * So offset stores the non-accumulated cycles. Thus the current 1804 * time (in shifted nanoseconds) is: 1805 * now = (offset * adj) + xtime_nsec 1806 * Now, even though we're adjusting the clock frequency, we have 1807 * to keep time consistent. In other words, we can't jump back 1808 * in time, and we also want to avoid jumping forward in time. 1809 * 1810 * So given the same offset value, we need the time to be the same 1811 * both before and after the freq adjustment. 1812 * now = (offset * adj_1) + xtime_nsec_1 1813 * now = (offset * adj_2) + xtime_nsec_2 1814 * So: 1815 * (offset * adj_1) + xtime_nsec_1 = 1816 * (offset * adj_2) + xtime_nsec_2 1817 * And we know: 1818 * adj_2 = adj_1 + 1 1819 * So: 1820 * (offset * adj_1) + xtime_nsec_1 = 1821 * (offset * (adj_1+1)) + xtime_nsec_2 1822 * (offset * adj_1) + xtime_nsec_1 = 1823 * (offset * adj_1) + offset + xtime_nsec_2 1824 * Canceling the sides: 1825 * xtime_nsec_1 = offset + xtime_nsec_2 1826 * Which gives us: 1827 * xtime_nsec_2 = xtime_nsec_1 - offset 1828 * Which simplfies to: 1829 * xtime_nsec -= offset 1830 * 1831 * XXX - TODO: Doc ntp_error calculation. 1832 */ 1833 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { 1834 /* NTP adjustment caused clocksource mult overflow */ 1835 WARN_ON_ONCE(1); 1836 return; 1837 } 1838 1839 tk->tkr_mono.mult += mult_adj; 1840 tk->xtime_interval += interval; 1841 tk->tkr_mono.xtime_nsec -= offset; 1842 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; 1843 } 1844 1845 /* 1846 * Calculate the multiplier adjustment needed to match the frequency 1847 * specified by NTP 1848 */ 1849 static __always_inline void timekeeping_freqadjust(struct timekeeper *tk, 1850 s64 offset) 1851 { 1852 s64 interval = tk->cycle_interval; 1853 s64 xinterval = tk->xtime_interval; 1854 u32 base = tk->tkr_mono.clock->mult; 1855 u32 max = tk->tkr_mono.clock->maxadj; 1856 u32 cur_adj = tk->tkr_mono.mult; 1857 s64 tick_error; 1858 bool negative; 1859 u32 adj_scale; 1860 1861 /* Remove any current error adj from freq calculation */ 1862 if (tk->ntp_err_mult) 1863 xinterval -= tk->cycle_interval; 1864 1865 tk->ntp_tick = ntp_tick_length(); 1866 1867 /* Calculate current error per tick */ 1868 tick_error = ntp_tick_length() >> tk->ntp_error_shift; 1869 tick_error -= (xinterval + tk->xtime_remainder); 1870 1871 /* Don't worry about correcting it if its small */ 1872 if (likely((tick_error >= 0) && (tick_error <= interval))) 1873 return; 1874 1875 /* preserve the direction of correction */ 1876 negative = (tick_error < 0); 1877 1878 /* If any adjustment would pass the max, just return */ 1879 if (negative && (cur_adj - 1) <= (base - max)) 1880 return; 1881 if (!negative && (cur_adj + 1) >= (base + max)) 1882 return; 1883 /* 1884 * Sort out the magnitude of the correction, but 1885 * avoid making so large a correction that we go 1886 * over the max adjustment. 1887 */ 1888 adj_scale = 0; 1889 tick_error = abs(tick_error); 1890 while (tick_error > interval) { 1891 u32 adj = 1 << (adj_scale + 1); 1892 1893 /* Check if adjustment gets us within 1 unit from the max */ 1894 if (negative && (cur_adj - adj) <= (base - max)) 1895 break; 1896 if (!negative && (cur_adj + adj) >= (base + max)) 1897 break; 1898 1899 adj_scale++; 1900 tick_error >>= 1; 1901 } 1902 1903 /* scale the corrections */ 1904 timekeeping_apply_adjustment(tk, offset, negative, adj_scale); 1905 } 1906 1907 /* 1908 * Adjust the timekeeper's multiplier to the correct frequency 1909 * and also to reduce the accumulated error value. 1910 */ 1911 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) 1912 { 1913 /* Correct for the current frequency error */ 1914 timekeeping_freqadjust(tk, offset); 1915 1916 /* Next make a small adjustment to fix any cumulative error */ 1917 if (!tk->ntp_err_mult && (tk->ntp_error > 0)) { 1918 tk->ntp_err_mult = 1; 1919 timekeeping_apply_adjustment(tk, offset, 0, 0); 1920 } else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) { 1921 /* Undo any existing error adjustment */ 1922 timekeeping_apply_adjustment(tk, offset, 1, 0); 1923 tk->ntp_err_mult = 0; 1924 } 1925 1926 if (unlikely(tk->tkr_mono.clock->maxadj && 1927 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) 1928 > tk->tkr_mono.clock->maxadj))) { 1929 printk_once(KERN_WARNING 1930 "Adjusting %s more than 11%% (%ld vs %ld)\n", 1931 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, 1932 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); 1933 } 1934 1935 /* 1936 * It may be possible that when we entered this function, xtime_nsec 1937 * was very small. Further, if we're slightly speeding the clocksource 1938 * in the code above, its possible the required corrective factor to 1939 * xtime_nsec could cause it to underflow. 1940 * 1941 * Now, since we already accumulated the second, cannot simply roll 1942 * the accumulated second back, since the NTP subsystem has been 1943 * notified via second_overflow. So instead we push xtime_nsec forward 1944 * by the amount we underflowed, and add that amount into the error. 1945 * 1946 * We'll correct this error next time through this function, when 1947 * xtime_nsec is not as small. 1948 */ 1949 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { 1950 s64 neg = -(s64)tk->tkr_mono.xtime_nsec; 1951 tk->tkr_mono.xtime_nsec = 0; 1952 tk->ntp_error += neg << tk->ntp_error_shift; 1953 } 1954 } 1955 1956 /** 1957 * accumulate_nsecs_to_secs - Accumulates nsecs into secs 1958 * 1959 * Helper function that accumulates the nsecs greater than a second 1960 * from the xtime_nsec field to the xtime_secs field. 1961 * It also calls into the NTP code to handle leapsecond processing. 1962 * 1963 */ 1964 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) 1965 { 1966 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 1967 unsigned int clock_set = 0; 1968 1969 while (tk->tkr_mono.xtime_nsec >= nsecps) { 1970 int leap; 1971 1972 tk->tkr_mono.xtime_nsec -= nsecps; 1973 tk->xtime_sec++; 1974 1975 /* Figure out if its a leap sec and apply if needed */ 1976 leap = second_overflow(tk->xtime_sec); 1977 if (unlikely(leap)) { 1978 struct timespec64 ts; 1979 1980 tk->xtime_sec += leap; 1981 1982 ts.tv_sec = leap; 1983 ts.tv_nsec = 0; 1984 tk_set_wall_to_mono(tk, 1985 timespec64_sub(tk->wall_to_monotonic, ts)); 1986 1987 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 1988 1989 clock_set = TK_CLOCK_WAS_SET; 1990 } 1991 } 1992 return clock_set; 1993 } 1994 1995 /** 1996 * logarithmic_accumulation - shifted accumulation of cycles 1997 * 1998 * This functions accumulates a shifted interval of cycles into 1999 * into a shifted interval nanoseconds. Allows for O(log) accumulation 2000 * loop. 2001 * 2002 * Returns the unconsumed cycles. 2003 */ 2004 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, 2005 u32 shift, unsigned int *clock_set) 2006 { 2007 u64 interval = tk->cycle_interval << shift; 2008 u64 snsec_per_sec; 2009 2010 /* If the offset is smaller than a shifted interval, do nothing */ 2011 if (offset < interval) 2012 return offset; 2013 2014 /* Accumulate one shifted interval */ 2015 offset -= interval; 2016 tk->tkr_mono.cycle_last += interval; 2017 tk->tkr_raw.cycle_last += interval; 2018 2019 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 2020 *clock_set |= accumulate_nsecs_to_secs(tk); 2021 2022 /* Accumulate raw time */ 2023 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 2024 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 2025 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { 2026 tk->tkr_raw.xtime_nsec -= snsec_per_sec; 2027 tk->raw_sec++; 2028 } 2029 2030 /* Accumulate error between NTP and clock interval */ 2031 tk->ntp_error += tk->ntp_tick << shift; 2032 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 2033 (tk->ntp_error_shift + shift); 2034 2035 return offset; 2036 } 2037 2038 /** 2039 * update_wall_time - Uses the current clocksource to increment the wall time 2040 * 2041 */ 2042 void update_wall_time(void) 2043 { 2044 struct timekeeper *real_tk = &tk_core.timekeeper; 2045 struct timekeeper *tk = &shadow_timekeeper; 2046 u64 offset; 2047 int shift = 0, maxshift; 2048 unsigned int clock_set = 0; 2049 unsigned long flags; 2050 2051 raw_spin_lock_irqsave(&timekeeper_lock, flags); 2052 2053 /* Make sure we're fully resumed: */ 2054 if (unlikely(timekeeping_suspended)) 2055 goto out; 2056 2057 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET 2058 offset = real_tk->cycle_interval; 2059 #else 2060 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), 2061 tk->tkr_mono.cycle_last, tk->tkr_mono.mask); 2062 #endif 2063 2064 /* Check if there's really nothing to do */ 2065 if (offset < real_tk->cycle_interval) 2066 goto out; 2067 2068 /* Do some additional sanity checking */ 2069 timekeeping_check_update(real_tk, offset); 2070 2071 /* 2072 * With NO_HZ we may have to accumulate many cycle_intervals 2073 * (think "ticks") worth of time at once. To do this efficiently, 2074 * we calculate the largest doubling multiple of cycle_intervals 2075 * that is smaller than the offset. We then accumulate that 2076 * chunk in one go, and then try to consume the next smaller 2077 * doubled multiple. 2078 */ 2079 shift = ilog2(offset) - ilog2(tk->cycle_interval); 2080 shift = max(0, shift); 2081 /* Bound shift to one less than what overflows tick_length */ 2082 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; 2083 shift = min(shift, maxshift); 2084 while (offset >= tk->cycle_interval) { 2085 offset = logarithmic_accumulation(tk, offset, shift, 2086 &clock_set); 2087 if (offset < tk->cycle_interval<<shift) 2088 shift--; 2089 } 2090 2091 /* correct the clock when NTP error is too big */ 2092 timekeeping_adjust(tk, offset); 2093 2094 /* 2095 * XXX This can be killed once everyone converts 2096 * to the new update_vsyscall. 2097 */ 2098 old_vsyscall_fixup(tk); 2099 2100 /* 2101 * Finally, make sure that after the rounding 2102 * xtime_nsec isn't larger than NSEC_PER_SEC 2103 */ 2104 clock_set |= accumulate_nsecs_to_secs(tk); 2105 2106 write_seqcount_begin(&tk_core.seq); 2107 /* 2108 * Update the real timekeeper. 2109 * 2110 * We could avoid this memcpy by switching pointers, but that 2111 * requires changes to all other timekeeper usage sites as 2112 * well, i.e. move the timekeeper pointer getter into the 2113 * spinlocked/seqcount protected sections. And we trade this 2114 * memcpy under the tk_core.seq against one before we start 2115 * updating. 2116 */ 2117 timekeeping_update(tk, clock_set); 2118 memcpy(real_tk, tk, sizeof(*tk)); 2119 /* The memcpy must come last. Do not put anything here! */ 2120 write_seqcount_end(&tk_core.seq); 2121 out: 2122 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 2123 if (clock_set) 2124 /* Have to call _delayed version, since in irq context*/ 2125 clock_was_set_delayed(); 2126 } 2127 2128 /** 2129 * getboottime64 - Return the real time of system boot. 2130 * @ts: pointer to the timespec64 to be set 2131 * 2132 * Returns the wall-time of boot in a timespec64. 2133 * 2134 * This is based on the wall_to_monotonic offset and the total suspend 2135 * time. Calls to settimeofday will affect the value returned (which 2136 * basically means that however wrong your real time clock is at boot time, 2137 * you get the right time here). 2138 */ 2139 void getboottime64(struct timespec64 *ts) 2140 { 2141 struct timekeeper *tk = &tk_core.timekeeper; 2142 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); 2143 2144 *ts = ktime_to_timespec64(t); 2145 } 2146 EXPORT_SYMBOL_GPL(getboottime64); 2147 2148 unsigned long get_seconds(void) 2149 { 2150 struct timekeeper *tk = &tk_core.timekeeper; 2151 2152 return tk->xtime_sec; 2153 } 2154 EXPORT_SYMBOL(get_seconds); 2155 2156 struct timespec __current_kernel_time(void) 2157 { 2158 struct timekeeper *tk = &tk_core.timekeeper; 2159 2160 return timespec64_to_timespec(tk_xtime(tk)); 2161 } 2162 2163 struct timespec64 current_kernel_time64(void) 2164 { 2165 struct timekeeper *tk = &tk_core.timekeeper; 2166 struct timespec64 now; 2167 unsigned long seq; 2168 2169 do { 2170 seq = read_seqcount_begin(&tk_core.seq); 2171 2172 now = tk_xtime(tk); 2173 } while (read_seqcount_retry(&tk_core.seq, seq)); 2174 2175 return now; 2176 } 2177 EXPORT_SYMBOL(current_kernel_time64); 2178 2179 struct timespec64 get_monotonic_coarse64(void) 2180 { 2181 struct timekeeper *tk = &tk_core.timekeeper; 2182 struct timespec64 now, mono; 2183 unsigned long seq; 2184 2185 do { 2186 seq = read_seqcount_begin(&tk_core.seq); 2187 2188 now = tk_xtime(tk); 2189 mono = tk->wall_to_monotonic; 2190 } while (read_seqcount_retry(&tk_core.seq, seq)); 2191 2192 set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec, 2193 now.tv_nsec + mono.tv_nsec); 2194 2195 return now; 2196 } 2197 EXPORT_SYMBOL(get_monotonic_coarse64); 2198 2199 /* 2200 * Must hold jiffies_lock 2201 */ 2202 void do_timer(unsigned long ticks) 2203 { 2204 jiffies_64 += ticks; 2205 calc_global_load(ticks); 2206 } 2207 2208 /** 2209 * ktime_get_update_offsets_now - hrtimer helper 2210 * @cwsseq: pointer to check and store the clock was set sequence number 2211 * @offs_real: pointer to storage for monotonic -> realtime offset 2212 * @offs_boot: pointer to storage for monotonic -> boottime offset 2213 * @offs_tai: pointer to storage for monotonic -> clock tai offset 2214 * 2215 * Returns current monotonic time and updates the offsets if the 2216 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are 2217 * different. 2218 * 2219 * Called from hrtimer_interrupt() or retrigger_next_event() 2220 */ 2221 ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real, 2222 ktime_t *offs_boot, ktime_t *offs_tai) 2223 { 2224 struct timekeeper *tk = &tk_core.timekeeper; 2225 unsigned int seq; 2226 ktime_t base; 2227 u64 nsecs; 2228 2229 do { 2230 seq = read_seqcount_begin(&tk_core.seq); 2231 2232 base = tk->tkr_mono.base; 2233 nsecs = timekeeping_get_ns(&tk->tkr_mono); 2234 base = ktime_add_ns(base, nsecs); 2235 2236 if (*cwsseq != tk->clock_was_set_seq) { 2237 *cwsseq = tk->clock_was_set_seq; 2238 *offs_real = tk->offs_real; 2239 *offs_boot = tk->offs_boot; 2240 *offs_tai = tk->offs_tai; 2241 } 2242 2243 /* Handle leapsecond insertion adjustments */ 2244 if (unlikely(base >= tk->next_leap_ktime)) 2245 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 2246 2247 } while (read_seqcount_retry(&tk_core.seq, seq)); 2248 2249 return base; 2250 } 2251 2252 /** 2253 * do_adjtimex() - Accessor function to NTP __do_adjtimex function 2254 */ 2255 int do_adjtimex(struct timex *txc) 2256 { 2257 struct timekeeper *tk = &tk_core.timekeeper; 2258 unsigned long flags; 2259 struct timespec64 ts; 2260 s32 orig_tai, tai; 2261 int ret; 2262 2263 /* Validate the data before disabling interrupts */ 2264 ret = ntp_validate_timex(txc); 2265 if (ret) 2266 return ret; 2267 2268 if (txc->modes & ADJ_SETOFFSET) { 2269 struct timespec delta; 2270 delta.tv_sec = txc->time.tv_sec; 2271 delta.tv_nsec = txc->time.tv_usec; 2272 if (!(txc->modes & ADJ_NANO)) 2273 delta.tv_nsec *= 1000; 2274 ret = timekeeping_inject_offset(&delta); 2275 if (ret) 2276 return ret; 2277 } 2278 2279 getnstimeofday64(&ts); 2280 2281 raw_spin_lock_irqsave(&timekeeper_lock, flags); 2282 write_seqcount_begin(&tk_core.seq); 2283 2284 orig_tai = tai = tk->tai_offset; 2285 ret = __do_adjtimex(txc, &ts, &tai); 2286 2287 if (tai != orig_tai) { 2288 __timekeeping_set_tai_offset(tk, tai); 2289 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); 2290 } 2291 tk_update_leap_state(tk); 2292 2293 write_seqcount_end(&tk_core.seq); 2294 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 2295 2296 if (tai != orig_tai) 2297 clock_was_set(); 2298 2299 ntp_notify_cmos_timer(); 2300 2301 return ret; 2302 } 2303 2304 #ifdef CONFIG_NTP_PPS 2305 /** 2306 * hardpps() - Accessor function to NTP __hardpps function 2307 */ 2308 void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts) 2309 { 2310 unsigned long flags; 2311 2312 raw_spin_lock_irqsave(&timekeeper_lock, flags); 2313 write_seqcount_begin(&tk_core.seq); 2314 2315 __hardpps(phase_ts, raw_ts); 2316 2317 write_seqcount_end(&tk_core.seq); 2318 raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 2319 } 2320 EXPORT_SYMBOL(hardpps); 2321 #endif 2322 2323 /** 2324 * xtime_update() - advances the timekeeping infrastructure 2325 * @ticks: number of ticks, that have elapsed since the last call. 2326 * 2327 * Must be called with interrupts disabled. 2328 */ 2329 void xtime_update(unsigned long ticks) 2330 { 2331 write_seqlock(&jiffies_lock); 2332 do_timer(ticks); 2333 write_sequnlock(&jiffies_lock); 2334 update_wall_time(); 2335 } 2336