1 /* 2 * linux/kernel/time/timekeeping.c 3 * 4 * Kernel timekeeping code and accessor functions 5 * 6 * This code was moved from linux/kernel/timer.c. 7 * Please see that file for copyright and history logs. 8 * 9 */ 10 11 #include <linux/module.h> 12 #include <linux/interrupt.h> 13 #include <linux/percpu.h> 14 #include <linux/init.h> 15 #include <linux/mm.h> 16 #include <linux/sched.h> 17 #include <linux/sysdev.h> 18 #include <linux/clocksource.h> 19 #include <linux/jiffies.h> 20 #include <linux/time.h> 21 #include <linux/tick.h> 22 #include <linux/stop_machine.h> 23 24 /* Structure holding internal timekeeping values. */ 25 struct timekeeper { 26 /* Current clocksource used for timekeeping. */ 27 struct clocksource *clock; 28 /* The shift value of the current clocksource. */ 29 int shift; 30 31 /* Number of clock cycles in one NTP interval. */ 32 cycle_t cycle_interval; 33 /* Number of clock shifted nano seconds in one NTP interval. */ 34 u64 xtime_interval; 35 /* Raw nano seconds accumulated per NTP interval. */ 36 u32 raw_interval; 37 38 /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */ 39 u64 xtime_nsec; 40 /* Difference between accumulated time and NTP time in ntp 41 * shifted nano seconds. */ 42 s64 ntp_error; 43 /* Shift conversion between clock shifted nano seconds and 44 * ntp shifted nano seconds. */ 45 int ntp_error_shift; 46 /* NTP adjusted clock multiplier */ 47 u32 mult; 48 }; 49 50 struct timekeeper timekeeper; 51 52 /** 53 * timekeeper_setup_internals - Set up internals to use clocksource clock. 54 * 55 * @clock: Pointer to clocksource. 56 * 57 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment 58 * pair and interval request. 59 * 60 * Unless you're the timekeeping code, you should not be using this! 61 */ 62 static void timekeeper_setup_internals(struct clocksource *clock) 63 { 64 cycle_t interval; 65 u64 tmp; 66 67 timekeeper.clock = clock; 68 clock->cycle_last = clock->read(clock); 69 70 /* Do the ns -> cycle conversion first, using original mult */ 71 tmp = NTP_INTERVAL_LENGTH; 72 tmp <<= clock->shift; 73 tmp += clock->mult/2; 74 do_div(tmp, clock->mult); 75 if (tmp == 0) 76 tmp = 1; 77 78 interval = (cycle_t) tmp; 79 timekeeper.cycle_interval = interval; 80 81 /* Go back from cycles -> shifted ns */ 82 timekeeper.xtime_interval = (u64) interval * clock->mult; 83 timekeeper.raw_interval = 84 ((u64) interval * clock->mult) >> clock->shift; 85 86 timekeeper.xtime_nsec = 0; 87 timekeeper.shift = clock->shift; 88 89 timekeeper.ntp_error = 0; 90 timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 91 92 /* 93 * The timekeeper keeps its own mult values for the currently 94 * active clocksource. These value will be adjusted via NTP 95 * to counteract clock drifting. 96 */ 97 timekeeper.mult = clock->mult; 98 } 99 100 /* Timekeeper helper functions. */ 101 static inline s64 timekeeping_get_ns(void) 102 { 103 cycle_t cycle_now, cycle_delta; 104 struct clocksource *clock; 105 106 /* read clocksource: */ 107 clock = timekeeper.clock; 108 cycle_now = clock->read(clock); 109 110 /* calculate the delta since the last update_wall_time: */ 111 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 112 113 /* return delta convert to nanoseconds using ntp adjusted mult. */ 114 return clocksource_cyc2ns(cycle_delta, timekeeper.mult, 115 timekeeper.shift); 116 } 117 118 static inline s64 timekeeping_get_ns_raw(void) 119 { 120 cycle_t cycle_now, cycle_delta; 121 struct clocksource *clock; 122 123 /* read clocksource: */ 124 clock = timekeeper.clock; 125 cycle_now = clock->read(clock); 126 127 /* calculate the delta since the last update_wall_time: */ 128 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 129 130 /* return delta convert to nanoseconds using ntp adjusted mult. */ 131 return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 132 } 133 134 /* 135 * This read-write spinlock protects us from races in SMP while 136 * playing with xtime. 137 */ 138 __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); 139 140 141 /* 142 * The current time 143 * wall_to_monotonic is what we need to add to xtime (or xtime corrected 144 * for sub jiffie times) to get to monotonic time. Monotonic is pegged 145 * at zero at system boot time, so wall_to_monotonic will be negative, 146 * however, we will ALWAYS keep the tv_nsec part positive so we can use 147 * the usual normalization. 148 * 149 * wall_to_monotonic is moved after resume from suspend for the monotonic 150 * time not to jump. We need to add total_sleep_time to wall_to_monotonic 151 * to get the real boot based time offset. 152 * 153 * - wall_to_monotonic is no longer the boot time, getboottime must be 154 * used instead. 155 */ 156 struct timespec xtime __attribute__ ((aligned (16))); 157 struct timespec wall_to_monotonic __attribute__ ((aligned (16))); 158 static struct timespec total_sleep_time; 159 160 /* 161 * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. 162 */ 163 struct timespec raw_time; 164 165 /* flag for if timekeeping is suspended */ 166 int __read_mostly timekeeping_suspended; 167 168 /* must hold xtime_lock */ 169 void timekeeping_leap_insert(int leapsecond) 170 { 171 xtime.tv_sec += leapsecond; 172 wall_to_monotonic.tv_sec -= leapsecond; 173 update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); 174 } 175 176 #ifdef CONFIG_GENERIC_TIME 177 178 /** 179 * timekeeping_forward_now - update clock to the current time 180 * 181 * Forward the current clock to update its state since the last call to 182 * update_wall_time(). This is useful before significant clock changes, 183 * as it avoids having to deal with this time offset explicitly. 184 */ 185 static void timekeeping_forward_now(void) 186 { 187 cycle_t cycle_now, cycle_delta; 188 struct clocksource *clock; 189 s64 nsec; 190 191 clock = timekeeper.clock; 192 cycle_now = clock->read(clock); 193 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 194 clock->cycle_last = cycle_now; 195 196 nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult, 197 timekeeper.shift); 198 199 /* If arch requires, add in gettimeoffset() */ 200 nsec += arch_gettimeoffset(); 201 202 timespec_add_ns(&xtime, nsec); 203 204 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); 205 timespec_add_ns(&raw_time, nsec); 206 } 207 208 /** 209 * getnstimeofday - Returns the time of day in a timespec 210 * @ts: pointer to the timespec to be set 211 * 212 * Returns the time of day in a timespec. 213 */ 214 void getnstimeofday(struct timespec *ts) 215 { 216 unsigned long seq; 217 s64 nsecs; 218 219 WARN_ON(timekeeping_suspended); 220 221 do { 222 seq = read_seqbegin(&xtime_lock); 223 224 *ts = xtime; 225 nsecs = timekeeping_get_ns(); 226 227 /* If arch requires, add in gettimeoffset() */ 228 nsecs += arch_gettimeoffset(); 229 230 } while (read_seqretry(&xtime_lock, seq)); 231 232 timespec_add_ns(ts, nsecs); 233 } 234 235 EXPORT_SYMBOL(getnstimeofday); 236 237 ktime_t ktime_get(void) 238 { 239 unsigned int seq; 240 s64 secs, nsecs; 241 242 WARN_ON(timekeeping_suspended); 243 244 do { 245 seq = read_seqbegin(&xtime_lock); 246 secs = xtime.tv_sec + wall_to_monotonic.tv_sec; 247 nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec; 248 nsecs += timekeeping_get_ns(); 249 250 } while (read_seqretry(&xtime_lock, seq)); 251 /* 252 * Use ktime_set/ktime_add_ns to create a proper ktime on 253 * 32-bit architectures without CONFIG_KTIME_SCALAR. 254 */ 255 return ktime_add_ns(ktime_set(secs, 0), nsecs); 256 } 257 EXPORT_SYMBOL_GPL(ktime_get); 258 259 /** 260 * ktime_get_ts - get the monotonic clock in timespec format 261 * @ts: pointer to timespec variable 262 * 263 * The function calculates the monotonic clock from the realtime 264 * clock and the wall_to_monotonic offset and stores the result 265 * in normalized timespec format in the variable pointed to by @ts. 266 */ 267 void ktime_get_ts(struct timespec *ts) 268 { 269 struct timespec tomono; 270 unsigned int seq; 271 s64 nsecs; 272 273 WARN_ON(timekeeping_suspended); 274 275 do { 276 seq = read_seqbegin(&xtime_lock); 277 *ts = xtime; 278 tomono = wall_to_monotonic; 279 nsecs = timekeeping_get_ns(); 280 281 } while (read_seqretry(&xtime_lock, seq)); 282 283 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, 284 ts->tv_nsec + tomono.tv_nsec + nsecs); 285 } 286 EXPORT_SYMBOL_GPL(ktime_get_ts); 287 288 /** 289 * do_gettimeofday - Returns the time of day in a timeval 290 * @tv: pointer to the timeval to be set 291 * 292 * NOTE: Users should be converted to using getnstimeofday() 293 */ 294 void do_gettimeofday(struct timeval *tv) 295 { 296 struct timespec now; 297 298 getnstimeofday(&now); 299 tv->tv_sec = now.tv_sec; 300 tv->tv_usec = now.tv_nsec/1000; 301 } 302 303 EXPORT_SYMBOL(do_gettimeofday); 304 /** 305 * do_settimeofday - Sets the time of day 306 * @tv: pointer to the timespec variable containing the new time 307 * 308 * Sets the time of day to the new time and update NTP and notify hrtimers 309 */ 310 int do_settimeofday(struct timespec *tv) 311 { 312 struct timespec ts_delta; 313 unsigned long flags; 314 315 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 316 return -EINVAL; 317 318 write_seqlock_irqsave(&xtime_lock, flags); 319 320 timekeeping_forward_now(); 321 322 ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec; 323 ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec; 324 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta); 325 326 xtime = *tv; 327 328 timekeeper.ntp_error = 0; 329 ntp_clear(); 330 331 update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); 332 333 write_sequnlock_irqrestore(&xtime_lock, flags); 334 335 /* signal hrtimers about time change */ 336 clock_was_set(); 337 338 return 0; 339 } 340 341 EXPORT_SYMBOL(do_settimeofday); 342 343 /** 344 * change_clocksource - Swaps clocksources if a new one is available 345 * 346 * Accumulates current time interval and initializes new clocksource 347 */ 348 static int change_clocksource(void *data) 349 { 350 struct clocksource *new, *old; 351 352 new = (struct clocksource *) data; 353 354 timekeeping_forward_now(); 355 if (!new->enable || new->enable(new) == 0) { 356 old = timekeeper.clock; 357 timekeeper_setup_internals(new); 358 if (old->disable) 359 old->disable(old); 360 } 361 return 0; 362 } 363 364 /** 365 * timekeeping_notify - Install a new clock source 366 * @clock: pointer to the clock source 367 * 368 * This function is called from clocksource.c after a new, better clock 369 * source has been registered. The caller holds the clocksource_mutex. 370 */ 371 void timekeeping_notify(struct clocksource *clock) 372 { 373 if (timekeeper.clock == clock) 374 return; 375 stop_machine(change_clocksource, clock, NULL); 376 tick_clock_notify(); 377 } 378 379 #else /* GENERIC_TIME */ 380 381 static inline void timekeeping_forward_now(void) { } 382 383 /** 384 * ktime_get - get the monotonic time in ktime_t format 385 * 386 * returns the time in ktime_t format 387 */ 388 ktime_t ktime_get(void) 389 { 390 struct timespec now; 391 392 ktime_get_ts(&now); 393 394 return timespec_to_ktime(now); 395 } 396 EXPORT_SYMBOL_GPL(ktime_get); 397 398 /** 399 * ktime_get_ts - get the monotonic clock in timespec format 400 * @ts: pointer to timespec variable 401 * 402 * The function calculates the monotonic clock from the realtime 403 * clock and the wall_to_monotonic offset and stores the result 404 * in normalized timespec format in the variable pointed to by @ts. 405 */ 406 void ktime_get_ts(struct timespec *ts) 407 { 408 struct timespec tomono; 409 unsigned long seq; 410 411 do { 412 seq = read_seqbegin(&xtime_lock); 413 getnstimeofday(ts); 414 tomono = wall_to_monotonic; 415 416 } while (read_seqretry(&xtime_lock, seq)); 417 418 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, 419 ts->tv_nsec + tomono.tv_nsec); 420 } 421 EXPORT_SYMBOL_GPL(ktime_get_ts); 422 423 #endif /* !GENERIC_TIME */ 424 425 /** 426 * ktime_get_real - get the real (wall-) time in ktime_t format 427 * 428 * returns the time in ktime_t format 429 */ 430 ktime_t ktime_get_real(void) 431 { 432 struct timespec now; 433 434 getnstimeofday(&now); 435 436 return timespec_to_ktime(now); 437 } 438 EXPORT_SYMBOL_GPL(ktime_get_real); 439 440 /** 441 * getrawmonotonic - Returns the raw monotonic time in a timespec 442 * @ts: pointer to the timespec to be set 443 * 444 * Returns the raw monotonic time (completely un-modified by ntp) 445 */ 446 void getrawmonotonic(struct timespec *ts) 447 { 448 unsigned long seq; 449 s64 nsecs; 450 451 do { 452 seq = read_seqbegin(&xtime_lock); 453 nsecs = timekeeping_get_ns_raw(); 454 *ts = raw_time; 455 456 } while (read_seqretry(&xtime_lock, seq)); 457 458 timespec_add_ns(ts, nsecs); 459 } 460 EXPORT_SYMBOL(getrawmonotonic); 461 462 463 /** 464 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 465 */ 466 int timekeeping_valid_for_hres(void) 467 { 468 unsigned long seq; 469 int ret; 470 471 do { 472 seq = read_seqbegin(&xtime_lock); 473 474 ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 475 476 } while (read_seqretry(&xtime_lock, seq)); 477 478 return ret; 479 } 480 481 /** 482 * timekeeping_max_deferment - Returns max time the clocksource can be deferred 483 * 484 * Caller must observe xtime_lock via read_seqbegin/read_seqretry to 485 * ensure that the clocksource does not change! 486 */ 487 u64 timekeeping_max_deferment(void) 488 { 489 return timekeeper.clock->max_idle_ns; 490 } 491 492 /** 493 * read_persistent_clock - Return time from the persistent clock. 494 * 495 * Weak dummy function for arches that do not yet support it. 496 * Reads the time from the battery backed persistent clock. 497 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported. 498 * 499 * XXX - Do be sure to remove it once all arches implement it. 500 */ 501 void __attribute__((weak)) read_persistent_clock(struct timespec *ts) 502 { 503 ts->tv_sec = 0; 504 ts->tv_nsec = 0; 505 } 506 507 /** 508 * read_boot_clock - Return time of the system start. 509 * 510 * Weak dummy function for arches that do not yet support it. 511 * Function to read the exact time the system has been started. 512 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported. 513 * 514 * XXX - Do be sure to remove it once all arches implement it. 515 */ 516 void __attribute__((weak)) read_boot_clock(struct timespec *ts) 517 { 518 ts->tv_sec = 0; 519 ts->tv_nsec = 0; 520 } 521 522 /* 523 * timekeeping_init - Initializes the clocksource and common timekeeping values 524 */ 525 void __init timekeeping_init(void) 526 { 527 struct clocksource *clock; 528 unsigned long flags; 529 struct timespec now, boot; 530 531 read_persistent_clock(&now); 532 read_boot_clock(&boot); 533 534 write_seqlock_irqsave(&xtime_lock, flags); 535 536 ntp_init(); 537 538 clock = clocksource_default_clock(); 539 if (clock->enable) 540 clock->enable(clock); 541 timekeeper_setup_internals(clock); 542 543 xtime.tv_sec = now.tv_sec; 544 xtime.tv_nsec = now.tv_nsec; 545 raw_time.tv_sec = 0; 546 raw_time.tv_nsec = 0; 547 if (boot.tv_sec == 0 && boot.tv_nsec == 0) { 548 boot.tv_sec = xtime.tv_sec; 549 boot.tv_nsec = xtime.tv_nsec; 550 } 551 set_normalized_timespec(&wall_to_monotonic, 552 -boot.tv_sec, -boot.tv_nsec); 553 total_sleep_time.tv_sec = 0; 554 total_sleep_time.tv_nsec = 0; 555 write_sequnlock_irqrestore(&xtime_lock, flags); 556 } 557 558 /* time in seconds when suspend began */ 559 static struct timespec timekeeping_suspend_time; 560 561 /** 562 * timekeeping_resume - Resumes the generic timekeeping subsystem. 563 * @dev: unused 564 * 565 * This is for the generic clocksource timekeeping. 566 * xtime/wall_to_monotonic/jiffies/etc are 567 * still managed by arch specific suspend/resume code. 568 */ 569 static int timekeeping_resume(struct sys_device *dev) 570 { 571 unsigned long flags; 572 struct timespec ts; 573 574 read_persistent_clock(&ts); 575 576 clocksource_resume(); 577 578 write_seqlock_irqsave(&xtime_lock, flags); 579 580 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { 581 ts = timespec_sub(ts, timekeeping_suspend_time); 582 xtime = timespec_add_safe(xtime, ts); 583 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts); 584 total_sleep_time = timespec_add_safe(total_sleep_time, ts); 585 } 586 /* re-base the last cycle value */ 587 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); 588 timekeeper.ntp_error = 0; 589 timekeeping_suspended = 0; 590 write_sequnlock_irqrestore(&xtime_lock, flags); 591 592 touch_softlockup_watchdog(); 593 594 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL); 595 596 /* Resume hrtimers */ 597 hres_timers_resume(); 598 599 return 0; 600 } 601 602 static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) 603 { 604 unsigned long flags; 605 606 read_persistent_clock(&timekeeping_suspend_time); 607 608 write_seqlock_irqsave(&xtime_lock, flags); 609 timekeeping_forward_now(); 610 timekeeping_suspended = 1; 611 write_sequnlock_irqrestore(&xtime_lock, flags); 612 613 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 614 clocksource_suspend(); 615 616 return 0; 617 } 618 619 /* sysfs resume/suspend bits for timekeeping */ 620 static struct sysdev_class timekeeping_sysclass = { 621 .name = "timekeeping", 622 .resume = timekeeping_resume, 623 .suspend = timekeeping_suspend, 624 }; 625 626 static struct sys_device device_timer = { 627 .id = 0, 628 .cls = &timekeeping_sysclass, 629 }; 630 631 static int __init timekeeping_init_device(void) 632 { 633 int error = sysdev_class_register(&timekeeping_sysclass); 634 if (!error) 635 error = sysdev_register(&device_timer); 636 return error; 637 } 638 639 device_initcall(timekeeping_init_device); 640 641 /* 642 * If the error is already larger, we look ahead even further 643 * to compensate for late or lost adjustments. 644 */ 645 static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval, 646 s64 *offset) 647 { 648 s64 tick_error, i; 649 u32 look_ahead, adj; 650 s32 error2, mult; 651 652 /* 653 * Use the current error value to determine how much to look ahead. 654 * The larger the error the slower we adjust for it to avoid problems 655 * with losing too many ticks, otherwise we would overadjust and 656 * produce an even larger error. The smaller the adjustment the 657 * faster we try to adjust for it, as lost ticks can do less harm 658 * here. This is tuned so that an error of about 1 msec is adjusted 659 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). 660 */ 661 error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); 662 error2 = abs(error2); 663 for (look_ahead = 0; error2 > 0; look_ahead++) 664 error2 >>= 2; 665 666 /* 667 * Now calculate the error in (1 << look_ahead) ticks, but first 668 * remove the single look ahead already included in the error. 669 */ 670 tick_error = tick_length >> (timekeeper.ntp_error_shift + 1); 671 tick_error -= timekeeper.xtime_interval >> 1; 672 error = ((error - tick_error) >> look_ahead) + tick_error; 673 674 /* Finally calculate the adjustment shift value. */ 675 i = *interval; 676 mult = 1; 677 if (error < 0) { 678 error = -error; 679 *interval = -*interval; 680 *offset = -*offset; 681 mult = -1; 682 } 683 for (adj = 0; error > i; adj++) 684 error >>= 1; 685 686 *interval <<= adj; 687 *offset <<= adj; 688 return mult << adj; 689 } 690 691 /* 692 * Adjust the multiplier to reduce the error value, 693 * this is optimized for the most common adjustments of -1,0,1, 694 * for other values we can do a bit more work. 695 */ 696 static void timekeeping_adjust(s64 offset) 697 { 698 s64 error, interval = timekeeper.cycle_interval; 699 int adj; 700 701 error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1); 702 if (error > interval) { 703 error >>= 2; 704 if (likely(error <= interval)) 705 adj = 1; 706 else 707 adj = timekeeping_bigadjust(error, &interval, &offset); 708 } else if (error < -interval) { 709 error >>= 2; 710 if (likely(error >= -interval)) { 711 adj = -1; 712 interval = -interval; 713 offset = -offset; 714 } else 715 adj = timekeeping_bigadjust(error, &interval, &offset); 716 } else 717 return; 718 719 timekeeper.mult += adj; 720 timekeeper.xtime_interval += interval; 721 timekeeper.xtime_nsec -= offset; 722 timekeeper.ntp_error -= (interval - offset) << 723 timekeeper.ntp_error_shift; 724 } 725 726 727 /** 728 * logarithmic_accumulation - shifted accumulation of cycles 729 * 730 * This functions accumulates a shifted interval of cycles into 731 * into a shifted interval nanoseconds. Allows for O(log) accumulation 732 * loop. 733 * 734 * Returns the unconsumed cycles. 735 */ 736 static cycle_t logarithmic_accumulation(cycle_t offset, int shift) 737 { 738 u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift; 739 740 /* If the offset is smaller then a shifted interval, do nothing */ 741 if (offset < timekeeper.cycle_interval<<shift) 742 return offset; 743 744 /* Accumulate one shifted interval */ 745 offset -= timekeeper.cycle_interval << shift; 746 timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift; 747 748 timekeeper.xtime_nsec += timekeeper.xtime_interval << shift; 749 while (timekeeper.xtime_nsec >= nsecps) { 750 timekeeper.xtime_nsec -= nsecps; 751 xtime.tv_sec++; 752 second_overflow(); 753 } 754 755 /* Accumulate into raw time */ 756 raw_time.tv_nsec += timekeeper.raw_interval << shift;; 757 while (raw_time.tv_nsec >= NSEC_PER_SEC) { 758 raw_time.tv_nsec -= NSEC_PER_SEC; 759 raw_time.tv_sec++; 760 } 761 762 /* Accumulate error between NTP and clock interval */ 763 timekeeper.ntp_error += tick_length << shift; 764 timekeeper.ntp_error -= timekeeper.xtime_interval << 765 (timekeeper.ntp_error_shift + shift); 766 767 return offset; 768 } 769 770 771 /** 772 * update_wall_time - Uses the current clocksource to increment the wall time 773 * 774 * Called from the timer interrupt, must hold a write on xtime_lock. 775 */ 776 void update_wall_time(void) 777 { 778 struct clocksource *clock; 779 cycle_t offset; 780 int shift = 0, maxshift; 781 782 /* Make sure we're fully resumed: */ 783 if (unlikely(timekeeping_suspended)) 784 return; 785 786 clock = timekeeper.clock; 787 #ifdef CONFIG_GENERIC_TIME 788 offset = (clock->read(clock) - clock->cycle_last) & clock->mask; 789 #else 790 offset = timekeeper.cycle_interval; 791 #endif 792 timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift; 793 794 /* 795 * With NO_HZ we may have to accumulate many cycle_intervals 796 * (think "ticks") worth of time at once. To do this efficiently, 797 * we calculate the largest doubling multiple of cycle_intervals 798 * that is smaller then the offset. We then accumulate that 799 * chunk in one go, and then try to consume the next smaller 800 * doubled multiple. 801 */ 802 shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); 803 shift = max(0, shift); 804 /* Bound shift to one less then what overflows tick_length */ 805 maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1; 806 shift = min(shift, maxshift); 807 while (offset >= timekeeper.cycle_interval) { 808 offset = logarithmic_accumulation(offset, shift); 809 if(offset < timekeeper.cycle_interval<<shift) 810 shift--; 811 } 812 813 /* correct the clock when NTP error is too big */ 814 timekeeping_adjust(offset); 815 816 /* 817 * Since in the loop above, we accumulate any amount of time 818 * in xtime_nsec over a second into xtime.tv_sec, its possible for 819 * xtime_nsec to be fairly small after the loop. Further, if we're 820 * slightly speeding the clocksource up in timekeeping_adjust(), 821 * its possible the required corrective factor to xtime_nsec could 822 * cause it to underflow. 823 * 824 * Now, we cannot simply roll the accumulated second back, since 825 * the NTP subsystem has been notified via second_overflow. So 826 * instead we push xtime_nsec forward by the amount we underflowed, 827 * and add that amount into the error. 828 * 829 * We'll correct this error next time through this function, when 830 * xtime_nsec is not as small. 831 */ 832 if (unlikely((s64)timekeeper.xtime_nsec < 0)) { 833 s64 neg = -(s64)timekeeper.xtime_nsec; 834 timekeeper.xtime_nsec = 0; 835 timekeeper.ntp_error += neg << timekeeper.ntp_error_shift; 836 } 837 838 839 /* 840 * Store full nanoseconds into xtime after rounding it up and 841 * add the remainder to the error difference. 842 */ 843 xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1; 844 timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift; 845 timekeeper.ntp_error += timekeeper.xtime_nsec << 846 timekeeper.ntp_error_shift; 847 848 /* 849 * Finally, make sure that after the rounding 850 * xtime.tv_nsec isn't larger then NSEC_PER_SEC 851 */ 852 if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) { 853 xtime.tv_nsec -= NSEC_PER_SEC; 854 xtime.tv_sec++; 855 second_overflow(); 856 } 857 858 /* check to see if there is a new clocksource to use */ 859 update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); 860 } 861 862 /** 863 * getboottime - Return the real time of system boot. 864 * @ts: pointer to the timespec to be set 865 * 866 * Returns the time of day in a timespec. 867 * 868 * This is based on the wall_to_monotonic offset and the total suspend 869 * time. Calls to settimeofday will affect the value returned (which 870 * basically means that however wrong your real time clock is at boot time, 871 * you get the right time here). 872 */ 873 void getboottime(struct timespec *ts) 874 { 875 struct timespec boottime = { 876 .tv_sec = wall_to_monotonic.tv_sec + total_sleep_time.tv_sec, 877 .tv_nsec = wall_to_monotonic.tv_nsec + total_sleep_time.tv_nsec 878 }; 879 880 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); 881 } 882 EXPORT_SYMBOL_GPL(getboottime); 883 884 /** 885 * monotonic_to_bootbased - Convert the monotonic time to boot based. 886 * @ts: pointer to the timespec to be converted 887 */ 888 void monotonic_to_bootbased(struct timespec *ts) 889 { 890 *ts = timespec_add_safe(*ts, total_sleep_time); 891 } 892 EXPORT_SYMBOL_GPL(monotonic_to_bootbased); 893 894 unsigned long get_seconds(void) 895 { 896 return xtime.tv_sec; 897 } 898 EXPORT_SYMBOL(get_seconds); 899 900 struct timespec __current_kernel_time(void) 901 { 902 return xtime; 903 } 904 905 struct timespec current_kernel_time(void) 906 { 907 struct timespec now; 908 unsigned long seq; 909 910 do { 911 seq = read_seqbegin(&xtime_lock); 912 913 now = xtime; 914 } while (read_seqretry(&xtime_lock, seq)); 915 916 return now; 917 } 918 EXPORT_SYMBOL(current_kernel_time); 919 920 struct timespec get_monotonic_coarse(void) 921 { 922 struct timespec now, mono; 923 unsigned long seq; 924 925 do { 926 seq = read_seqbegin(&xtime_lock); 927 928 now = xtime; 929 mono = wall_to_monotonic; 930 } while (read_seqretry(&xtime_lock, seq)); 931 932 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, 933 now.tv_nsec + mono.tv_nsec); 934 return now; 935 } 936