1 /* 2 * linux/kernel/time/timekeeping.c 3 * 4 * Kernel timekeeping code and accessor functions 5 * 6 * This code was moved from linux/kernel/timer.c. 7 * Please see that file for copyright and history logs. 8 * 9 */ 10 11 #include <linux/module.h> 12 #include <linux/interrupt.h> 13 #include <linux/percpu.h> 14 #include <linux/init.h> 15 #include <linux/mm.h> 16 #include <linux/sysdev.h> 17 #include <linux/clocksource.h> 18 #include <linux/jiffies.h> 19 #include <linux/time.h> 20 #include <linux/tick.h> 21 22 23 /* 24 * This read-write spinlock protects us from races in SMP while 25 * playing with xtime and avenrun. 26 */ 27 __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); 28 29 30 /* 31 * The current time 32 * wall_to_monotonic is what we need to add to xtime (or xtime corrected 33 * for sub jiffie times) to get to monotonic time. Monotonic is pegged 34 * at zero at system boot time, so wall_to_monotonic will be negative, 35 * however, we will ALWAYS keep the tv_nsec part positive so we can use 36 * the usual normalization. 37 * 38 * wall_to_monotonic is moved after resume from suspend for the monotonic 39 * time not to jump. We need to add total_sleep_time to wall_to_monotonic 40 * to get the real boot based time offset. 41 * 42 * - wall_to_monotonic is no longer the boot time, getboottime must be 43 * used instead. 44 */ 45 struct timespec xtime __attribute__ ((aligned (16))); 46 struct timespec wall_to_monotonic __attribute__ ((aligned (16))); 47 static unsigned long total_sleep_time; /* seconds */ 48 49 static struct timespec xtime_cache __attribute__ ((aligned (16))); 50 void update_xtime_cache(u64 nsec) 51 { 52 xtime_cache = xtime; 53 timespec_add_ns(&xtime_cache, nsec); 54 } 55 56 struct clocksource *clock; 57 58 59 #ifdef CONFIG_GENERIC_TIME 60 /** 61 * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook 62 * 63 * private function, must hold xtime_lock lock when being 64 * called. Returns the number of nanoseconds since the 65 * last call to update_wall_time() (adjusted by NTP scaling) 66 */ 67 static inline s64 __get_nsec_offset(void) 68 { 69 cycle_t cycle_now, cycle_delta; 70 s64 ns_offset; 71 72 /* read clocksource: */ 73 cycle_now = clocksource_read(clock); 74 75 /* calculate the delta since the last update_wall_time: */ 76 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 77 78 /* convert to nanoseconds: */ 79 ns_offset = cyc2ns(clock, cycle_delta); 80 81 return ns_offset; 82 } 83 84 /** 85 * getnstimeofday - Returns the time of day in a timespec 86 * @ts: pointer to the timespec to be set 87 * 88 * Returns the time of day in a timespec. 89 */ 90 void getnstimeofday(struct timespec *ts) 91 { 92 unsigned long seq; 93 s64 nsecs; 94 95 do { 96 seq = read_seqbegin(&xtime_lock); 97 98 *ts = xtime; 99 nsecs = __get_nsec_offset(); 100 101 } while (read_seqretry(&xtime_lock, seq)); 102 103 timespec_add_ns(ts, nsecs); 104 } 105 106 EXPORT_SYMBOL(getnstimeofday); 107 108 /** 109 * do_gettimeofday - Returns the time of day in a timeval 110 * @tv: pointer to the timeval to be set 111 * 112 * NOTE: Users should be converted to using getnstimeofday() 113 */ 114 void do_gettimeofday(struct timeval *tv) 115 { 116 struct timespec now; 117 118 getnstimeofday(&now); 119 tv->tv_sec = now.tv_sec; 120 tv->tv_usec = now.tv_nsec/1000; 121 } 122 123 EXPORT_SYMBOL(do_gettimeofday); 124 /** 125 * do_settimeofday - Sets the time of day 126 * @tv: pointer to the timespec variable containing the new time 127 * 128 * Sets the time of day to the new time and update NTP and notify hrtimers 129 */ 130 int do_settimeofday(struct timespec *tv) 131 { 132 unsigned long flags; 133 time_t wtm_sec, sec = tv->tv_sec; 134 long wtm_nsec, nsec = tv->tv_nsec; 135 136 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 137 return -EINVAL; 138 139 write_seqlock_irqsave(&xtime_lock, flags); 140 141 nsec -= __get_nsec_offset(); 142 143 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); 144 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); 145 146 set_normalized_timespec(&xtime, sec, nsec); 147 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); 148 update_xtime_cache(0); 149 150 clock->error = 0; 151 ntp_clear(); 152 153 update_vsyscall(&xtime, clock); 154 155 write_sequnlock_irqrestore(&xtime_lock, flags); 156 157 /* signal hrtimers about time change */ 158 clock_was_set(); 159 160 return 0; 161 } 162 163 EXPORT_SYMBOL(do_settimeofday); 164 165 /** 166 * change_clocksource - Swaps clocksources if a new one is available 167 * 168 * Accumulates current time interval and initializes new clocksource 169 */ 170 static void change_clocksource(void) 171 { 172 struct clocksource *new; 173 cycle_t now; 174 u64 nsec; 175 176 new = clocksource_get_next(); 177 178 if (clock == new) 179 return; 180 181 new->cycle_last = 0; 182 now = clocksource_read(new); 183 nsec = __get_nsec_offset(); 184 timespec_add_ns(&xtime, nsec); 185 186 clock = new; 187 clock->cycle_last = now; 188 189 clock->error = 0; 190 clock->xtime_nsec = 0; 191 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); 192 193 tick_clock_notify(); 194 195 /* 196 * We're holding xtime lock and waking up klogd would deadlock 197 * us on enqueue. So no printing! 198 printk(KERN_INFO "Time: %s clocksource has been installed.\n", 199 clock->name); 200 */ 201 } 202 #else 203 static inline void change_clocksource(void) { } 204 static inline s64 __get_nsec_offset(void) { return 0; } 205 #endif 206 207 /** 208 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 209 */ 210 int timekeeping_valid_for_hres(void) 211 { 212 unsigned long seq; 213 int ret; 214 215 do { 216 seq = read_seqbegin(&xtime_lock); 217 218 ret = clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 219 220 } while (read_seqretry(&xtime_lock, seq)); 221 222 return ret; 223 } 224 225 /** 226 * read_persistent_clock - Return time in seconds from the persistent clock. 227 * 228 * Weak dummy function for arches that do not yet support it. 229 * Returns seconds from epoch using the battery backed persistent clock. 230 * Returns zero if unsupported. 231 * 232 * XXX - Do be sure to remove it once all arches implement it. 233 */ 234 unsigned long __attribute__((weak)) read_persistent_clock(void) 235 { 236 return 0; 237 } 238 239 /* 240 * timekeeping_init - Initializes the clocksource and common timekeeping values 241 */ 242 void __init timekeeping_init(void) 243 { 244 unsigned long flags; 245 unsigned long sec = read_persistent_clock(); 246 247 write_seqlock_irqsave(&xtime_lock, flags); 248 249 ntp_init(); 250 251 clock = clocksource_get_next(); 252 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); 253 clock->cycle_last = clocksource_read(clock); 254 255 xtime.tv_sec = sec; 256 xtime.tv_nsec = 0; 257 set_normalized_timespec(&wall_to_monotonic, 258 -xtime.tv_sec, -xtime.tv_nsec); 259 update_xtime_cache(0); 260 total_sleep_time = 0; 261 write_sequnlock_irqrestore(&xtime_lock, flags); 262 } 263 264 /* flag for if timekeeping is suspended */ 265 static int timekeeping_suspended; 266 /* time in seconds when suspend began */ 267 static unsigned long timekeeping_suspend_time; 268 /* xtime offset when we went into suspend */ 269 static s64 timekeeping_suspend_nsecs; 270 271 /** 272 * timekeeping_resume - Resumes the generic timekeeping subsystem. 273 * @dev: unused 274 * 275 * This is for the generic clocksource timekeeping. 276 * xtime/wall_to_monotonic/jiffies/etc are 277 * still managed by arch specific suspend/resume code. 278 */ 279 static int timekeeping_resume(struct sys_device *dev) 280 { 281 unsigned long flags; 282 unsigned long now = read_persistent_clock(); 283 284 clocksource_resume(); 285 286 write_seqlock_irqsave(&xtime_lock, flags); 287 288 if (now && (now > timekeeping_suspend_time)) { 289 unsigned long sleep_length = now - timekeeping_suspend_time; 290 291 xtime.tv_sec += sleep_length; 292 wall_to_monotonic.tv_sec -= sleep_length; 293 total_sleep_time += sleep_length; 294 } 295 /* Make sure that we have the correct xtime reference */ 296 timespec_add_ns(&xtime, timekeeping_suspend_nsecs); 297 update_xtime_cache(0); 298 /* re-base the last cycle value */ 299 clock->cycle_last = 0; 300 clock->cycle_last = clocksource_read(clock); 301 clock->error = 0; 302 timekeeping_suspended = 0; 303 write_sequnlock_irqrestore(&xtime_lock, flags); 304 305 touch_softlockup_watchdog(); 306 307 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL); 308 309 /* Resume hrtimers */ 310 hres_timers_resume(); 311 312 return 0; 313 } 314 315 static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) 316 { 317 unsigned long flags; 318 319 timekeeping_suspend_time = read_persistent_clock(); 320 321 write_seqlock_irqsave(&xtime_lock, flags); 322 /* Get the current xtime offset */ 323 timekeeping_suspend_nsecs = __get_nsec_offset(); 324 timekeeping_suspended = 1; 325 write_sequnlock_irqrestore(&xtime_lock, flags); 326 327 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 328 329 return 0; 330 } 331 332 /* sysfs resume/suspend bits for timekeeping */ 333 static struct sysdev_class timekeeping_sysclass = { 334 .name = "timekeeping", 335 .resume = timekeeping_resume, 336 .suspend = timekeeping_suspend, 337 }; 338 339 static struct sys_device device_timer = { 340 .id = 0, 341 .cls = &timekeeping_sysclass, 342 }; 343 344 static int __init timekeeping_init_device(void) 345 { 346 int error = sysdev_class_register(&timekeeping_sysclass); 347 if (!error) 348 error = sysdev_register(&device_timer); 349 return error; 350 } 351 352 device_initcall(timekeeping_init_device); 353 354 /* 355 * If the error is already larger, we look ahead even further 356 * to compensate for late or lost adjustments. 357 */ 358 static __always_inline int clocksource_bigadjust(s64 error, s64 *interval, 359 s64 *offset) 360 { 361 s64 tick_error, i; 362 u32 look_ahead, adj; 363 s32 error2, mult; 364 365 /* 366 * Use the current error value to determine how much to look ahead. 367 * The larger the error the slower we adjust for it to avoid problems 368 * with losing too many ticks, otherwise we would overadjust and 369 * produce an even larger error. The smaller the adjustment the 370 * faster we try to adjust for it, as lost ticks can do less harm 371 * here. This is tuned so that an error of about 1 msec is adjusted 372 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). 373 */ 374 error2 = clock->error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); 375 error2 = abs(error2); 376 for (look_ahead = 0; error2 > 0; look_ahead++) 377 error2 >>= 2; 378 379 /* 380 * Now calculate the error in (1 << look_ahead) ticks, but first 381 * remove the single look ahead already included in the error. 382 */ 383 tick_error = tick_length >> (NTP_SCALE_SHIFT - clock->shift + 1); 384 tick_error -= clock->xtime_interval >> 1; 385 error = ((error - tick_error) >> look_ahead) + tick_error; 386 387 /* Finally calculate the adjustment shift value. */ 388 i = *interval; 389 mult = 1; 390 if (error < 0) { 391 error = -error; 392 *interval = -*interval; 393 *offset = -*offset; 394 mult = -1; 395 } 396 for (adj = 0; error > i; adj++) 397 error >>= 1; 398 399 *interval <<= adj; 400 *offset <<= adj; 401 return mult << adj; 402 } 403 404 /* 405 * Adjust the multiplier to reduce the error value, 406 * this is optimized for the most common adjustments of -1,0,1, 407 * for other values we can do a bit more work. 408 */ 409 static void clocksource_adjust(s64 offset) 410 { 411 s64 error, interval = clock->cycle_interval; 412 int adj; 413 414 error = clock->error >> (NTP_SCALE_SHIFT - clock->shift - 1); 415 if (error > interval) { 416 error >>= 2; 417 if (likely(error <= interval)) 418 adj = 1; 419 else 420 adj = clocksource_bigadjust(error, &interval, &offset); 421 } else if (error < -interval) { 422 error >>= 2; 423 if (likely(error >= -interval)) { 424 adj = -1; 425 interval = -interval; 426 offset = -offset; 427 } else 428 adj = clocksource_bigadjust(error, &interval, &offset); 429 } else 430 return; 431 432 clock->mult += adj; 433 clock->xtime_interval += interval; 434 clock->xtime_nsec -= offset; 435 clock->error -= (interval - offset) << 436 (NTP_SCALE_SHIFT - clock->shift); 437 } 438 439 /** 440 * update_wall_time - Uses the current clocksource to increment the wall time 441 * 442 * Called from the timer interrupt, must hold a write on xtime_lock. 443 */ 444 void update_wall_time(void) 445 { 446 cycle_t offset; 447 448 /* Make sure we're fully resumed: */ 449 if (unlikely(timekeeping_suspended)) 450 return; 451 452 #ifdef CONFIG_GENERIC_TIME 453 offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask; 454 #else 455 offset = clock->cycle_interval; 456 #endif 457 clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; 458 459 /* normally this loop will run just once, however in the 460 * case of lost or late ticks, it will accumulate correctly. 461 */ 462 while (offset >= clock->cycle_interval) { 463 /* accumulate one interval */ 464 clock->xtime_nsec += clock->xtime_interval; 465 clock->cycle_last += clock->cycle_interval; 466 offset -= clock->cycle_interval; 467 468 if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { 469 clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; 470 xtime.tv_sec++; 471 second_overflow(); 472 } 473 474 /* accumulate error between NTP and clock interval */ 475 clock->error += tick_length; 476 clock->error -= clock->xtime_interval << (NTP_SCALE_SHIFT - clock->shift); 477 } 478 479 /* correct the clock when NTP error is too big */ 480 clocksource_adjust(offset); 481 482 /* store full nanoseconds into xtime */ 483 xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; 484 clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; 485 486 update_xtime_cache(cyc2ns(clock, offset)); 487 488 /* check to see if there is a new clocksource to use */ 489 change_clocksource(); 490 update_vsyscall(&xtime, clock); 491 } 492 493 /** 494 * getboottime - Return the real time of system boot. 495 * @ts: pointer to the timespec to be set 496 * 497 * Returns the time of day in a timespec. 498 * 499 * This is based on the wall_to_monotonic offset and the total suspend 500 * time. Calls to settimeofday will affect the value returned (which 501 * basically means that however wrong your real time clock is at boot time, 502 * you get the right time here). 503 */ 504 void getboottime(struct timespec *ts) 505 { 506 set_normalized_timespec(ts, 507 - (wall_to_monotonic.tv_sec + total_sleep_time), 508 - wall_to_monotonic.tv_nsec); 509 } 510 511 /** 512 * monotonic_to_bootbased - Convert the monotonic time to boot based. 513 * @ts: pointer to the timespec to be converted 514 */ 515 void monotonic_to_bootbased(struct timespec *ts) 516 { 517 ts->tv_sec += total_sleep_time; 518 } 519 520 unsigned long get_seconds(void) 521 { 522 return xtime_cache.tv_sec; 523 } 524 EXPORT_SYMBOL(get_seconds); 525 526 527 struct timespec current_kernel_time(void) 528 { 529 struct timespec now; 530 unsigned long seq; 531 532 do { 533 seq = read_seqbegin(&xtime_lock); 534 535 now = xtime_cache; 536 } while (read_seqretry(&xtime_lock, seq)); 537 538 return now; 539 } 540 EXPORT_SYMBOL(current_kernel_time); 541