1 /* 2 * linux/kernel/hrtimer.c 3 * 4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner 7 * 8 * High-resolution kernel timers 9 * 10 * In contrast to the low-resolution timeout API implemented in 11 * kernel/timer.c, hrtimers provide finer resolution and accuracy 12 * depending on system configuration and capabilities. 13 * 14 * These timers are currently used for: 15 * - itimers 16 * - POSIX timers 17 * - nanosleep 18 * - precise in-kernel timing 19 * 20 * Started by: Thomas Gleixner and Ingo Molnar 21 * 22 * Credits: 23 * based on kernel/timer.c 24 * 25 * Help, testing, suggestions, bugfixes, improvements were 26 * provided by: 27 * 28 * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel 29 * et. al. 30 * 31 * For licencing details see kernel-base/COPYING 32 */ 33 34 #include <linux/cpu.h> 35 #include <linux/export.h> 36 #include <linux/percpu.h> 37 #include <linux/hrtimer.h> 38 #include <linux/notifier.h> 39 #include <linux/syscalls.h> 40 #include <linux/kallsyms.h> 41 #include <linux/interrupt.h> 42 #include <linux/tick.h> 43 #include <linux/seq_file.h> 44 #include <linux/err.h> 45 #include <linux/debugobjects.h> 46 #include <linux/sched.h> 47 #include <linux/sched/sysctl.h> 48 #include <linux/sched/rt.h> 49 #include <linux/sched/deadline.h> 50 #include <linux/timer.h> 51 #include <linux/freezer.h> 52 53 #include <asm/uaccess.h> 54 55 #include <trace/events/timer.h> 56 57 #include "tick-internal.h" 58 59 /* 60 * The timer bases: 61 * 62 * There are more clockids than hrtimer bases. Thus, we index 63 * into the timer bases by the hrtimer_base_type enum. When trying 64 * to reach a base using a clockid, hrtimer_clockid_to_base() 65 * is used to convert from clockid to the proper hrtimer_base_type. 66 */ 67 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = 68 { 69 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock), 70 .seq = SEQCNT_ZERO(hrtimer_bases.seq), 71 .clock_base = 72 { 73 { 74 .index = HRTIMER_BASE_MONOTONIC, 75 .clockid = CLOCK_MONOTONIC, 76 .get_time = &ktime_get, 77 }, 78 { 79 .index = HRTIMER_BASE_REALTIME, 80 .clockid = CLOCK_REALTIME, 81 .get_time = &ktime_get_real, 82 }, 83 { 84 .index = HRTIMER_BASE_BOOTTIME, 85 .clockid = CLOCK_BOOTTIME, 86 .get_time = &ktime_get_boottime, 87 }, 88 { 89 .index = HRTIMER_BASE_TAI, 90 .clockid = CLOCK_TAI, 91 .get_time = &ktime_get_clocktai, 92 }, 93 } 94 }; 95 96 static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = { 97 [CLOCK_REALTIME] = HRTIMER_BASE_REALTIME, 98 [CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC, 99 [CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME, 100 [CLOCK_TAI] = HRTIMER_BASE_TAI, 101 }; 102 103 static inline int hrtimer_clockid_to_base(clockid_t clock_id) 104 { 105 return hrtimer_clock_to_base_table[clock_id]; 106 } 107 108 /* 109 * Functions and macros which are different for UP/SMP systems are kept in a 110 * single place 111 */ 112 #ifdef CONFIG_SMP 113 114 /* 115 * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base() 116 * such that hrtimer_callback_running() can unconditionally dereference 117 * timer->base->cpu_base 118 */ 119 static struct hrtimer_cpu_base migration_cpu_base = { 120 .seq = SEQCNT_ZERO(migration_cpu_base), 121 .clock_base = { { .cpu_base = &migration_cpu_base, }, }, 122 }; 123 124 #define migration_base migration_cpu_base.clock_base[0] 125 126 /* 127 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock 128 * means that all timers which are tied to this base via timer->base are 129 * locked, and the base itself is locked too. 130 * 131 * So __run_timers/migrate_timers can safely modify all timers which could 132 * be found on the lists/queues. 133 * 134 * When the timer's base is locked, and the timer removed from list, it is 135 * possible to set timer->base = &migration_base and drop the lock: the timer 136 * remains locked. 137 */ 138 static 139 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, 140 unsigned long *flags) 141 { 142 struct hrtimer_clock_base *base; 143 144 for (;;) { 145 base = timer->base; 146 if (likely(base != &migration_base)) { 147 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); 148 if (likely(base == timer->base)) 149 return base; 150 /* The timer has migrated to another CPU: */ 151 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); 152 } 153 cpu_relax(); 154 } 155 } 156 157 /* 158 * With HIGHRES=y we do not migrate the timer when it is expiring 159 * before the next event on the target cpu because we cannot reprogram 160 * the target cpu hardware and we would cause it to fire late. 161 * 162 * Called with cpu_base->lock of target cpu held. 163 */ 164 static int 165 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) 166 { 167 #ifdef CONFIG_HIGH_RES_TIMERS 168 ktime_t expires; 169 170 if (!new_base->cpu_base->hres_active) 171 return 0; 172 173 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 174 return expires.tv64 <= new_base->cpu_base->expires_next.tv64; 175 #else 176 return 0; 177 #endif 178 } 179 180 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 181 static inline 182 struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, 183 int pinned) 184 { 185 if (pinned || !base->migration_enabled) 186 return base; 187 return &per_cpu(hrtimer_bases, get_nohz_timer_target()); 188 } 189 #else 190 static inline 191 struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, 192 int pinned) 193 { 194 return base; 195 } 196 #endif 197 198 /* 199 * We switch the timer base to a power-optimized selected CPU target, 200 * if: 201 * - NO_HZ_COMMON is enabled 202 * - timer migration is enabled 203 * - the timer callback is not running 204 * - the timer is not the first expiring timer on the new target 205 * 206 * If one of the above requirements is not fulfilled we move the timer 207 * to the current CPU or leave it on the previously assigned CPU if 208 * the timer callback is currently running. 209 */ 210 static inline struct hrtimer_clock_base * 211 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, 212 int pinned) 213 { 214 struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base; 215 struct hrtimer_clock_base *new_base; 216 int basenum = base->index; 217 218 this_cpu_base = this_cpu_ptr(&hrtimer_bases); 219 new_cpu_base = get_target_base(this_cpu_base, pinned); 220 again: 221 new_base = &new_cpu_base->clock_base[basenum]; 222 223 if (base != new_base) { 224 /* 225 * We are trying to move timer to new_base. 226 * However we can't change timer's base while it is running, 227 * so we keep it on the same CPU. No hassle vs. reprogramming 228 * the event source in the high resolution case. The softirq 229 * code will take care of this when the timer function has 230 * completed. There is no conflict as we hold the lock until 231 * the timer is enqueued. 232 */ 233 if (unlikely(hrtimer_callback_running(timer))) 234 return base; 235 236 /* See the comment in lock_hrtimer_base() */ 237 timer->base = &migration_base; 238 raw_spin_unlock(&base->cpu_base->lock); 239 raw_spin_lock(&new_base->cpu_base->lock); 240 241 if (new_cpu_base != this_cpu_base && 242 hrtimer_check_target(timer, new_base)) { 243 raw_spin_unlock(&new_base->cpu_base->lock); 244 raw_spin_lock(&base->cpu_base->lock); 245 new_cpu_base = this_cpu_base; 246 timer->base = base; 247 goto again; 248 } 249 timer->base = new_base; 250 } else { 251 if (new_cpu_base != this_cpu_base && 252 hrtimer_check_target(timer, new_base)) { 253 new_cpu_base = this_cpu_base; 254 goto again; 255 } 256 } 257 return new_base; 258 } 259 260 #else /* CONFIG_SMP */ 261 262 static inline struct hrtimer_clock_base * 263 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 264 { 265 struct hrtimer_clock_base *base = timer->base; 266 267 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); 268 269 return base; 270 } 271 272 # define switch_hrtimer_base(t, b, p) (b) 273 274 #endif /* !CONFIG_SMP */ 275 276 /* 277 * Functions for the union type storage format of ktime_t which are 278 * too large for inlining: 279 */ 280 #if BITS_PER_LONG < 64 281 /* 282 * Divide a ktime value by a nanosecond value 283 */ 284 s64 __ktime_divns(const ktime_t kt, s64 div) 285 { 286 int sft = 0; 287 s64 dclc; 288 u64 tmp; 289 290 dclc = ktime_to_ns(kt); 291 tmp = dclc < 0 ? -dclc : dclc; 292 293 /* Make sure the divisor is less than 2^32: */ 294 while (div >> 32) { 295 sft++; 296 div >>= 1; 297 } 298 tmp >>= sft; 299 do_div(tmp, (unsigned long) div); 300 return dclc < 0 ? -tmp : tmp; 301 } 302 EXPORT_SYMBOL_GPL(__ktime_divns); 303 #endif /* BITS_PER_LONG >= 64 */ 304 305 /* 306 * Add two ktime values and do a safety check for overflow: 307 */ 308 ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) 309 { 310 ktime_t res = ktime_add(lhs, rhs); 311 312 /* 313 * We use KTIME_SEC_MAX here, the maximum timeout which we can 314 * return to user space in a timespec: 315 */ 316 if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64) 317 res = ktime_set(KTIME_SEC_MAX, 0); 318 319 return res; 320 } 321 322 EXPORT_SYMBOL_GPL(ktime_add_safe); 323 324 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS 325 326 static struct debug_obj_descr hrtimer_debug_descr; 327 328 static void *hrtimer_debug_hint(void *addr) 329 { 330 return ((struct hrtimer *) addr)->function; 331 } 332 333 /* 334 * fixup_init is called when: 335 * - an active object is initialized 336 */ 337 static int hrtimer_fixup_init(void *addr, enum debug_obj_state state) 338 { 339 struct hrtimer *timer = addr; 340 341 switch (state) { 342 case ODEBUG_STATE_ACTIVE: 343 hrtimer_cancel(timer); 344 debug_object_init(timer, &hrtimer_debug_descr); 345 return 1; 346 default: 347 return 0; 348 } 349 } 350 351 /* 352 * fixup_activate is called when: 353 * - an active object is activated 354 * - an unknown object is activated (might be a statically initialized object) 355 */ 356 static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state) 357 { 358 switch (state) { 359 360 case ODEBUG_STATE_NOTAVAILABLE: 361 WARN_ON_ONCE(1); 362 return 0; 363 364 case ODEBUG_STATE_ACTIVE: 365 WARN_ON(1); 366 367 default: 368 return 0; 369 } 370 } 371 372 /* 373 * fixup_free is called when: 374 * - an active object is freed 375 */ 376 static int hrtimer_fixup_free(void *addr, enum debug_obj_state state) 377 { 378 struct hrtimer *timer = addr; 379 380 switch (state) { 381 case ODEBUG_STATE_ACTIVE: 382 hrtimer_cancel(timer); 383 debug_object_free(timer, &hrtimer_debug_descr); 384 return 1; 385 default: 386 return 0; 387 } 388 } 389 390 static struct debug_obj_descr hrtimer_debug_descr = { 391 .name = "hrtimer", 392 .debug_hint = hrtimer_debug_hint, 393 .fixup_init = hrtimer_fixup_init, 394 .fixup_activate = hrtimer_fixup_activate, 395 .fixup_free = hrtimer_fixup_free, 396 }; 397 398 static inline void debug_hrtimer_init(struct hrtimer *timer) 399 { 400 debug_object_init(timer, &hrtimer_debug_descr); 401 } 402 403 static inline void debug_hrtimer_activate(struct hrtimer *timer) 404 { 405 debug_object_activate(timer, &hrtimer_debug_descr); 406 } 407 408 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) 409 { 410 debug_object_deactivate(timer, &hrtimer_debug_descr); 411 } 412 413 static inline void debug_hrtimer_free(struct hrtimer *timer) 414 { 415 debug_object_free(timer, &hrtimer_debug_descr); 416 } 417 418 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, 419 enum hrtimer_mode mode); 420 421 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, 422 enum hrtimer_mode mode) 423 { 424 debug_object_init_on_stack(timer, &hrtimer_debug_descr); 425 __hrtimer_init(timer, clock_id, mode); 426 } 427 EXPORT_SYMBOL_GPL(hrtimer_init_on_stack); 428 429 void destroy_hrtimer_on_stack(struct hrtimer *timer) 430 { 431 debug_object_free(timer, &hrtimer_debug_descr); 432 } 433 434 #else 435 static inline void debug_hrtimer_init(struct hrtimer *timer) { } 436 static inline void debug_hrtimer_activate(struct hrtimer *timer) { } 437 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } 438 #endif 439 440 static inline void 441 debug_init(struct hrtimer *timer, clockid_t clockid, 442 enum hrtimer_mode mode) 443 { 444 debug_hrtimer_init(timer); 445 trace_hrtimer_init(timer, clockid, mode); 446 } 447 448 static inline void debug_activate(struct hrtimer *timer) 449 { 450 debug_hrtimer_activate(timer); 451 trace_hrtimer_start(timer); 452 } 453 454 static inline void debug_deactivate(struct hrtimer *timer) 455 { 456 debug_hrtimer_deactivate(timer); 457 trace_hrtimer_cancel(timer); 458 } 459 460 #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS) 461 static inline void hrtimer_update_next_timer(struct hrtimer_cpu_base *cpu_base, 462 struct hrtimer *timer) 463 { 464 #ifdef CONFIG_HIGH_RES_TIMERS 465 cpu_base->next_timer = timer; 466 #endif 467 } 468 469 static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base) 470 { 471 struct hrtimer_clock_base *base = cpu_base->clock_base; 472 ktime_t expires, expires_next = { .tv64 = KTIME_MAX }; 473 unsigned int active = cpu_base->active_bases; 474 475 hrtimer_update_next_timer(cpu_base, NULL); 476 for (; active; base++, active >>= 1) { 477 struct timerqueue_node *next; 478 struct hrtimer *timer; 479 480 if (!(active & 0x01)) 481 continue; 482 483 next = timerqueue_getnext(&base->active); 484 timer = container_of(next, struct hrtimer, node); 485 expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 486 if (expires.tv64 < expires_next.tv64) { 487 expires_next = expires; 488 hrtimer_update_next_timer(cpu_base, timer); 489 } 490 } 491 /* 492 * clock_was_set() might have changed base->offset of any of 493 * the clock bases so the result might be negative. Fix it up 494 * to prevent a false positive in clockevents_program_event(). 495 */ 496 if (expires_next.tv64 < 0) 497 expires_next.tv64 = 0; 498 return expires_next; 499 } 500 #endif 501 502 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) 503 { 504 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; 505 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; 506 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 507 508 return ktime_get_update_offsets_now(&base->clock_was_set_seq, 509 offs_real, offs_boot, offs_tai); 510 } 511 512 /* High resolution timer related functions */ 513 #ifdef CONFIG_HIGH_RES_TIMERS 514 515 /* 516 * High resolution timer enabled ? 517 */ 518 static int hrtimer_hres_enabled __read_mostly = 1; 519 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC; 520 EXPORT_SYMBOL_GPL(hrtimer_resolution); 521 522 /* 523 * Enable / Disable high resolution mode 524 */ 525 static int __init setup_hrtimer_hres(char *str) 526 { 527 if (!strcmp(str, "off")) 528 hrtimer_hres_enabled = 0; 529 else if (!strcmp(str, "on")) 530 hrtimer_hres_enabled = 1; 531 else 532 return 0; 533 return 1; 534 } 535 536 __setup("highres=", setup_hrtimer_hres); 537 538 /* 539 * hrtimer_high_res_enabled - query, if the highres mode is enabled 540 */ 541 static inline int hrtimer_is_hres_enabled(void) 542 { 543 return hrtimer_hres_enabled; 544 } 545 546 /* 547 * Is the high resolution mode active ? 548 */ 549 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) 550 { 551 return cpu_base->hres_active; 552 } 553 554 static inline int hrtimer_hres_active(void) 555 { 556 return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases)); 557 } 558 559 /* 560 * Reprogram the event source with checking both queues for the 561 * next event 562 * Called with interrupts disabled and base->lock held 563 */ 564 static void 565 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) 566 { 567 ktime_t expires_next; 568 569 if (!cpu_base->hres_active) 570 return; 571 572 expires_next = __hrtimer_get_next_event(cpu_base); 573 574 if (skip_equal && expires_next.tv64 == cpu_base->expires_next.tv64) 575 return; 576 577 cpu_base->expires_next.tv64 = expires_next.tv64; 578 579 /* 580 * If a hang was detected in the last timer interrupt then we 581 * leave the hang delay active in the hardware. We want the 582 * system to make progress. That also prevents the following 583 * scenario: 584 * T1 expires 50ms from now 585 * T2 expires 5s from now 586 * 587 * T1 is removed, so this code is called and would reprogram 588 * the hardware to 5s from now. Any hrtimer_start after that 589 * will not reprogram the hardware due to hang_detected being 590 * set. So we'd effectivly block all timers until the T2 event 591 * fires. 592 */ 593 if (cpu_base->hang_detected) 594 return; 595 596 tick_program_event(cpu_base->expires_next, 1); 597 } 598 599 /* 600 * When a timer is enqueued and expires earlier than the already enqueued 601 * timers, we have to check, whether it expires earlier than the timer for 602 * which the clock event device was armed. 603 * 604 * Called with interrupts disabled and base->cpu_base.lock held 605 */ 606 static void hrtimer_reprogram(struct hrtimer *timer, 607 struct hrtimer_clock_base *base) 608 { 609 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 610 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 611 612 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); 613 614 /* 615 * If the timer is not on the current cpu, we cannot reprogram 616 * the other cpus clock event device. 617 */ 618 if (base->cpu_base != cpu_base) 619 return; 620 621 /* 622 * If the hrtimer interrupt is running, then it will 623 * reevaluate the clock bases and reprogram the clock event 624 * device. The callbacks are always executed in hard interrupt 625 * context so we don't need an extra check for a running 626 * callback. 627 */ 628 if (cpu_base->in_hrtirq) 629 return; 630 631 /* 632 * CLOCK_REALTIME timer might be requested with an absolute 633 * expiry time which is less than base->offset. Set it to 0. 634 */ 635 if (expires.tv64 < 0) 636 expires.tv64 = 0; 637 638 if (expires.tv64 >= cpu_base->expires_next.tv64) 639 return; 640 641 /* Update the pointer to the next expiring timer */ 642 cpu_base->next_timer = timer; 643 644 /* 645 * If a hang was detected in the last timer interrupt then we 646 * do not schedule a timer which is earlier than the expiry 647 * which we enforced in the hang detection. We want the system 648 * to make progress. 649 */ 650 if (cpu_base->hang_detected) 651 return; 652 653 /* 654 * Program the timer hardware. We enforce the expiry for 655 * events which are already in the past. 656 */ 657 cpu_base->expires_next = expires; 658 tick_program_event(expires, 1); 659 } 660 661 /* 662 * Initialize the high resolution related parts of cpu_base 663 */ 664 static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) 665 { 666 base->expires_next.tv64 = KTIME_MAX; 667 base->hres_active = 0; 668 } 669 670 /* 671 * Retrigger next event is called after clock was set 672 * 673 * Called with interrupts disabled via on_each_cpu() 674 */ 675 static void retrigger_next_event(void *arg) 676 { 677 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 678 679 if (!base->hres_active) 680 return; 681 682 raw_spin_lock(&base->lock); 683 hrtimer_update_base(base); 684 hrtimer_force_reprogram(base, 0); 685 raw_spin_unlock(&base->lock); 686 } 687 688 /* 689 * Switch to high resolution mode 690 */ 691 static void hrtimer_switch_to_hres(void) 692 { 693 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 694 695 if (tick_init_highres()) { 696 printk(KERN_WARNING "Could not switch to high resolution " 697 "mode on CPU %d\n", base->cpu); 698 return; 699 } 700 base->hres_active = 1; 701 hrtimer_resolution = HIGH_RES_NSEC; 702 703 tick_setup_sched_timer(); 704 /* "Retrigger" the interrupt to get things going */ 705 retrigger_next_event(NULL); 706 } 707 708 static void clock_was_set_work(struct work_struct *work) 709 { 710 clock_was_set(); 711 } 712 713 static DECLARE_WORK(hrtimer_work, clock_was_set_work); 714 715 /* 716 * Called from timekeeping and resume code to reprogramm the hrtimer 717 * interrupt device on all cpus. 718 */ 719 void clock_was_set_delayed(void) 720 { 721 schedule_work(&hrtimer_work); 722 } 723 724 #else 725 726 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *b) { return 0; } 727 static inline int hrtimer_hres_active(void) { return 0; } 728 static inline int hrtimer_is_hres_enabled(void) { return 0; } 729 static inline void hrtimer_switch_to_hres(void) { } 730 static inline void 731 hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } 732 static inline int hrtimer_reprogram(struct hrtimer *timer, 733 struct hrtimer_clock_base *base) 734 { 735 return 0; 736 } 737 static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } 738 static inline void retrigger_next_event(void *arg) { } 739 740 #endif /* CONFIG_HIGH_RES_TIMERS */ 741 742 /* 743 * Clock realtime was set 744 * 745 * Change the offset of the realtime clock vs. the monotonic 746 * clock. 747 * 748 * We might have to reprogram the high resolution timer interrupt. On 749 * SMP we call the architecture specific code to retrigger _all_ high 750 * resolution timer interrupts. On UP we just disable interrupts and 751 * call the high resolution interrupt code. 752 */ 753 void clock_was_set(void) 754 { 755 #ifdef CONFIG_HIGH_RES_TIMERS 756 /* Retrigger the CPU local events everywhere */ 757 on_each_cpu(retrigger_next_event, NULL, 1); 758 #endif 759 timerfd_clock_was_set(); 760 } 761 762 /* 763 * During resume we might have to reprogram the high resolution timer 764 * interrupt on all online CPUs. However, all other CPUs will be 765 * stopped with IRQs interrupts disabled so the clock_was_set() call 766 * must be deferred. 767 */ 768 void hrtimers_resume(void) 769 { 770 WARN_ONCE(!irqs_disabled(), 771 KERN_INFO "hrtimers_resume() called with IRQs enabled!"); 772 773 /* Retrigger on the local CPU */ 774 retrigger_next_event(NULL); 775 /* And schedule a retrigger for all others */ 776 clock_was_set_delayed(); 777 } 778 779 static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer) 780 { 781 #ifdef CONFIG_TIMER_STATS 782 if (timer->start_site) 783 return; 784 timer->start_site = __builtin_return_address(0); 785 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); 786 timer->start_pid = current->pid; 787 #endif 788 } 789 790 static inline void timer_stats_hrtimer_clear_start_info(struct hrtimer *timer) 791 { 792 #ifdef CONFIG_TIMER_STATS 793 timer->start_site = NULL; 794 #endif 795 } 796 797 static inline void timer_stats_account_hrtimer(struct hrtimer *timer) 798 { 799 #ifdef CONFIG_TIMER_STATS 800 if (likely(!timer_stats_active)) 801 return; 802 timer_stats_update_stats(timer, timer->start_pid, timer->start_site, 803 timer->function, timer->start_comm, 0); 804 #endif 805 } 806 807 /* 808 * Counterpart to lock_hrtimer_base above: 809 */ 810 static inline 811 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 812 { 813 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); 814 } 815 816 /** 817 * hrtimer_forward - forward the timer expiry 818 * @timer: hrtimer to forward 819 * @now: forward past this time 820 * @interval: the interval to forward 821 * 822 * Forward the timer expiry so it will expire in the future. 823 * Returns the number of overruns. 824 * 825 * Can be safely called from the callback function of @timer. If 826 * called from other contexts @timer must neither be enqueued nor 827 * running the callback and the caller needs to take care of 828 * serialization. 829 * 830 * Note: This only updates the timer expiry value and does not requeue 831 * the timer. 832 */ 833 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) 834 { 835 u64 orun = 1; 836 ktime_t delta; 837 838 delta = ktime_sub(now, hrtimer_get_expires(timer)); 839 840 if (delta.tv64 < 0) 841 return 0; 842 843 if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED)) 844 return 0; 845 846 if (interval.tv64 < hrtimer_resolution) 847 interval.tv64 = hrtimer_resolution; 848 849 if (unlikely(delta.tv64 >= interval.tv64)) { 850 s64 incr = ktime_to_ns(interval); 851 852 orun = ktime_divns(delta, incr); 853 hrtimer_add_expires_ns(timer, incr * orun); 854 if (hrtimer_get_expires_tv64(timer) > now.tv64) 855 return orun; 856 /* 857 * This (and the ktime_add() below) is the 858 * correction for exact: 859 */ 860 orun++; 861 } 862 hrtimer_add_expires(timer, interval); 863 864 return orun; 865 } 866 EXPORT_SYMBOL_GPL(hrtimer_forward); 867 868 /* 869 * enqueue_hrtimer - internal function to (re)start a timer 870 * 871 * The timer is inserted in expiry order. Insertion into the 872 * red black tree is O(log(n)). Must hold the base lock. 873 * 874 * Returns 1 when the new timer is the leftmost timer in the tree. 875 */ 876 static int enqueue_hrtimer(struct hrtimer *timer, 877 struct hrtimer_clock_base *base) 878 { 879 debug_activate(timer); 880 881 base->cpu_base->active_bases |= 1 << base->index; 882 883 timer->state = HRTIMER_STATE_ENQUEUED; 884 885 return timerqueue_add(&base->active, &timer->node); 886 } 887 888 /* 889 * __remove_hrtimer - internal function to remove a timer 890 * 891 * Caller must hold the base lock. 892 * 893 * High resolution timer mode reprograms the clock event device when the 894 * timer is the one which expires next. The caller can disable this by setting 895 * reprogram to zero. This is useful, when the context does a reprogramming 896 * anyway (e.g. timer interrupt) 897 */ 898 static void __remove_hrtimer(struct hrtimer *timer, 899 struct hrtimer_clock_base *base, 900 u8 newstate, int reprogram) 901 { 902 struct hrtimer_cpu_base *cpu_base = base->cpu_base; 903 u8 state = timer->state; 904 905 timer->state = newstate; 906 if (!(state & HRTIMER_STATE_ENQUEUED)) 907 return; 908 909 if (!timerqueue_del(&base->active, &timer->node)) 910 cpu_base->active_bases &= ~(1 << base->index); 911 912 #ifdef CONFIG_HIGH_RES_TIMERS 913 /* 914 * Note: If reprogram is false we do not update 915 * cpu_base->next_timer. This happens when we remove the first 916 * timer on a remote cpu. No harm as we never dereference 917 * cpu_base->next_timer. So the worst thing what can happen is 918 * an superflous call to hrtimer_force_reprogram() on the 919 * remote cpu later on if the same timer gets enqueued again. 920 */ 921 if (reprogram && timer == cpu_base->next_timer) 922 hrtimer_force_reprogram(cpu_base, 1); 923 #endif 924 } 925 926 /* 927 * remove hrtimer, called with base lock held 928 */ 929 static inline int 930 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart) 931 { 932 if (hrtimer_is_queued(timer)) { 933 u8 state = timer->state; 934 int reprogram; 935 936 /* 937 * Remove the timer and force reprogramming when high 938 * resolution mode is active and the timer is on the current 939 * CPU. If we remove a timer on another CPU, reprogramming is 940 * skipped. The interrupt event on this CPU is fired and 941 * reprogramming happens in the interrupt handler. This is a 942 * rare case and less expensive than a smp call. 943 */ 944 debug_deactivate(timer); 945 timer_stats_hrtimer_clear_start_info(timer); 946 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 947 948 if (!restart) 949 state = HRTIMER_STATE_INACTIVE; 950 951 __remove_hrtimer(timer, base, state, reprogram); 952 return 1; 953 } 954 return 0; 955 } 956 957 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, 958 const enum hrtimer_mode mode) 959 { 960 #ifdef CONFIG_TIME_LOW_RES 961 /* 962 * CONFIG_TIME_LOW_RES indicates that the system has no way to return 963 * granular time values. For relative timers we add hrtimer_resolution 964 * (i.e. one jiffie) to prevent short timeouts. 965 */ 966 timer->is_rel = mode & HRTIMER_MODE_REL; 967 if (timer->is_rel) 968 tim = ktime_add_safe(tim, ktime_set(0, hrtimer_resolution)); 969 #endif 970 return tim; 971 } 972 973 /** 974 * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU 975 * @timer: the timer to be added 976 * @tim: expiry time 977 * @delta_ns: "slack" range for the timer 978 * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or 979 * relative (HRTIMER_MODE_REL) 980 */ 981 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, 982 unsigned long delta_ns, const enum hrtimer_mode mode) 983 { 984 struct hrtimer_clock_base *base, *new_base; 985 unsigned long flags; 986 int leftmost; 987 988 base = lock_hrtimer_base(timer, &flags); 989 990 /* Remove an active timer from the queue: */ 991 remove_hrtimer(timer, base, true); 992 993 if (mode & HRTIMER_MODE_REL) 994 tim = ktime_add_safe(tim, base->get_time()); 995 996 tim = hrtimer_update_lowres(timer, tim, mode); 997 998 hrtimer_set_expires_range_ns(timer, tim, delta_ns); 999 1000 /* Switch the timer base, if necessary: */ 1001 new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); 1002 1003 timer_stats_hrtimer_set_start_info(timer); 1004 1005 leftmost = enqueue_hrtimer(timer, new_base); 1006 if (!leftmost) 1007 goto unlock; 1008 1009 if (!hrtimer_is_hres_active(timer)) { 1010 /* 1011 * Kick to reschedule the next tick to handle the new timer 1012 * on dynticks target. 1013 */ 1014 if (new_base->cpu_base->nohz_active) 1015 wake_up_nohz_cpu(new_base->cpu_base->cpu); 1016 } else { 1017 hrtimer_reprogram(timer, new_base); 1018 } 1019 unlock: 1020 unlock_hrtimer_base(timer, &flags); 1021 } 1022 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); 1023 1024 /** 1025 * hrtimer_try_to_cancel - try to deactivate a timer 1026 * @timer: hrtimer to stop 1027 * 1028 * Returns: 1029 * 0 when the timer was not active 1030 * 1 when the timer was active 1031 * -1 when the timer is currently excuting the callback function and 1032 * cannot be stopped 1033 */ 1034 int hrtimer_try_to_cancel(struct hrtimer *timer) 1035 { 1036 struct hrtimer_clock_base *base; 1037 unsigned long flags; 1038 int ret = -1; 1039 1040 /* 1041 * Check lockless first. If the timer is not active (neither 1042 * enqueued nor running the callback, nothing to do here. The 1043 * base lock does not serialize against a concurrent enqueue, 1044 * so we can avoid taking it. 1045 */ 1046 if (!hrtimer_active(timer)) 1047 return 0; 1048 1049 base = lock_hrtimer_base(timer, &flags); 1050 1051 if (!hrtimer_callback_running(timer)) 1052 ret = remove_hrtimer(timer, base, false); 1053 1054 unlock_hrtimer_base(timer, &flags); 1055 1056 return ret; 1057 1058 } 1059 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); 1060 1061 /** 1062 * hrtimer_cancel - cancel a timer and wait for the handler to finish. 1063 * @timer: the timer to be cancelled 1064 * 1065 * Returns: 1066 * 0 when the timer was not active 1067 * 1 when the timer was active 1068 */ 1069 int hrtimer_cancel(struct hrtimer *timer) 1070 { 1071 for (;;) { 1072 int ret = hrtimer_try_to_cancel(timer); 1073 1074 if (ret >= 0) 1075 return ret; 1076 cpu_relax(); 1077 } 1078 } 1079 EXPORT_SYMBOL_GPL(hrtimer_cancel); 1080 1081 /** 1082 * hrtimer_get_remaining - get remaining time for the timer 1083 * @timer: the timer to read 1084 * @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y 1085 */ 1086 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust) 1087 { 1088 unsigned long flags; 1089 ktime_t rem; 1090 1091 lock_hrtimer_base(timer, &flags); 1092 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust) 1093 rem = hrtimer_expires_remaining_adjusted(timer); 1094 else 1095 rem = hrtimer_expires_remaining(timer); 1096 unlock_hrtimer_base(timer, &flags); 1097 1098 return rem; 1099 } 1100 EXPORT_SYMBOL_GPL(__hrtimer_get_remaining); 1101 1102 #ifdef CONFIG_NO_HZ_COMMON 1103 /** 1104 * hrtimer_get_next_event - get the time until next expiry event 1105 * 1106 * Returns the next expiry time or KTIME_MAX if no timer is pending. 1107 */ 1108 u64 hrtimer_get_next_event(void) 1109 { 1110 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1111 u64 expires = KTIME_MAX; 1112 unsigned long flags; 1113 1114 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1115 1116 if (!__hrtimer_hres_active(cpu_base)) 1117 expires = __hrtimer_get_next_event(cpu_base).tv64; 1118 1119 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1120 1121 return expires; 1122 } 1123 #endif 1124 1125 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, 1126 enum hrtimer_mode mode) 1127 { 1128 struct hrtimer_cpu_base *cpu_base; 1129 int base; 1130 1131 memset(timer, 0, sizeof(struct hrtimer)); 1132 1133 cpu_base = raw_cpu_ptr(&hrtimer_bases); 1134 1135 if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) 1136 clock_id = CLOCK_MONOTONIC; 1137 1138 base = hrtimer_clockid_to_base(clock_id); 1139 timer->base = &cpu_base->clock_base[base]; 1140 timerqueue_init(&timer->node); 1141 1142 #ifdef CONFIG_TIMER_STATS 1143 timer->start_site = NULL; 1144 timer->start_pid = -1; 1145 memset(timer->start_comm, 0, TASK_COMM_LEN); 1146 #endif 1147 } 1148 1149 /** 1150 * hrtimer_init - initialize a timer to the given clock 1151 * @timer: the timer to be initialized 1152 * @clock_id: the clock to be used 1153 * @mode: timer mode abs/rel 1154 */ 1155 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, 1156 enum hrtimer_mode mode) 1157 { 1158 debug_init(timer, clock_id, mode); 1159 __hrtimer_init(timer, clock_id, mode); 1160 } 1161 EXPORT_SYMBOL_GPL(hrtimer_init); 1162 1163 /* 1164 * A timer is active, when it is enqueued into the rbtree or the 1165 * callback function is running or it's in the state of being migrated 1166 * to another cpu. 1167 * 1168 * It is important for this function to not return a false negative. 1169 */ 1170 bool hrtimer_active(const struct hrtimer *timer) 1171 { 1172 struct hrtimer_cpu_base *cpu_base; 1173 unsigned int seq; 1174 1175 do { 1176 cpu_base = READ_ONCE(timer->base->cpu_base); 1177 seq = raw_read_seqcount_begin(&cpu_base->seq); 1178 1179 if (timer->state != HRTIMER_STATE_INACTIVE || 1180 cpu_base->running == timer) 1181 return true; 1182 1183 } while (read_seqcount_retry(&cpu_base->seq, seq) || 1184 cpu_base != READ_ONCE(timer->base->cpu_base)); 1185 1186 return false; 1187 } 1188 EXPORT_SYMBOL_GPL(hrtimer_active); 1189 1190 /* 1191 * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3 1192 * distinct sections: 1193 * 1194 * - queued: the timer is queued 1195 * - callback: the timer is being ran 1196 * - post: the timer is inactive or (re)queued 1197 * 1198 * On the read side we ensure we observe timer->state and cpu_base->running 1199 * from the same section, if anything changed while we looked at it, we retry. 1200 * This includes timer->base changing because sequence numbers alone are 1201 * insufficient for that. 1202 * 1203 * The sequence numbers are required because otherwise we could still observe 1204 * a false negative if the read side got smeared over multiple consequtive 1205 * __run_hrtimer() invocations. 1206 */ 1207 1208 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, 1209 struct hrtimer_clock_base *base, 1210 struct hrtimer *timer, ktime_t *now) 1211 { 1212 enum hrtimer_restart (*fn)(struct hrtimer *); 1213 int restart; 1214 1215 lockdep_assert_held(&cpu_base->lock); 1216 1217 debug_deactivate(timer); 1218 cpu_base->running = timer; 1219 1220 /* 1221 * Separate the ->running assignment from the ->state assignment. 1222 * 1223 * As with a regular write barrier, this ensures the read side in 1224 * hrtimer_active() cannot observe cpu_base->running == NULL && 1225 * timer->state == INACTIVE. 1226 */ 1227 raw_write_seqcount_barrier(&cpu_base->seq); 1228 1229 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0); 1230 timer_stats_account_hrtimer(timer); 1231 fn = timer->function; 1232 1233 /* 1234 * Clear the 'is relative' flag for the TIME_LOW_RES case. If the 1235 * timer is restarted with a period then it becomes an absolute 1236 * timer. If its not restarted it does not matter. 1237 */ 1238 if (IS_ENABLED(CONFIG_TIME_LOW_RES)) 1239 timer->is_rel = false; 1240 1241 /* 1242 * Because we run timers from hardirq context, there is no chance 1243 * they get migrated to another cpu, therefore its safe to unlock 1244 * the timer base. 1245 */ 1246 raw_spin_unlock(&cpu_base->lock); 1247 trace_hrtimer_expire_entry(timer, now); 1248 restart = fn(timer); 1249 trace_hrtimer_expire_exit(timer); 1250 raw_spin_lock(&cpu_base->lock); 1251 1252 /* 1253 * Note: We clear the running state after enqueue_hrtimer and 1254 * we do not reprogramm the event hardware. Happens either in 1255 * hrtimer_start_range_ns() or in hrtimer_interrupt() 1256 * 1257 * Note: Because we dropped the cpu_base->lock above, 1258 * hrtimer_start_range_ns() can have popped in and enqueued the timer 1259 * for us already. 1260 */ 1261 if (restart != HRTIMER_NORESTART && 1262 !(timer->state & HRTIMER_STATE_ENQUEUED)) 1263 enqueue_hrtimer(timer, base); 1264 1265 /* 1266 * Separate the ->running assignment from the ->state assignment. 1267 * 1268 * As with a regular write barrier, this ensures the read side in 1269 * hrtimer_active() cannot observe cpu_base->running == NULL && 1270 * timer->state == INACTIVE. 1271 */ 1272 raw_write_seqcount_barrier(&cpu_base->seq); 1273 1274 WARN_ON_ONCE(cpu_base->running != timer); 1275 cpu_base->running = NULL; 1276 } 1277 1278 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now) 1279 { 1280 struct hrtimer_clock_base *base = cpu_base->clock_base; 1281 unsigned int active = cpu_base->active_bases; 1282 1283 for (; active; base++, active >>= 1) { 1284 struct timerqueue_node *node; 1285 ktime_t basenow; 1286 1287 if (!(active & 0x01)) 1288 continue; 1289 1290 basenow = ktime_add(now, base->offset); 1291 1292 while ((node = timerqueue_getnext(&base->active))) { 1293 struct hrtimer *timer; 1294 1295 timer = container_of(node, struct hrtimer, node); 1296 1297 /* 1298 * The immediate goal for using the softexpires is 1299 * minimizing wakeups, not running timers at the 1300 * earliest interrupt after their soft expiration. 1301 * This allows us to avoid using a Priority Search 1302 * Tree, which can answer a stabbing querry for 1303 * overlapping intervals and instead use the simple 1304 * BST we already have. 1305 * We don't add extra wakeups by delaying timers that 1306 * are right-of a not yet expired timer, because that 1307 * timer will have to trigger a wakeup anyway. 1308 */ 1309 if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) 1310 break; 1311 1312 __run_hrtimer(cpu_base, base, timer, &basenow); 1313 } 1314 } 1315 } 1316 1317 #ifdef CONFIG_HIGH_RES_TIMERS 1318 1319 /* 1320 * High resolution timer interrupt 1321 * Called with interrupts disabled 1322 */ 1323 void hrtimer_interrupt(struct clock_event_device *dev) 1324 { 1325 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1326 ktime_t expires_next, now, entry_time, delta; 1327 int retries = 0; 1328 1329 BUG_ON(!cpu_base->hres_active); 1330 cpu_base->nr_events++; 1331 dev->next_event.tv64 = KTIME_MAX; 1332 1333 raw_spin_lock(&cpu_base->lock); 1334 entry_time = now = hrtimer_update_base(cpu_base); 1335 retry: 1336 cpu_base->in_hrtirq = 1; 1337 /* 1338 * We set expires_next to KTIME_MAX here with cpu_base->lock 1339 * held to prevent that a timer is enqueued in our queue via 1340 * the migration code. This does not affect enqueueing of 1341 * timers which run their callback and need to be requeued on 1342 * this CPU. 1343 */ 1344 cpu_base->expires_next.tv64 = KTIME_MAX; 1345 1346 __hrtimer_run_queues(cpu_base, now); 1347 1348 /* Reevaluate the clock bases for the next expiry */ 1349 expires_next = __hrtimer_get_next_event(cpu_base); 1350 /* 1351 * Store the new expiry value so the migration code can verify 1352 * against it. 1353 */ 1354 cpu_base->expires_next = expires_next; 1355 cpu_base->in_hrtirq = 0; 1356 raw_spin_unlock(&cpu_base->lock); 1357 1358 /* Reprogramming necessary ? */ 1359 if (!tick_program_event(expires_next, 0)) { 1360 cpu_base->hang_detected = 0; 1361 return; 1362 } 1363 1364 /* 1365 * The next timer was already expired due to: 1366 * - tracing 1367 * - long lasting callbacks 1368 * - being scheduled away when running in a VM 1369 * 1370 * We need to prevent that we loop forever in the hrtimer 1371 * interrupt routine. We give it 3 attempts to avoid 1372 * overreacting on some spurious event. 1373 * 1374 * Acquire base lock for updating the offsets and retrieving 1375 * the current time. 1376 */ 1377 raw_spin_lock(&cpu_base->lock); 1378 now = hrtimer_update_base(cpu_base); 1379 cpu_base->nr_retries++; 1380 if (++retries < 3) 1381 goto retry; 1382 /* 1383 * Give the system a chance to do something else than looping 1384 * here. We stored the entry time, so we know exactly how long 1385 * we spent here. We schedule the next event this amount of 1386 * time away. 1387 */ 1388 cpu_base->nr_hangs++; 1389 cpu_base->hang_detected = 1; 1390 raw_spin_unlock(&cpu_base->lock); 1391 delta = ktime_sub(now, entry_time); 1392 if ((unsigned int)delta.tv64 > cpu_base->max_hang_time) 1393 cpu_base->max_hang_time = (unsigned int) delta.tv64; 1394 /* 1395 * Limit it to a sensible value as we enforce a longer 1396 * delay. Give the CPU at least 100ms to catch up. 1397 */ 1398 if (delta.tv64 > 100 * NSEC_PER_MSEC) 1399 expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC); 1400 else 1401 expires_next = ktime_add(now, delta); 1402 tick_program_event(expires_next, 1); 1403 printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", 1404 ktime_to_ns(delta)); 1405 } 1406 1407 /* 1408 * local version of hrtimer_peek_ahead_timers() called with interrupts 1409 * disabled. 1410 */ 1411 static inline void __hrtimer_peek_ahead_timers(void) 1412 { 1413 struct tick_device *td; 1414 1415 if (!hrtimer_hres_active()) 1416 return; 1417 1418 td = this_cpu_ptr(&tick_cpu_device); 1419 if (td && td->evtdev) 1420 hrtimer_interrupt(td->evtdev); 1421 } 1422 1423 #else /* CONFIG_HIGH_RES_TIMERS */ 1424 1425 static inline void __hrtimer_peek_ahead_timers(void) { } 1426 1427 #endif /* !CONFIG_HIGH_RES_TIMERS */ 1428 1429 /* 1430 * Called from run_local_timers in hardirq context every jiffy 1431 */ 1432 void hrtimer_run_queues(void) 1433 { 1434 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1435 ktime_t now; 1436 1437 if (__hrtimer_hres_active(cpu_base)) 1438 return; 1439 1440 /* 1441 * This _is_ ugly: We have to check periodically, whether we 1442 * can switch to highres and / or nohz mode. The clocksource 1443 * switch happens with xtime_lock held. Notification from 1444 * there only sets the check bit in the tick_oneshot code, 1445 * otherwise we might deadlock vs. xtime_lock. 1446 */ 1447 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) { 1448 hrtimer_switch_to_hres(); 1449 return; 1450 } 1451 1452 raw_spin_lock(&cpu_base->lock); 1453 now = hrtimer_update_base(cpu_base); 1454 __hrtimer_run_queues(cpu_base, now); 1455 raw_spin_unlock(&cpu_base->lock); 1456 } 1457 1458 /* 1459 * Sleep related functions: 1460 */ 1461 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) 1462 { 1463 struct hrtimer_sleeper *t = 1464 container_of(timer, struct hrtimer_sleeper, timer); 1465 struct task_struct *task = t->task; 1466 1467 t->task = NULL; 1468 if (task) 1469 wake_up_process(task); 1470 1471 return HRTIMER_NORESTART; 1472 } 1473 1474 void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) 1475 { 1476 sl->timer.function = hrtimer_wakeup; 1477 sl->task = task; 1478 } 1479 EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); 1480 1481 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) 1482 { 1483 hrtimer_init_sleeper(t, current); 1484 1485 do { 1486 set_current_state(TASK_INTERRUPTIBLE); 1487 hrtimer_start_expires(&t->timer, mode); 1488 1489 if (likely(t->task)) 1490 freezable_schedule(); 1491 1492 hrtimer_cancel(&t->timer); 1493 mode = HRTIMER_MODE_ABS; 1494 1495 } while (t->task && !signal_pending(current)); 1496 1497 __set_current_state(TASK_RUNNING); 1498 1499 return t->task == NULL; 1500 } 1501 1502 static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp) 1503 { 1504 struct timespec rmt; 1505 ktime_t rem; 1506 1507 rem = hrtimer_expires_remaining(timer); 1508 if (rem.tv64 <= 0) 1509 return 0; 1510 rmt = ktime_to_timespec(rem); 1511 1512 if (copy_to_user(rmtp, &rmt, sizeof(*rmtp))) 1513 return -EFAULT; 1514 1515 return 1; 1516 } 1517 1518 long __sched hrtimer_nanosleep_restart(struct restart_block *restart) 1519 { 1520 struct hrtimer_sleeper t; 1521 struct timespec __user *rmtp; 1522 int ret = 0; 1523 1524 hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid, 1525 HRTIMER_MODE_ABS); 1526 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); 1527 1528 if (do_nanosleep(&t, HRTIMER_MODE_ABS)) 1529 goto out; 1530 1531 rmtp = restart->nanosleep.rmtp; 1532 if (rmtp) { 1533 ret = update_rmtp(&t.timer, rmtp); 1534 if (ret <= 0) 1535 goto out; 1536 } 1537 1538 /* The other values in restart are already filled in */ 1539 ret = -ERESTART_RESTARTBLOCK; 1540 out: 1541 destroy_hrtimer_on_stack(&t.timer); 1542 return ret; 1543 } 1544 1545 long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, 1546 const enum hrtimer_mode mode, const clockid_t clockid) 1547 { 1548 struct restart_block *restart; 1549 struct hrtimer_sleeper t; 1550 int ret = 0; 1551 unsigned long slack; 1552 1553 slack = current->timer_slack_ns; 1554 if (dl_task(current) || rt_task(current)) 1555 slack = 0; 1556 1557 hrtimer_init_on_stack(&t.timer, clockid, mode); 1558 hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); 1559 if (do_nanosleep(&t, mode)) 1560 goto out; 1561 1562 /* Absolute timers do not update the rmtp value and restart: */ 1563 if (mode == HRTIMER_MODE_ABS) { 1564 ret = -ERESTARTNOHAND; 1565 goto out; 1566 } 1567 1568 if (rmtp) { 1569 ret = update_rmtp(&t.timer, rmtp); 1570 if (ret <= 0) 1571 goto out; 1572 } 1573 1574 restart = ¤t->restart_block; 1575 restart->fn = hrtimer_nanosleep_restart; 1576 restart->nanosleep.clockid = t.timer.base->clockid; 1577 restart->nanosleep.rmtp = rmtp; 1578 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); 1579 1580 ret = -ERESTART_RESTARTBLOCK; 1581 out: 1582 destroy_hrtimer_on_stack(&t.timer); 1583 return ret; 1584 } 1585 1586 SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, 1587 struct timespec __user *, rmtp) 1588 { 1589 struct timespec tu; 1590 1591 if (copy_from_user(&tu, rqtp, sizeof(tu))) 1592 return -EFAULT; 1593 1594 if (!timespec_valid(&tu)) 1595 return -EINVAL; 1596 1597 return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); 1598 } 1599 1600 /* 1601 * Functions related to boot-time initialization: 1602 */ 1603 static void init_hrtimers_cpu(int cpu) 1604 { 1605 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); 1606 int i; 1607 1608 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 1609 cpu_base->clock_base[i].cpu_base = cpu_base; 1610 timerqueue_init_head(&cpu_base->clock_base[i].active); 1611 } 1612 1613 cpu_base->cpu = cpu; 1614 hrtimer_init_hres(cpu_base); 1615 } 1616 1617 #ifdef CONFIG_HOTPLUG_CPU 1618 1619 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, 1620 struct hrtimer_clock_base *new_base) 1621 { 1622 struct hrtimer *timer; 1623 struct timerqueue_node *node; 1624 1625 while ((node = timerqueue_getnext(&old_base->active))) { 1626 timer = container_of(node, struct hrtimer, node); 1627 BUG_ON(hrtimer_callback_running(timer)); 1628 debug_deactivate(timer); 1629 1630 /* 1631 * Mark it as ENQUEUED not INACTIVE otherwise the 1632 * timer could be seen as !active and just vanish away 1633 * under us on another CPU 1634 */ 1635 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); 1636 timer->base = new_base; 1637 /* 1638 * Enqueue the timers on the new cpu. This does not 1639 * reprogram the event device in case the timer 1640 * expires before the earliest on this CPU, but we run 1641 * hrtimer_interrupt after we migrated everything to 1642 * sort out already expired timers and reprogram the 1643 * event device. 1644 */ 1645 enqueue_hrtimer(timer, new_base); 1646 } 1647 } 1648 1649 static void migrate_hrtimers(int scpu) 1650 { 1651 struct hrtimer_cpu_base *old_base, *new_base; 1652 int i; 1653 1654 BUG_ON(cpu_online(scpu)); 1655 tick_cancel_sched_timer(scpu); 1656 1657 local_irq_disable(); 1658 old_base = &per_cpu(hrtimer_bases, scpu); 1659 new_base = this_cpu_ptr(&hrtimer_bases); 1660 /* 1661 * The caller is globally serialized and nobody else 1662 * takes two locks at once, deadlock is not possible. 1663 */ 1664 raw_spin_lock(&new_base->lock); 1665 raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); 1666 1667 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 1668 migrate_hrtimer_list(&old_base->clock_base[i], 1669 &new_base->clock_base[i]); 1670 } 1671 1672 raw_spin_unlock(&old_base->lock); 1673 raw_spin_unlock(&new_base->lock); 1674 1675 /* Check, if we got expired work to do */ 1676 __hrtimer_peek_ahead_timers(); 1677 local_irq_enable(); 1678 } 1679 1680 #endif /* CONFIG_HOTPLUG_CPU */ 1681 1682 static int hrtimer_cpu_notify(struct notifier_block *self, 1683 unsigned long action, void *hcpu) 1684 { 1685 int scpu = (long)hcpu; 1686 1687 switch (action) { 1688 1689 case CPU_UP_PREPARE: 1690 case CPU_UP_PREPARE_FROZEN: 1691 init_hrtimers_cpu(scpu); 1692 break; 1693 1694 #ifdef CONFIG_HOTPLUG_CPU 1695 case CPU_DEAD: 1696 case CPU_DEAD_FROZEN: 1697 migrate_hrtimers(scpu); 1698 break; 1699 #endif 1700 1701 default: 1702 break; 1703 } 1704 1705 return NOTIFY_OK; 1706 } 1707 1708 static struct notifier_block hrtimers_nb = { 1709 .notifier_call = hrtimer_cpu_notify, 1710 }; 1711 1712 void __init hrtimers_init(void) 1713 { 1714 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, 1715 (void *)(long)smp_processor_id()); 1716 register_cpu_notifier(&hrtimers_nb); 1717 } 1718 1719 /** 1720 * schedule_hrtimeout_range_clock - sleep until timeout 1721 * @expires: timeout value (ktime_t) 1722 * @delta: slack in expires timeout (ktime_t) 1723 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1724 * @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME 1725 */ 1726 int __sched 1727 schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta, 1728 const enum hrtimer_mode mode, int clock) 1729 { 1730 struct hrtimer_sleeper t; 1731 1732 /* 1733 * Optimize when a zero timeout value is given. It does not 1734 * matter whether this is an absolute or a relative time. 1735 */ 1736 if (expires && !expires->tv64) { 1737 __set_current_state(TASK_RUNNING); 1738 return 0; 1739 } 1740 1741 /* 1742 * A NULL parameter means "infinite" 1743 */ 1744 if (!expires) { 1745 schedule(); 1746 return -EINTR; 1747 } 1748 1749 hrtimer_init_on_stack(&t.timer, clock, mode); 1750 hrtimer_set_expires_range_ns(&t.timer, *expires, delta); 1751 1752 hrtimer_init_sleeper(&t, current); 1753 1754 hrtimer_start_expires(&t.timer, mode); 1755 1756 if (likely(t.task)) 1757 schedule(); 1758 1759 hrtimer_cancel(&t.timer); 1760 destroy_hrtimer_on_stack(&t.timer); 1761 1762 __set_current_state(TASK_RUNNING); 1763 1764 return !t.task ? 0 : -EINTR; 1765 } 1766 1767 /** 1768 * schedule_hrtimeout_range - sleep until timeout 1769 * @expires: timeout value (ktime_t) 1770 * @delta: slack in expires timeout (ktime_t) 1771 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1772 * 1773 * Make the current task sleep until the given expiry time has 1774 * elapsed. The routine will return immediately unless 1775 * the current task state has been set (see set_current_state()). 1776 * 1777 * The @delta argument gives the kernel the freedom to schedule the 1778 * actual wakeup to a time that is both power and performance friendly. 1779 * The kernel give the normal best effort behavior for "@expires+@delta", 1780 * but may decide to fire the timer earlier, but no earlier than @expires. 1781 * 1782 * You can set the task state as follows - 1783 * 1784 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to 1785 * pass before the routine returns. 1786 * 1787 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is 1788 * delivered to the current task. 1789 * 1790 * The current task state is guaranteed to be TASK_RUNNING when this 1791 * routine returns. 1792 * 1793 * Returns 0 when the timer has expired otherwise -EINTR 1794 */ 1795 int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, 1796 const enum hrtimer_mode mode) 1797 { 1798 return schedule_hrtimeout_range_clock(expires, delta, mode, 1799 CLOCK_MONOTONIC); 1800 } 1801 EXPORT_SYMBOL_GPL(schedule_hrtimeout_range); 1802 1803 /** 1804 * schedule_hrtimeout - sleep until timeout 1805 * @expires: timeout value (ktime_t) 1806 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1807 * 1808 * Make the current task sleep until the given expiry time has 1809 * elapsed. The routine will return immediately unless 1810 * the current task state has been set (see set_current_state()). 1811 * 1812 * You can set the task state as follows - 1813 * 1814 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to 1815 * pass before the routine returns. 1816 * 1817 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is 1818 * delivered to the current task. 1819 * 1820 * The current task state is guaranteed to be TASK_RUNNING when this 1821 * routine returns. 1822 * 1823 * Returns 0 when the timer has expired otherwise -EINTR 1824 */ 1825 int __sched schedule_hrtimeout(ktime_t *expires, 1826 const enum hrtimer_mode mode) 1827 { 1828 return schedule_hrtimeout_range(expires, 0, mode); 1829 } 1830 EXPORT_SYMBOL_GPL(schedule_hrtimeout); 1831