1 /* 2 * linux/kernel/time/tick-sched.c 3 * 4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner 7 * 8 * No idle tick implementation for low and high resolution timers 9 * 10 * Started by: Thomas Gleixner and Ingo Molnar 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 #include <linux/cpu.h> 15 #include <linux/err.h> 16 #include <linux/hrtimer.h> 17 #include <linux/interrupt.h> 18 #include <linux/kernel_stat.h> 19 #include <linux/percpu.h> 20 #include <linux/profile.h> 21 #include <linux/sched.h> 22 #include <linux/tick.h> 23 24 #include <asm/irq_regs.h> 25 26 #include "tick-internal.h" 27 28 /* 29 * Per cpu nohz control structure 30 */ 31 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); 32 33 /* 34 * The time, when the last jiffy update happened. Protected by xtime_lock. 35 */ 36 static ktime_t last_jiffies_update; 37 38 struct tick_sched *tick_get_tick_sched(int cpu) 39 { 40 return &per_cpu(tick_cpu_sched, cpu); 41 } 42 43 /* 44 * Must be called with interrupts disabled ! 45 */ 46 static void tick_do_update_jiffies64(ktime_t now) 47 { 48 unsigned long ticks = 0; 49 ktime_t delta; 50 51 /* Reevalute with xtime_lock held */ 52 write_seqlock(&xtime_lock); 53 54 delta = ktime_sub(now, last_jiffies_update); 55 if (delta.tv64 >= tick_period.tv64) { 56 57 delta = ktime_sub(delta, tick_period); 58 last_jiffies_update = ktime_add(last_jiffies_update, 59 tick_period); 60 61 /* Slow path for long timeouts */ 62 if (unlikely(delta.tv64 >= tick_period.tv64)) { 63 s64 incr = ktime_to_ns(tick_period); 64 65 ticks = ktime_divns(delta, incr); 66 67 last_jiffies_update = ktime_add_ns(last_jiffies_update, 68 incr * ticks); 69 } 70 do_timer(++ticks); 71 } 72 write_sequnlock(&xtime_lock); 73 } 74 75 /* 76 * Initialize and return retrieve the jiffies update. 77 */ 78 static ktime_t tick_init_jiffy_update(void) 79 { 80 ktime_t period; 81 82 write_seqlock(&xtime_lock); 83 /* Did we start the jiffies update yet ? */ 84 if (last_jiffies_update.tv64 == 0) 85 last_jiffies_update = tick_next_period; 86 period = last_jiffies_update; 87 write_sequnlock(&xtime_lock); 88 return period; 89 } 90 91 /* 92 * NOHZ - aka dynamic tick functionality 93 */ 94 #ifdef CONFIG_NO_HZ 95 /* 96 * NO HZ enabled ? 97 */ 98 static int tick_nohz_enabled __read_mostly = 1; 99 100 /* 101 * Enable / Disable tickless mode 102 */ 103 static int __init setup_tick_nohz(char *str) 104 { 105 if (!strcmp(str, "off")) 106 tick_nohz_enabled = 0; 107 else if (!strcmp(str, "on")) 108 tick_nohz_enabled = 1; 109 else 110 return 0; 111 return 1; 112 } 113 114 __setup("nohz=", setup_tick_nohz); 115 116 /** 117 * tick_nohz_update_jiffies - update jiffies when idle was interrupted 118 * 119 * Called from interrupt entry when the CPU was idle 120 * 121 * In case the sched_tick was stopped on this CPU, we have to check if jiffies 122 * must be updated. Otherwise an interrupt handler could use a stale jiffy 123 * value. We do this unconditionally on any cpu, as we don't know whether the 124 * cpu, which has the update task assigned is in a long sleep. 125 */ 126 void tick_nohz_update_jiffies(void) 127 { 128 int cpu = smp_processor_id(); 129 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 130 unsigned long flags; 131 ktime_t now; 132 133 if (!ts->tick_stopped) 134 return; 135 136 cpu_clear(cpu, nohz_cpu_mask); 137 now = ktime_get(); 138 139 local_irq_save(flags); 140 tick_do_update_jiffies64(now); 141 local_irq_restore(flags); 142 } 143 144 /** 145 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task 146 * 147 * When the next event is more than a tick into the future, stop the idle tick 148 * Called either from the idle loop or from irq_exit() when an idle period was 149 * just interrupted by an interrupt which did not cause a reschedule. 150 */ 151 void tick_nohz_stop_sched_tick(void) 152 { 153 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; 154 struct tick_sched *ts; 155 ktime_t last_update, expires, now, delta; 156 int cpu; 157 158 local_irq_save(flags); 159 160 cpu = smp_processor_id(); 161 ts = &per_cpu(tick_cpu_sched, cpu); 162 163 /* 164 * If this cpu is offline and it is the one which updates 165 * jiffies, then give up the assignment and let it be taken by 166 * the cpu which runs the tick timer next. If we don't drop 167 * this here the jiffies might be stale and do_timer() never 168 * invoked. 169 */ 170 if (unlikely(!cpu_online(cpu))) { 171 if (cpu == tick_do_timer_cpu) 172 tick_do_timer_cpu = -1; 173 } 174 175 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 176 goto end; 177 178 if (need_resched()) 179 goto end; 180 181 cpu = smp_processor_id(); 182 if (unlikely(local_softirq_pending())) { 183 static int ratelimit; 184 185 if (ratelimit < 10) { 186 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", 187 local_softirq_pending()); 188 ratelimit++; 189 } 190 } 191 192 now = ktime_get(); 193 /* 194 * When called from irq_exit we need to account the idle sleep time 195 * correctly. 196 */ 197 if (ts->tick_stopped) { 198 delta = ktime_sub(now, ts->idle_entrytime); 199 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); 200 } 201 202 ts->idle_entrytime = now; 203 ts->idle_calls++; 204 205 /* Read jiffies and the time when jiffies were updated last */ 206 do { 207 seq = read_seqbegin(&xtime_lock); 208 last_update = last_jiffies_update; 209 last_jiffies = jiffies; 210 } while (read_seqretry(&xtime_lock, seq)); 211 212 /* Get the next timer wheel timer */ 213 next_jiffies = get_next_timer_interrupt(last_jiffies); 214 delta_jiffies = next_jiffies - last_jiffies; 215 216 if (rcu_needs_cpu(cpu)) 217 delta_jiffies = 1; 218 /* 219 * Do not stop the tick, if we are only one off 220 * or if the cpu is required for rcu 221 */ 222 if (!ts->tick_stopped && delta_jiffies == 1) 223 goto out; 224 225 /* Schedule the tick, if we are at least one jiffie off */ 226 if ((long)delta_jiffies >= 1) { 227 228 if (delta_jiffies > 1) 229 cpu_set(cpu, nohz_cpu_mask); 230 /* 231 * nohz_stop_sched_tick can be called several times before 232 * the nohz_restart_sched_tick is called. This happens when 233 * interrupts arrive which do not cause a reschedule. In the 234 * first call we save the current tick time, so we can restart 235 * the scheduler tick in nohz_restart_sched_tick. 236 */ 237 if (!ts->tick_stopped) { 238 if (select_nohz_load_balancer(1)) { 239 /* 240 * sched tick not stopped! 241 */ 242 cpu_clear(cpu, nohz_cpu_mask); 243 goto out; 244 } 245 246 ts->idle_tick = ts->sched_timer.expires; 247 ts->tick_stopped = 1; 248 ts->idle_jiffies = last_jiffies; 249 } 250 251 /* 252 * If this cpu is the one which updates jiffies, then 253 * give up the assignment and let it be taken by the 254 * cpu which runs the tick timer next, which might be 255 * this cpu as well. If we don't drop this here the 256 * jiffies might be stale and do_timer() never 257 * invoked. 258 */ 259 if (cpu == tick_do_timer_cpu) 260 tick_do_timer_cpu = -1; 261 262 ts->idle_sleeps++; 263 264 /* 265 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that 266 * there is no timer pending or at least extremly far 267 * into the future (12 days for HZ=1000). In this case 268 * we simply stop the tick timer: 269 */ 270 if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) { 271 ts->idle_expires.tv64 = KTIME_MAX; 272 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) 273 hrtimer_cancel(&ts->sched_timer); 274 goto out; 275 } 276 277 /* 278 * calculate the expiry time for the next timer wheel 279 * timer 280 */ 281 expires = ktime_add_ns(last_update, tick_period.tv64 * 282 delta_jiffies); 283 ts->idle_expires = expires; 284 285 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 286 hrtimer_start(&ts->sched_timer, expires, 287 HRTIMER_MODE_ABS); 288 /* Check, if the timer was already in the past */ 289 if (hrtimer_active(&ts->sched_timer)) 290 goto out; 291 } else if(!tick_program_event(expires, 0)) 292 goto out; 293 /* 294 * We are past the event already. So we crossed a 295 * jiffie boundary. Update jiffies and raise the 296 * softirq. 297 */ 298 tick_do_update_jiffies64(ktime_get()); 299 cpu_clear(cpu, nohz_cpu_mask); 300 } 301 raise_softirq_irqoff(TIMER_SOFTIRQ); 302 out: 303 ts->next_jiffies = next_jiffies; 304 ts->last_jiffies = last_jiffies; 305 end: 306 local_irq_restore(flags); 307 } 308 309 /** 310 * nohz_restart_sched_tick - restart the idle tick from the idle task 311 * 312 * Restart the idle tick when the CPU is woken up from idle 313 */ 314 void tick_nohz_restart_sched_tick(void) 315 { 316 int cpu = smp_processor_id(); 317 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 318 unsigned long ticks; 319 ktime_t now, delta; 320 321 if (!ts->tick_stopped) 322 return; 323 324 /* Update jiffies first */ 325 now = ktime_get(); 326 327 local_irq_disable(); 328 select_nohz_load_balancer(0); 329 tick_do_update_jiffies64(now); 330 cpu_clear(cpu, nohz_cpu_mask); 331 332 /* Account the idle time */ 333 delta = ktime_sub(now, ts->idle_entrytime); 334 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); 335 336 /* 337 * We stopped the tick in idle. Update process times would miss the 338 * time we slept as update_process_times does only a 1 tick 339 * accounting. Enforce that this is accounted to idle ! 340 */ 341 ticks = jiffies - ts->idle_jiffies; 342 /* 343 * We might be one off. Do not randomly account a huge number of ticks! 344 */ 345 if (ticks && ticks < LONG_MAX) { 346 add_preempt_count(HARDIRQ_OFFSET); 347 account_system_time(current, HARDIRQ_OFFSET, 348 jiffies_to_cputime(ticks)); 349 sub_preempt_count(HARDIRQ_OFFSET); 350 } 351 352 /* 353 * Cancel the scheduled timer and restore the tick 354 */ 355 ts->tick_stopped = 0; 356 hrtimer_cancel(&ts->sched_timer); 357 ts->sched_timer.expires = ts->idle_tick; 358 359 while (1) { 360 /* Forward the time to expire in the future */ 361 hrtimer_forward(&ts->sched_timer, now, tick_period); 362 363 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 364 hrtimer_start(&ts->sched_timer, 365 ts->sched_timer.expires, 366 HRTIMER_MODE_ABS); 367 /* Check, if the timer was already in the past */ 368 if (hrtimer_active(&ts->sched_timer)) 369 break; 370 } else { 371 if (!tick_program_event(ts->sched_timer.expires, 0)) 372 break; 373 } 374 /* Update jiffies and reread time */ 375 tick_do_update_jiffies64(now); 376 now = ktime_get(); 377 } 378 local_irq_enable(); 379 } 380 381 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now) 382 { 383 hrtimer_forward(&ts->sched_timer, now, tick_period); 384 return tick_program_event(ts->sched_timer.expires, 0); 385 } 386 387 /* 388 * The nohz low res interrupt handler 389 */ 390 static void tick_nohz_handler(struct clock_event_device *dev) 391 { 392 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 393 struct pt_regs *regs = get_irq_regs(); 394 int cpu = smp_processor_id(); 395 ktime_t now = ktime_get(); 396 397 dev->next_event.tv64 = KTIME_MAX; 398 399 /* 400 * Check if the do_timer duty was dropped. We don't care about 401 * concurrency: This happens only when the cpu in charge went 402 * into a long sleep. If two cpus happen to assign themself to 403 * this duty, then the jiffies update is still serialized by 404 * xtime_lock. 405 */ 406 if (unlikely(tick_do_timer_cpu == -1)) 407 tick_do_timer_cpu = cpu; 408 409 /* Check, if the jiffies need an update */ 410 if (tick_do_timer_cpu == cpu) 411 tick_do_update_jiffies64(now); 412 413 /* 414 * When we are idle and the tick is stopped, we have to touch 415 * the watchdog as we might not schedule for a really long 416 * time. This happens on complete idle SMP systems while 417 * waiting on the login prompt. We also increment the "start 418 * of idle" jiffy stamp so the idle accounting adjustment we 419 * do when we go busy again does not account too much ticks. 420 */ 421 if (ts->tick_stopped) { 422 touch_softlockup_watchdog(); 423 ts->idle_jiffies++; 424 } 425 426 update_process_times(user_mode(regs)); 427 profile_tick(CPU_PROFILING); 428 429 /* Do not restart, when we are in the idle loop */ 430 if (ts->tick_stopped) 431 return; 432 433 while (tick_nohz_reprogram(ts, now)) { 434 now = ktime_get(); 435 tick_do_update_jiffies64(now); 436 } 437 } 438 439 /** 440 * tick_nohz_switch_to_nohz - switch to nohz mode 441 */ 442 static void tick_nohz_switch_to_nohz(void) 443 { 444 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 445 ktime_t next; 446 447 if (!tick_nohz_enabled) 448 return; 449 450 local_irq_disable(); 451 if (tick_switch_to_oneshot(tick_nohz_handler)) { 452 local_irq_enable(); 453 return; 454 } 455 456 ts->nohz_mode = NOHZ_MODE_LOWRES; 457 458 /* 459 * Recycle the hrtimer in ts, so we can share the 460 * hrtimer_forward with the highres code. 461 */ 462 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 463 /* Get the next period */ 464 next = tick_init_jiffy_update(); 465 466 for (;;) { 467 ts->sched_timer.expires = next; 468 if (!tick_program_event(next, 0)) 469 break; 470 next = ktime_add(next, tick_period); 471 } 472 local_irq_enable(); 473 474 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n", 475 smp_processor_id()); 476 } 477 478 #else 479 480 static inline void tick_nohz_switch_to_nohz(void) { } 481 482 #endif /* NO_HZ */ 483 484 /* 485 * High resolution timer specific code 486 */ 487 #ifdef CONFIG_HIGH_RES_TIMERS 488 /* 489 * We rearm the timer until we get disabled by the idle code 490 * Called with interrupts disabled and timer->base->cpu_base->lock held. 491 */ 492 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) 493 { 494 struct tick_sched *ts = 495 container_of(timer, struct tick_sched, sched_timer); 496 struct hrtimer_cpu_base *base = timer->base->cpu_base; 497 struct pt_regs *regs = get_irq_regs(); 498 ktime_t now = ktime_get(); 499 int cpu = smp_processor_id(); 500 501 #ifdef CONFIG_NO_HZ 502 /* 503 * Check if the do_timer duty was dropped. We don't care about 504 * concurrency: This happens only when the cpu in charge went 505 * into a long sleep. If two cpus happen to assign themself to 506 * this duty, then the jiffies update is still serialized by 507 * xtime_lock. 508 */ 509 if (unlikely(tick_do_timer_cpu == -1)) 510 tick_do_timer_cpu = cpu; 511 #endif 512 513 /* Check, if the jiffies need an update */ 514 if (tick_do_timer_cpu == cpu) 515 tick_do_update_jiffies64(now); 516 517 /* 518 * Do not call, when we are not in irq context and have 519 * no valid regs pointer 520 */ 521 if (regs) { 522 /* 523 * When we are idle and the tick is stopped, we have to touch 524 * the watchdog as we might not schedule for a really long 525 * time. This happens on complete idle SMP systems while 526 * waiting on the login prompt. We also increment the "start of 527 * idle" jiffy stamp so the idle accounting adjustment we do 528 * when we go busy again does not account too much ticks. 529 */ 530 if (ts->tick_stopped) { 531 touch_softlockup_watchdog(); 532 ts->idle_jiffies++; 533 } 534 /* 535 * update_process_times() might take tasklist_lock, hence 536 * drop the base lock. sched-tick hrtimers are per-CPU and 537 * never accessible by userspace APIs, so this is safe to do. 538 */ 539 spin_unlock(&base->lock); 540 update_process_times(user_mode(regs)); 541 profile_tick(CPU_PROFILING); 542 spin_lock(&base->lock); 543 } 544 545 /* Do not restart, when we are in the idle loop */ 546 if (ts->tick_stopped) 547 return HRTIMER_NORESTART; 548 549 hrtimer_forward(timer, now, tick_period); 550 551 return HRTIMER_RESTART; 552 } 553 554 /** 555 * tick_setup_sched_timer - setup the tick emulation timer 556 */ 557 void tick_setup_sched_timer(void) 558 { 559 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 560 ktime_t now = ktime_get(); 561 u64 offset; 562 563 /* 564 * Emulate tick processing via per-CPU hrtimers: 565 */ 566 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 567 ts->sched_timer.function = tick_sched_timer; 568 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; 569 570 /* Get the next period (per cpu) */ 571 ts->sched_timer.expires = tick_init_jiffy_update(); 572 offset = ktime_to_ns(tick_period) >> 1; 573 do_div(offset, num_possible_cpus()); 574 offset *= smp_processor_id(); 575 ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset); 576 577 for (;;) { 578 hrtimer_forward(&ts->sched_timer, now, tick_period); 579 hrtimer_start(&ts->sched_timer, ts->sched_timer.expires, 580 HRTIMER_MODE_ABS); 581 /* Check, if the timer was already in the past */ 582 if (hrtimer_active(&ts->sched_timer)) 583 break; 584 now = ktime_get(); 585 } 586 587 #ifdef CONFIG_NO_HZ 588 if (tick_nohz_enabled) 589 ts->nohz_mode = NOHZ_MODE_HIGHRES; 590 #endif 591 } 592 593 void tick_cancel_sched_timer(int cpu) 594 { 595 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 596 597 if (ts->sched_timer.base) 598 hrtimer_cancel(&ts->sched_timer); 599 ts->tick_stopped = 0; 600 ts->nohz_mode = NOHZ_MODE_INACTIVE; 601 } 602 #endif /* HIGH_RES_TIMERS */ 603 604 /** 605 * Async notification about clocksource changes 606 */ 607 void tick_clock_notify(void) 608 { 609 int cpu; 610 611 for_each_possible_cpu(cpu) 612 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks); 613 } 614 615 /* 616 * Async notification about clock event changes 617 */ 618 void tick_oneshot_notify(void) 619 { 620 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 621 622 set_bit(0, &ts->check_clocks); 623 } 624 625 /** 626 * Check, if a change happened, which makes oneshot possible. 627 * 628 * Called cyclic from the hrtimer softirq (driven by the timer 629 * softirq) allow_nohz signals, that we can switch into low-res nohz 630 * mode, because high resolution timers are disabled (either compile 631 * or runtime). 632 */ 633 int tick_check_oneshot_change(int allow_nohz) 634 { 635 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 636 637 if (!test_and_clear_bit(0, &ts->check_clocks)) 638 return 0; 639 640 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) 641 return 0; 642 643 if (!timekeeping_is_continuous() || !tick_is_oneshot_available()) 644 return 0; 645 646 if (!allow_nohz) 647 return 1; 648 649 tick_nohz_switch_to_nohz(); 650 return 0; 651 } 652