1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner 6 * 7 * No idle tick implementation for low and high resolution timers 8 * 9 * Started by: Thomas Gleixner and Ingo Molnar 10 */ 11 #include <linux/cpu.h> 12 #include <linux/err.h> 13 #include <linux/hrtimer.h> 14 #include <linux/interrupt.h> 15 #include <linux/kernel_stat.h> 16 #include <linux/percpu.h> 17 #include <linux/nmi.h> 18 #include <linux/profile.h> 19 #include <linux/sched/signal.h> 20 #include <linux/sched/clock.h> 21 #include <linux/sched/stat.h> 22 #include <linux/sched/nohz.h> 23 #include <linux/sched/loadavg.h> 24 #include <linux/module.h> 25 #include <linux/irq_work.h> 26 #include <linux/posix-timers.h> 27 #include <linux/context_tracking.h> 28 #include <linux/mm.h> 29 30 #include <asm/irq_regs.h> 31 32 #include "tick-internal.h" 33 34 #include <trace/events/timer.h> 35 36 /* 37 * Per-CPU nohz control structure 38 */ 39 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); 40 41 struct tick_sched *tick_get_tick_sched(int cpu) 42 { 43 return &per_cpu(tick_cpu_sched, cpu); 44 } 45 46 #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS) 47 /* 48 * The time, when the last jiffy update happened. Write access must hold 49 * jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a 50 * consistent view of jiffies and last_jiffies_update. 51 */ 52 static ktime_t last_jiffies_update; 53 54 /* 55 * Must be called with interrupts disabled ! 56 */ 57 static void tick_do_update_jiffies64(ktime_t now) 58 { 59 unsigned long ticks = 1; 60 ktime_t delta, nextp; 61 62 /* 63 * 64bit can do a quick check without holding jiffies lock and 64 * without looking at the sequence count. The smp_load_acquire() 65 * pairs with the update done later in this function. 66 * 67 * 32bit cannot do that because the store of tick_next_period 68 * consists of two 32bit stores and the first store could move it 69 * to a random point in the future. 70 */ 71 if (IS_ENABLED(CONFIG_64BIT)) { 72 if (ktime_before(now, smp_load_acquire(&tick_next_period))) 73 return; 74 } else { 75 unsigned int seq; 76 77 /* 78 * Avoid contention on jiffies_lock and protect the quick 79 * check with the sequence count. 80 */ 81 do { 82 seq = read_seqcount_begin(&jiffies_seq); 83 nextp = tick_next_period; 84 } while (read_seqcount_retry(&jiffies_seq, seq)); 85 86 if (ktime_before(now, nextp)) 87 return; 88 } 89 90 /* Quick check failed, i.e. update is required. */ 91 raw_spin_lock(&jiffies_lock); 92 /* 93 * Reevaluate with the lock held. Another CPU might have done the 94 * update already. 95 */ 96 if (ktime_before(now, tick_next_period)) { 97 raw_spin_unlock(&jiffies_lock); 98 return; 99 } 100 101 write_seqcount_begin(&jiffies_seq); 102 103 delta = ktime_sub(now, tick_next_period); 104 if (unlikely(delta >= TICK_NSEC)) { 105 /* Slow path for long idle sleep times */ 106 s64 incr = TICK_NSEC; 107 108 ticks += ktime_divns(delta, incr); 109 110 last_jiffies_update = ktime_add_ns(last_jiffies_update, 111 incr * ticks); 112 } else { 113 last_jiffies_update = ktime_add_ns(last_jiffies_update, 114 TICK_NSEC); 115 } 116 117 /* Advance jiffies to complete the jiffies_seq protected job */ 118 jiffies_64 += ticks; 119 120 /* 121 * Keep the tick_next_period variable up to date. 122 */ 123 nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC); 124 125 if (IS_ENABLED(CONFIG_64BIT)) { 126 /* 127 * Pairs with smp_load_acquire() in the lockless quick 128 * check above and ensures that the update to jiffies_64 is 129 * not reordered vs. the store to tick_next_period, neither 130 * by the compiler nor by the CPU. 131 */ 132 smp_store_release(&tick_next_period, nextp); 133 } else { 134 /* 135 * A plain store is good enough on 32bit as the quick check 136 * above is protected by the sequence count. 137 */ 138 tick_next_period = nextp; 139 } 140 141 /* 142 * Release the sequence count. calc_global_load() below is not 143 * protected by it, but jiffies_lock needs to be held to prevent 144 * concurrent invocations. 145 */ 146 write_seqcount_end(&jiffies_seq); 147 148 calc_global_load(); 149 150 raw_spin_unlock(&jiffies_lock); 151 update_wall_time(); 152 } 153 154 /* 155 * Initialize and return retrieve the jiffies update. 156 */ 157 static ktime_t tick_init_jiffy_update(void) 158 { 159 ktime_t period; 160 161 raw_spin_lock(&jiffies_lock); 162 write_seqcount_begin(&jiffies_seq); 163 /* Did we start the jiffies update yet ? */ 164 if (last_jiffies_update == 0) 165 last_jiffies_update = tick_next_period; 166 period = last_jiffies_update; 167 write_seqcount_end(&jiffies_seq); 168 raw_spin_unlock(&jiffies_lock); 169 return period; 170 } 171 172 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) 173 { 174 int cpu = smp_processor_id(); 175 176 #ifdef CONFIG_NO_HZ_COMMON 177 /* 178 * Check if the do_timer duty was dropped. We don't care about 179 * concurrency: This happens only when the CPU in charge went 180 * into a long sleep. If two CPUs happen to assign themselves to 181 * this duty, then the jiffies update is still serialized by 182 * jiffies_lock. 183 * 184 * If nohz_full is enabled, this should not happen because the 185 * tick_do_timer_cpu never relinquishes. 186 */ 187 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) { 188 #ifdef CONFIG_NO_HZ_FULL 189 WARN_ON(tick_nohz_full_running); 190 #endif 191 tick_do_timer_cpu = cpu; 192 } 193 #endif 194 195 /* Check, if the jiffies need an update */ 196 if (tick_do_timer_cpu == cpu) 197 tick_do_update_jiffies64(now); 198 199 if (ts->inidle) 200 ts->got_idle_tick = 1; 201 } 202 203 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) 204 { 205 #ifdef CONFIG_NO_HZ_COMMON 206 /* 207 * When we are idle and the tick is stopped, we have to touch 208 * the watchdog as we might not schedule for a really long 209 * time. This happens on complete idle SMP systems while 210 * waiting on the login prompt. We also increment the "start of 211 * idle" jiffy stamp so the idle accounting adjustment we do 212 * when we go busy again does not account too much ticks. 213 */ 214 if (ts->tick_stopped) { 215 touch_softlockup_watchdog_sched(); 216 if (is_idle_task(current)) 217 ts->idle_jiffies++; 218 /* 219 * In case the current tick fired too early past its expected 220 * expiration, make sure we don't bypass the next clock reprogramming 221 * to the same deadline. 222 */ 223 ts->next_tick = 0; 224 } 225 #endif 226 update_process_times(user_mode(regs)); 227 profile_tick(CPU_PROFILING); 228 } 229 #endif 230 231 #ifdef CONFIG_NO_HZ_FULL 232 cpumask_var_t tick_nohz_full_mask; 233 bool tick_nohz_full_running; 234 EXPORT_SYMBOL_GPL(tick_nohz_full_running); 235 static atomic_t tick_dep_mask; 236 237 static bool check_tick_dependency(atomic_t *dep) 238 { 239 int val = atomic_read(dep); 240 241 if (val & TICK_DEP_MASK_POSIX_TIMER) { 242 trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER); 243 return true; 244 } 245 246 if (val & TICK_DEP_MASK_PERF_EVENTS) { 247 trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS); 248 return true; 249 } 250 251 if (val & TICK_DEP_MASK_SCHED) { 252 trace_tick_stop(0, TICK_DEP_MASK_SCHED); 253 return true; 254 } 255 256 if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) { 257 trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE); 258 return true; 259 } 260 261 if (val & TICK_DEP_MASK_RCU) { 262 trace_tick_stop(0, TICK_DEP_MASK_RCU); 263 return true; 264 } 265 266 return false; 267 } 268 269 static bool can_stop_full_tick(int cpu, struct tick_sched *ts) 270 { 271 lockdep_assert_irqs_disabled(); 272 273 if (unlikely(!cpu_online(cpu))) 274 return false; 275 276 if (check_tick_dependency(&tick_dep_mask)) 277 return false; 278 279 if (check_tick_dependency(&ts->tick_dep_mask)) 280 return false; 281 282 if (check_tick_dependency(¤t->tick_dep_mask)) 283 return false; 284 285 if (check_tick_dependency(¤t->signal->tick_dep_mask)) 286 return false; 287 288 return true; 289 } 290 291 static void nohz_full_kick_func(struct irq_work *work) 292 { 293 /* Empty, the tick restart happens on tick_nohz_irq_exit() */ 294 } 295 296 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = 297 IRQ_WORK_INIT_HARD(nohz_full_kick_func); 298 299 /* 300 * Kick this CPU if it's full dynticks in order to force it to 301 * re-evaluate its dependency on the tick and restart it if necessary. 302 * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(), 303 * is NMI safe. 304 */ 305 static void tick_nohz_full_kick(void) 306 { 307 if (!tick_nohz_full_cpu(smp_processor_id())) 308 return; 309 310 irq_work_queue(this_cpu_ptr(&nohz_full_kick_work)); 311 } 312 313 /* 314 * Kick the CPU if it's full dynticks in order to force it to 315 * re-evaluate its dependency on the tick and restart it if necessary. 316 */ 317 void tick_nohz_full_kick_cpu(int cpu) 318 { 319 if (!tick_nohz_full_cpu(cpu)) 320 return; 321 322 irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu); 323 } 324 325 /* 326 * Kick all full dynticks CPUs in order to force these to re-evaluate 327 * their dependency on the tick and restart it if necessary. 328 */ 329 static void tick_nohz_full_kick_all(void) 330 { 331 int cpu; 332 333 if (!tick_nohz_full_running) 334 return; 335 336 preempt_disable(); 337 for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask) 338 tick_nohz_full_kick_cpu(cpu); 339 preempt_enable(); 340 } 341 342 static void tick_nohz_dep_set_all(atomic_t *dep, 343 enum tick_dep_bits bit) 344 { 345 int prev; 346 347 prev = atomic_fetch_or(BIT(bit), dep); 348 if (!prev) 349 tick_nohz_full_kick_all(); 350 } 351 352 /* 353 * Set a global tick dependency. Used by perf events that rely on freq and 354 * by unstable clock. 355 */ 356 void tick_nohz_dep_set(enum tick_dep_bits bit) 357 { 358 tick_nohz_dep_set_all(&tick_dep_mask, bit); 359 } 360 361 void tick_nohz_dep_clear(enum tick_dep_bits bit) 362 { 363 atomic_andnot(BIT(bit), &tick_dep_mask); 364 } 365 366 /* 367 * Set per-CPU tick dependency. Used by scheduler and perf events in order to 368 * manage events throttling. 369 */ 370 void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit) 371 { 372 int prev; 373 struct tick_sched *ts; 374 375 ts = per_cpu_ptr(&tick_cpu_sched, cpu); 376 377 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); 378 if (!prev) { 379 preempt_disable(); 380 /* Perf needs local kick that is NMI safe */ 381 if (cpu == smp_processor_id()) { 382 tick_nohz_full_kick(); 383 } else { 384 /* Remote irq work not NMI-safe */ 385 if (!WARN_ON_ONCE(in_nmi())) 386 tick_nohz_full_kick_cpu(cpu); 387 } 388 preempt_enable(); 389 } 390 } 391 EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu); 392 393 void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit) 394 { 395 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); 396 397 atomic_andnot(BIT(bit), &ts->tick_dep_mask); 398 } 399 EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu); 400 401 /* 402 * Set a per-task tick dependency. RCU need this. Also posix CPU timers 403 * in order to elapse per task timers. 404 */ 405 void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit) 406 { 407 if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) { 408 if (tsk == current) { 409 preempt_disable(); 410 tick_nohz_full_kick(); 411 preempt_enable(); 412 } else { 413 /* 414 * Some future tick_nohz_full_kick_task() 415 * should optimize this. 416 */ 417 tick_nohz_full_kick_all(); 418 } 419 } 420 } 421 EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task); 422 423 void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit) 424 { 425 atomic_andnot(BIT(bit), &tsk->tick_dep_mask); 426 } 427 EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task); 428 429 /* 430 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse 431 * per process timers. 432 */ 433 void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit) 434 { 435 tick_nohz_dep_set_all(&sig->tick_dep_mask, bit); 436 } 437 438 void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit) 439 { 440 atomic_andnot(BIT(bit), &sig->tick_dep_mask); 441 } 442 443 /* 444 * Re-evaluate the need for the tick as we switch the current task. 445 * It might need the tick due to per task/process properties: 446 * perf events, posix CPU timers, ... 447 */ 448 void __tick_nohz_task_switch(void) 449 { 450 unsigned long flags; 451 struct tick_sched *ts; 452 453 local_irq_save(flags); 454 455 if (!tick_nohz_full_cpu(smp_processor_id())) 456 goto out; 457 458 ts = this_cpu_ptr(&tick_cpu_sched); 459 460 if (ts->tick_stopped) { 461 if (atomic_read(¤t->tick_dep_mask) || 462 atomic_read(¤t->signal->tick_dep_mask)) 463 tick_nohz_full_kick(); 464 } 465 out: 466 local_irq_restore(flags); 467 } 468 469 /* Get the boot-time nohz CPU list from the kernel parameters. */ 470 void __init tick_nohz_full_setup(cpumask_var_t cpumask) 471 { 472 alloc_bootmem_cpumask_var(&tick_nohz_full_mask); 473 cpumask_copy(tick_nohz_full_mask, cpumask); 474 tick_nohz_full_running = true; 475 } 476 EXPORT_SYMBOL_GPL(tick_nohz_full_setup); 477 478 static int tick_nohz_cpu_down(unsigned int cpu) 479 { 480 /* 481 * The tick_do_timer_cpu CPU handles housekeeping duty (unbound 482 * timers, workqueues, timekeeping, ...) on behalf of full dynticks 483 * CPUs. It must remain online when nohz full is enabled. 484 */ 485 if (tick_nohz_full_running && tick_do_timer_cpu == cpu) 486 return -EBUSY; 487 return 0; 488 } 489 490 void __init tick_nohz_init(void) 491 { 492 int cpu, ret; 493 494 if (!tick_nohz_full_running) 495 return; 496 497 /* 498 * Full dynticks uses irq work to drive the tick rescheduling on safe 499 * locking contexts. But then we need irq work to raise its own 500 * interrupts to avoid circular dependency on the tick 501 */ 502 if (!arch_irq_work_has_interrupt()) { 503 pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n"); 504 cpumask_clear(tick_nohz_full_mask); 505 tick_nohz_full_running = false; 506 return; 507 } 508 509 if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) && 510 !IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) { 511 cpu = smp_processor_id(); 512 513 if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { 514 pr_warn("NO_HZ: Clearing %d from nohz_full range " 515 "for timekeeping\n", cpu); 516 cpumask_clear_cpu(cpu, tick_nohz_full_mask); 517 } 518 } 519 520 for_each_cpu(cpu, tick_nohz_full_mask) 521 context_tracking_cpu_set(cpu); 522 523 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 524 "kernel/nohz:predown", NULL, 525 tick_nohz_cpu_down); 526 WARN_ON(ret < 0); 527 pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n", 528 cpumask_pr_args(tick_nohz_full_mask)); 529 } 530 #endif 531 532 /* 533 * NOHZ - aka dynamic tick functionality 534 */ 535 #ifdef CONFIG_NO_HZ_COMMON 536 /* 537 * NO HZ enabled ? 538 */ 539 bool tick_nohz_enabled __read_mostly = true; 540 unsigned long tick_nohz_active __read_mostly; 541 /* 542 * Enable / Disable tickless mode 543 */ 544 static int __init setup_tick_nohz(char *str) 545 { 546 return (kstrtobool(str, &tick_nohz_enabled) == 0); 547 } 548 549 __setup("nohz=", setup_tick_nohz); 550 551 bool tick_nohz_tick_stopped(void) 552 { 553 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 554 555 return ts->tick_stopped; 556 } 557 558 bool tick_nohz_tick_stopped_cpu(int cpu) 559 { 560 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); 561 562 return ts->tick_stopped; 563 } 564 565 /** 566 * tick_nohz_update_jiffies - update jiffies when idle was interrupted 567 * 568 * Called from interrupt entry when the CPU was idle 569 * 570 * In case the sched_tick was stopped on this CPU, we have to check if jiffies 571 * must be updated. Otherwise an interrupt handler could use a stale jiffy 572 * value. We do this unconditionally on any CPU, as we don't know whether the 573 * CPU, which has the update task assigned is in a long sleep. 574 */ 575 static void tick_nohz_update_jiffies(ktime_t now) 576 { 577 unsigned long flags; 578 579 __this_cpu_write(tick_cpu_sched.idle_waketime, now); 580 581 local_irq_save(flags); 582 tick_do_update_jiffies64(now); 583 local_irq_restore(flags); 584 585 touch_softlockup_watchdog_sched(); 586 } 587 588 /* 589 * Updates the per-CPU time idle statistics counters 590 */ 591 static void 592 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) 593 { 594 ktime_t delta; 595 596 if (ts->idle_active) { 597 delta = ktime_sub(now, ts->idle_entrytime); 598 if (nr_iowait_cpu(cpu) > 0) 599 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); 600 else 601 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); 602 ts->idle_entrytime = now; 603 } 604 605 if (last_update_time) 606 *last_update_time = ktime_to_us(now); 607 608 } 609 610 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) 611 { 612 update_ts_time_stats(smp_processor_id(), ts, now, NULL); 613 ts->idle_active = 0; 614 615 sched_clock_idle_wakeup_event(); 616 } 617 618 static void tick_nohz_start_idle(struct tick_sched *ts) 619 { 620 ts->idle_entrytime = ktime_get(); 621 ts->idle_active = 1; 622 sched_clock_idle_sleep_event(); 623 } 624 625 /** 626 * get_cpu_idle_time_us - get the total idle time of a CPU 627 * @cpu: CPU number to query 628 * @last_update_time: variable to store update time in. Do not update 629 * counters if NULL. 630 * 631 * Return the cumulative idle time (since boot) for a given 632 * CPU, in microseconds. 633 * 634 * This time is measured via accounting rather than sampling, 635 * and is as accurate as ktime_get() is. 636 * 637 * This function returns -1 if NOHZ is not enabled. 638 */ 639 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) 640 { 641 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 642 ktime_t now, idle; 643 644 if (!tick_nohz_active) 645 return -1; 646 647 now = ktime_get(); 648 if (last_update_time) { 649 update_ts_time_stats(cpu, ts, now, last_update_time); 650 idle = ts->idle_sleeptime; 651 } else { 652 if (ts->idle_active && !nr_iowait_cpu(cpu)) { 653 ktime_t delta = ktime_sub(now, ts->idle_entrytime); 654 655 idle = ktime_add(ts->idle_sleeptime, delta); 656 } else { 657 idle = ts->idle_sleeptime; 658 } 659 } 660 661 return ktime_to_us(idle); 662 663 } 664 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); 665 666 /** 667 * get_cpu_iowait_time_us - get the total iowait time of a CPU 668 * @cpu: CPU number to query 669 * @last_update_time: variable to store update time in. Do not update 670 * counters if NULL. 671 * 672 * Return the cumulative iowait time (since boot) for a given 673 * CPU, in microseconds. 674 * 675 * This time is measured via accounting rather than sampling, 676 * and is as accurate as ktime_get() is. 677 * 678 * This function returns -1 if NOHZ is not enabled. 679 */ 680 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) 681 { 682 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 683 ktime_t now, iowait; 684 685 if (!tick_nohz_active) 686 return -1; 687 688 now = ktime_get(); 689 if (last_update_time) { 690 update_ts_time_stats(cpu, ts, now, last_update_time); 691 iowait = ts->iowait_sleeptime; 692 } else { 693 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { 694 ktime_t delta = ktime_sub(now, ts->idle_entrytime); 695 696 iowait = ktime_add(ts->iowait_sleeptime, delta); 697 } else { 698 iowait = ts->iowait_sleeptime; 699 } 700 } 701 702 return ktime_to_us(iowait); 703 } 704 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); 705 706 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) 707 { 708 hrtimer_cancel(&ts->sched_timer); 709 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); 710 711 /* Forward the time to expire in the future */ 712 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); 713 714 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 715 hrtimer_start_expires(&ts->sched_timer, 716 HRTIMER_MODE_ABS_PINNED_HARD); 717 } else { 718 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); 719 } 720 721 /* 722 * Reset to make sure next tick stop doesn't get fooled by past 723 * cached clock deadline. 724 */ 725 ts->next_tick = 0; 726 } 727 728 static inline bool local_timer_softirq_pending(void) 729 { 730 return local_softirq_pending() & BIT(TIMER_SOFTIRQ); 731 } 732 733 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) 734 { 735 u64 basemono, next_tick, next_tmr, next_rcu, delta, expires; 736 unsigned long basejiff; 737 unsigned int seq; 738 739 /* Read jiffies and the time when jiffies were updated last */ 740 do { 741 seq = read_seqcount_begin(&jiffies_seq); 742 basemono = last_jiffies_update; 743 basejiff = jiffies; 744 } while (read_seqcount_retry(&jiffies_seq, seq)); 745 ts->last_jiffies = basejiff; 746 ts->timer_expires_base = basemono; 747 748 /* 749 * Keep the periodic tick, when RCU, architecture or irq_work 750 * requests it. 751 * Aside of that check whether the local timer softirq is 752 * pending. If so its a bad idea to call get_next_timer_interrupt() 753 * because there is an already expired timer, so it will request 754 * immeditate expiry, which rearms the hardware timer with a 755 * minimal delta which brings us back to this place 756 * immediately. Lather, rinse and repeat... 757 */ 758 if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() || 759 irq_work_needs_cpu() || local_timer_softirq_pending()) { 760 next_tick = basemono + TICK_NSEC; 761 } else { 762 /* 763 * Get the next pending timer. If high resolution 764 * timers are enabled this only takes the timer wheel 765 * timers into account. If high resolution timers are 766 * disabled this also looks at the next expiring 767 * hrtimer. 768 */ 769 next_tmr = get_next_timer_interrupt(basejiff, basemono); 770 ts->next_timer = next_tmr; 771 /* Take the next rcu event into account */ 772 next_tick = next_rcu < next_tmr ? next_rcu : next_tmr; 773 } 774 775 /* 776 * If the tick is due in the next period, keep it ticking or 777 * force prod the timer. 778 */ 779 delta = next_tick - basemono; 780 if (delta <= (u64)TICK_NSEC) { 781 /* 782 * Tell the timer code that the base is not idle, i.e. undo 783 * the effect of get_next_timer_interrupt(): 784 */ 785 timer_clear_idle(); 786 /* 787 * We've not stopped the tick yet, and there's a timer in the 788 * next period, so no point in stopping it either, bail. 789 */ 790 if (!ts->tick_stopped) { 791 ts->timer_expires = 0; 792 goto out; 793 } 794 } 795 796 /* 797 * If this CPU is the one which had the do_timer() duty last, we limit 798 * the sleep time to the timekeeping max_deferment value. 799 * Otherwise we can sleep as long as we want. 800 */ 801 delta = timekeeping_max_deferment(); 802 if (cpu != tick_do_timer_cpu && 803 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) 804 delta = KTIME_MAX; 805 806 /* Calculate the next expiry time */ 807 if (delta < (KTIME_MAX - basemono)) 808 expires = basemono + delta; 809 else 810 expires = KTIME_MAX; 811 812 ts->timer_expires = min_t(u64, expires, next_tick); 813 814 out: 815 return ts->timer_expires; 816 } 817 818 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) 819 { 820 struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 821 u64 basemono = ts->timer_expires_base; 822 u64 expires = ts->timer_expires; 823 ktime_t tick = expires; 824 825 /* Make sure we won't be trying to stop it twice in a row. */ 826 ts->timer_expires_base = 0; 827 828 /* 829 * If this CPU is the one which updates jiffies, then give up 830 * the assignment and let it be taken by the CPU which runs 831 * the tick timer next, which might be this CPU as well. If we 832 * don't drop this here the jiffies might be stale and 833 * do_timer() never invoked. Keep track of the fact that it 834 * was the one which had the do_timer() duty last. 835 */ 836 if (cpu == tick_do_timer_cpu) { 837 tick_do_timer_cpu = TICK_DO_TIMER_NONE; 838 ts->do_timer_last = 1; 839 } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) { 840 ts->do_timer_last = 0; 841 } 842 843 /* Skip reprogram of event if its not changed */ 844 if (ts->tick_stopped && (expires == ts->next_tick)) { 845 /* Sanity check: make sure clockevent is actually programmed */ 846 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) 847 return; 848 849 WARN_ON_ONCE(1); 850 printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n", 851 basemono, ts->next_tick, dev->next_event, 852 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); 853 } 854 855 /* 856 * nohz_stop_sched_tick can be called several times before 857 * the nohz_restart_sched_tick is called. This happens when 858 * interrupts arrive which do not cause a reschedule. In the 859 * first call we save the current tick time, so we can restart 860 * the scheduler tick in nohz_restart_sched_tick. 861 */ 862 if (!ts->tick_stopped) { 863 calc_load_nohz_start(); 864 quiet_vmstat(); 865 866 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); 867 ts->tick_stopped = 1; 868 trace_tick_stop(1, TICK_DEP_MASK_NONE); 869 } 870 871 ts->next_tick = tick; 872 873 /* 874 * If the expiration time == KTIME_MAX, then we simply stop 875 * the tick timer. 876 */ 877 if (unlikely(expires == KTIME_MAX)) { 878 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) 879 hrtimer_cancel(&ts->sched_timer); 880 return; 881 } 882 883 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 884 hrtimer_start(&ts->sched_timer, tick, 885 HRTIMER_MODE_ABS_PINNED_HARD); 886 } else { 887 hrtimer_set_expires(&ts->sched_timer, tick); 888 tick_program_event(tick, 1); 889 } 890 } 891 892 static void tick_nohz_retain_tick(struct tick_sched *ts) 893 { 894 ts->timer_expires_base = 0; 895 } 896 897 #ifdef CONFIG_NO_HZ_FULL 898 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu) 899 { 900 if (tick_nohz_next_event(ts, cpu)) 901 tick_nohz_stop_tick(ts, cpu); 902 else 903 tick_nohz_retain_tick(ts); 904 } 905 #endif /* CONFIG_NO_HZ_FULL */ 906 907 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) 908 { 909 /* Update jiffies first */ 910 tick_do_update_jiffies64(now); 911 912 /* 913 * Clear the timer idle flag, so we avoid IPIs on remote queueing and 914 * the clock forward checks in the enqueue path: 915 */ 916 timer_clear_idle(); 917 918 calc_load_nohz_stop(); 919 touch_softlockup_watchdog_sched(); 920 /* 921 * Cancel the scheduled timer and restore the tick 922 */ 923 ts->tick_stopped = 0; 924 ts->idle_exittime = now; 925 926 tick_nohz_restart(ts, now); 927 } 928 929 static void tick_nohz_full_update_tick(struct tick_sched *ts) 930 { 931 #ifdef CONFIG_NO_HZ_FULL 932 int cpu = smp_processor_id(); 933 934 if (!tick_nohz_full_cpu(cpu)) 935 return; 936 937 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) 938 return; 939 940 if (can_stop_full_tick(cpu, ts)) 941 tick_nohz_stop_sched_tick(ts, cpu); 942 else if (ts->tick_stopped) 943 tick_nohz_restart_sched_tick(ts, ktime_get()); 944 #endif 945 } 946 947 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) 948 { 949 /* 950 * If this CPU is offline and it is the one which updates 951 * jiffies, then give up the assignment and let it be taken by 952 * the CPU which runs the tick timer next. If we don't drop 953 * this here the jiffies might be stale and do_timer() never 954 * invoked. 955 */ 956 if (unlikely(!cpu_online(cpu))) { 957 if (cpu == tick_do_timer_cpu) 958 tick_do_timer_cpu = TICK_DO_TIMER_NONE; 959 /* 960 * Make sure the CPU doesn't get fooled by obsolete tick 961 * deadline if it comes back online later. 962 */ 963 ts->next_tick = 0; 964 return false; 965 } 966 967 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 968 return false; 969 970 if (need_resched()) 971 return false; 972 973 if (unlikely(local_softirq_pending())) { 974 static int ratelimit; 975 976 if (ratelimit < 10 && 977 (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { 978 pr_warn("NOHZ tick-stop error: Non-RCU local softirq work is pending, handler #%02x!!!\n", 979 (unsigned int) local_softirq_pending()); 980 ratelimit++; 981 } 982 return false; 983 } 984 985 if (tick_nohz_full_enabled()) { 986 /* 987 * Keep the tick alive to guarantee timekeeping progression 988 * if there are full dynticks CPUs around 989 */ 990 if (tick_do_timer_cpu == cpu) 991 return false; 992 /* 993 * Boot safety: make sure the timekeeping duty has been 994 * assigned before entering dyntick-idle mode, 995 * tick_do_timer_cpu is TICK_DO_TIMER_BOOT 996 */ 997 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_BOOT)) 998 return false; 999 1000 /* Should not happen for nohz-full */ 1001 if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) 1002 return false; 1003 } 1004 1005 return true; 1006 } 1007 1008 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) 1009 { 1010 ktime_t expires; 1011 int cpu = smp_processor_id(); 1012 1013 /* 1014 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the 1015 * tick timer expiration time is known already. 1016 */ 1017 if (ts->timer_expires_base) 1018 expires = ts->timer_expires; 1019 else if (can_stop_idle_tick(cpu, ts)) 1020 expires = tick_nohz_next_event(ts, cpu); 1021 else 1022 return; 1023 1024 ts->idle_calls++; 1025 1026 if (expires > 0LL) { 1027 int was_stopped = ts->tick_stopped; 1028 1029 tick_nohz_stop_tick(ts, cpu); 1030 1031 ts->idle_sleeps++; 1032 ts->idle_expires = expires; 1033 1034 if (!was_stopped && ts->tick_stopped) { 1035 ts->idle_jiffies = ts->last_jiffies; 1036 nohz_balance_enter_idle(cpu); 1037 } 1038 } else { 1039 tick_nohz_retain_tick(ts); 1040 } 1041 } 1042 1043 /** 1044 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task 1045 * 1046 * When the next event is more than a tick into the future, stop the idle tick 1047 */ 1048 void tick_nohz_idle_stop_tick(void) 1049 { 1050 __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched)); 1051 } 1052 1053 void tick_nohz_idle_retain_tick(void) 1054 { 1055 tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched)); 1056 /* 1057 * Undo the effect of get_next_timer_interrupt() called from 1058 * tick_nohz_next_event(). 1059 */ 1060 timer_clear_idle(); 1061 } 1062 1063 /** 1064 * tick_nohz_idle_enter - prepare for entering idle on the current CPU 1065 * 1066 * Called when we start the idle loop. 1067 */ 1068 void tick_nohz_idle_enter(void) 1069 { 1070 struct tick_sched *ts; 1071 1072 lockdep_assert_irqs_enabled(); 1073 1074 local_irq_disable(); 1075 1076 ts = this_cpu_ptr(&tick_cpu_sched); 1077 1078 WARN_ON_ONCE(ts->timer_expires_base); 1079 1080 ts->inidle = 1; 1081 tick_nohz_start_idle(ts); 1082 1083 local_irq_enable(); 1084 } 1085 1086 /** 1087 * tick_nohz_irq_exit - update next tick event from interrupt exit 1088 * 1089 * When an interrupt fires while we are idle and it doesn't cause 1090 * a reschedule, it may still add, modify or delete a timer, enqueue 1091 * an RCU callback, etc... 1092 * So we need to re-calculate and reprogram the next tick event. 1093 */ 1094 void tick_nohz_irq_exit(void) 1095 { 1096 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1097 1098 if (ts->inidle) 1099 tick_nohz_start_idle(ts); 1100 else 1101 tick_nohz_full_update_tick(ts); 1102 } 1103 1104 /** 1105 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run 1106 */ 1107 bool tick_nohz_idle_got_tick(void) 1108 { 1109 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1110 1111 if (ts->got_idle_tick) { 1112 ts->got_idle_tick = 0; 1113 return true; 1114 } 1115 return false; 1116 } 1117 1118 /** 1119 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer 1120 * or the tick, whatever that expires first. Note that, if the tick has been 1121 * stopped, it returns the next hrtimer. 1122 * 1123 * Called from power state control code with interrupts disabled 1124 */ 1125 ktime_t tick_nohz_get_next_hrtimer(void) 1126 { 1127 return __this_cpu_read(tick_cpu_device.evtdev)->next_event; 1128 } 1129 1130 /** 1131 * tick_nohz_get_sleep_length - return the expected length of the current sleep 1132 * @delta_next: duration until the next event if the tick cannot be stopped 1133 * 1134 * Called from power state control code with interrupts disabled 1135 */ 1136 ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next) 1137 { 1138 struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 1139 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1140 int cpu = smp_processor_id(); 1141 /* 1142 * The idle entry time is expected to be a sufficient approximation of 1143 * the current time at this point. 1144 */ 1145 ktime_t now = ts->idle_entrytime; 1146 ktime_t next_event; 1147 1148 WARN_ON_ONCE(!ts->inidle); 1149 1150 *delta_next = ktime_sub(dev->next_event, now); 1151 1152 if (!can_stop_idle_tick(cpu, ts)) 1153 return *delta_next; 1154 1155 next_event = tick_nohz_next_event(ts, cpu); 1156 if (!next_event) 1157 return *delta_next; 1158 1159 /* 1160 * If the next highres timer to expire is earlier than next_event, the 1161 * idle governor needs to know that. 1162 */ 1163 next_event = min_t(u64, next_event, 1164 hrtimer_next_event_without(&ts->sched_timer)); 1165 1166 return ktime_sub(next_event, now); 1167 } 1168 1169 /** 1170 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value 1171 * for a particular CPU. 1172 * 1173 * Called from the schedutil frequency scaling governor in scheduler context. 1174 */ 1175 unsigned long tick_nohz_get_idle_calls_cpu(int cpu) 1176 { 1177 struct tick_sched *ts = tick_get_tick_sched(cpu); 1178 1179 return ts->idle_calls; 1180 } 1181 1182 /** 1183 * tick_nohz_get_idle_calls - return the current idle calls counter value 1184 * 1185 * Called from the schedutil frequency scaling governor in scheduler context. 1186 */ 1187 unsigned long tick_nohz_get_idle_calls(void) 1188 { 1189 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1190 1191 return ts->idle_calls; 1192 } 1193 1194 static void tick_nohz_account_idle_ticks(struct tick_sched *ts) 1195 { 1196 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 1197 unsigned long ticks; 1198 1199 if (vtime_accounting_enabled_this_cpu()) 1200 return; 1201 /* 1202 * We stopped the tick in idle. Update process times would miss the 1203 * time we slept as update_process_times does only a 1 tick 1204 * accounting. Enforce that this is accounted to idle ! 1205 */ 1206 ticks = jiffies - ts->idle_jiffies; 1207 /* 1208 * We might be one off. Do not randomly account a huge number of ticks! 1209 */ 1210 if (ticks && ticks < LONG_MAX) 1211 account_idle_ticks(ticks); 1212 #endif 1213 } 1214 1215 static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now) 1216 { 1217 tick_nohz_restart_sched_tick(ts, now); 1218 tick_nohz_account_idle_ticks(ts); 1219 } 1220 1221 void tick_nohz_idle_restart_tick(void) 1222 { 1223 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1224 1225 if (ts->tick_stopped) 1226 __tick_nohz_idle_restart_tick(ts, ktime_get()); 1227 } 1228 1229 /** 1230 * tick_nohz_idle_exit - restart the idle tick from the idle task 1231 * 1232 * Restart the idle tick when the CPU is woken up from idle 1233 * This also exit the RCU extended quiescent state. The CPU 1234 * can use RCU again after this function is called. 1235 */ 1236 void tick_nohz_idle_exit(void) 1237 { 1238 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1239 bool idle_active, tick_stopped; 1240 ktime_t now; 1241 1242 local_irq_disable(); 1243 1244 WARN_ON_ONCE(!ts->inidle); 1245 WARN_ON_ONCE(ts->timer_expires_base); 1246 1247 ts->inidle = 0; 1248 idle_active = ts->idle_active; 1249 tick_stopped = ts->tick_stopped; 1250 1251 if (idle_active || tick_stopped) 1252 now = ktime_get(); 1253 1254 if (idle_active) 1255 tick_nohz_stop_idle(ts, now); 1256 1257 if (tick_stopped) 1258 __tick_nohz_idle_restart_tick(ts, now); 1259 1260 local_irq_enable(); 1261 } 1262 1263 /* 1264 * The nohz low res interrupt handler 1265 */ 1266 static void tick_nohz_handler(struct clock_event_device *dev) 1267 { 1268 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1269 struct pt_regs *regs = get_irq_regs(); 1270 ktime_t now = ktime_get(); 1271 1272 dev->next_event = KTIME_MAX; 1273 1274 tick_sched_do_timer(ts, now); 1275 tick_sched_handle(ts, regs); 1276 1277 /* No need to reprogram if we are running tickless */ 1278 if (unlikely(ts->tick_stopped)) 1279 return; 1280 1281 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); 1282 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); 1283 } 1284 1285 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) 1286 { 1287 if (!tick_nohz_enabled) 1288 return; 1289 ts->nohz_mode = mode; 1290 /* One update is enough */ 1291 if (!test_and_set_bit(0, &tick_nohz_active)) 1292 timers_update_nohz(); 1293 } 1294 1295 /** 1296 * tick_nohz_switch_to_nohz - switch to nohz mode 1297 */ 1298 static void tick_nohz_switch_to_nohz(void) 1299 { 1300 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1301 ktime_t next; 1302 1303 if (!tick_nohz_enabled) 1304 return; 1305 1306 if (tick_switch_to_oneshot(tick_nohz_handler)) 1307 return; 1308 1309 /* 1310 * Recycle the hrtimer in ts, so we can share the 1311 * hrtimer_forward with the highres code. 1312 */ 1313 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); 1314 /* Get the next period */ 1315 next = tick_init_jiffy_update(); 1316 1317 hrtimer_set_expires(&ts->sched_timer, next); 1318 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); 1319 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); 1320 tick_nohz_activate(ts, NOHZ_MODE_LOWRES); 1321 } 1322 1323 static inline void tick_nohz_irq_enter(void) 1324 { 1325 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1326 ktime_t now; 1327 1328 if (!ts->idle_active && !ts->tick_stopped) 1329 return; 1330 now = ktime_get(); 1331 if (ts->idle_active) 1332 tick_nohz_stop_idle(ts, now); 1333 if (ts->tick_stopped) 1334 tick_nohz_update_jiffies(now); 1335 } 1336 1337 #else 1338 1339 static inline void tick_nohz_switch_to_nohz(void) { } 1340 static inline void tick_nohz_irq_enter(void) { } 1341 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { } 1342 1343 #endif /* CONFIG_NO_HZ_COMMON */ 1344 1345 /* 1346 * Called from irq_enter to notify about the possible interruption of idle() 1347 */ 1348 void tick_irq_enter(void) 1349 { 1350 tick_check_oneshot_broadcast_this_cpu(); 1351 tick_nohz_irq_enter(); 1352 } 1353 1354 /* 1355 * High resolution timer specific code 1356 */ 1357 #ifdef CONFIG_HIGH_RES_TIMERS 1358 /* 1359 * We rearm the timer until we get disabled by the idle code. 1360 * Called with interrupts disabled. 1361 */ 1362 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) 1363 { 1364 struct tick_sched *ts = 1365 container_of(timer, struct tick_sched, sched_timer); 1366 struct pt_regs *regs = get_irq_regs(); 1367 ktime_t now = ktime_get(); 1368 1369 tick_sched_do_timer(ts, now); 1370 1371 /* 1372 * Do not call, when we are not in irq context and have 1373 * no valid regs pointer 1374 */ 1375 if (regs) 1376 tick_sched_handle(ts, regs); 1377 else 1378 ts->next_tick = 0; 1379 1380 /* No need to reprogram if we are in idle or full dynticks mode */ 1381 if (unlikely(ts->tick_stopped)) 1382 return HRTIMER_NORESTART; 1383 1384 hrtimer_forward(timer, now, TICK_NSEC); 1385 1386 return HRTIMER_RESTART; 1387 } 1388 1389 static int sched_skew_tick; 1390 1391 static int __init skew_tick(char *str) 1392 { 1393 get_option(&str, &sched_skew_tick); 1394 1395 return 0; 1396 } 1397 early_param("skew_tick", skew_tick); 1398 1399 /** 1400 * tick_setup_sched_timer - setup the tick emulation timer 1401 */ 1402 void tick_setup_sched_timer(void) 1403 { 1404 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1405 ktime_t now = ktime_get(); 1406 1407 /* 1408 * Emulate tick processing via per-CPU hrtimers: 1409 */ 1410 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); 1411 ts->sched_timer.function = tick_sched_timer; 1412 1413 /* Get the next period (per-CPU) */ 1414 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); 1415 1416 /* Offset the tick to avert jiffies_lock contention. */ 1417 if (sched_skew_tick) { 1418 u64 offset = TICK_NSEC >> 1; 1419 do_div(offset, num_possible_cpus()); 1420 offset *= smp_processor_id(); 1421 hrtimer_add_expires_ns(&ts->sched_timer, offset); 1422 } 1423 1424 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); 1425 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); 1426 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); 1427 } 1428 #endif /* HIGH_RES_TIMERS */ 1429 1430 #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS 1431 void tick_cancel_sched_timer(int cpu) 1432 { 1433 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 1434 1435 # ifdef CONFIG_HIGH_RES_TIMERS 1436 if (ts->sched_timer.base) 1437 hrtimer_cancel(&ts->sched_timer); 1438 # endif 1439 1440 memset(ts, 0, sizeof(*ts)); 1441 } 1442 #endif 1443 1444 /** 1445 * Async notification about clocksource changes 1446 */ 1447 void tick_clock_notify(void) 1448 { 1449 int cpu; 1450 1451 for_each_possible_cpu(cpu) 1452 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks); 1453 } 1454 1455 /* 1456 * Async notification about clock event changes 1457 */ 1458 void tick_oneshot_notify(void) 1459 { 1460 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1461 1462 set_bit(0, &ts->check_clocks); 1463 } 1464 1465 /** 1466 * Check, if a change happened, which makes oneshot possible. 1467 * 1468 * Called cyclic from the hrtimer softirq (driven by the timer 1469 * softirq) allow_nohz signals, that we can switch into low-res nohz 1470 * mode, because high resolution timers are disabled (either compile 1471 * or runtime). Called with interrupts disabled. 1472 */ 1473 int tick_check_oneshot_change(int allow_nohz) 1474 { 1475 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1476 1477 if (!test_and_clear_bit(0, &ts->check_clocks)) 1478 return 0; 1479 1480 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) 1481 return 0; 1482 1483 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available()) 1484 return 0; 1485 1486 if (!allow_nohz) 1487 return 1; 1488 1489 tick_nohz_switch_to_nohz(); 1490 return 0; 1491 } 1492