1 /* 2 * linux/kernel/time/tick-broadcast.c 3 * 4 * This file contains functions which emulate a local clock-event 5 * device via a broadcast event source. 6 * 7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner 10 * 11 * This code is licenced under the GPL version 2. For details see 12 * kernel-base/COPYING. 13 */ 14 #include <linux/cpu.h> 15 #include <linux/err.h> 16 #include <linux/hrtimer.h> 17 #include <linux/interrupt.h> 18 #include <linux/percpu.h> 19 #include <linux/profile.h> 20 #include <linux/sched.h> 21 #include <linux/smp.h> 22 #include <linux/module.h> 23 24 #include "tick-internal.h" 25 26 /* 27 * Broadcast support for broken x86 hardware, where the local apic 28 * timer stops in C3 state. 29 */ 30 31 static struct tick_device tick_broadcast_device; 32 static cpumask_var_t tick_broadcast_mask; 33 static cpumask_var_t tick_broadcast_on; 34 static cpumask_var_t tmpmask; 35 static DEFINE_RAW_SPINLOCK(tick_broadcast_lock); 36 static int tick_broadcast_force; 37 38 #ifdef CONFIG_TICK_ONESHOT 39 static void tick_broadcast_clear_oneshot(int cpu); 40 #else 41 static inline void tick_broadcast_clear_oneshot(int cpu) { } 42 #endif 43 44 /* 45 * Debugging: see timer_list.c 46 */ 47 struct tick_device *tick_get_broadcast_device(void) 48 { 49 return &tick_broadcast_device; 50 } 51 52 struct cpumask *tick_get_broadcast_mask(void) 53 { 54 return tick_broadcast_mask; 55 } 56 57 /* 58 * Start the device in periodic mode 59 */ 60 static void tick_broadcast_start_periodic(struct clock_event_device *bc) 61 { 62 if (bc) 63 tick_setup_periodic(bc, 1); 64 } 65 66 /* 67 * Check, if the device can be utilized as broadcast device: 68 */ 69 static bool tick_check_broadcast_device(struct clock_event_device *curdev, 70 struct clock_event_device *newdev) 71 { 72 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) || 73 (newdev->features & CLOCK_EVT_FEAT_PERCPU) || 74 (newdev->features & CLOCK_EVT_FEAT_C3STOP)) 75 return false; 76 77 if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT && 78 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) 79 return false; 80 81 return !curdev || newdev->rating > curdev->rating; 82 } 83 84 /* 85 * Conditionally install/replace broadcast device 86 */ 87 void tick_install_broadcast_device(struct clock_event_device *dev) 88 { 89 struct clock_event_device *cur = tick_broadcast_device.evtdev; 90 91 if (!tick_check_broadcast_device(cur, dev)) 92 return; 93 94 if (!try_module_get(dev->owner)) 95 return; 96 97 clockevents_exchange_device(cur, dev); 98 if (cur) 99 cur->event_handler = clockevents_handle_noop; 100 tick_broadcast_device.evtdev = dev; 101 if (!cpumask_empty(tick_broadcast_mask)) 102 tick_broadcast_start_periodic(dev); 103 /* 104 * Inform all cpus about this. We might be in a situation 105 * where we did not switch to oneshot mode because the per cpu 106 * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack 107 * of a oneshot capable broadcast device. Without that 108 * notification the systems stays stuck in periodic mode 109 * forever. 110 */ 111 if (dev->features & CLOCK_EVT_FEAT_ONESHOT) 112 tick_clock_notify(); 113 } 114 115 /* 116 * Check, if the device is the broadcast device 117 */ 118 int tick_is_broadcast_device(struct clock_event_device *dev) 119 { 120 return (dev && tick_broadcast_device.evtdev == dev); 121 } 122 123 int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq) 124 { 125 int ret = -ENODEV; 126 127 if (tick_is_broadcast_device(dev)) { 128 raw_spin_lock(&tick_broadcast_lock); 129 ret = __clockevents_update_freq(dev, freq); 130 raw_spin_unlock(&tick_broadcast_lock); 131 } 132 return ret; 133 } 134 135 136 static void err_broadcast(const struct cpumask *mask) 137 { 138 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n"); 139 } 140 141 static void tick_device_setup_broadcast_func(struct clock_event_device *dev) 142 { 143 if (!dev->broadcast) 144 dev->broadcast = tick_broadcast; 145 if (!dev->broadcast) { 146 pr_warn_once("%s depends on broadcast, but no broadcast function available\n", 147 dev->name); 148 dev->broadcast = err_broadcast; 149 } 150 } 151 152 /* 153 * Check, if the device is disfunctional and a place holder, which 154 * needs to be handled by the broadcast device. 155 */ 156 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) 157 { 158 struct clock_event_device *bc = tick_broadcast_device.evtdev; 159 unsigned long flags; 160 int ret; 161 162 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 163 164 /* 165 * Devices might be registered with both periodic and oneshot 166 * mode disabled. This signals, that the device needs to be 167 * operated from the broadcast device and is a placeholder for 168 * the cpu local device. 169 */ 170 if (!tick_device_is_functional(dev)) { 171 dev->event_handler = tick_handle_periodic; 172 tick_device_setup_broadcast_func(dev); 173 cpumask_set_cpu(cpu, tick_broadcast_mask); 174 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 175 tick_broadcast_start_periodic(bc); 176 else 177 tick_broadcast_setup_oneshot(bc); 178 ret = 1; 179 } else { 180 /* 181 * Clear the broadcast bit for this cpu if the 182 * device is not power state affected. 183 */ 184 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 185 cpumask_clear_cpu(cpu, tick_broadcast_mask); 186 else 187 tick_device_setup_broadcast_func(dev); 188 189 /* 190 * Clear the broadcast bit if the CPU is not in 191 * periodic broadcast on state. 192 */ 193 if (!cpumask_test_cpu(cpu, tick_broadcast_on)) 194 cpumask_clear_cpu(cpu, tick_broadcast_mask); 195 196 switch (tick_broadcast_device.mode) { 197 case TICKDEV_MODE_ONESHOT: 198 /* 199 * If the system is in oneshot mode we can 200 * unconditionally clear the oneshot mask bit, 201 * because the CPU is running and therefore 202 * not in an idle state which causes the power 203 * state affected device to stop. Let the 204 * caller initialize the device. 205 */ 206 tick_broadcast_clear_oneshot(cpu); 207 ret = 0; 208 break; 209 210 case TICKDEV_MODE_PERIODIC: 211 /* 212 * If the system is in periodic mode, check 213 * whether the broadcast device can be 214 * switched off now. 215 */ 216 if (cpumask_empty(tick_broadcast_mask) && bc) 217 clockevents_shutdown(bc); 218 /* 219 * If we kept the cpu in the broadcast mask, 220 * tell the caller to leave the per cpu device 221 * in shutdown state. The periodic interrupt 222 * is delivered by the broadcast device. 223 */ 224 ret = cpumask_test_cpu(cpu, tick_broadcast_mask); 225 break; 226 default: 227 /* Nothing to do */ 228 ret = 0; 229 break; 230 } 231 } 232 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 233 return ret; 234 } 235 236 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 237 int tick_receive_broadcast(void) 238 { 239 struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 240 struct clock_event_device *evt = td->evtdev; 241 242 if (!evt) 243 return -ENODEV; 244 245 if (!evt->event_handler) 246 return -EINVAL; 247 248 evt->event_handler(evt); 249 return 0; 250 } 251 #endif 252 253 /* 254 * Broadcast the event to the cpus, which are set in the mask (mangled). 255 */ 256 static void tick_do_broadcast(struct cpumask *mask) 257 { 258 int cpu = smp_processor_id(); 259 struct tick_device *td; 260 261 /* 262 * Check, if the current cpu is in the mask 263 */ 264 if (cpumask_test_cpu(cpu, mask)) { 265 cpumask_clear_cpu(cpu, mask); 266 td = &per_cpu(tick_cpu_device, cpu); 267 td->evtdev->event_handler(td->evtdev); 268 } 269 270 if (!cpumask_empty(mask)) { 271 /* 272 * It might be necessary to actually check whether the devices 273 * have different broadcast functions. For now, just use the 274 * one of the first device. This works as long as we have this 275 * misfeature only on x86 (lapic) 276 */ 277 td = &per_cpu(tick_cpu_device, cpumask_first(mask)); 278 td->evtdev->broadcast(mask); 279 } 280 } 281 282 /* 283 * Periodic broadcast: 284 * - invoke the broadcast handlers 285 */ 286 static void tick_do_periodic_broadcast(void) 287 { 288 cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); 289 tick_do_broadcast(tmpmask); 290 } 291 292 /* 293 * Event handler for periodic broadcast ticks 294 */ 295 static void tick_handle_periodic_broadcast(struct clock_event_device *dev) 296 { 297 ktime_t next; 298 299 raw_spin_lock(&tick_broadcast_lock); 300 301 tick_do_periodic_broadcast(); 302 303 /* 304 * The device is in periodic mode. No reprogramming necessary: 305 */ 306 if (dev->mode == CLOCK_EVT_MODE_PERIODIC) 307 goto unlock; 308 309 /* 310 * Setup the next period for devices, which do not have 311 * periodic mode. We read dev->next_event first and add to it 312 * when the event already expired. clockevents_program_event() 313 * sets dev->next_event only when the event is really 314 * programmed to the device. 315 */ 316 for (next = dev->next_event; ;) { 317 next = ktime_add(next, tick_period); 318 319 if (!clockevents_program_event(dev, next, false)) 320 goto unlock; 321 tick_do_periodic_broadcast(); 322 } 323 unlock: 324 raw_spin_unlock(&tick_broadcast_lock); 325 } 326 327 /* 328 * Powerstate information: The system enters/leaves a state, where 329 * affected devices might stop 330 */ 331 static void tick_do_broadcast_on_off(unsigned long *reason) 332 { 333 struct clock_event_device *bc, *dev; 334 struct tick_device *td; 335 unsigned long flags; 336 int cpu, bc_stopped; 337 338 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 339 340 cpu = smp_processor_id(); 341 td = &per_cpu(tick_cpu_device, cpu); 342 dev = td->evtdev; 343 bc = tick_broadcast_device.evtdev; 344 345 /* 346 * Is the device not affected by the powerstate ? 347 */ 348 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) 349 goto out; 350 351 if (!tick_device_is_functional(dev)) 352 goto out; 353 354 bc_stopped = cpumask_empty(tick_broadcast_mask); 355 356 switch (*reason) { 357 case CLOCK_EVT_NOTIFY_BROADCAST_ON: 358 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: 359 cpumask_set_cpu(cpu, tick_broadcast_on); 360 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { 361 if (tick_broadcast_device.mode == 362 TICKDEV_MODE_PERIODIC) 363 clockevents_shutdown(dev); 364 } 365 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) 366 tick_broadcast_force = 1; 367 break; 368 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 369 if (tick_broadcast_force) 370 break; 371 cpumask_clear_cpu(cpu, tick_broadcast_on); 372 if (!tick_device_is_functional(dev)) 373 break; 374 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { 375 if (tick_broadcast_device.mode == 376 TICKDEV_MODE_PERIODIC) 377 tick_setup_periodic(dev, 0); 378 } 379 break; 380 } 381 382 if (cpumask_empty(tick_broadcast_mask)) { 383 if (!bc_stopped) 384 clockevents_shutdown(bc); 385 } else if (bc_stopped) { 386 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 387 tick_broadcast_start_periodic(bc); 388 else 389 tick_broadcast_setup_oneshot(bc); 390 } 391 out: 392 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 393 } 394 395 /* 396 * Powerstate information: The system enters/leaves a state, where 397 * affected devices might stop. 398 */ 399 void tick_broadcast_on_off(unsigned long reason, int *oncpu) 400 { 401 if (!cpumask_test_cpu(*oncpu, cpu_online_mask)) 402 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 403 "offline CPU #%d\n", *oncpu); 404 else 405 tick_do_broadcast_on_off(&reason); 406 } 407 408 /* 409 * Set the periodic handler depending on broadcast on/off 410 */ 411 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast) 412 { 413 if (!broadcast) 414 dev->event_handler = tick_handle_periodic; 415 else 416 dev->event_handler = tick_handle_periodic_broadcast; 417 } 418 419 /* 420 * Remove a CPU from broadcasting 421 */ 422 void tick_shutdown_broadcast(unsigned int *cpup) 423 { 424 struct clock_event_device *bc; 425 unsigned long flags; 426 unsigned int cpu = *cpup; 427 428 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 429 430 bc = tick_broadcast_device.evtdev; 431 cpumask_clear_cpu(cpu, tick_broadcast_mask); 432 cpumask_clear_cpu(cpu, tick_broadcast_on); 433 434 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 435 if (bc && cpumask_empty(tick_broadcast_mask)) 436 clockevents_shutdown(bc); 437 } 438 439 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 440 } 441 442 void tick_suspend_broadcast(void) 443 { 444 struct clock_event_device *bc; 445 unsigned long flags; 446 447 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 448 449 bc = tick_broadcast_device.evtdev; 450 if (bc) 451 clockevents_shutdown(bc); 452 453 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 454 } 455 456 int tick_resume_broadcast(void) 457 { 458 struct clock_event_device *bc; 459 unsigned long flags; 460 int broadcast = 0; 461 462 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 463 464 bc = tick_broadcast_device.evtdev; 465 466 if (bc) { 467 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME); 468 469 switch (tick_broadcast_device.mode) { 470 case TICKDEV_MODE_PERIODIC: 471 if (!cpumask_empty(tick_broadcast_mask)) 472 tick_broadcast_start_periodic(bc); 473 broadcast = cpumask_test_cpu(smp_processor_id(), 474 tick_broadcast_mask); 475 break; 476 case TICKDEV_MODE_ONESHOT: 477 if (!cpumask_empty(tick_broadcast_mask)) 478 broadcast = tick_resume_broadcast_oneshot(bc); 479 break; 480 } 481 } 482 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 483 484 return broadcast; 485 } 486 487 488 #ifdef CONFIG_TICK_ONESHOT 489 490 static cpumask_var_t tick_broadcast_oneshot_mask; 491 static cpumask_var_t tick_broadcast_pending_mask; 492 static cpumask_var_t tick_broadcast_force_mask; 493 494 /* 495 * Exposed for debugging: see timer_list.c 496 */ 497 struct cpumask *tick_get_broadcast_oneshot_mask(void) 498 { 499 return tick_broadcast_oneshot_mask; 500 } 501 502 /* 503 * Called before going idle with interrupts disabled. Checks whether a 504 * broadcast event from the other core is about to happen. We detected 505 * that in tick_broadcast_oneshot_control(). The callsite can use this 506 * to avoid a deep idle transition as we are about to get the 507 * broadcast IPI right away. 508 */ 509 int tick_check_broadcast_expired(void) 510 { 511 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask); 512 } 513 514 /* 515 * Set broadcast interrupt affinity 516 */ 517 static void tick_broadcast_set_affinity(struct clock_event_device *bc, 518 const struct cpumask *cpumask) 519 { 520 if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ)) 521 return; 522 523 if (cpumask_equal(bc->cpumask, cpumask)) 524 return; 525 526 bc->cpumask = cpumask; 527 irq_set_affinity(bc->irq, bc->cpumask); 528 } 529 530 static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu, 531 ktime_t expires, int force) 532 { 533 int ret; 534 535 if (bc->mode != CLOCK_EVT_MODE_ONESHOT) 536 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 537 538 ret = clockevents_program_event(bc, expires, force); 539 if (!ret) 540 tick_broadcast_set_affinity(bc, cpumask_of(cpu)); 541 return ret; 542 } 543 544 int tick_resume_broadcast_oneshot(struct clock_event_device *bc) 545 { 546 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 547 return 0; 548 } 549 550 /* 551 * Called from irq_enter() when idle was interrupted to reenable the 552 * per cpu device. 553 */ 554 void tick_check_oneshot_broadcast_this_cpu(void) 555 { 556 if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) { 557 struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 558 559 /* 560 * We might be in the middle of switching over from 561 * periodic to oneshot. If the CPU has not yet 562 * switched over, leave the device alone. 563 */ 564 if (td->mode == TICKDEV_MODE_ONESHOT) { 565 clockevents_set_mode(td->evtdev, 566 CLOCK_EVT_MODE_ONESHOT); 567 } 568 } 569 } 570 571 /* 572 * Handle oneshot mode broadcasting 573 */ 574 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 575 { 576 struct tick_device *td; 577 ktime_t now, next_event; 578 int cpu, next_cpu = 0; 579 580 raw_spin_lock(&tick_broadcast_lock); 581 again: 582 dev->next_event.tv64 = KTIME_MAX; 583 next_event.tv64 = KTIME_MAX; 584 cpumask_clear(tmpmask); 585 now = ktime_get(); 586 /* Find all expired events */ 587 for_each_cpu(cpu, tick_broadcast_oneshot_mask) { 588 td = &per_cpu(tick_cpu_device, cpu); 589 if (td->evtdev->next_event.tv64 <= now.tv64) { 590 cpumask_set_cpu(cpu, tmpmask); 591 /* 592 * Mark the remote cpu in the pending mask, so 593 * it can avoid reprogramming the cpu local 594 * timer in tick_broadcast_oneshot_control(). 595 */ 596 cpumask_set_cpu(cpu, tick_broadcast_pending_mask); 597 } else if (td->evtdev->next_event.tv64 < next_event.tv64) { 598 next_event.tv64 = td->evtdev->next_event.tv64; 599 next_cpu = cpu; 600 } 601 } 602 603 /* 604 * Remove the current cpu from the pending mask. The event is 605 * delivered immediately in tick_do_broadcast() ! 606 */ 607 cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask); 608 609 /* Take care of enforced broadcast requests */ 610 cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask); 611 cpumask_clear(tick_broadcast_force_mask); 612 613 /* 614 * Sanity check. Catch the case where we try to broadcast to 615 * offline cpus. 616 */ 617 if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask))) 618 cpumask_and(tmpmask, tmpmask, cpu_online_mask); 619 620 /* 621 * Wakeup the cpus which have an expired event. 622 */ 623 tick_do_broadcast(tmpmask); 624 625 /* 626 * Two reasons for reprogram: 627 * 628 * - The global event did not expire any CPU local 629 * events. This happens in dyntick mode, as the maximum PIT 630 * delta is quite small. 631 * 632 * - There are pending events on sleeping CPUs which were not 633 * in the event mask 634 */ 635 if (next_event.tv64 != KTIME_MAX) { 636 /* 637 * Rearm the broadcast device. If event expired, 638 * repeat the above 639 */ 640 if (tick_broadcast_set_event(dev, next_cpu, next_event, 0)) 641 goto again; 642 } 643 raw_spin_unlock(&tick_broadcast_lock); 644 } 645 646 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu) 647 { 648 if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER)) 649 return 0; 650 if (bc->next_event.tv64 == KTIME_MAX) 651 return 0; 652 return bc->bound_on == cpu ? -EBUSY : 0; 653 } 654 655 static void broadcast_shutdown_local(struct clock_event_device *bc, 656 struct clock_event_device *dev) 657 { 658 /* 659 * For hrtimer based broadcasting we cannot shutdown the cpu 660 * local device if our own event is the first one to expire or 661 * if we own the broadcast timer. 662 */ 663 if (bc->features & CLOCK_EVT_FEAT_HRTIMER) { 664 if (broadcast_needs_cpu(bc, smp_processor_id())) 665 return; 666 if (dev->next_event.tv64 < bc->next_event.tv64) 667 return; 668 } 669 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 670 } 671 672 static void broadcast_move_bc(int deadcpu) 673 { 674 struct clock_event_device *bc = tick_broadcast_device.evtdev; 675 676 if (!bc || !broadcast_needs_cpu(bc, deadcpu)) 677 return; 678 /* This moves the broadcast assignment to this cpu */ 679 clockevents_program_event(bc, bc->next_event, 1); 680 } 681 682 /* 683 * Powerstate information: The system enters/leaves a state, where 684 * affected devices might stop 685 * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups. 686 */ 687 int tick_broadcast_oneshot_control(unsigned long reason) 688 { 689 struct clock_event_device *bc, *dev; 690 struct tick_device *td; 691 unsigned long flags; 692 ktime_t now; 693 int cpu, ret = 0; 694 695 /* 696 * Periodic mode does not care about the enter/exit of power 697 * states 698 */ 699 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 700 return 0; 701 702 /* 703 * We are called with preemtion disabled from the depth of the 704 * idle code, so we can't be moved away. 705 */ 706 cpu = smp_processor_id(); 707 td = &per_cpu(tick_cpu_device, cpu); 708 dev = td->evtdev; 709 710 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 711 return 0; 712 713 bc = tick_broadcast_device.evtdev; 714 715 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 716 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 717 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { 718 WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask)); 719 broadcast_shutdown_local(bc, dev); 720 /* 721 * We only reprogram the broadcast timer if we 722 * did not mark ourself in the force mask and 723 * if the cpu local event is earlier than the 724 * broadcast event. If the current CPU is in 725 * the force mask, then we are going to be 726 * woken by the IPI right away. 727 */ 728 if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) && 729 dev->next_event.tv64 < bc->next_event.tv64) 730 tick_broadcast_set_event(bc, cpu, dev->next_event, 1); 731 } 732 /* 733 * If the current CPU owns the hrtimer broadcast 734 * mechanism, it cannot go deep idle and we remove the 735 * CPU from the broadcast mask. We don't have to go 736 * through the EXIT path as the local timer is not 737 * shutdown. 738 */ 739 ret = broadcast_needs_cpu(bc, cpu); 740 if (ret) 741 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 742 } else { 743 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { 744 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 745 /* 746 * The cpu which was handling the broadcast 747 * timer marked this cpu in the broadcast 748 * pending mask and fired the broadcast 749 * IPI. So we are going to handle the expired 750 * event anyway via the broadcast IPI 751 * handler. No need to reprogram the timer 752 * with an already expired event. 753 */ 754 if (cpumask_test_and_clear_cpu(cpu, 755 tick_broadcast_pending_mask)) 756 goto out; 757 758 /* 759 * Bail out if there is no next event. 760 */ 761 if (dev->next_event.tv64 == KTIME_MAX) 762 goto out; 763 /* 764 * If the pending bit is not set, then we are 765 * either the CPU handling the broadcast 766 * interrupt or we got woken by something else. 767 * 768 * We are not longer in the broadcast mask, so 769 * if the cpu local expiry time is already 770 * reached, we would reprogram the cpu local 771 * timer with an already expired event. 772 * 773 * This can lead to a ping-pong when we return 774 * to idle and therefor rearm the broadcast 775 * timer before the cpu local timer was able 776 * to fire. This happens because the forced 777 * reprogramming makes sure that the event 778 * will happen in the future and depending on 779 * the min_delta setting this might be far 780 * enough out that the ping-pong starts. 781 * 782 * If the cpu local next_event has expired 783 * then we know that the broadcast timer 784 * next_event has expired as well and 785 * broadcast is about to be handled. So we 786 * avoid reprogramming and enforce that the 787 * broadcast handler, which did not run yet, 788 * will invoke the cpu local handler. 789 * 790 * We cannot call the handler directly from 791 * here, because we might be in a NOHZ phase 792 * and we did not go through the irq_enter() 793 * nohz fixups. 794 */ 795 now = ktime_get(); 796 if (dev->next_event.tv64 <= now.tv64) { 797 cpumask_set_cpu(cpu, tick_broadcast_force_mask); 798 goto out; 799 } 800 /* 801 * We got woken by something else. Reprogram 802 * the cpu local timer device. 803 */ 804 tick_program_event(dev->next_event, 1); 805 } 806 } 807 out: 808 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 809 return ret; 810 } 811 812 /* 813 * Reset the one shot broadcast for a cpu 814 * 815 * Called with tick_broadcast_lock held 816 */ 817 static void tick_broadcast_clear_oneshot(int cpu) 818 { 819 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 820 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask); 821 } 822 823 static void tick_broadcast_init_next_event(struct cpumask *mask, 824 ktime_t expires) 825 { 826 struct tick_device *td; 827 int cpu; 828 829 for_each_cpu(cpu, mask) { 830 td = &per_cpu(tick_cpu_device, cpu); 831 if (td->evtdev) 832 td->evtdev->next_event = expires; 833 } 834 } 835 836 /** 837 * tick_broadcast_setup_oneshot - setup the broadcast device 838 */ 839 void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 840 { 841 int cpu = smp_processor_id(); 842 843 /* Set it up only once ! */ 844 if (bc->event_handler != tick_handle_oneshot_broadcast) { 845 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; 846 847 bc->event_handler = tick_handle_oneshot_broadcast; 848 849 /* 850 * We must be careful here. There might be other CPUs 851 * waiting for periodic broadcast. We need to set the 852 * oneshot_mask bits for those and program the 853 * broadcast device to fire. 854 */ 855 cpumask_copy(tmpmask, tick_broadcast_mask); 856 cpumask_clear_cpu(cpu, tmpmask); 857 cpumask_or(tick_broadcast_oneshot_mask, 858 tick_broadcast_oneshot_mask, tmpmask); 859 860 if (was_periodic && !cpumask_empty(tmpmask)) { 861 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 862 tick_broadcast_init_next_event(tmpmask, 863 tick_next_period); 864 tick_broadcast_set_event(bc, cpu, tick_next_period, 1); 865 } else 866 bc->next_event.tv64 = KTIME_MAX; 867 } else { 868 /* 869 * The first cpu which switches to oneshot mode sets 870 * the bit for all other cpus which are in the general 871 * (periodic) broadcast mask. So the bit is set and 872 * would prevent the first broadcast enter after this 873 * to program the bc device. 874 */ 875 tick_broadcast_clear_oneshot(cpu); 876 } 877 } 878 879 /* 880 * Select oneshot operating mode for the broadcast device 881 */ 882 void tick_broadcast_switch_to_oneshot(void) 883 { 884 struct clock_event_device *bc; 885 unsigned long flags; 886 887 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 888 889 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; 890 bc = tick_broadcast_device.evtdev; 891 if (bc) 892 tick_broadcast_setup_oneshot(bc); 893 894 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 895 } 896 897 898 /* 899 * Remove a dead CPU from broadcasting 900 */ 901 void tick_shutdown_broadcast_oneshot(unsigned int *cpup) 902 { 903 unsigned long flags; 904 unsigned int cpu = *cpup; 905 906 raw_spin_lock_irqsave(&tick_broadcast_lock, flags); 907 908 /* 909 * Clear the broadcast masks for the dead cpu, but do not stop 910 * the broadcast device! 911 */ 912 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); 913 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask); 914 cpumask_clear_cpu(cpu, tick_broadcast_force_mask); 915 916 broadcast_move_bc(cpu); 917 918 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); 919 } 920 921 /* 922 * Check, whether the broadcast device is in one shot mode 923 */ 924 int tick_broadcast_oneshot_active(void) 925 { 926 return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT; 927 } 928 929 /* 930 * Check whether the broadcast device supports oneshot. 931 */ 932 bool tick_broadcast_oneshot_available(void) 933 { 934 struct clock_event_device *bc = tick_broadcast_device.evtdev; 935 936 return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; 937 } 938 939 #endif 940 941 void __init tick_broadcast_init(void) 942 { 943 zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT); 944 zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT); 945 zalloc_cpumask_var(&tmpmask, GFP_NOWAIT); 946 #ifdef CONFIG_TICK_ONESHOT 947 zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); 948 zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT); 949 zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT); 950 #endif 951 } 952