1 /* 2 * linux/kernel/time/tick-broadcast.c 3 * 4 * This file contains functions which emulate a local clock-event 5 * device via a broadcast event source. 6 * 7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> 8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner 10 * 11 * This code is licenced under the GPL version 2. For details see 12 * kernel-base/COPYING. 13 */ 14 #include <linux/cpu.h> 15 #include <linux/err.h> 16 #include <linux/hrtimer.h> 17 #include <linux/irq.h> 18 #include <linux/percpu.h> 19 #include <linux/profile.h> 20 #include <linux/sched.h> 21 #include <linux/tick.h> 22 23 #include "tick-internal.h" 24 25 /* 26 * Broadcast support for broken x86 hardware, where the local apic 27 * timer stops in C3 state. 28 */ 29 30 struct tick_device tick_broadcast_device; 31 static cpumask_t tick_broadcast_mask; 32 static DEFINE_SPINLOCK(tick_broadcast_lock); 33 34 #ifdef CONFIG_TICK_ONESHOT 35 static void tick_broadcast_clear_oneshot(int cpu); 36 #else 37 static inline void tick_broadcast_clear_oneshot(int cpu) { } 38 #endif 39 40 /* 41 * Debugging: see timer_list.c 42 */ 43 struct tick_device *tick_get_broadcast_device(void) 44 { 45 return &tick_broadcast_device; 46 } 47 48 cpumask_t *tick_get_broadcast_mask(void) 49 { 50 return &tick_broadcast_mask; 51 } 52 53 /* 54 * Start the device in periodic mode 55 */ 56 static void tick_broadcast_start_periodic(struct clock_event_device *bc) 57 { 58 if (bc) 59 tick_setup_periodic(bc, 1); 60 } 61 62 /* 63 * Check, if the device can be utilized as broadcast device: 64 */ 65 int tick_check_broadcast_device(struct clock_event_device *dev) 66 { 67 if ((tick_broadcast_device.evtdev && 68 tick_broadcast_device.evtdev->rating >= dev->rating) || 69 (dev->features & CLOCK_EVT_FEAT_C3STOP)) 70 return 0; 71 72 clockevents_exchange_device(NULL, dev); 73 tick_broadcast_device.evtdev = dev; 74 if (!cpus_empty(tick_broadcast_mask)) 75 tick_broadcast_start_periodic(dev); 76 return 1; 77 } 78 79 /* 80 * Check, if the device is the broadcast device 81 */ 82 int tick_is_broadcast_device(struct clock_event_device *dev) 83 { 84 return (dev && tick_broadcast_device.evtdev == dev); 85 } 86 87 /* 88 * Check, if the device is disfunctional and a place holder, which 89 * needs to be handled by the broadcast device. 90 */ 91 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) 92 { 93 unsigned long flags; 94 int ret = 0; 95 96 spin_lock_irqsave(&tick_broadcast_lock, flags); 97 98 /* 99 * Devices might be registered with both periodic and oneshot 100 * mode disabled. This signals, that the device needs to be 101 * operated from the broadcast device and is a placeholder for 102 * the cpu local device. 103 */ 104 if (!tick_device_is_functional(dev)) { 105 dev->event_handler = tick_handle_periodic; 106 cpu_set(cpu, tick_broadcast_mask); 107 tick_broadcast_start_periodic(tick_broadcast_device.evtdev); 108 ret = 1; 109 } else { 110 /* 111 * When the new device is not affected by the stop 112 * feature and the cpu is marked in the broadcast mask 113 * then clear the broadcast bit. 114 */ 115 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { 116 int cpu = smp_processor_id(); 117 118 cpu_clear(cpu, tick_broadcast_mask); 119 tick_broadcast_clear_oneshot(cpu); 120 } 121 } 122 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 123 return ret; 124 } 125 126 /* 127 * Broadcast the event to the cpus, which are set in the mask 128 */ 129 int tick_do_broadcast(cpumask_t mask) 130 { 131 int ret = 0, cpu = smp_processor_id(); 132 struct tick_device *td; 133 134 /* 135 * Check, if the current cpu is in the mask 136 */ 137 if (cpu_isset(cpu, mask)) { 138 cpu_clear(cpu, mask); 139 td = &per_cpu(tick_cpu_device, cpu); 140 td->evtdev->event_handler(td->evtdev); 141 ret = 1; 142 } 143 144 if (!cpus_empty(mask)) { 145 /* 146 * It might be necessary to actually check whether the devices 147 * have different broadcast functions. For now, just use the 148 * one of the first device. This works as long as we have this 149 * misfeature only on x86 (lapic) 150 */ 151 cpu = first_cpu(mask); 152 td = &per_cpu(tick_cpu_device, cpu); 153 td->evtdev->broadcast(mask); 154 ret = 1; 155 } 156 return ret; 157 } 158 159 /* 160 * Periodic broadcast: 161 * - invoke the broadcast handlers 162 */ 163 static void tick_do_periodic_broadcast(void) 164 { 165 cpumask_t mask; 166 167 spin_lock(&tick_broadcast_lock); 168 169 cpus_and(mask, cpu_online_map, tick_broadcast_mask); 170 tick_do_broadcast(mask); 171 172 spin_unlock(&tick_broadcast_lock); 173 } 174 175 /* 176 * Event handler for periodic broadcast ticks 177 */ 178 static void tick_handle_periodic_broadcast(struct clock_event_device *dev) 179 { 180 tick_do_periodic_broadcast(); 181 182 /* 183 * The device is in periodic mode. No reprogramming necessary: 184 */ 185 if (dev->mode == CLOCK_EVT_MODE_PERIODIC) 186 return; 187 188 /* 189 * Setup the next period for devices, which do not have 190 * periodic mode: 191 */ 192 for (;;) { 193 ktime_t next = ktime_add(dev->next_event, tick_period); 194 195 if (!clockevents_program_event(dev, next, ktime_get())) 196 return; 197 tick_do_periodic_broadcast(); 198 } 199 } 200 201 /* 202 * Powerstate information: The system enters/leaves a state, where 203 * affected devices might stop 204 */ 205 static void tick_do_broadcast_on_off(void *why) 206 { 207 struct clock_event_device *bc, *dev; 208 struct tick_device *td; 209 unsigned long flags, *reason = why; 210 int cpu; 211 212 spin_lock_irqsave(&tick_broadcast_lock, flags); 213 214 cpu = smp_processor_id(); 215 td = &per_cpu(tick_cpu_device, cpu); 216 dev = td->evtdev; 217 bc = tick_broadcast_device.evtdev; 218 219 /* 220 * Is the device not affected by the powerstate ? 221 */ 222 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) 223 goto out; 224 225 if (!tick_device_is_functional(dev)) 226 goto out; 227 228 switch (*reason) { 229 case CLOCK_EVT_NOTIFY_BROADCAST_ON: 230 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: 231 if (!cpu_isset(cpu, tick_broadcast_mask)) { 232 cpu_set(cpu, tick_broadcast_mask); 233 if (td->mode == TICKDEV_MODE_PERIODIC) 234 clockevents_set_mode(dev, 235 CLOCK_EVT_MODE_SHUTDOWN); 236 } 237 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) 238 dev->features |= CLOCK_EVT_FEAT_DUMMY; 239 break; 240 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 241 if (cpu_isset(cpu, tick_broadcast_mask)) { 242 cpu_clear(cpu, tick_broadcast_mask); 243 if (td->mode == TICKDEV_MODE_PERIODIC) 244 tick_setup_periodic(dev, 0); 245 } 246 break; 247 } 248 249 if (cpus_empty(tick_broadcast_mask)) 250 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 251 else { 252 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 253 tick_broadcast_start_periodic(bc); 254 else 255 tick_broadcast_setup_oneshot(bc); 256 } 257 out: 258 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 259 } 260 261 /* 262 * Powerstate information: The system enters/leaves a state, where 263 * affected devices might stop. 264 */ 265 void tick_broadcast_on_off(unsigned long reason, int *oncpu) 266 { 267 if (!cpu_isset(*oncpu, cpu_online_map)) 268 printk(KERN_ERR "tick-braodcast: ignoring broadcast for " 269 "offline CPU #%d\n", *oncpu); 270 else 271 smp_call_function_single(*oncpu, tick_do_broadcast_on_off, 272 &reason, 1, 1); 273 } 274 275 /* 276 * Set the periodic handler depending on broadcast on/off 277 */ 278 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast) 279 { 280 if (!broadcast) 281 dev->event_handler = tick_handle_periodic; 282 else 283 dev->event_handler = tick_handle_periodic_broadcast; 284 } 285 286 /* 287 * Remove a CPU from broadcasting 288 */ 289 void tick_shutdown_broadcast(unsigned int *cpup) 290 { 291 struct clock_event_device *bc; 292 unsigned long flags; 293 unsigned int cpu = *cpup; 294 295 spin_lock_irqsave(&tick_broadcast_lock, flags); 296 297 bc = tick_broadcast_device.evtdev; 298 cpu_clear(cpu, tick_broadcast_mask); 299 300 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 301 if (bc && cpus_empty(tick_broadcast_mask)) 302 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 303 } 304 305 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 306 } 307 308 void tick_suspend_broadcast(void) 309 { 310 struct clock_event_device *bc; 311 unsigned long flags; 312 313 spin_lock_irqsave(&tick_broadcast_lock, flags); 314 315 bc = tick_broadcast_device.evtdev; 316 if (bc) 317 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); 318 319 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 320 } 321 322 int tick_resume_broadcast(void) 323 { 324 struct clock_event_device *bc; 325 unsigned long flags; 326 int broadcast = 0; 327 328 spin_lock_irqsave(&tick_broadcast_lock, flags); 329 330 bc = tick_broadcast_device.evtdev; 331 332 if (bc) { 333 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME); 334 335 switch (tick_broadcast_device.mode) { 336 case TICKDEV_MODE_PERIODIC: 337 if(!cpus_empty(tick_broadcast_mask)) 338 tick_broadcast_start_periodic(bc); 339 broadcast = cpu_isset(smp_processor_id(), 340 tick_broadcast_mask); 341 break; 342 case TICKDEV_MODE_ONESHOT: 343 broadcast = tick_resume_broadcast_oneshot(bc); 344 break; 345 } 346 } 347 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 348 349 return broadcast; 350 } 351 352 353 #ifdef CONFIG_TICK_ONESHOT 354 355 static cpumask_t tick_broadcast_oneshot_mask; 356 357 /* 358 * Debugging: see timer_list.c 359 */ 360 cpumask_t *tick_get_broadcast_oneshot_mask(void) 361 { 362 return &tick_broadcast_oneshot_mask; 363 } 364 365 static int tick_broadcast_set_event(ktime_t expires, int force) 366 { 367 struct clock_event_device *bc = tick_broadcast_device.evtdev; 368 ktime_t now = ktime_get(); 369 int res; 370 371 for(;;) { 372 res = clockevents_program_event(bc, expires, now); 373 if (!res || !force) 374 return res; 375 now = ktime_get(); 376 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns)); 377 } 378 } 379 380 int tick_resume_broadcast_oneshot(struct clock_event_device *bc) 381 { 382 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 383 return 0; 384 } 385 386 /* 387 * Reprogram the broadcast device: 388 * 389 * Called with tick_broadcast_lock held and interrupts disabled. 390 */ 391 static int tick_broadcast_reprogram(void) 392 { 393 ktime_t expires = { .tv64 = KTIME_MAX }; 394 struct tick_device *td; 395 int cpu; 396 397 /* 398 * Find the event which expires next: 399 */ 400 for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; 401 cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { 402 td = &per_cpu(tick_cpu_device, cpu); 403 if (td->evtdev->next_event.tv64 < expires.tv64) 404 expires = td->evtdev->next_event; 405 } 406 407 if (expires.tv64 == KTIME_MAX) 408 return 0; 409 410 return tick_broadcast_set_event(expires, 0); 411 } 412 413 /* 414 * Handle oneshot mode broadcasting 415 */ 416 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 417 { 418 struct tick_device *td; 419 cpumask_t mask; 420 ktime_t now; 421 int cpu; 422 423 spin_lock(&tick_broadcast_lock); 424 again: 425 dev->next_event.tv64 = KTIME_MAX; 426 mask = CPU_MASK_NONE; 427 now = ktime_get(); 428 /* Find all expired events */ 429 for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; 430 cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { 431 td = &per_cpu(tick_cpu_device, cpu); 432 if (td->evtdev->next_event.tv64 <= now.tv64) 433 cpu_set(cpu, mask); 434 } 435 436 /* 437 * Wakeup the cpus which have an expired event. The broadcast 438 * device is reprogrammed in the return from idle code. 439 */ 440 if (!tick_do_broadcast(mask)) { 441 /* 442 * The global event did not expire any CPU local 443 * events. This happens in dyntick mode, as the 444 * maximum PIT delta is quite small. 445 */ 446 if (tick_broadcast_reprogram()) 447 goto again; 448 } 449 spin_unlock(&tick_broadcast_lock); 450 } 451 452 /* 453 * Powerstate information: The system enters/leaves a state, where 454 * affected devices might stop 455 */ 456 void tick_broadcast_oneshot_control(unsigned long reason) 457 { 458 struct clock_event_device *bc, *dev; 459 struct tick_device *td; 460 unsigned long flags; 461 int cpu; 462 463 spin_lock_irqsave(&tick_broadcast_lock, flags); 464 465 /* 466 * Periodic mode does not care about the enter/exit of power 467 * states 468 */ 469 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) 470 goto out; 471 472 bc = tick_broadcast_device.evtdev; 473 cpu = smp_processor_id(); 474 td = &per_cpu(tick_cpu_device, cpu); 475 dev = td->evtdev; 476 477 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) 478 goto out; 479 480 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 481 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 482 cpu_set(cpu, tick_broadcast_oneshot_mask); 483 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 484 if (dev->next_event.tv64 < bc->next_event.tv64) 485 tick_broadcast_set_event(dev->next_event, 1); 486 } 487 } else { 488 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 489 cpu_clear(cpu, tick_broadcast_oneshot_mask); 490 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 491 if (dev->next_event.tv64 != KTIME_MAX) 492 tick_program_event(dev->next_event, 1); 493 } 494 } 495 496 out: 497 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 498 } 499 500 /* 501 * Reset the one shot broadcast for a cpu 502 * 503 * Called with tick_broadcast_lock held 504 */ 505 static void tick_broadcast_clear_oneshot(int cpu) 506 { 507 cpu_clear(cpu, tick_broadcast_oneshot_mask); 508 } 509 510 /** 511 * tick_broadcast_setup_highres - setup the broadcast device for highres 512 */ 513 void tick_broadcast_setup_oneshot(struct clock_event_device *bc) 514 { 515 bc->event_handler = tick_handle_oneshot_broadcast; 516 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 517 bc->next_event.tv64 = KTIME_MAX; 518 } 519 520 /* 521 * Select oneshot operating mode for the broadcast device 522 */ 523 void tick_broadcast_switch_to_oneshot(void) 524 { 525 struct clock_event_device *bc; 526 unsigned long flags; 527 528 spin_lock_irqsave(&tick_broadcast_lock, flags); 529 530 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT; 531 bc = tick_broadcast_device.evtdev; 532 if (bc) 533 tick_broadcast_setup_oneshot(bc); 534 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 535 } 536 537 538 /* 539 * Remove a dead CPU from broadcasting 540 */ 541 void tick_shutdown_broadcast_oneshot(unsigned int *cpup) 542 { 543 unsigned long flags; 544 unsigned int cpu = *cpup; 545 546 spin_lock_irqsave(&tick_broadcast_lock, flags); 547 548 /* 549 * Clear the broadcast mask flag for the dead cpu, but do not 550 * stop the broadcast device! 551 */ 552 cpu_clear(cpu, tick_broadcast_oneshot_mask); 553 554 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 555 } 556 557 #endif 558