1 /* 2 * Derived from arch/i386/kernel/irq.c 3 * Copyright (C) 1992 Linus Torvalds 4 * Adapted from arch/i386 by Gary Thomas 5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 6 * Updated and modified by Cort Dougan <cort@fsmlabs.com> 7 * Copyright (C) 1996-2001 Cort Dougan 8 * Adapted for Power Macintosh by Paul Mackerras 9 * Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au) 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU General Public License 13 * as published by the Free Software Foundation; either version 14 * 2 of the License, or (at your option) any later version. 15 * 16 * This file contains the code used by various IRQ handling routines: 17 * asking for different IRQ's should be done through these routines 18 * instead of just grabbing them. Thus setups with different IRQ numbers 19 * shouldn't result in any weird surprises, and installing new handlers 20 * should be easier. 21 * 22 * The MPC8xx has an interrupt mask in the SIU. If a bit is set, the 23 * interrupt is _enabled_. As expected, IRQ0 is bit 0 in the 32-bit 24 * mask register (of which only 16 are defined), hence the weird shifting 25 * and complement of the cached_irq_mask. I want to be able to stuff 26 * this right into the SIU SMASK register. 27 * Many of the prep/chrp functions are conditional compiled on CONFIG_8xx 28 * to reduce code space and undefined function references. 29 */ 30 31 #undef DEBUG 32 33 #include <linux/export.h> 34 #include <linux/threads.h> 35 #include <linux/kernel_stat.h> 36 #include <linux/signal.h> 37 #include <linux/sched.h> 38 #include <linux/ptrace.h> 39 #include <linux/ioport.h> 40 #include <linux/interrupt.h> 41 #include <linux/timex.h> 42 #include <linux/init.h> 43 #include <linux/slab.h> 44 #include <linux/delay.h> 45 #include <linux/irq.h> 46 #include <linux/seq_file.h> 47 #include <linux/cpumask.h> 48 #include <linux/profile.h> 49 #include <linux/bitops.h> 50 #include <linux/list.h> 51 #include <linux/radix-tree.h> 52 #include <linux/mutex.h> 53 #include <linux/bootmem.h> 54 #include <linux/pci.h> 55 #include <linux/debugfs.h> 56 #include <linux/of.h> 57 #include <linux/of_irq.h> 58 59 #include <asm/uaccess.h> 60 #include <asm/io.h> 61 #include <asm/pgtable.h> 62 #include <asm/irq.h> 63 #include <asm/cache.h> 64 #include <asm/prom.h> 65 #include <asm/ptrace.h> 66 #include <asm/machdep.h> 67 #include <asm/udbg.h> 68 #include <asm/smp.h> 69 #include <asm/debug.h> 70 71 #ifdef CONFIG_PPC64 72 #include <asm/paca.h> 73 #include <asm/firmware.h> 74 #include <asm/lv1call.h> 75 #endif 76 #define CREATE_TRACE_POINTS 77 #include <asm/trace.h> 78 79 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); 80 EXPORT_PER_CPU_SYMBOL(irq_stat); 81 82 int __irq_offset_value; 83 84 #ifdef CONFIG_PPC32 85 EXPORT_SYMBOL(__irq_offset_value); 86 atomic_t ppc_n_lost_interrupts; 87 88 #ifdef CONFIG_TAU_INT 89 extern int tau_initialized; 90 extern int tau_interrupts(int); 91 #endif 92 #endif /* CONFIG_PPC32 */ 93 94 #ifdef CONFIG_PPC64 95 96 int distribute_irqs = 1; 97 98 static inline notrace unsigned long get_irq_happened(void) 99 { 100 unsigned long happened; 101 102 __asm__ __volatile__("lbz %0,%1(13)" 103 : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened))); 104 105 return happened; 106 } 107 108 static inline notrace void set_soft_enabled(unsigned long enable) 109 { 110 __asm__ __volatile__("stb %0,%1(13)" 111 : : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled))); 112 } 113 114 static inline notrace int decrementer_check_overflow(void) 115 { 116 u64 now = get_tb_or_rtc(); 117 u64 *next_tb = &__get_cpu_var(decrementers_next_tb); 118 119 return now >= *next_tb; 120 } 121 122 /* This is called whenever we are re-enabling interrupts 123 * and returns either 0 (nothing to do) or 500/900/280/a00/e80 if 124 * there's an EE, DEC or DBELL to generate. 125 * 126 * This is called in two contexts: From arch_local_irq_restore() 127 * before soft-enabling interrupts, and from the exception exit 128 * path when returning from an interrupt from a soft-disabled to 129 * a soft enabled context. In both case we have interrupts hard 130 * disabled. 131 * 132 * We take care of only clearing the bits we handled in the 133 * PACA irq_happened field since we can only re-emit one at a 134 * time and we don't want to "lose" one. 135 */ 136 notrace unsigned int __check_irq_replay(void) 137 { 138 /* 139 * We use local_paca rather than get_paca() to avoid all 140 * the debug_smp_processor_id() business in this low level 141 * function 142 */ 143 unsigned char happened = local_paca->irq_happened; 144 145 /* Clear bit 0 which we wouldn't clear otherwise */ 146 local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; 147 148 /* 149 * Force the delivery of pending soft-disabled interrupts on PS3. 150 * Any HV call will have this side effect. 151 */ 152 if (firmware_has_feature(FW_FEATURE_PS3_LV1)) { 153 u64 tmp, tmp2; 154 lv1_get_version_info(&tmp, &tmp2); 155 } 156 157 /* 158 * We may have missed a decrementer interrupt. We check the 159 * decrementer itself rather than the paca irq_happened field 160 * in case we also had a rollover while hard disabled 161 */ 162 local_paca->irq_happened &= ~PACA_IRQ_DEC; 163 if ((happened & PACA_IRQ_DEC) || decrementer_check_overflow()) 164 return 0x900; 165 166 /* Finally check if an external interrupt happened */ 167 local_paca->irq_happened &= ~PACA_IRQ_EE; 168 if (happened & PACA_IRQ_EE) 169 return 0x500; 170 171 #ifdef CONFIG_PPC_BOOK3E 172 /* Finally check if an EPR external interrupt happened 173 * this bit is typically set if we need to handle another 174 * "edge" interrupt from within the MPIC "EPR" handler 175 */ 176 local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE; 177 if (happened & PACA_IRQ_EE_EDGE) 178 return 0x500; 179 180 local_paca->irq_happened &= ~PACA_IRQ_DBELL; 181 if (happened & PACA_IRQ_DBELL) 182 return 0x280; 183 #else 184 local_paca->irq_happened &= ~PACA_IRQ_DBELL; 185 if (happened & PACA_IRQ_DBELL) { 186 if (cpu_has_feature(CPU_FTR_HVMODE)) 187 return 0xe80; 188 return 0xa00; 189 } 190 #endif /* CONFIG_PPC_BOOK3E */ 191 192 /* There should be nothing left ! */ 193 BUG_ON(local_paca->irq_happened != 0); 194 195 return 0; 196 } 197 198 notrace void arch_local_irq_restore(unsigned long en) 199 { 200 unsigned char irq_happened; 201 unsigned int replay; 202 203 /* Write the new soft-enabled value */ 204 set_soft_enabled(en); 205 if (!en) 206 return; 207 /* 208 * From this point onward, we can take interrupts, preempt, 209 * etc... unless we got hard-disabled. We check if an event 210 * happened. If none happened, we know we can just return. 211 * 212 * We may have preempted before the check below, in which case 213 * we are checking the "new" CPU instead of the old one. This 214 * is only a problem if an event happened on the "old" CPU. 215 * 216 * External interrupt events will have caused interrupts to 217 * be hard-disabled, so there is no problem, we 218 * cannot have preempted. 219 */ 220 irq_happened = get_irq_happened(); 221 if (!irq_happened) 222 return; 223 224 /* 225 * We need to hard disable to get a trusted value from 226 * __check_irq_replay(). We also need to soft-disable 227 * again to avoid warnings in there due to the use of 228 * per-cpu variables. 229 * 230 * We know that if the value in irq_happened is exactly 0x01 231 * then we are already hard disabled (there are other less 232 * common cases that we'll ignore for now), so we skip the 233 * (expensive) mtmsrd. 234 */ 235 if (unlikely(irq_happened != PACA_IRQ_HARD_DIS)) 236 __hard_irq_disable(); 237 #ifdef CONFIG_TRACE_IRQFLAGS 238 else { 239 /* 240 * We should already be hard disabled here. We had bugs 241 * where that wasn't the case so let's dbl check it and 242 * warn if we are wrong. Only do that when IRQ tracing 243 * is enabled as mfmsr() can be costly. 244 */ 245 if (WARN_ON(mfmsr() & MSR_EE)) 246 __hard_irq_disable(); 247 } 248 #endif /* CONFIG_TRACE_IRQFLAG */ 249 250 set_soft_enabled(0); 251 252 /* 253 * Check if anything needs to be re-emitted. We haven't 254 * soft-enabled yet to avoid warnings in decrementer_check_overflow 255 * accessing per-cpu variables 256 */ 257 replay = __check_irq_replay(); 258 259 /* We can soft-enable now */ 260 set_soft_enabled(1); 261 262 /* 263 * And replay if we have to. This will return with interrupts 264 * hard-enabled. 265 */ 266 if (replay) { 267 __replay_interrupt(replay); 268 return; 269 } 270 271 /* Finally, let's ensure we are hard enabled */ 272 __hard_irq_enable(); 273 } 274 EXPORT_SYMBOL(arch_local_irq_restore); 275 276 /* 277 * This is specifically called by assembly code to re-enable interrupts 278 * if they are currently disabled. This is typically called before 279 * schedule() or do_signal() when returning to userspace. We do it 280 * in C to avoid the burden of dealing with lockdep etc... 281 * 282 * NOTE: This is called with interrupts hard disabled but not marked 283 * as such in paca->irq_happened, so we need to resync this. 284 */ 285 void notrace restore_interrupts(void) 286 { 287 if (irqs_disabled()) { 288 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 289 local_irq_enable(); 290 } else 291 __hard_irq_enable(); 292 } 293 294 /* 295 * This is a helper to use when about to go into idle low-power 296 * when the latter has the side effect of re-enabling interrupts 297 * (such as calling H_CEDE under pHyp). 298 * 299 * You call this function with interrupts soft-disabled (this is 300 * already the case when ppc_md.power_save is called). The function 301 * will return whether to enter power save or just return. 302 * 303 * In the former case, it will have notified lockdep of interrupts 304 * being re-enabled and generally sanitized the lazy irq state, 305 * and in the latter case it will leave with interrupts hard 306 * disabled and marked as such, so the local_irq_enable() call 307 * in cpu_idle() will properly re-enable everything. 308 */ 309 bool prep_irq_for_idle(void) 310 { 311 /* 312 * First we need to hard disable to ensure no interrupt 313 * occurs before we effectively enter the low power state 314 */ 315 hard_irq_disable(); 316 317 /* 318 * If anything happened while we were soft-disabled, 319 * we return now and do not enter the low power state. 320 */ 321 if (lazy_irq_pending()) 322 return false; 323 324 /* Tell lockdep we are about to re-enable */ 325 trace_hardirqs_on(); 326 327 /* 328 * Mark interrupts as soft-enabled and clear the 329 * PACA_IRQ_HARD_DIS from the pending mask since we 330 * are about to hard enable as well as a side effect 331 * of entering the low power state. 332 */ 333 local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; 334 local_paca->soft_enabled = 1; 335 336 /* Tell the caller to enter the low power state */ 337 return true; 338 } 339 340 #endif /* CONFIG_PPC64 */ 341 342 int arch_show_interrupts(struct seq_file *p, int prec) 343 { 344 int j; 345 346 #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT) 347 if (tau_initialized) { 348 seq_printf(p, "%*s: ", prec, "TAU"); 349 for_each_online_cpu(j) 350 seq_printf(p, "%10u ", tau_interrupts(j)); 351 seq_puts(p, " PowerPC Thermal Assist (cpu temp)\n"); 352 } 353 #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */ 354 355 seq_printf(p, "%*s: ", prec, "LOC"); 356 for_each_online_cpu(j) 357 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event); 358 seq_printf(p, " Local timer interrupts for timer event device\n"); 359 360 seq_printf(p, "%*s: ", prec, "LOC"); 361 for_each_online_cpu(j) 362 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others); 363 seq_printf(p, " Local timer interrupts for others\n"); 364 365 seq_printf(p, "%*s: ", prec, "SPU"); 366 for_each_online_cpu(j) 367 seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs); 368 seq_printf(p, " Spurious interrupts\n"); 369 370 seq_printf(p, "%*s: ", prec, "PMI"); 371 for_each_online_cpu(j) 372 seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs); 373 seq_printf(p, " Performance monitoring interrupts\n"); 374 375 seq_printf(p, "%*s: ", prec, "MCE"); 376 for_each_online_cpu(j) 377 seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions); 378 seq_printf(p, " Machine check exceptions\n"); 379 380 #ifdef CONFIG_PPC_DOORBELL 381 if (cpu_has_feature(CPU_FTR_DBELL)) { 382 seq_printf(p, "%*s: ", prec, "DBL"); 383 for_each_online_cpu(j) 384 seq_printf(p, "%10u ", per_cpu(irq_stat, j).doorbell_irqs); 385 seq_printf(p, " Doorbell interrupts\n"); 386 } 387 #endif 388 389 return 0; 390 } 391 392 /* 393 * /proc/stat helpers 394 */ 395 u64 arch_irq_stat_cpu(unsigned int cpu) 396 { 397 u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event; 398 399 sum += per_cpu(irq_stat, cpu).pmu_irqs; 400 sum += per_cpu(irq_stat, cpu).mce_exceptions; 401 sum += per_cpu(irq_stat, cpu).spurious_irqs; 402 sum += per_cpu(irq_stat, cpu).timer_irqs_others; 403 #ifdef CONFIG_PPC_DOORBELL 404 sum += per_cpu(irq_stat, cpu).doorbell_irqs; 405 #endif 406 407 return sum; 408 } 409 410 #ifdef CONFIG_HOTPLUG_CPU 411 void migrate_irqs(void) 412 { 413 struct irq_desc *desc; 414 unsigned int irq; 415 static int warned; 416 cpumask_var_t mask; 417 const struct cpumask *map = cpu_online_mask; 418 419 alloc_cpumask_var(&mask, GFP_KERNEL); 420 421 for_each_irq_desc(irq, desc) { 422 struct irq_data *data; 423 struct irq_chip *chip; 424 425 data = irq_desc_get_irq_data(desc); 426 if (irqd_is_per_cpu(data)) 427 continue; 428 429 chip = irq_data_get_irq_chip(data); 430 431 cpumask_and(mask, data->affinity, map); 432 if (cpumask_any(mask) >= nr_cpu_ids) { 433 printk("Breaking affinity for irq %i\n", irq); 434 cpumask_copy(mask, map); 435 } 436 if (chip->irq_set_affinity) 437 chip->irq_set_affinity(data, mask, true); 438 else if (desc->action && !(warned++)) 439 printk("Cannot set affinity for irq %i\n", irq); 440 } 441 442 free_cpumask_var(mask); 443 444 local_irq_enable(); 445 mdelay(1); 446 local_irq_disable(); 447 } 448 #endif 449 450 static inline void check_stack_overflow(void) 451 { 452 #ifdef CONFIG_DEBUG_STACKOVERFLOW 453 long sp; 454 455 sp = __get_SP() & (THREAD_SIZE-1); 456 457 /* check for stack overflow: is there less than 2KB free? */ 458 if (unlikely(sp < (sizeof(struct thread_info) + 2048))) { 459 printk("do_IRQ: stack overflow: %ld\n", 460 sp - sizeof(struct thread_info)); 461 dump_stack(); 462 } 463 #endif 464 } 465 466 void __do_irq(struct pt_regs *regs) 467 { 468 unsigned int irq; 469 470 irq_enter(); 471 472 trace_irq_entry(regs); 473 474 check_stack_overflow(); 475 476 /* 477 * Query the platform PIC for the interrupt & ack it. 478 * 479 * This will typically lower the interrupt line to the CPU 480 */ 481 irq = ppc_md.get_irq(); 482 483 /* We can hard enable interrupts now to allow perf interrupts */ 484 may_hard_irq_enable(); 485 486 /* And finally process it */ 487 if (unlikely(irq == NO_IRQ)) 488 __get_cpu_var(irq_stat).spurious_irqs++; 489 else 490 generic_handle_irq(irq); 491 492 trace_irq_exit(regs); 493 494 irq_exit(); 495 } 496 497 void do_IRQ(struct pt_regs *regs) 498 { 499 struct pt_regs *old_regs = set_irq_regs(regs); 500 struct thread_info *curtp, *irqtp, *sirqtp; 501 502 /* Switch to the irq stack to handle this */ 503 curtp = current_thread_info(); 504 irqtp = hardirq_ctx[raw_smp_processor_id()]; 505 sirqtp = softirq_ctx[raw_smp_processor_id()]; 506 507 /* Already there ? */ 508 if (unlikely(curtp == irqtp || curtp == sirqtp)) { 509 __do_irq(regs); 510 set_irq_regs(old_regs); 511 return; 512 } 513 514 /* Prepare the thread_info in the irq stack */ 515 irqtp->task = curtp->task; 516 irqtp->flags = 0; 517 518 /* Copy the preempt_count so that the [soft]irq checks work. */ 519 irqtp->preempt_count = curtp->preempt_count; 520 521 /* Switch stack and call */ 522 call_do_irq(regs, irqtp); 523 524 /* Restore stack limit */ 525 irqtp->task = NULL; 526 527 /* Copy back updates to the thread_info */ 528 if (irqtp->flags) 529 set_bits(irqtp->flags, &curtp->flags); 530 531 set_irq_regs(old_regs); 532 } 533 534 void __init init_IRQ(void) 535 { 536 if (ppc_md.init_IRQ) 537 ppc_md.init_IRQ(); 538 539 exc_lvl_ctx_init(); 540 541 irq_ctx_init(); 542 } 543 544 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) 545 struct thread_info *critirq_ctx[NR_CPUS] __read_mostly; 546 struct thread_info *dbgirq_ctx[NR_CPUS] __read_mostly; 547 struct thread_info *mcheckirq_ctx[NR_CPUS] __read_mostly; 548 549 void exc_lvl_ctx_init(void) 550 { 551 struct thread_info *tp; 552 int i, cpu_nr; 553 554 for_each_possible_cpu(i) { 555 #ifdef CONFIG_PPC64 556 cpu_nr = i; 557 #else 558 #ifdef CONFIG_SMP 559 cpu_nr = get_hard_smp_processor_id(i); 560 #else 561 cpu_nr = 0; 562 #endif 563 #endif 564 565 memset((void *)critirq_ctx[cpu_nr], 0, THREAD_SIZE); 566 tp = critirq_ctx[cpu_nr]; 567 tp->cpu = cpu_nr; 568 tp->preempt_count = 0; 569 570 #ifdef CONFIG_BOOKE 571 memset((void *)dbgirq_ctx[cpu_nr], 0, THREAD_SIZE); 572 tp = dbgirq_ctx[cpu_nr]; 573 tp->cpu = cpu_nr; 574 tp->preempt_count = 0; 575 576 memset((void *)mcheckirq_ctx[cpu_nr], 0, THREAD_SIZE); 577 tp = mcheckirq_ctx[cpu_nr]; 578 tp->cpu = cpu_nr; 579 tp->preempt_count = HARDIRQ_OFFSET; 580 #endif 581 } 582 } 583 #endif 584 585 struct thread_info *softirq_ctx[NR_CPUS] __read_mostly; 586 struct thread_info *hardirq_ctx[NR_CPUS] __read_mostly; 587 588 void irq_ctx_init(void) 589 { 590 struct thread_info *tp; 591 int i; 592 593 for_each_possible_cpu(i) { 594 memset((void *)softirq_ctx[i], 0, THREAD_SIZE); 595 tp = softirq_ctx[i]; 596 tp->cpu = i; 597 598 memset((void *)hardirq_ctx[i], 0, THREAD_SIZE); 599 tp = hardirq_ctx[i]; 600 tp->cpu = i; 601 } 602 } 603 604 void do_softirq_own_stack(void) 605 { 606 struct thread_info *curtp, *irqtp; 607 608 curtp = current_thread_info(); 609 irqtp = softirq_ctx[smp_processor_id()]; 610 irqtp->task = curtp->task; 611 irqtp->flags = 0; 612 call_do_softirq(irqtp); 613 irqtp->task = NULL; 614 615 /* Set any flag that may have been set on the 616 * alternate stack 617 */ 618 if (irqtp->flags) 619 set_bits(irqtp->flags, &curtp->flags); 620 } 621 622 irq_hw_number_t virq_to_hw(unsigned int virq) 623 { 624 struct irq_data *irq_data = irq_get_irq_data(virq); 625 return WARN_ON(!irq_data) ? 0 : irq_data->hwirq; 626 } 627 EXPORT_SYMBOL_GPL(virq_to_hw); 628 629 #ifdef CONFIG_SMP 630 int irq_choose_cpu(const struct cpumask *mask) 631 { 632 int cpuid; 633 634 if (cpumask_equal(mask, cpu_online_mask)) { 635 static int irq_rover; 636 static DEFINE_RAW_SPINLOCK(irq_rover_lock); 637 unsigned long flags; 638 639 /* Round-robin distribution... */ 640 do_round_robin: 641 raw_spin_lock_irqsave(&irq_rover_lock, flags); 642 643 irq_rover = cpumask_next(irq_rover, cpu_online_mask); 644 if (irq_rover >= nr_cpu_ids) 645 irq_rover = cpumask_first(cpu_online_mask); 646 647 cpuid = irq_rover; 648 649 raw_spin_unlock_irqrestore(&irq_rover_lock, flags); 650 } else { 651 cpuid = cpumask_first_and(mask, cpu_online_mask); 652 if (cpuid >= nr_cpu_ids) 653 goto do_round_robin; 654 } 655 656 return get_hard_smp_processor_id(cpuid); 657 } 658 #else 659 int irq_choose_cpu(const struct cpumask *mask) 660 { 661 return hard_smp_processor_id(); 662 } 663 #endif 664 665 int arch_early_irq_init(void) 666 { 667 return 0; 668 } 669 670 #ifdef CONFIG_PPC64 671 static int __init setup_noirqdistrib(char *str) 672 { 673 distribute_irqs = 0; 674 return 1; 675 } 676 677 __setup("noirqdistrib", setup_noirqdistrib); 678 #endif /* CONFIG_PPC64 */ 679