1 /* 2 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar 3 * 4 * This file contains the lowest level x86-specific interrupt 5 * entry, irq-stacks and irq statistics code. All the remaining 6 * irq logic is done by the generic kernel/irq/ code and 7 * by the x86-specific irq controller code. (e.g. i8259.c and 8 * io_apic.c.) 9 */ 10 11 #include <linux/module.h> 12 #include <linux/seq_file.h> 13 #include <linux/interrupt.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/notifier.h> 16 #include <linux/cpu.h> 17 #include <linux/delay.h> 18 19 #include <asm/apic.h> 20 #include <asm/uaccess.h> 21 22 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); 23 EXPORT_PER_CPU_SYMBOL(irq_stat); 24 25 DEFINE_PER_CPU(struct pt_regs *, irq_regs); 26 EXPORT_PER_CPU_SYMBOL(irq_regs); 27 28 /* 29 * 'what should we do if we get a hw irq event on an illegal vector'. 30 * each architecture has to answer this themselves. 31 */ 32 void ack_bad_irq(unsigned int irq) 33 { 34 printk(KERN_ERR "unexpected IRQ trap at vector %02x\n", irq); 35 36 #ifdef CONFIG_X86_LOCAL_APIC 37 /* 38 * Currently unexpected vectors happen only on SMP and APIC. 39 * We _must_ ack these because every local APIC has only N 40 * irq slots per priority level, and a 'hanging, unacked' IRQ 41 * holds up an irq slot - in excessive cases (when multiple 42 * unexpected vectors occur) that might lock up the APIC 43 * completely. 44 * But only ack when the APIC is enabled -AK 45 */ 46 if (cpu_has_apic) 47 ack_APIC_irq(); 48 #endif 49 } 50 51 #ifdef CONFIG_DEBUG_STACKOVERFLOW 52 /* Debugging check for stack overflow: is there less than 1KB free? */ 53 static int check_stack_overflow(void) 54 { 55 long sp; 56 57 __asm__ __volatile__("andl %%esp,%0" : 58 "=r" (sp) : "0" (THREAD_SIZE - 1)); 59 60 return sp < (sizeof(struct thread_info) + STACK_WARN); 61 } 62 63 static void print_stack_overflow(void) 64 { 65 printk(KERN_WARNING "low stack detected by irq handler\n"); 66 dump_stack(); 67 } 68 69 #else 70 static inline int check_stack_overflow(void) { return 0; } 71 static inline void print_stack_overflow(void) { } 72 #endif 73 74 #ifdef CONFIG_4KSTACKS 75 /* 76 * per-CPU IRQ handling contexts (thread information and stack) 77 */ 78 union irq_ctx { 79 struct thread_info tinfo; 80 u32 stack[THREAD_SIZE/sizeof(u32)]; 81 }; 82 83 static union irq_ctx *hardirq_ctx[NR_CPUS] __read_mostly; 84 static union irq_ctx *softirq_ctx[NR_CPUS] __read_mostly; 85 86 static char softirq_stack[NR_CPUS * THREAD_SIZE] __page_aligned_bss; 87 static char hardirq_stack[NR_CPUS * THREAD_SIZE] __page_aligned_bss; 88 89 static void call_on_stack(void *func, void *stack) 90 { 91 asm volatile("xchgl %%ebx,%%esp \n" 92 "call *%%edi \n" 93 "movl %%ebx,%%esp \n" 94 : "=b" (stack) 95 : "0" (stack), 96 "D"(func) 97 : "memory", "cc", "edx", "ecx", "eax"); 98 } 99 100 static inline int 101 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) 102 { 103 union irq_ctx *curctx, *irqctx; 104 u32 *isp, arg1, arg2; 105 106 curctx = (union irq_ctx *) current_thread_info(); 107 irqctx = hardirq_ctx[smp_processor_id()]; 108 109 /* 110 * this is where we switch to the IRQ stack. However, if we are 111 * already using the IRQ stack (because we interrupted a hardirq 112 * handler) we can't do that and just have to keep using the 113 * current stack (which is the irq stack already after all) 114 */ 115 if (unlikely(curctx == irqctx)) 116 return 0; 117 118 /* build the stack frame on the IRQ stack */ 119 isp = (u32 *) ((char*)irqctx + sizeof(*irqctx)); 120 irqctx->tinfo.task = curctx->tinfo.task; 121 irqctx->tinfo.previous_esp = current_stack_pointer; 122 123 /* 124 * Copy the softirq bits in preempt_count so that the 125 * softirq checks work in the hardirq context. 126 */ 127 irqctx->tinfo.preempt_count = 128 (irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) | 129 (curctx->tinfo.preempt_count & SOFTIRQ_MASK); 130 131 if (unlikely(overflow)) 132 call_on_stack(print_stack_overflow, isp); 133 134 asm volatile("xchgl %%ebx,%%esp \n" 135 "call *%%edi \n" 136 "movl %%ebx,%%esp \n" 137 : "=a" (arg1), "=d" (arg2), "=b" (isp) 138 : "0" (irq), "1" (desc), "2" (isp), 139 "D" (desc->handle_irq) 140 : "memory", "cc", "ecx"); 141 return 1; 142 } 143 144 /* 145 * allocate per-cpu stacks for hardirq and for softirq processing 146 */ 147 void __cpuinit irq_ctx_init(int cpu) 148 { 149 union irq_ctx *irqctx; 150 151 if (hardirq_ctx[cpu]) 152 return; 153 154 irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE]; 155 irqctx->tinfo.task = NULL; 156 irqctx->tinfo.exec_domain = NULL; 157 irqctx->tinfo.cpu = cpu; 158 irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; 159 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); 160 161 hardirq_ctx[cpu] = irqctx; 162 163 irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE]; 164 irqctx->tinfo.task = NULL; 165 irqctx->tinfo.exec_domain = NULL; 166 irqctx->tinfo.cpu = cpu; 167 irqctx->tinfo.preempt_count = 0; 168 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); 169 170 softirq_ctx[cpu] = irqctx; 171 172 printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n", 173 cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); 174 } 175 176 void irq_ctx_exit(int cpu) 177 { 178 hardirq_ctx[cpu] = NULL; 179 } 180 181 asmlinkage void do_softirq(void) 182 { 183 unsigned long flags; 184 struct thread_info *curctx; 185 union irq_ctx *irqctx; 186 u32 *isp; 187 188 if (in_interrupt()) 189 return; 190 191 local_irq_save(flags); 192 193 if (local_softirq_pending()) { 194 curctx = current_thread_info(); 195 irqctx = softirq_ctx[smp_processor_id()]; 196 irqctx->tinfo.task = curctx->task; 197 irqctx->tinfo.previous_esp = current_stack_pointer; 198 199 /* build the stack frame on the softirq stack */ 200 isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); 201 202 call_on_stack(__do_softirq, isp); 203 /* 204 * Shouldnt happen, we returned above if in_interrupt(): 205 */ 206 WARN_ON_ONCE(softirq_count()); 207 } 208 209 local_irq_restore(flags); 210 } 211 212 #else 213 static inline int 214 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) { return 0; } 215 #endif 216 217 /* 218 * do_IRQ handles all normal device IRQ's (the special 219 * SMP cross-CPU interrupts have their own specific 220 * handlers). 221 */ 222 unsigned int do_IRQ(struct pt_regs *regs) 223 { 224 struct pt_regs *old_regs; 225 /* high bit used in ret_from_ code */ 226 int overflow, irq = ~regs->orig_ax; 227 struct irq_desc *desc = irq_desc + irq; 228 229 if (unlikely((unsigned)irq >= NR_IRQS)) { 230 printk(KERN_EMERG "%s: cannot handle IRQ %d\n", 231 __func__, irq); 232 BUG(); 233 } 234 235 old_regs = set_irq_regs(regs); 236 irq_enter(); 237 238 overflow = check_stack_overflow(); 239 240 if (!execute_on_irq_stack(overflow, desc, irq)) { 241 if (unlikely(overflow)) 242 print_stack_overflow(); 243 desc->handle_irq(irq, desc); 244 } 245 246 irq_exit(); 247 set_irq_regs(old_regs); 248 return 1; 249 } 250 251 /* 252 * Interrupt statistics: 253 */ 254 255 atomic_t irq_err_count; 256 257 /* 258 * /proc/interrupts printing: 259 */ 260 261 int show_interrupts(struct seq_file *p, void *v) 262 { 263 int i = *(loff_t *) v, j; 264 struct irqaction * action; 265 unsigned long flags; 266 267 if (i == 0) { 268 seq_printf(p, " "); 269 for_each_online_cpu(j) 270 seq_printf(p, "CPU%-8d",j); 271 seq_putc(p, '\n'); 272 } 273 274 if (i < NR_IRQS) { 275 unsigned any_count = 0; 276 277 spin_lock_irqsave(&irq_desc[i].lock, flags); 278 #ifndef CONFIG_SMP 279 any_count = kstat_irqs(i); 280 #else 281 for_each_online_cpu(j) 282 any_count |= kstat_cpu(j).irqs[i]; 283 #endif 284 action = irq_desc[i].action; 285 if (!action && !any_count) 286 goto skip; 287 seq_printf(p, "%3d: ",i); 288 #ifndef CONFIG_SMP 289 seq_printf(p, "%10u ", kstat_irqs(i)); 290 #else 291 for_each_online_cpu(j) 292 seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); 293 #endif 294 seq_printf(p, " %8s", irq_desc[i].chip->name); 295 seq_printf(p, "-%-8s", irq_desc[i].name); 296 297 if (action) { 298 seq_printf(p, " %s", action->name); 299 while ((action = action->next) != NULL) 300 seq_printf(p, ", %s", action->name); 301 } 302 303 seq_putc(p, '\n'); 304 skip: 305 spin_unlock_irqrestore(&irq_desc[i].lock, flags); 306 } else if (i == NR_IRQS) { 307 seq_printf(p, "NMI: "); 308 for_each_online_cpu(j) 309 seq_printf(p, "%10u ", nmi_count(j)); 310 seq_printf(p, " Non-maskable interrupts\n"); 311 #ifdef CONFIG_X86_LOCAL_APIC 312 seq_printf(p, "LOC: "); 313 for_each_online_cpu(j) 314 seq_printf(p, "%10u ", 315 per_cpu(irq_stat,j).apic_timer_irqs); 316 seq_printf(p, " Local timer interrupts\n"); 317 #endif 318 #ifdef CONFIG_SMP 319 seq_printf(p, "RES: "); 320 for_each_online_cpu(j) 321 seq_printf(p, "%10u ", 322 per_cpu(irq_stat,j).irq_resched_count); 323 seq_printf(p, " Rescheduling interrupts\n"); 324 seq_printf(p, "CAL: "); 325 for_each_online_cpu(j) 326 seq_printf(p, "%10u ", 327 per_cpu(irq_stat,j).irq_call_count); 328 seq_printf(p, " function call interrupts\n"); 329 seq_printf(p, "TLB: "); 330 for_each_online_cpu(j) 331 seq_printf(p, "%10u ", 332 per_cpu(irq_stat,j).irq_tlb_count); 333 seq_printf(p, " TLB shootdowns\n"); 334 #endif 335 #ifdef CONFIG_X86_MCE 336 seq_printf(p, "TRM: "); 337 for_each_online_cpu(j) 338 seq_printf(p, "%10u ", 339 per_cpu(irq_stat,j).irq_thermal_count); 340 seq_printf(p, " Thermal event interrupts\n"); 341 #endif 342 #ifdef CONFIG_X86_LOCAL_APIC 343 seq_printf(p, "SPU: "); 344 for_each_online_cpu(j) 345 seq_printf(p, "%10u ", 346 per_cpu(irq_stat,j).irq_spurious_count); 347 seq_printf(p, " Spurious interrupts\n"); 348 #endif 349 seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); 350 #if defined(CONFIG_X86_IO_APIC) 351 seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count)); 352 #endif 353 } 354 return 0; 355 } 356 357 /* 358 * /proc/stat helpers 359 */ 360 u64 arch_irq_stat_cpu(unsigned int cpu) 361 { 362 u64 sum = nmi_count(cpu); 363 364 #ifdef CONFIG_X86_LOCAL_APIC 365 sum += per_cpu(irq_stat, cpu).apic_timer_irqs; 366 #endif 367 #ifdef CONFIG_SMP 368 sum += per_cpu(irq_stat, cpu).irq_resched_count; 369 sum += per_cpu(irq_stat, cpu).irq_call_count; 370 sum += per_cpu(irq_stat, cpu).irq_tlb_count; 371 #endif 372 #ifdef CONFIG_X86_MCE 373 sum += per_cpu(irq_stat, cpu).irq_thermal_count; 374 #endif 375 #ifdef CONFIG_X86_LOCAL_APIC 376 sum += per_cpu(irq_stat, cpu).irq_spurious_count; 377 #endif 378 return sum; 379 } 380 381 u64 arch_irq_stat(void) 382 { 383 u64 sum = atomic_read(&irq_err_count); 384 385 #ifdef CONFIG_X86_IO_APIC 386 sum += atomic_read(&irq_mis_count); 387 #endif 388 return sum; 389 } 390 391 #ifdef CONFIG_HOTPLUG_CPU 392 #include <mach_apic.h> 393 394 void fixup_irqs(cpumask_t map) 395 { 396 unsigned int irq; 397 static int warned; 398 399 for (irq = 0; irq < NR_IRQS; irq++) { 400 cpumask_t mask; 401 if (irq == 2) 402 continue; 403 404 cpus_and(mask, irq_desc[irq].affinity, map); 405 if (any_online_cpu(mask) == NR_CPUS) { 406 printk("Breaking affinity for irq %i\n", irq); 407 mask = map; 408 } 409 if (irq_desc[irq].chip->set_affinity) 410 irq_desc[irq].chip->set_affinity(irq, mask); 411 else if (irq_desc[irq].action && !(warned++)) 412 printk("Cannot set affinity for irq %i\n", irq); 413 } 414 415 #if 0 416 barrier(); 417 /* Ingo Molnar says: "after the IO-APIC masks have been redirected 418 [note the nop - the interrupt-enable boundary on x86 is two 419 instructions from sti] - to flush out pending hardirqs and 420 IPIs. After this point nothing is supposed to reach this CPU." */ 421 __asm__ __volatile__("sti; nop; cli"); 422 barrier(); 423 #else 424 /* That doesn't seem sufficient. Give it 1ms. */ 425 local_irq_enable(); 426 mdelay(1); 427 local_irq_disable(); 428 #endif 429 } 430 #endif 431 432