1 /* 2 * I/O SAPIC support. 3 * 4 * Copyright (C) 1999 Intel Corp. 5 * Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com> 6 * Copyright (C) 2000-2002 J.I. Lee <jung-ik.lee@intel.com> 7 * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co. 8 * David Mosberger-Tang <davidm@hpl.hp.com> 9 * Copyright (C) 1999 VA Linux Systems 10 * Copyright (C) 1999,2000 Walt Drummond <drummond@valinux.com> 11 * 12 * 00/04/19 D. Mosberger Rewritten to mirror more closely the x86 I/O APIC code. 13 * In particular, we now have separate handlers for edge 14 * and level triggered interrupts. 15 * 00/10/27 Asit Mallick, Goutham Rao <goutham.rao@intel.com> IRQ vector allocation 16 * PCI to vector mapping, shared PCI interrupts. 17 * 00/10/27 D. Mosberger Document things a bit more to make them more understandable. 18 * Clean up much of the old IOSAPIC cruft. 19 * 01/07/27 J.I. Lee PCI irq routing, Platform/Legacy interrupts and fixes for 20 * ACPI S5(SoftOff) support. 21 * 02/01/23 J.I. Lee iosapic pgm fixes for PCI irq routing from _PRT 22 * 02/01/07 E. Focht <efocht@ess.nec.de> Redirectable interrupt vectors in 23 * iosapic_set_affinity(), initializations for 24 * /proc/irq/#/smp_affinity 25 * 02/04/02 P. Diefenbaugh Cleaned up ACPI PCI IRQ routing. 26 * 02/04/18 J.I. Lee bug fix in iosapic_init_pci_irq 27 * 02/04/30 J.I. Lee bug fix in find_iosapic to fix ACPI PCI IRQ to IOSAPIC mapping 28 * error 29 * 02/07/29 T. Kochi Allocate interrupt vectors dynamically 30 * 02/08/04 T. Kochi Cleaned up terminology (irq, global system interrupt, vector, etc.) 31 * 02/09/20 D. Mosberger Simplified by taking advantage of ACPI's pci_irq code. 32 * 03/02/19 B. Helgaas Make pcat_compat system-wide, not per-IOSAPIC. 33 * Remove iosapic_address & gsi_base from external interfaces. 34 * Rationalize __init/__devinit attributes. 35 * 04/12/04 Ashok Raj <ashok.raj@intel.com> Intel Corporation 2004 36 * Updated to work with irq migration necessary for CPU Hotplug 37 */ 38 /* 39 * Here is what the interrupt logic between a PCI device and the kernel looks like: 40 * 41 * (1) A PCI device raises one of the four interrupt pins (INTA, INTB, INTC, INTD). The 42 * device is uniquely identified by its bus--, and slot-number (the function 43 * number does not matter here because all functions share the same interrupt 44 * lines). 45 * 46 * (2) The motherboard routes the interrupt line to a pin on a IOSAPIC controller. 47 * Multiple interrupt lines may have to share the same IOSAPIC pin (if they're level 48 * triggered and use the same polarity). Each interrupt line has a unique Global 49 * System Interrupt (GSI) number which can be calculated as the sum of the controller's 50 * base GSI number and the IOSAPIC pin number to which the line connects. 51 * 52 * (3) The IOSAPIC uses an internal routing table entries (RTEs) to map the IOSAPIC pin 53 * into the IA-64 interrupt vector. This interrupt vector is then sent to the CPU. 54 * 55 * (4) The kernel recognizes an interrupt as an IRQ. The IRQ interface is used as 56 * architecture-independent interrupt handling mechanism in Linux. As an 57 * IRQ is a number, we have to have IA-64 interrupt vector number <-> IRQ number 58 * mapping. On smaller systems, we use one-to-one mapping between IA-64 vector and 59 * IRQ. A platform can implement platform_irq_to_vector(irq) and 60 * platform_local_vector_to_irq(vector) APIs to differentiate the mapping. 61 * Please see also include/asm-ia64/hw_irq.h for those APIs. 62 * 63 * To sum up, there are three levels of mappings involved: 64 * 65 * PCI pin -> global system interrupt (GSI) -> IA-64 vector <-> IRQ 66 * 67 * Note: The term "IRQ" is loosely used everywhere in Linux kernel to describe interrupts. 68 * Now we use "IRQ" only for Linux IRQ's. ISA IRQ (isa_irq) is the only exception in this 69 * source code. 70 */ 71 #include <linux/config.h> 72 73 #include <linux/acpi.h> 74 #include <linux/init.h> 75 #include <linux/irq.h> 76 #include <linux/kernel.h> 77 #include <linux/list.h> 78 #include <linux/pci.h> 79 #include <linux/smp.h> 80 #include <linux/smp_lock.h> 81 #include <linux/string.h> 82 #include <linux/bootmem.h> 83 84 #include <asm/delay.h> 85 #include <asm/hw_irq.h> 86 #include <asm/io.h> 87 #include <asm/iosapic.h> 88 #include <asm/machvec.h> 89 #include <asm/processor.h> 90 #include <asm/ptrace.h> 91 #include <asm/system.h> 92 93 94 #undef DEBUG_INTERRUPT_ROUTING 95 96 #ifdef DEBUG_INTERRUPT_ROUTING 97 #define DBG(fmt...) printk(fmt) 98 #else 99 #define DBG(fmt...) 100 #endif 101 102 #define NR_PREALLOCATE_RTE_ENTRIES (PAGE_SIZE / sizeof(struct iosapic_rte_info)) 103 #define RTE_PREALLOCATED (1) 104 105 static DEFINE_SPINLOCK(iosapic_lock); 106 107 /* These tables map IA-64 vectors to the IOSAPIC pin that generates this vector. */ 108 109 struct iosapic_rte_info { 110 struct list_head rte_list; /* node in list of RTEs sharing the same vector */ 111 char __iomem *addr; /* base address of IOSAPIC */ 112 unsigned int gsi_base; /* first GSI assigned to this IOSAPIC */ 113 char rte_index; /* IOSAPIC RTE index */ 114 int refcnt; /* reference counter */ 115 unsigned int flags; /* flags */ 116 } ____cacheline_aligned; 117 118 static struct iosapic_intr_info { 119 struct list_head rtes; /* RTEs using this vector (empty => not an IOSAPIC interrupt) */ 120 int count; /* # of RTEs that shares this vector */ 121 u32 low32; /* current value of low word of Redirection table entry */ 122 unsigned int dest; /* destination CPU physical ID */ 123 unsigned char dmode : 3; /* delivery mode (see iosapic.h) */ 124 unsigned char polarity: 1; /* interrupt polarity (see iosapic.h) */ 125 unsigned char trigger : 1; /* trigger mode (see iosapic.h) */ 126 } iosapic_intr_info[IA64_NUM_VECTORS]; 127 128 static struct iosapic { 129 char __iomem *addr; /* base address of IOSAPIC */ 130 unsigned int gsi_base; /* first GSI assigned to this IOSAPIC */ 131 unsigned short num_rte; /* number of RTE in this IOSAPIC */ 132 int rtes_inuse; /* # of RTEs in use on this IOSAPIC */ 133 #ifdef CONFIG_NUMA 134 unsigned short node; /* numa node association via pxm */ 135 #endif 136 } iosapic_lists[NR_IOSAPICS]; 137 138 static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */ 139 140 static int iosapic_kmalloc_ok; 141 static LIST_HEAD(free_rte_list); 142 143 /* 144 * Find an IOSAPIC associated with a GSI 145 */ 146 static inline int 147 find_iosapic (unsigned int gsi) 148 { 149 int i; 150 151 for (i = 0; i < NR_IOSAPICS; i++) { 152 if ((unsigned) (gsi - iosapic_lists[i].gsi_base) < iosapic_lists[i].num_rte) 153 return i; 154 } 155 156 return -1; 157 } 158 159 static inline int 160 _gsi_to_vector (unsigned int gsi) 161 { 162 struct iosapic_intr_info *info; 163 struct iosapic_rte_info *rte; 164 165 for (info = iosapic_intr_info; info < iosapic_intr_info + IA64_NUM_VECTORS; ++info) 166 list_for_each_entry(rte, &info->rtes, rte_list) 167 if (rte->gsi_base + rte->rte_index == gsi) 168 return info - iosapic_intr_info; 169 return -1; 170 } 171 172 /* 173 * Translate GSI number to the corresponding IA-64 interrupt vector. If no 174 * entry exists, return -1. 175 */ 176 inline int 177 gsi_to_vector (unsigned int gsi) 178 { 179 return _gsi_to_vector(gsi); 180 } 181 182 int 183 gsi_to_irq (unsigned int gsi) 184 { 185 unsigned long flags; 186 int irq; 187 /* 188 * XXX fix me: this assumes an identity mapping vetween IA-64 vector and Linux irq 189 * numbers... 190 */ 191 spin_lock_irqsave(&iosapic_lock, flags); 192 { 193 irq = _gsi_to_vector(gsi); 194 } 195 spin_unlock_irqrestore(&iosapic_lock, flags); 196 197 return irq; 198 } 199 200 static struct iosapic_rte_info *gsi_vector_to_rte(unsigned int gsi, unsigned int vec) 201 { 202 struct iosapic_rte_info *rte; 203 204 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) 205 if (rte->gsi_base + rte->rte_index == gsi) 206 return rte; 207 return NULL; 208 } 209 210 static void 211 set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask) 212 { 213 unsigned long pol, trigger, dmode; 214 u32 low32, high32; 215 char __iomem *addr; 216 int rte_index; 217 char redir; 218 struct iosapic_rte_info *rte; 219 220 DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest); 221 222 rte = gsi_vector_to_rte(gsi, vector); 223 if (!rte) 224 return; /* not an IOSAPIC interrupt */ 225 226 rte_index = rte->rte_index; 227 addr = rte->addr; 228 pol = iosapic_intr_info[vector].polarity; 229 trigger = iosapic_intr_info[vector].trigger; 230 dmode = iosapic_intr_info[vector].dmode; 231 232 redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0; 233 234 #ifdef CONFIG_SMP 235 { 236 unsigned int irq; 237 238 for (irq = 0; irq < NR_IRQS; ++irq) 239 if (irq_to_vector(irq) == vector) { 240 set_irq_affinity_info(irq, (int)(dest & 0xffff), redir); 241 break; 242 } 243 } 244 #endif 245 246 low32 = ((pol << IOSAPIC_POLARITY_SHIFT) | 247 (trigger << IOSAPIC_TRIGGER_SHIFT) | 248 (dmode << IOSAPIC_DELIVERY_SHIFT) | 249 ((mask ? 1 : 0) << IOSAPIC_MASK_SHIFT) | 250 vector); 251 252 /* dest contains both id and eid */ 253 high32 = (dest << IOSAPIC_DEST_SHIFT); 254 255 iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32); 256 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); 257 iosapic_intr_info[vector].low32 = low32; 258 iosapic_intr_info[vector].dest = dest; 259 } 260 261 static void 262 nop (unsigned int vector) 263 { 264 /* do nothing... */ 265 } 266 267 static void 268 mask_irq (unsigned int irq) 269 { 270 unsigned long flags; 271 char __iomem *addr; 272 u32 low32; 273 int rte_index; 274 ia64_vector vec = irq_to_vector(irq); 275 struct iosapic_rte_info *rte; 276 277 if (list_empty(&iosapic_intr_info[vec].rtes)) 278 return; /* not an IOSAPIC interrupt! */ 279 280 spin_lock_irqsave(&iosapic_lock, flags); 281 { 282 /* set only the mask bit */ 283 low32 = iosapic_intr_info[vec].low32 |= IOSAPIC_MASK; 284 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) { 285 addr = rte->addr; 286 rte_index = rte->rte_index; 287 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); 288 } 289 } 290 spin_unlock_irqrestore(&iosapic_lock, flags); 291 } 292 293 static void 294 unmask_irq (unsigned int irq) 295 { 296 unsigned long flags; 297 char __iomem *addr; 298 u32 low32; 299 int rte_index; 300 ia64_vector vec = irq_to_vector(irq); 301 struct iosapic_rte_info *rte; 302 303 if (list_empty(&iosapic_intr_info[vec].rtes)) 304 return; /* not an IOSAPIC interrupt! */ 305 306 spin_lock_irqsave(&iosapic_lock, flags); 307 { 308 low32 = iosapic_intr_info[vec].low32 &= ~IOSAPIC_MASK; 309 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) { 310 addr = rte->addr; 311 rte_index = rte->rte_index; 312 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); 313 } 314 } 315 spin_unlock_irqrestore(&iosapic_lock, flags); 316 } 317 318 319 static void 320 iosapic_set_affinity (unsigned int irq, cpumask_t mask) 321 { 322 #ifdef CONFIG_SMP 323 unsigned long flags; 324 u32 high32, low32; 325 int dest, rte_index; 326 char __iomem *addr; 327 int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0; 328 ia64_vector vec; 329 struct iosapic_rte_info *rte; 330 331 irq &= (~IA64_IRQ_REDIRECTED); 332 vec = irq_to_vector(irq); 333 334 if (cpus_empty(mask)) 335 return; 336 337 dest = cpu_physical_id(first_cpu(mask)); 338 339 if (list_empty(&iosapic_intr_info[vec].rtes)) 340 return; /* not an IOSAPIC interrupt */ 341 342 set_irq_affinity_info(irq, dest, redir); 343 344 /* dest contains both id and eid */ 345 high32 = dest << IOSAPIC_DEST_SHIFT; 346 347 spin_lock_irqsave(&iosapic_lock, flags); 348 { 349 low32 = iosapic_intr_info[vec].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT); 350 351 if (redir) 352 /* change delivery mode to lowest priority */ 353 low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT); 354 else 355 /* change delivery mode to fixed */ 356 low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); 357 358 iosapic_intr_info[vec].low32 = low32; 359 iosapic_intr_info[vec].dest = dest; 360 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) { 361 addr = rte->addr; 362 rte_index = rte->rte_index; 363 iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32); 364 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); 365 } 366 } 367 spin_unlock_irqrestore(&iosapic_lock, flags); 368 #endif 369 } 370 371 /* 372 * Handlers for level-triggered interrupts. 373 */ 374 375 static unsigned int 376 iosapic_startup_level_irq (unsigned int irq) 377 { 378 unmask_irq(irq); 379 return 0; 380 } 381 382 static void 383 iosapic_end_level_irq (unsigned int irq) 384 { 385 ia64_vector vec = irq_to_vector(irq); 386 struct iosapic_rte_info *rte; 387 388 move_irq(irq); 389 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) 390 iosapic_eoi(rte->addr, vec); 391 } 392 393 #define iosapic_shutdown_level_irq mask_irq 394 #define iosapic_enable_level_irq unmask_irq 395 #define iosapic_disable_level_irq mask_irq 396 #define iosapic_ack_level_irq nop 397 398 struct hw_interrupt_type irq_type_iosapic_level = { 399 .typename = "IO-SAPIC-level", 400 .startup = iosapic_startup_level_irq, 401 .shutdown = iosapic_shutdown_level_irq, 402 .enable = iosapic_enable_level_irq, 403 .disable = iosapic_disable_level_irq, 404 .ack = iosapic_ack_level_irq, 405 .end = iosapic_end_level_irq, 406 .set_affinity = iosapic_set_affinity 407 }; 408 409 /* 410 * Handlers for edge-triggered interrupts. 411 */ 412 413 static unsigned int 414 iosapic_startup_edge_irq (unsigned int irq) 415 { 416 unmask_irq(irq); 417 /* 418 * IOSAPIC simply drops interrupts pended while the 419 * corresponding pin was masked, so we can't know if an 420 * interrupt is pending already. Let's hope not... 421 */ 422 return 0; 423 } 424 425 static void 426 iosapic_ack_edge_irq (unsigned int irq) 427 { 428 irq_desc_t *idesc = irq_descp(irq); 429 430 move_irq(irq); 431 /* 432 * Once we have recorded IRQ_PENDING already, we can mask the 433 * interrupt for real. This prevents IRQ storms from unhandled 434 * devices. 435 */ 436 if ((idesc->status & (IRQ_PENDING|IRQ_DISABLED)) == (IRQ_PENDING|IRQ_DISABLED)) 437 mask_irq(irq); 438 } 439 440 #define iosapic_enable_edge_irq unmask_irq 441 #define iosapic_disable_edge_irq nop 442 #define iosapic_end_edge_irq nop 443 444 struct hw_interrupt_type irq_type_iosapic_edge = { 445 .typename = "IO-SAPIC-edge", 446 .startup = iosapic_startup_edge_irq, 447 .shutdown = iosapic_disable_edge_irq, 448 .enable = iosapic_enable_edge_irq, 449 .disable = iosapic_disable_edge_irq, 450 .ack = iosapic_ack_edge_irq, 451 .end = iosapic_end_edge_irq, 452 .set_affinity = iosapic_set_affinity 453 }; 454 455 unsigned int 456 iosapic_version (char __iomem *addr) 457 { 458 /* 459 * IOSAPIC Version Register return 32 bit structure like: 460 * { 461 * unsigned int version : 8; 462 * unsigned int reserved1 : 8; 463 * unsigned int max_redir : 8; 464 * unsigned int reserved2 : 8; 465 * } 466 */ 467 return iosapic_read(addr, IOSAPIC_VERSION); 468 } 469 470 static int iosapic_find_sharable_vector (unsigned long trigger, unsigned long pol) 471 { 472 int i, vector = -1, min_count = -1; 473 struct iosapic_intr_info *info; 474 475 /* 476 * shared vectors for edge-triggered interrupts are not 477 * supported yet 478 */ 479 if (trigger == IOSAPIC_EDGE) 480 return -1; 481 482 for (i = IA64_FIRST_DEVICE_VECTOR; i <= IA64_LAST_DEVICE_VECTOR; i++) { 483 info = &iosapic_intr_info[i]; 484 if (info->trigger == trigger && info->polarity == pol && 485 (info->dmode == IOSAPIC_FIXED || info->dmode == IOSAPIC_LOWEST_PRIORITY)) { 486 if (min_count == -1 || info->count < min_count) { 487 vector = i; 488 min_count = info->count; 489 } 490 } 491 } 492 if (vector < 0) 493 panic("%s: out of interrupt vectors!\n", __FUNCTION__); 494 495 return vector; 496 } 497 498 /* 499 * if the given vector is already owned by other, 500 * assign a new vector for the other and make the vector available 501 */ 502 static void __init 503 iosapic_reassign_vector (int vector) 504 { 505 int new_vector; 506 507 if (!list_empty(&iosapic_intr_info[vector].rtes)) { 508 new_vector = assign_irq_vector(AUTO_ASSIGN); 509 printk(KERN_INFO "Reassigning vector %d to %d\n", vector, new_vector); 510 memcpy(&iosapic_intr_info[new_vector], &iosapic_intr_info[vector], 511 sizeof(struct iosapic_intr_info)); 512 INIT_LIST_HEAD(&iosapic_intr_info[new_vector].rtes); 513 list_move(iosapic_intr_info[vector].rtes.next, &iosapic_intr_info[new_vector].rtes); 514 memset(&iosapic_intr_info[vector], 0, sizeof(struct iosapic_intr_info)); 515 iosapic_intr_info[vector].low32 = IOSAPIC_MASK; 516 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); 517 } 518 } 519 520 static struct iosapic_rte_info *iosapic_alloc_rte (void) 521 { 522 int i; 523 struct iosapic_rte_info *rte; 524 int preallocated = 0; 525 526 if (!iosapic_kmalloc_ok && list_empty(&free_rte_list)) { 527 rte = alloc_bootmem(sizeof(struct iosapic_rte_info) * NR_PREALLOCATE_RTE_ENTRIES); 528 if (!rte) 529 return NULL; 530 for (i = 0; i < NR_PREALLOCATE_RTE_ENTRIES; i++, rte++) 531 list_add(&rte->rte_list, &free_rte_list); 532 } 533 534 if (!list_empty(&free_rte_list)) { 535 rte = list_entry(free_rte_list.next, struct iosapic_rte_info, rte_list); 536 list_del(&rte->rte_list); 537 preallocated++; 538 } else { 539 rte = kmalloc(sizeof(struct iosapic_rte_info), GFP_ATOMIC); 540 if (!rte) 541 return NULL; 542 } 543 544 memset(rte, 0, sizeof(struct iosapic_rte_info)); 545 if (preallocated) 546 rte->flags |= RTE_PREALLOCATED; 547 548 return rte; 549 } 550 551 static void iosapic_free_rte (struct iosapic_rte_info *rte) 552 { 553 if (rte->flags & RTE_PREALLOCATED) 554 list_add_tail(&rte->rte_list, &free_rte_list); 555 else 556 kfree(rte); 557 } 558 559 static inline int vector_is_shared (int vector) 560 { 561 return (iosapic_intr_info[vector].count > 1); 562 } 563 564 static void 565 register_intr (unsigned int gsi, int vector, unsigned char delivery, 566 unsigned long polarity, unsigned long trigger) 567 { 568 irq_desc_t *idesc; 569 struct hw_interrupt_type *irq_type; 570 int rte_index; 571 int index; 572 unsigned long gsi_base; 573 void __iomem *iosapic_address; 574 struct iosapic_rte_info *rte; 575 576 index = find_iosapic(gsi); 577 if (index < 0) { 578 printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n", __FUNCTION__, gsi); 579 return; 580 } 581 582 iosapic_address = iosapic_lists[index].addr; 583 gsi_base = iosapic_lists[index].gsi_base; 584 585 rte = gsi_vector_to_rte(gsi, vector); 586 if (!rte) { 587 rte = iosapic_alloc_rte(); 588 if (!rte) { 589 printk(KERN_WARNING "%s: cannot allocate memory\n", __FUNCTION__); 590 return; 591 } 592 593 rte_index = gsi - gsi_base; 594 rte->rte_index = rte_index; 595 rte->addr = iosapic_address; 596 rte->gsi_base = gsi_base; 597 rte->refcnt++; 598 list_add_tail(&rte->rte_list, &iosapic_intr_info[vector].rtes); 599 iosapic_intr_info[vector].count++; 600 iosapic_lists[index].rtes_inuse++; 601 } 602 else if (vector_is_shared(vector)) { 603 struct iosapic_intr_info *info = &iosapic_intr_info[vector]; 604 if (info->trigger != trigger || info->polarity != polarity) { 605 printk (KERN_WARNING "%s: cannot override the interrupt\n", __FUNCTION__); 606 return; 607 } 608 } 609 610 iosapic_intr_info[vector].polarity = polarity; 611 iosapic_intr_info[vector].dmode = delivery; 612 iosapic_intr_info[vector].trigger = trigger; 613 614 if (trigger == IOSAPIC_EDGE) 615 irq_type = &irq_type_iosapic_edge; 616 else 617 irq_type = &irq_type_iosapic_level; 618 619 idesc = irq_descp(vector); 620 if (idesc->handler != irq_type) { 621 if (idesc->handler != &no_irq_type) 622 printk(KERN_WARNING "%s: changing vector %d from %s to %s\n", 623 __FUNCTION__, vector, idesc->handler->typename, irq_type->typename); 624 idesc->handler = irq_type; 625 } 626 } 627 628 static unsigned int 629 get_target_cpu (unsigned int gsi, int vector) 630 { 631 #ifdef CONFIG_SMP 632 static int cpu = -1; 633 634 /* 635 * In case of vector shared by multiple RTEs, all RTEs that 636 * share the vector need to use the same destination CPU. 637 */ 638 if (!list_empty(&iosapic_intr_info[vector].rtes)) 639 return iosapic_intr_info[vector].dest; 640 641 /* 642 * If the platform supports redirection via XTP, let it 643 * distribute interrupts. 644 */ 645 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 646 return cpu_physical_id(smp_processor_id()); 647 648 /* 649 * Some interrupts (ACPI SCI, for instance) are registered 650 * before the BSP is marked as online. 651 */ 652 if (!cpu_online(smp_processor_id())) 653 return cpu_physical_id(smp_processor_id()); 654 655 #ifdef CONFIG_NUMA 656 { 657 int num_cpus, cpu_index, iosapic_index, numa_cpu, i = 0; 658 cpumask_t cpu_mask; 659 660 iosapic_index = find_iosapic(gsi); 661 if (iosapic_index < 0 || 662 iosapic_lists[iosapic_index].node == MAX_NUMNODES) 663 goto skip_numa_setup; 664 665 cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node); 666 667 for_each_cpu_mask(numa_cpu, cpu_mask) { 668 if (!cpu_online(numa_cpu)) 669 cpu_clear(numa_cpu, cpu_mask); 670 } 671 672 num_cpus = cpus_weight(cpu_mask); 673 674 if (!num_cpus) 675 goto skip_numa_setup; 676 677 /* Use vector assigment to distribute across cpus in node */ 678 cpu_index = vector % num_cpus; 679 680 for (numa_cpu = first_cpu(cpu_mask) ; i < cpu_index ; i++) 681 numa_cpu = next_cpu(numa_cpu, cpu_mask); 682 683 if (numa_cpu != NR_CPUS) 684 return cpu_physical_id(numa_cpu); 685 } 686 skip_numa_setup: 687 #endif 688 /* 689 * Otherwise, round-robin interrupt vectors across all the 690 * processors. (It'd be nice if we could be smarter in the 691 * case of NUMA.) 692 */ 693 do { 694 if (++cpu >= NR_CPUS) 695 cpu = 0; 696 } while (!cpu_online(cpu)); 697 698 return cpu_physical_id(cpu); 699 #else 700 return cpu_physical_id(smp_processor_id()); 701 #endif 702 } 703 704 /* 705 * ACPI can describe IOSAPIC interrupts via static tables and namespace 706 * methods. This provides an interface to register those interrupts and 707 * program the IOSAPIC RTE. 708 */ 709 int 710 iosapic_register_intr (unsigned int gsi, 711 unsigned long polarity, unsigned long trigger) 712 { 713 int vector, mask = 1; 714 unsigned int dest; 715 unsigned long flags; 716 struct iosapic_rte_info *rte; 717 u32 low32; 718 again: 719 /* 720 * If this GSI has already been registered (i.e., it's a 721 * shared interrupt, or we lost a race to register it), 722 * don't touch the RTE. 723 */ 724 spin_lock_irqsave(&iosapic_lock, flags); 725 { 726 vector = gsi_to_vector(gsi); 727 if (vector > 0) { 728 rte = gsi_vector_to_rte(gsi, vector); 729 rte->refcnt++; 730 spin_unlock_irqrestore(&iosapic_lock, flags); 731 return vector; 732 } 733 } 734 spin_unlock_irqrestore(&iosapic_lock, flags); 735 736 /* If vector is running out, we try to find a sharable vector */ 737 vector = assign_irq_vector_nopanic(AUTO_ASSIGN); 738 if (vector < 0) 739 vector = iosapic_find_sharable_vector(trigger, polarity); 740 741 spin_lock_irqsave(&irq_descp(vector)->lock, flags); 742 spin_lock(&iosapic_lock); 743 { 744 if (gsi_to_vector(gsi) > 0) { 745 if (list_empty(&iosapic_intr_info[vector].rtes)) 746 free_irq_vector(vector); 747 spin_unlock(&iosapic_lock); 748 spin_unlock_irqrestore(&irq_descp(vector)->lock, flags); 749 goto again; 750 } 751 752 dest = get_target_cpu(gsi, vector); 753 register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, 754 polarity, trigger); 755 756 /* 757 * If the vector is shared and already unmasked for 758 * other interrupt sources, don't mask it. 759 */ 760 low32 = iosapic_intr_info[vector].low32; 761 if (vector_is_shared(vector) && !(low32 & IOSAPIC_MASK)) 762 mask = 0; 763 set_rte(gsi, vector, dest, mask); 764 } 765 spin_unlock(&iosapic_lock); 766 spin_unlock_irqrestore(&irq_descp(vector)->lock, flags); 767 768 printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n", 769 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 770 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 771 cpu_logical_id(dest), dest, vector); 772 773 return vector; 774 } 775 776 #ifdef CONFIG_ACPI_DEALLOCATE_IRQ 777 void 778 iosapic_unregister_intr (unsigned int gsi) 779 { 780 unsigned long flags; 781 int irq, vector, index; 782 irq_desc_t *idesc; 783 u32 low32; 784 unsigned long trigger, polarity; 785 unsigned int dest; 786 struct iosapic_rte_info *rte; 787 788 /* 789 * If the irq associated with the gsi is not found, 790 * iosapic_unregister_intr() is unbalanced. We need to check 791 * this again after getting locks. 792 */ 793 irq = gsi_to_irq(gsi); 794 if (irq < 0) { 795 printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n", gsi); 796 WARN_ON(1); 797 return; 798 } 799 vector = irq_to_vector(irq); 800 801 idesc = irq_descp(irq); 802 spin_lock_irqsave(&idesc->lock, flags); 803 spin_lock(&iosapic_lock); 804 { 805 if ((rte = gsi_vector_to_rte(gsi, vector)) == NULL) { 806 printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n", gsi); 807 WARN_ON(1); 808 goto out; 809 } 810 811 if (--rte->refcnt > 0) 812 goto out; 813 814 /* Mask the interrupt */ 815 low32 = iosapic_intr_info[vector].low32 | IOSAPIC_MASK; 816 iosapic_write(rte->addr, IOSAPIC_RTE_LOW(rte->rte_index), low32); 817 818 /* Remove the rte entry from the list */ 819 list_del(&rte->rte_list); 820 iosapic_intr_info[vector].count--; 821 iosapic_free_rte(rte); 822 index = find_iosapic(gsi); 823 iosapic_lists[index].rtes_inuse--; 824 WARN_ON(iosapic_lists[index].rtes_inuse < 0); 825 826 trigger = iosapic_intr_info[vector].trigger; 827 polarity = iosapic_intr_info[vector].polarity; 828 dest = iosapic_intr_info[vector].dest; 829 printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n", 830 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 831 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 832 cpu_logical_id(dest), dest, vector); 833 834 if (list_empty(&iosapic_intr_info[vector].rtes)) { 835 /* Sanity check */ 836 BUG_ON(iosapic_intr_info[vector].count); 837 838 /* Clear the interrupt controller descriptor */ 839 idesc->handler = &no_irq_type; 840 841 /* Clear the interrupt information */ 842 memset(&iosapic_intr_info[vector], 0, sizeof(struct iosapic_intr_info)); 843 iosapic_intr_info[vector].low32 |= IOSAPIC_MASK; 844 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); 845 846 if (idesc->action) { 847 printk(KERN_ERR "interrupt handlers still exist on IRQ %u\n", irq); 848 WARN_ON(1); 849 } 850 851 /* Free the interrupt vector */ 852 free_irq_vector(vector); 853 } 854 } 855 out: 856 spin_unlock(&iosapic_lock); 857 spin_unlock_irqrestore(&idesc->lock, flags); 858 } 859 #endif /* CONFIG_ACPI_DEALLOCATE_IRQ */ 860 861 /* 862 * ACPI calls this when it finds an entry for a platform interrupt. 863 * Note that the irq_base and IOSAPIC address must be set in iosapic_init(). 864 */ 865 int __init 866 iosapic_register_platform_intr (u32 int_type, unsigned int gsi, 867 int iosapic_vector, u16 eid, u16 id, 868 unsigned long polarity, unsigned long trigger) 869 { 870 static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"}; 871 unsigned char delivery; 872 int vector, mask = 0; 873 unsigned int dest = ((id << 8) | eid) & 0xffff; 874 875 switch (int_type) { 876 case ACPI_INTERRUPT_PMI: 877 vector = iosapic_vector; 878 /* 879 * since PMI vector is alloc'd by FW(ACPI) not by kernel, 880 * we need to make sure the vector is available 881 */ 882 iosapic_reassign_vector(vector); 883 delivery = IOSAPIC_PMI; 884 break; 885 case ACPI_INTERRUPT_INIT: 886 vector = assign_irq_vector(AUTO_ASSIGN); 887 delivery = IOSAPIC_INIT; 888 break; 889 case ACPI_INTERRUPT_CPEI: 890 vector = IA64_CPE_VECTOR; 891 delivery = IOSAPIC_LOWEST_PRIORITY; 892 mask = 1; 893 break; 894 default: 895 printk(KERN_ERR "iosapic_register_platform_irq(): invalid int type 0x%x\n", int_type); 896 return -1; 897 } 898 899 register_intr(gsi, vector, delivery, polarity, trigger); 900 901 printk(KERN_INFO "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n", 902 int_type < ARRAY_SIZE(name) ? name[int_type] : "unknown", 903 int_type, gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 904 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 905 cpu_logical_id(dest), dest, vector); 906 907 set_rte(gsi, vector, dest, mask); 908 return vector; 909 } 910 911 912 /* 913 * ACPI calls this when it finds an entry for a legacy ISA IRQ override. 914 * Note that the gsi_base and IOSAPIC address must be set in iosapic_init(). 915 */ 916 void __init 917 iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi, 918 unsigned long polarity, 919 unsigned long trigger) 920 { 921 int vector; 922 unsigned int dest = cpu_physical_id(smp_processor_id()); 923 924 vector = isa_irq_to_vector(isa_irq); 925 926 register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); 927 928 DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", 929 isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level", 930 polarity == IOSAPIC_POL_HIGH ? "high" : "low", 931 cpu_logical_id(dest), dest, vector); 932 933 set_rte(gsi, vector, dest, 1); 934 } 935 936 void __init 937 iosapic_system_init (int system_pcat_compat) 938 { 939 int vector; 940 941 for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) { 942 iosapic_intr_info[vector].low32 = IOSAPIC_MASK; 943 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); /* mark as unused */ 944 } 945 946 pcat_compat = system_pcat_compat; 947 if (pcat_compat) { 948 /* 949 * Disable the compatibility mode interrupts (8259 style), needs IN/OUT support 950 * enabled. 951 */ 952 printk(KERN_INFO "%s: Disabling PC-AT compatible 8259 interrupts\n", __FUNCTION__); 953 outb(0xff, 0xA1); 954 outb(0xff, 0x21); 955 } 956 } 957 958 static inline int 959 iosapic_alloc (void) 960 { 961 int index; 962 963 for (index = 0; index < NR_IOSAPICS; index++) 964 if (!iosapic_lists[index].addr) 965 return index; 966 967 printk(KERN_WARNING "%s: failed to allocate iosapic\n", __FUNCTION__); 968 return -1; 969 } 970 971 static inline void 972 iosapic_free (int index) 973 { 974 memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0])); 975 } 976 977 static inline int 978 iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver) 979 { 980 int index; 981 unsigned int gsi_end, base, end; 982 983 /* check gsi range */ 984 gsi_end = gsi_base + ((ver >> 16) & 0xff); 985 for (index = 0; index < NR_IOSAPICS; index++) { 986 if (!iosapic_lists[index].addr) 987 continue; 988 989 base = iosapic_lists[index].gsi_base; 990 end = base + iosapic_lists[index].num_rte - 1; 991 992 if (gsi_base < base && gsi_end < base) 993 continue;/* OK */ 994 995 if (gsi_base > end && gsi_end > end) 996 continue; /* OK */ 997 998 return -EBUSY; 999 } 1000 return 0; 1001 } 1002 1003 int __devinit 1004 iosapic_init (unsigned long phys_addr, unsigned int gsi_base) 1005 { 1006 int num_rte, err, index; 1007 unsigned int isa_irq, ver; 1008 char __iomem *addr; 1009 unsigned long flags; 1010 1011 spin_lock_irqsave(&iosapic_lock, flags); 1012 { 1013 addr = ioremap(phys_addr, 0); 1014 ver = iosapic_version(addr); 1015 1016 if ((err = iosapic_check_gsi_range(gsi_base, ver))) { 1017 iounmap(addr); 1018 spin_unlock_irqrestore(&iosapic_lock, flags); 1019 return err; 1020 } 1021 1022 /* 1023 * The MAX_REDIR register holds the highest input pin 1024 * number (starting from 0). 1025 * We add 1 so that we can use it for number of pins (= RTEs) 1026 */ 1027 num_rte = ((ver >> 16) & 0xff) + 1; 1028 1029 index = iosapic_alloc(); 1030 iosapic_lists[index].addr = addr; 1031 iosapic_lists[index].gsi_base = gsi_base; 1032 iosapic_lists[index].num_rte = num_rte; 1033 #ifdef CONFIG_NUMA 1034 iosapic_lists[index].node = MAX_NUMNODES; 1035 #endif 1036 } 1037 spin_unlock_irqrestore(&iosapic_lock, flags); 1038 1039 if ((gsi_base == 0) && pcat_compat) { 1040 /* 1041 * Map the legacy ISA devices into the IOSAPIC data. Some of these may 1042 * get reprogrammed later on with data from the ACPI Interrupt Source 1043 * Override table. 1044 */ 1045 for (isa_irq = 0; isa_irq < 16; ++isa_irq) 1046 iosapic_override_isa_irq(isa_irq, isa_irq, IOSAPIC_POL_HIGH, IOSAPIC_EDGE); 1047 } 1048 return 0; 1049 } 1050 1051 #ifdef CONFIG_HOTPLUG 1052 int 1053 iosapic_remove (unsigned int gsi_base) 1054 { 1055 int index, err = 0; 1056 unsigned long flags; 1057 1058 spin_lock_irqsave(&iosapic_lock, flags); 1059 { 1060 index = find_iosapic(gsi_base); 1061 if (index < 0) { 1062 printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n", 1063 __FUNCTION__, gsi_base); 1064 goto out; 1065 } 1066 1067 if (iosapic_lists[index].rtes_inuse) { 1068 err = -EBUSY; 1069 printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n", 1070 __FUNCTION__, gsi_base); 1071 goto out; 1072 } 1073 1074 iounmap(iosapic_lists[index].addr); 1075 iosapic_free(index); 1076 } 1077 out: 1078 spin_unlock_irqrestore(&iosapic_lock, flags); 1079 return err; 1080 } 1081 #endif /* CONFIG_HOTPLUG */ 1082 1083 #ifdef CONFIG_NUMA 1084 void __devinit 1085 map_iosapic_to_node(unsigned int gsi_base, int node) 1086 { 1087 int index; 1088 1089 index = find_iosapic(gsi_base); 1090 if (index < 0) { 1091 printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n", 1092 __FUNCTION__, gsi_base); 1093 return; 1094 } 1095 iosapic_lists[index].node = node; 1096 return; 1097 } 1098 #endif 1099 1100 static int __init iosapic_enable_kmalloc (void) 1101 { 1102 iosapic_kmalloc_ok = 1; 1103 return 0; 1104 } 1105 core_initcall (iosapic_enable_kmalloc); 1106