1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel IO-APIC support for multi-Pentium hosts. 4 * 5 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo 6 * 7 * Many thanks to Stig Venaas for trying out countless experimental 8 * patches and reporting/debugging problems patiently! 9 * 10 * (c) 1999, Multiple IO-APIC support, developed by 11 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and 12 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>, 13 * further tested and cleaned up by Zach Brown <zab@redhat.com> 14 * and Ingo Molnar <mingo@redhat.com> 15 * 16 * Fixes 17 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 18 * thanks to Eric Gilmore 19 * and Rolf G. Tews 20 * for testing these extensively 21 * Paul Diefenbaugh : Added full ACPI support 22 * 23 * Historical information which is worth to be preserved: 24 * 25 * - SiS APIC rmw bug: 26 * 27 * We used to have a workaround for a bug in SiS chips which 28 * required to rewrite the index register for a read-modify-write 29 * operation as the chip lost the index information which was 30 * setup for the read already. We cache the data now, so that 31 * workaround has been removed. 32 */ 33 34 #include <linux/mm.h> 35 #include <linux/interrupt.h> 36 #include <linux/irq.h> 37 #include <linux/init.h> 38 #include <linux/delay.h> 39 #include <linux/sched.h> 40 #include <linux/pci.h> 41 #include <linux/mc146818rtc.h> 42 #include <linux/compiler.h> 43 #include <linux/acpi.h> 44 #include <linux/export.h> 45 #include <linux/syscore_ops.h> 46 #include <linux/freezer.h> 47 #include <linux/kthread.h> 48 #include <linux/jiffies.h> /* time_after() */ 49 #include <linux/slab.h> 50 #include <linux/memblock.h> 51 52 #include <asm/irqdomain.h> 53 #include <asm/io.h> 54 #include <asm/smp.h> 55 #include <asm/cpu.h> 56 #include <asm/desc.h> 57 #include <asm/proto.h> 58 #include <asm/acpi.h> 59 #include <asm/dma.h> 60 #include <asm/timer.h> 61 #include <asm/time.h> 62 #include <asm/i8259.h> 63 #include <asm/setup.h> 64 #include <asm/irq_remapping.h> 65 #include <asm/hw_irq.h> 66 67 #include <asm/apic.h> 68 69 #define for_each_ioapic(idx) \ 70 for ((idx) = 0; (idx) < nr_ioapics; (idx)++) 71 #define for_each_ioapic_reverse(idx) \ 72 for ((idx) = nr_ioapics - 1; (idx) >= 0; (idx)--) 73 #define for_each_pin(idx, pin) \ 74 for ((pin) = 0; (pin) < ioapics[(idx)].nr_registers; (pin)++) 75 #define for_each_ioapic_pin(idx, pin) \ 76 for_each_ioapic((idx)) \ 77 for_each_pin((idx), (pin)) 78 #define for_each_irq_pin(entry, head) \ 79 list_for_each_entry(entry, &head, list) 80 81 static DEFINE_RAW_SPINLOCK(ioapic_lock); 82 static DEFINE_MUTEX(ioapic_mutex); 83 static unsigned int ioapic_dynirq_base; 84 static int ioapic_initialized; 85 86 struct irq_pin_list { 87 struct list_head list; 88 int apic, pin; 89 }; 90 91 struct mp_chip_data { 92 struct list_head irq_2_pin; 93 struct IO_APIC_route_entry entry; 94 int trigger; 95 int polarity; 96 u32 count; 97 bool isa_irq; 98 }; 99 100 struct mp_ioapic_gsi { 101 u32 gsi_base; 102 u32 gsi_end; 103 }; 104 105 static struct ioapic { 106 /* 107 * # of IRQ routing registers 108 */ 109 int nr_registers; 110 /* 111 * Saved state during suspend/resume, or while enabling intr-remap. 112 */ 113 struct IO_APIC_route_entry *saved_registers; 114 /* I/O APIC config */ 115 struct mpc_ioapic mp_config; 116 /* IO APIC gsi routing info */ 117 struct mp_ioapic_gsi gsi_config; 118 struct ioapic_domain_cfg irqdomain_cfg; 119 struct irq_domain *irqdomain; 120 struct resource *iomem_res; 121 } ioapics[MAX_IO_APICS]; 122 123 #define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver 124 125 int mpc_ioapic_id(int ioapic_idx) 126 { 127 return ioapics[ioapic_idx].mp_config.apicid; 128 } 129 130 unsigned int mpc_ioapic_addr(int ioapic_idx) 131 { 132 return ioapics[ioapic_idx].mp_config.apicaddr; 133 } 134 135 static inline struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx) 136 { 137 return &ioapics[ioapic_idx].gsi_config; 138 } 139 140 static inline int mp_ioapic_pin_count(int ioapic) 141 { 142 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic); 143 144 return gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1; 145 } 146 147 static inline u32 mp_pin_to_gsi(int ioapic, int pin) 148 { 149 return mp_ioapic_gsi_routing(ioapic)->gsi_base + pin; 150 } 151 152 static inline bool mp_is_legacy_irq(int irq) 153 { 154 return irq >= 0 && irq < nr_legacy_irqs(); 155 } 156 157 static inline struct irq_domain *mp_ioapic_irqdomain(int ioapic) 158 { 159 return ioapics[ioapic].irqdomain; 160 } 161 162 int nr_ioapics; 163 164 /* The one past the highest gsi number used */ 165 u32 gsi_top; 166 167 /* MP IRQ source entries */ 168 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES]; 169 170 /* # of MP IRQ source entries */ 171 int mp_irq_entries; 172 173 #ifdef CONFIG_EISA 174 int mp_bus_id_to_type[MAX_MP_BUSSES]; 175 #endif 176 177 DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); 178 179 int skip_ioapic_setup; 180 181 /** 182 * disable_ioapic_support() - disables ioapic support at runtime 183 */ 184 void disable_ioapic_support(void) 185 { 186 #ifdef CONFIG_PCI 187 noioapicquirk = 1; 188 noioapicreroute = -1; 189 #endif 190 skip_ioapic_setup = 1; 191 } 192 193 static int __init parse_noapic(char *str) 194 { 195 /* disable IO-APIC */ 196 disable_ioapic_support(); 197 return 0; 198 } 199 early_param("noapic", parse_noapic); 200 201 /* Will be called in mpparse/acpi/sfi codes for saving IRQ info */ 202 void mp_save_irq(struct mpc_intsrc *m) 203 { 204 int i; 205 206 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x," 207 " IRQ %02x, APIC ID %x, APIC INT %02x\n", 208 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus, 209 m->srcbusirq, m->dstapic, m->dstirq); 210 211 for (i = 0; i < mp_irq_entries; i++) { 212 if (!memcmp(&mp_irqs[i], m, sizeof(*m))) 213 return; 214 } 215 216 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m)); 217 if (++mp_irq_entries == MAX_IRQ_SOURCES) 218 panic("Max # of irq sources exceeded!!\n"); 219 } 220 221 static void alloc_ioapic_saved_registers(int idx) 222 { 223 size_t size; 224 225 if (ioapics[idx].saved_registers) 226 return; 227 228 size = sizeof(struct IO_APIC_route_entry) * ioapics[idx].nr_registers; 229 ioapics[idx].saved_registers = kzalloc(size, GFP_KERNEL); 230 if (!ioapics[idx].saved_registers) 231 pr_err("IOAPIC %d: suspend/resume impossible!\n", idx); 232 } 233 234 static void free_ioapic_saved_registers(int idx) 235 { 236 kfree(ioapics[idx].saved_registers); 237 ioapics[idx].saved_registers = NULL; 238 } 239 240 int __init arch_early_ioapic_init(void) 241 { 242 int i; 243 244 if (!nr_legacy_irqs()) 245 io_apic_irqs = ~0UL; 246 247 for_each_ioapic(i) 248 alloc_ioapic_saved_registers(i); 249 250 return 0; 251 } 252 253 struct io_apic { 254 unsigned int index; 255 unsigned int unused[3]; 256 unsigned int data; 257 unsigned int unused2[11]; 258 unsigned int eoi; 259 }; 260 261 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) 262 { 263 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx) 264 + (mpc_ioapic_addr(idx) & ~PAGE_MASK); 265 } 266 267 static inline void io_apic_eoi(unsigned int apic, unsigned int vector) 268 { 269 struct io_apic __iomem *io_apic = io_apic_base(apic); 270 writel(vector, &io_apic->eoi); 271 } 272 273 unsigned int native_io_apic_read(unsigned int apic, unsigned int reg) 274 { 275 struct io_apic __iomem *io_apic = io_apic_base(apic); 276 writel(reg, &io_apic->index); 277 return readl(&io_apic->data); 278 } 279 280 static void io_apic_write(unsigned int apic, unsigned int reg, 281 unsigned int value) 282 { 283 struct io_apic __iomem *io_apic = io_apic_base(apic); 284 285 writel(reg, &io_apic->index); 286 writel(value, &io_apic->data); 287 } 288 289 union entry_union { 290 struct { u32 w1, w2; }; 291 struct IO_APIC_route_entry entry; 292 }; 293 294 static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin) 295 { 296 union entry_union eu; 297 298 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin); 299 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin); 300 301 return eu.entry; 302 } 303 304 static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin) 305 { 306 union entry_union eu; 307 unsigned long flags; 308 309 raw_spin_lock_irqsave(&ioapic_lock, flags); 310 eu.entry = __ioapic_read_entry(apic, pin); 311 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 312 313 return eu.entry; 314 } 315 316 /* 317 * When we write a new IO APIC routing entry, we need to write the high 318 * word first! If the mask bit in the low word is clear, we will enable 319 * the interrupt, and we need to make sure the entry is fully populated 320 * before that happens. 321 */ 322 static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 323 { 324 union entry_union eu = {{0, 0}}; 325 326 eu.entry = e; 327 io_apic_write(apic, 0x11 + 2*pin, eu.w2); 328 io_apic_write(apic, 0x10 + 2*pin, eu.w1); 329 } 330 331 static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 332 { 333 unsigned long flags; 334 335 raw_spin_lock_irqsave(&ioapic_lock, flags); 336 __ioapic_write_entry(apic, pin, e); 337 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 338 } 339 340 /* 341 * When we mask an IO APIC routing entry, we need to write the low 342 * word first, in order to set the mask bit before we change the 343 * high bits! 344 */ 345 static void ioapic_mask_entry(int apic, int pin) 346 { 347 unsigned long flags; 348 union entry_union eu = { .entry.mask = IOAPIC_MASKED }; 349 350 raw_spin_lock_irqsave(&ioapic_lock, flags); 351 io_apic_write(apic, 0x10 + 2*pin, eu.w1); 352 io_apic_write(apic, 0x11 + 2*pin, eu.w2); 353 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 354 } 355 356 /* 357 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are 358 * shared ISA-space IRQs, so we have to support them. We are super 359 * fast in the common case, and fast for shared ISA-space IRQs. 360 */ 361 static int __add_pin_to_irq_node(struct mp_chip_data *data, 362 int node, int apic, int pin) 363 { 364 struct irq_pin_list *entry; 365 366 /* don't allow duplicates */ 367 for_each_irq_pin(entry, data->irq_2_pin) 368 if (entry->apic == apic && entry->pin == pin) 369 return 0; 370 371 entry = kzalloc_node(sizeof(struct irq_pin_list), GFP_ATOMIC, node); 372 if (!entry) { 373 pr_err("can not alloc irq_pin_list (%d,%d,%d)\n", 374 node, apic, pin); 375 return -ENOMEM; 376 } 377 entry->apic = apic; 378 entry->pin = pin; 379 list_add_tail(&entry->list, &data->irq_2_pin); 380 381 return 0; 382 } 383 384 static void __remove_pin_from_irq(struct mp_chip_data *data, int apic, int pin) 385 { 386 struct irq_pin_list *tmp, *entry; 387 388 list_for_each_entry_safe(entry, tmp, &data->irq_2_pin, list) 389 if (entry->apic == apic && entry->pin == pin) { 390 list_del(&entry->list); 391 kfree(entry); 392 return; 393 } 394 } 395 396 static void add_pin_to_irq_node(struct mp_chip_data *data, 397 int node, int apic, int pin) 398 { 399 if (__add_pin_to_irq_node(data, node, apic, pin)) 400 panic("IO-APIC: failed to add irq-pin. Can not proceed\n"); 401 } 402 403 /* 404 * Reroute an IRQ to a different pin. 405 */ 406 static void __init replace_pin_at_irq_node(struct mp_chip_data *data, int node, 407 int oldapic, int oldpin, 408 int newapic, int newpin) 409 { 410 struct irq_pin_list *entry; 411 412 for_each_irq_pin(entry, data->irq_2_pin) { 413 if (entry->apic == oldapic && entry->pin == oldpin) { 414 entry->apic = newapic; 415 entry->pin = newpin; 416 /* every one is different, right? */ 417 return; 418 } 419 } 420 421 /* old apic/pin didn't exist, so just add new ones */ 422 add_pin_to_irq_node(data, node, newapic, newpin); 423 } 424 425 static void io_apic_modify_irq(struct mp_chip_data *data, 426 int mask_and, int mask_or, 427 void (*final)(struct irq_pin_list *entry)) 428 { 429 union entry_union eu; 430 struct irq_pin_list *entry; 431 432 eu.entry = data->entry; 433 eu.w1 &= mask_and; 434 eu.w1 |= mask_or; 435 data->entry = eu.entry; 436 437 for_each_irq_pin(entry, data->irq_2_pin) { 438 io_apic_write(entry->apic, 0x10 + 2 * entry->pin, eu.w1); 439 if (final) 440 final(entry); 441 } 442 } 443 444 static void io_apic_sync(struct irq_pin_list *entry) 445 { 446 /* 447 * Synchronize the IO-APIC and the CPU by doing 448 * a dummy read from the IO-APIC 449 */ 450 struct io_apic __iomem *io_apic; 451 452 io_apic = io_apic_base(entry->apic); 453 readl(&io_apic->data); 454 } 455 456 static void mask_ioapic_irq(struct irq_data *irq_data) 457 { 458 struct mp_chip_data *data = irq_data->chip_data; 459 unsigned long flags; 460 461 raw_spin_lock_irqsave(&ioapic_lock, flags); 462 io_apic_modify_irq(data, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync); 463 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 464 } 465 466 static void __unmask_ioapic(struct mp_chip_data *data) 467 { 468 io_apic_modify_irq(data, ~IO_APIC_REDIR_MASKED, 0, NULL); 469 } 470 471 static void unmask_ioapic_irq(struct irq_data *irq_data) 472 { 473 struct mp_chip_data *data = irq_data->chip_data; 474 unsigned long flags; 475 476 raw_spin_lock_irqsave(&ioapic_lock, flags); 477 __unmask_ioapic(data); 478 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 479 } 480 481 /* 482 * IO-APIC versions below 0x20 don't support EOI register. 483 * For the record, here is the information about various versions: 484 * 0Xh 82489DX 485 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant 486 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant 487 * 30h-FFh Reserved 488 * 489 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic 490 * version as 0x2. This is an error with documentation and these ICH chips 491 * use io-apic's of version 0x20. 492 * 493 * For IO-APIC's with EOI register, we use that to do an explicit EOI. 494 * Otherwise, we simulate the EOI message manually by changing the trigger 495 * mode to edge and then back to level, with RTE being masked during this. 496 */ 497 static void __eoi_ioapic_pin(int apic, int pin, int vector) 498 { 499 if (mpc_ioapic_ver(apic) >= 0x20) { 500 io_apic_eoi(apic, vector); 501 } else { 502 struct IO_APIC_route_entry entry, entry1; 503 504 entry = entry1 = __ioapic_read_entry(apic, pin); 505 506 /* 507 * Mask the entry and change the trigger mode to edge. 508 */ 509 entry1.mask = IOAPIC_MASKED; 510 entry1.trigger = IOAPIC_EDGE; 511 512 __ioapic_write_entry(apic, pin, entry1); 513 514 /* 515 * Restore the previous level triggered entry. 516 */ 517 __ioapic_write_entry(apic, pin, entry); 518 } 519 } 520 521 static void eoi_ioapic_pin(int vector, struct mp_chip_data *data) 522 { 523 unsigned long flags; 524 struct irq_pin_list *entry; 525 526 raw_spin_lock_irqsave(&ioapic_lock, flags); 527 for_each_irq_pin(entry, data->irq_2_pin) 528 __eoi_ioapic_pin(entry->apic, entry->pin, vector); 529 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 530 } 531 532 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) 533 { 534 struct IO_APIC_route_entry entry; 535 536 /* Check delivery_mode to be sure we're not clearing an SMI pin */ 537 entry = ioapic_read_entry(apic, pin); 538 if (entry.delivery_mode == APIC_DELIVERY_MODE_SMI) 539 return; 540 541 /* 542 * Make sure the entry is masked and re-read the contents to check 543 * if it is a level triggered pin and if the remote-IRR is set. 544 */ 545 if (entry.mask == IOAPIC_UNMASKED) { 546 entry.mask = IOAPIC_MASKED; 547 ioapic_write_entry(apic, pin, entry); 548 entry = ioapic_read_entry(apic, pin); 549 } 550 551 if (entry.irr) { 552 unsigned long flags; 553 554 /* 555 * Make sure the trigger mode is set to level. Explicit EOI 556 * doesn't clear the remote-IRR if the trigger mode is not 557 * set to level. 558 */ 559 if (entry.trigger == IOAPIC_EDGE) { 560 entry.trigger = IOAPIC_LEVEL; 561 ioapic_write_entry(apic, pin, entry); 562 } 563 raw_spin_lock_irqsave(&ioapic_lock, flags); 564 __eoi_ioapic_pin(apic, pin, entry.vector); 565 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 566 } 567 568 /* 569 * Clear the rest of the bits in the IO-APIC RTE except for the mask 570 * bit. 571 */ 572 ioapic_mask_entry(apic, pin); 573 entry = ioapic_read_entry(apic, pin); 574 if (entry.irr) 575 pr_err("Unable to reset IRR for apic: %d, pin :%d\n", 576 mpc_ioapic_id(apic), pin); 577 } 578 579 void clear_IO_APIC (void) 580 { 581 int apic, pin; 582 583 for_each_ioapic_pin(apic, pin) 584 clear_IO_APIC_pin(apic, pin); 585 } 586 587 #ifdef CONFIG_X86_32 588 /* 589 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to 590 * specific CPU-side IRQs. 591 */ 592 593 #define MAX_PIRQS 8 594 static int pirq_entries[MAX_PIRQS] = { 595 [0 ... MAX_PIRQS - 1] = -1 596 }; 597 598 static int __init ioapic_pirq_setup(char *str) 599 { 600 int i, max; 601 int ints[MAX_PIRQS+1]; 602 603 get_options(str, ARRAY_SIZE(ints), ints); 604 605 apic_printk(APIC_VERBOSE, KERN_INFO 606 "PIRQ redirection, working around broken MP-BIOS.\n"); 607 max = MAX_PIRQS; 608 if (ints[0] < MAX_PIRQS) 609 max = ints[0]; 610 611 for (i = 0; i < max; i++) { 612 apic_printk(APIC_VERBOSE, KERN_DEBUG 613 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]); 614 /* 615 * PIRQs are mapped upside down, usually. 616 */ 617 pirq_entries[MAX_PIRQS-i-1] = ints[i+1]; 618 } 619 return 1; 620 } 621 622 __setup("pirq=", ioapic_pirq_setup); 623 #endif /* CONFIG_X86_32 */ 624 625 /* 626 * Saves all the IO-APIC RTE's 627 */ 628 int save_ioapic_entries(void) 629 { 630 int apic, pin; 631 int err = 0; 632 633 for_each_ioapic(apic) { 634 if (!ioapics[apic].saved_registers) { 635 err = -ENOMEM; 636 continue; 637 } 638 639 for_each_pin(apic, pin) 640 ioapics[apic].saved_registers[pin] = 641 ioapic_read_entry(apic, pin); 642 } 643 644 return err; 645 } 646 647 /* 648 * Mask all IO APIC entries. 649 */ 650 void mask_ioapic_entries(void) 651 { 652 int apic, pin; 653 654 for_each_ioapic(apic) { 655 if (!ioapics[apic].saved_registers) 656 continue; 657 658 for_each_pin(apic, pin) { 659 struct IO_APIC_route_entry entry; 660 661 entry = ioapics[apic].saved_registers[pin]; 662 if (entry.mask == IOAPIC_UNMASKED) { 663 entry.mask = IOAPIC_MASKED; 664 ioapic_write_entry(apic, pin, entry); 665 } 666 } 667 } 668 } 669 670 /* 671 * Restore IO APIC entries which was saved in the ioapic structure. 672 */ 673 int restore_ioapic_entries(void) 674 { 675 int apic, pin; 676 677 for_each_ioapic(apic) { 678 if (!ioapics[apic].saved_registers) 679 continue; 680 681 for_each_pin(apic, pin) 682 ioapic_write_entry(apic, pin, 683 ioapics[apic].saved_registers[pin]); 684 } 685 return 0; 686 } 687 688 /* 689 * Find the IRQ entry number of a certain pin. 690 */ 691 static int find_irq_entry(int ioapic_idx, int pin, int type) 692 { 693 int i; 694 695 for (i = 0; i < mp_irq_entries; i++) 696 if (mp_irqs[i].irqtype == type && 697 (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) || 698 mp_irqs[i].dstapic == MP_APIC_ALL) && 699 mp_irqs[i].dstirq == pin) 700 return i; 701 702 return -1; 703 } 704 705 /* 706 * Find the pin to which IRQ[irq] (ISA) is connected 707 */ 708 static int __init find_isa_irq_pin(int irq, int type) 709 { 710 int i; 711 712 for (i = 0; i < mp_irq_entries; i++) { 713 int lbus = mp_irqs[i].srcbus; 714 715 if (test_bit(lbus, mp_bus_not_pci) && 716 (mp_irqs[i].irqtype == type) && 717 (mp_irqs[i].srcbusirq == irq)) 718 719 return mp_irqs[i].dstirq; 720 } 721 return -1; 722 } 723 724 static int __init find_isa_irq_apic(int irq, int type) 725 { 726 int i; 727 728 for (i = 0; i < mp_irq_entries; i++) { 729 int lbus = mp_irqs[i].srcbus; 730 731 if (test_bit(lbus, mp_bus_not_pci) && 732 (mp_irqs[i].irqtype == type) && 733 (mp_irqs[i].srcbusirq == irq)) 734 break; 735 } 736 737 if (i < mp_irq_entries) { 738 int ioapic_idx; 739 740 for_each_ioapic(ioapic_idx) 741 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic) 742 return ioapic_idx; 743 } 744 745 return -1; 746 } 747 748 #ifdef CONFIG_EISA 749 /* 750 * EISA Edge/Level control register, ELCR 751 */ 752 static int EISA_ELCR(unsigned int irq) 753 { 754 if (irq < nr_legacy_irqs()) { 755 unsigned int port = 0x4d0 + (irq >> 3); 756 return (inb(port) >> (irq & 7)) & 1; 757 } 758 apic_printk(APIC_VERBOSE, KERN_INFO 759 "Broken MPtable reports ISA irq %d\n", irq); 760 return 0; 761 } 762 763 #endif 764 765 /* ISA interrupts are always active high edge triggered, 766 * when listed as conforming in the MP table. */ 767 768 #define default_ISA_trigger(idx) (IOAPIC_EDGE) 769 #define default_ISA_polarity(idx) (IOAPIC_POL_HIGH) 770 771 /* EISA interrupts are always polarity zero and can be edge or level 772 * trigger depending on the ELCR value. If an interrupt is listed as 773 * EISA conforming in the MP table, that means its trigger type must 774 * be read in from the ELCR */ 775 776 #define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq)) 777 #define default_EISA_polarity(idx) default_ISA_polarity(idx) 778 779 /* PCI interrupts are always active low level triggered, 780 * when listed as conforming in the MP table. */ 781 782 #define default_PCI_trigger(idx) (IOAPIC_LEVEL) 783 #define default_PCI_polarity(idx) (IOAPIC_POL_LOW) 784 785 static int irq_polarity(int idx) 786 { 787 int bus = mp_irqs[idx].srcbus; 788 789 /* 790 * Determine IRQ line polarity (high active or low active): 791 */ 792 switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) { 793 case MP_IRQPOL_DEFAULT: 794 /* conforms to spec, ie. bus-type dependent polarity */ 795 if (test_bit(bus, mp_bus_not_pci)) 796 return default_ISA_polarity(idx); 797 else 798 return default_PCI_polarity(idx); 799 case MP_IRQPOL_ACTIVE_HIGH: 800 return IOAPIC_POL_HIGH; 801 case MP_IRQPOL_RESERVED: 802 pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n"); 803 fallthrough; 804 case MP_IRQPOL_ACTIVE_LOW: 805 default: /* Pointless default required due to do gcc stupidity */ 806 return IOAPIC_POL_LOW; 807 } 808 } 809 810 #ifdef CONFIG_EISA 811 static int eisa_irq_trigger(int idx, int bus, int trigger) 812 { 813 switch (mp_bus_id_to_type[bus]) { 814 case MP_BUS_PCI: 815 case MP_BUS_ISA: 816 return trigger; 817 case MP_BUS_EISA: 818 return default_EISA_trigger(idx); 819 } 820 pr_warn("IOAPIC: Invalid srcbus: %d defaulting to level\n", bus); 821 return IOAPIC_LEVEL; 822 } 823 #else 824 static inline int eisa_irq_trigger(int idx, int bus, int trigger) 825 { 826 return trigger; 827 } 828 #endif 829 830 static int irq_trigger(int idx) 831 { 832 int bus = mp_irqs[idx].srcbus; 833 int trigger; 834 835 /* 836 * Determine IRQ trigger mode (edge or level sensitive): 837 */ 838 switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) { 839 case MP_IRQTRIG_DEFAULT: 840 /* conforms to spec, ie. bus-type dependent trigger mode */ 841 if (test_bit(bus, mp_bus_not_pci)) 842 trigger = default_ISA_trigger(idx); 843 else 844 trigger = default_PCI_trigger(idx); 845 /* Take EISA into account */ 846 return eisa_irq_trigger(idx, bus, trigger); 847 case MP_IRQTRIG_EDGE: 848 return IOAPIC_EDGE; 849 case MP_IRQTRIG_RESERVED: 850 pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n"); 851 fallthrough; 852 case MP_IRQTRIG_LEVEL: 853 default: /* Pointless default required due to do gcc stupidity */ 854 return IOAPIC_LEVEL; 855 } 856 } 857 858 void ioapic_set_alloc_attr(struct irq_alloc_info *info, int node, 859 int trigger, int polarity) 860 { 861 init_irq_alloc_info(info, NULL); 862 info->type = X86_IRQ_ALLOC_TYPE_IOAPIC; 863 info->ioapic.node = node; 864 info->ioapic.trigger = trigger; 865 info->ioapic.polarity = polarity; 866 info->ioapic.valid = 1; 867 } 868 869 #ifndef CONFIG_ACPI 870 int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity); 871 #endif 872 873 static void ioapic_copy_alloc_attr(struct irq_alloc_info *dst, 874 struct irq_alloc_info *src, 875 u32 gsi, int ioapic_idx, int pin) 876 { 877 int trigger, polarity; 878 879 copy_irq_alloc_info(dst, src); 880 dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC; 881 dst->devid = mpc_ioapic_id(ioapic_idx); 882 dst->ioapic.pin = pin; 883 dst->ioapic.valid = 1; 884 if (src && src->ioapic.valid) { 885 dst->ioapic.node = src->ioapic.node; 886 dst->ioapic.trigger = src->ioapic.trigger; 887 dst->ioapic.polarity = src->ioapic.polarity; 888 } else { 889 dst->ioapic.node = NUMA_NO_NODE; 890 if (acpi_get_override_irq(gsi, &trigger, &polarity) >= 0) { 891 dst->ioapic.trigger = trigger; 892 dst->ioapic.polarity = polarity; 893 } else { 894 /* 895 * PCI interrupts are always active low level 896 * triggered. 897 */ 898 dst->ioapic.trigger = IOAPIC_LEVEL; 899 dst->ioapic.polarity = IOAPIC_POL_LOW; 900 } 901 } 902 } 903 904 static int ioapic_alloc_attr_node(struct irq_alloc_info *info) 905 { 906 return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE; 907 } 908 909 static void mp_register_handler(unsigned int irq, unsigned long trigger) 910 { 911 irq_flow_handler_t hdl; 912 bool fasteoi; 913 914 if (trigger) { 915 irq_set_status_flags(irq, IRQ_LEVEL); 916 fasteoi = true; 917 } else { 918 irq_clear_status_flags(irq, IRQ_LEVEL); 919 fasteoi = false; 920 } 921 922 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq; 923 __irq_set_handler(irq, hdl, 0, fasteoi ? "fasteoi" : "edge"); 924 } 925 926 static bool mp_check_pin_attr(int irq, struct irq_alloc_info *info) 927 { 928 struct mp_chip_data *data = irq_get_chip_data(irq); 929 930 /* 931 * setup_IO_APIC_irqs() programs all legacy IRQs with default trigger 932 * and polarity attirbutes. So allow the first user to reprogram the 933 * pin with real trigger and polarity attributes. 934 */ 935 if (irq < nr_legacy_irqs() && data->count == 1) { 936 if (info->ioapic.trigger != data->trigger) 937 mp_register_handler(irq, info->ioapic.trigger); 938 data->entry.trigger = data->trigger = info->ioapic.trigger; 939 data->entry.polarity = data->polarity = info->ioapic.polarity; 940 } 941 942 return data->trigger == info->ioapic.trigger && 943 data->polarity == info->ioapic.polarity; 944 } 945 946 static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi, 947 struct irq_alloc_info *info) 948 { 949 bool legacy = false; 950 int irq = -1; 951 int type = ioapics[ioapic].irqdomain_cfg.type; 952 953 switch (type) { 954 case IOAPIC_DOMAIN_LEGACY: 955 /* 956 * Dynamically allocate IRQ number for non-ISA IRQs in the first 957 * 16 GSIs on some weird platforms. 958 */ 959 if (!ioapic_initialized || gsi >= nr_legacy_irqs()) 960 irq = gsi; 961 legacy = mp_is_legacy_irq(irq); 962 break; 963 case IOAPIC_DOMAIN_STRICT: 964 irq = gsi; 965 break; 966 case IOAPIC_DOMAIN_DYNAMIC: 967 break; 968 default: 969 WARN(1, "ioapic: unknown irqdomain type %d\n", type); 970 return -1; 971 } 972 973 return __irq_domain_alloc_irqs(domain, irq, 1, 974 ioapic_alloc_attr_node(info), 975 info, legacy, NULL); 976 } 977 978 /* 979 * Need special handling for ISA IRQs because there may be multiple IOAPIC pins 980 * sharing the same ISA IRQ number and irqdomain only supports 1:1 mapping 981 * between IOAPIC pin and IRQ number. A typical IOAPIC has 24 pins, pin 0-15 are 982 * used for legacy IRQs and pin 16-23 are used for PCI IRQs (PIRQ A-H). 983 * When ACPI is disabled, only legacy IRQ numbers (IRQ0-15) are available, and 984 * some BIOSes may use MP Interrupt Source records to override IRQ numbers for 985 * PIRQs instead of reprogramming the interrupt routing logic. Thus there may be 986 * multiple pins sharing the same legacy IRQ number when ACPI is disabled. 987 */ 988 static int alloc_isa_irq_from_domain(struct irq_domain *domain, 989 int irq, int ioapic, int pin, 990 struct irq_alloc_info *info) 991 { 992 struct mp_chip_data *data; 993 struct irq_data *irq_data = irq_get_irq_data(irq); 994 int node = ioapic_alloc_attr_node(info); 995 996 /* 997 * Legacy ISA IRQ has already been allocated, just add pin to 998 * the pin list assoicated with this IRQ and program the IOAPIC 999 * entry. The IOAPIC entry 1000 */ 1001 if (irq_data && irq_data->parent_data) { 1002 if (!mp_check_pin_attr(irq, info)) 1003 return -EBUSY; 1004 if (__add_pin_to_irq_node(irq_data->chip_data, node, ioapic, 1005 info->ioapic.pin)) 1006 return -ENOMEM; 1007 } else { 1008 info->flags |= X86_IRQ_ALLOC_LEGACY; 1009 irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true, 1010 NULL); 1011 if (irq >= 0) { 1012 irq_data = irq_domain_get_irq_data(domain, irq); 1013 data = irq_data->chip_data; 1014 data->isa_irq = true; 1015 } 1016 } 1017 1018 return irq; 1019 } 1020 1021 static int mp_map_pin_to_irq(u32 gsi, int idx, int ioapic, int pin, 1022 unsigned int flags, struct irq_alloc_info *info) 1023 { 1024 int irq; 1025 bool legacy = false; 1026 struct irq_alloc_info tmp; 1027 struct mp_chip_data *data; 1028 struct irq_domain *domain = mp_ioapic_irqdomain(ioapic); 1029 1030 if (!domain) 1031 return -ENOSYS; 1032 1033 if (idx >= 0 && test_bit(mp_irqs[idx].srcbus, mp_bus_not_pci)) { 1034 irq = mp_irqs[idx].srcbusirq; 1035 legacy = mp_is_legacy_irq(irq); 1036 } 1037 1038 mutex_lock(&ioapic_mutex); 1039 if (!(flags & IOAPIC_MAP_ALLOC)) { 1040 if (!legacy) { 1041 irq = irq_find_mapping(domain, pin); 1042 if (irq == 0) 1043 irq = -ENOENT; 1044 } 1045 } else { 1046 ioapic_copy_alloc_attr(&tmp, info, gsi, ioapic, pin); 1047 if (legacy) 1048 irq = alloc_isa_irq_from_domain(domain, irq, 1049 ioapic, pin, &tmp); 1050 else if ((irq = irq_find_mapping(domain, pin)) == 0) 1051 irq = alloc_irq_from_domain(domain, ioapic, gsi, &tmp); 1052 else if (!mp_check_pin_attr(irq, &tmp)) 1053 irq = -EBUSY; 1054 if (irq >= 0) { 1055 data = irq_get_chip_data(irq); 1056 data->count++; 1057 } 1058 } 1059 mutex_unlock(&ioapic_mutex); 1060 1061 return irq; 1062 } 1063 1064 static int pin_2_irq(int idx, int ioapic, int pin, unsigned int flags) 1065 { 1066 u32 gsi = mp_pin_to_gsi(ioapic, pin); 1067 1068 /* 1069 * Debugging check, we are in big trouble if this message pops up! 1070 */ 1071 if (mp_irqs[idx].dstirq != pin) 1072 pr_err("broken BIOS or MPTABLE parser, ayiee!!\n"); 1073 1074 #ifdef CONFIG_X86_32 1075 /* 1076 * PCI IRQ command line redirection. Yes, limits are hardcoded. 1077 */ 1078 if ((pin >= 16) && (pin <= 23)) { 1079 if (pirq_entries[pin-16] != -1) { 1080 if (!pirq_entries[pin-16]) { 1081 apic_printk(APIC_VERBOSE, KERN_DEBUG 1082 "disabling PIRQ%d\n", pin-16); 1083 } else { 1084 int irq = pirq_entries[pin-16]; 1085 apic_printk(APIC_VERBOSE, KERN_DEBUG 1086 "using PIRQ%d -> IRQ %d\n", 1087 pin-16, irq); 1088 return irq; 1089 } 1090 } 1091 } 1092 #endif 1093 1094 return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, NULL); 1095 } 1096 1097 int mp_map_gsi_to_irq(u32 gsi, unsigned int flags, struct irq_alloc_info *info) 1098 { 1099 int ioapic, pin, idx; 1100 1101 ioapic = mp_find_ioapic(gsi); 1102 if (ioapic < 0) 1103 return -ENODEV; 1104 1105 pin = mp_find_ioapic_pin(ioapic, gsi); 1106 idx = find_irq_entry(ioapic, pin, mp_INT); 1107 if ((flags & IOAPIC_MAP_CHECK) && idx < 0) 1108 return -ENODEV; 1109 1110 return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, info); 1111 } 1112 1113 void mp_unmap_irq(int irq) 1114 { 1115 struct irq_data *irq_data = irq_get_irq_data(irq); 1116 struct mp_chip_data *data; 1117 1118 if (!irq_data || !irq_data->domain) 1119 return; 1120 1121 data = irq_data->chip_data; 1122 if (!data || data->isa_irq) 1123 return; 1124 1125 mutex_lock(&ioapic_mutex); 1126 if (--data->count == 0) 1127 irq_domain_free_irqs(irq, 1); 1128 mutex_unlock(&ioapic_mutex); 1129 } 1130 1131 /* 1132 * Find a specific PCI IRQ entry. 1133 * Not an __init, possibly needed by modules 1134 */ 1135 int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin) 1136 { 1137 int irq, i, best_ioapic = -1, best_idx = -1; 1138 1139 apic_printk(APIC_DEBUG, 1140 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n", 1141 bus, slot, pin); 1142 if (test_bit(bus, mp_bus_not_pci)) { 1143 apic_printk(APIC_VERBOSE, 1144 "PCI BIOS passed nonexistent PCI bus %d!\n", bus); 1145 return -1; 1146 } 1147 1148 for (i = 0; i < mp_irq_entries; i++) { 1149 int lbus = mp_irqs[i].srcbus; 1150 int ioapic_idx, found = 0; 1151 1152 if (bus != lbus || mp_irqs[i].irqtype != mp_INT || 1153 slot != ((mp_irqs[i].srcbusirq >> 2) & 0x1f)) 1154 continue; 1155 1156 for_each_ioapic(ioapic_idx) 1157 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic || 1158 mp_irqs[i].dstapic == MP_APIC_ALL) { 1159 found = 1; 1160 break; 1161 } 1162 if (!found) 1163 continue; 1164 1165 /* Skip ISA IRQs */ 1166 irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq, 0); 1167 if (irq > 0 && !IO_APIC_IRQ(irq)) 1168 continue; 1169 1170 if (pin == (mp_irqs[i].srcbusirq & 3)) { 1171 best_idx = i; 1172 best_ioapic = ioapic_idx; 1173 goto out; 1174 } 1175 1176 /* 1177 * Use the first all-but-pin matching entry as a 1178 * best-guess fuzzy result for broken mptables. 1179 */ 1180 if (best_idx < 0) { 1181 best_idx = i; 1182 best_ioapic = ioapic_idx; 1183 } 1184 } 1185 if (best_idx < 0) 1186 return -1; 1187 1188 out: 1189 return pin_2_irq(best_idx, best_ioapic, mp_irqs[best_idx].dstirq, 1190 IOAPIC_MAP_ALLOC); 1191 } 1192 EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector); 1193 1194 static struct irq_chip ioapic_chip, ioapic_ir_chip; 1195 1196 static void __init setup_IO_APIC_irqs(void) 1197 { 1198 unsigned int ioapic, pin; 1199 int idx; 1200 1201 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1202 1203 for_each_ioapic_pin(ioapic, pin) { 1204 idx = find_irq_entry(ioapic, pin, mp_INT); 1205 if (idx < 0) 1206 apic_printk(APIC_VERBOSE, 1207 KERN_DEBUG " apic %d pin %d not connected\n", 1208 mpc_ioapic_id(ioapic), pin); 1209 else 1210 pin_2_irq(idx, ioapic, pin, 1211 ioapic ? 0 : IOAPIC_MAP_ALLOC); 1212 } 1213 } 1214 1215 void ioapic_zap_locks(void) 1216 { 1217 raw_spin_lock_init(&ioapic_lock); 1218 } 1219 1220 static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries) 1221 { 1222 int i; 1223 char buf[256]; 1224 struct IO_APIC_route_entry entry; 1225 struct IR_IO_APIC_route_entry *ir_entry = (void *)&entry; 1226 1227 printk(KERN_DEBUG "IOAPIC %d:\n", apic); 1228 for (i = 0; i <= nr_entries; i++) { 1229 entry = ioapic_read_entry(apic, i); 1230 snprintf(buf, sizeof(buf), 1231 " pin%02x, %s, %s, %s, V(%02X), IRR(%1d), S(%1d)", 1232 i, 1233 entry.mask == IOAPIC_MASKED ? "disabled" : "enabled ", 1234 entry.trigger == IOAPIC_LEVEL ? "level" : "edge ", 1235 entry.polarity == IOAPIC_POL_LOW ? "low " : "high", 1236 entry.vector, entry.irr, entry.delivery_status); 1237 if (ir_entry->format) 1238 printk(KERN_DEBUG "%s, remapped, I(%04X), Z(%X)\n", 1239 buf, (ir_entry->index2 << 15) | ir_entry->index, 1240 ir_entry->zero); 1241 else 1242 printk(KERN_DEBUG "%s, %s, D(%02X), M(%1d)\n", 1243 buf, 1244 entry.dest_mode == IOAPIC_DEST_MODE_LOGICAL ? 1245 "logical " : "physical", 1246 entry.dest, entry.delivery_mode); 1247 } 1248 } 1249 1250 static void __init print_IO_APIC(int ioapic_idx) 1251 { 1252 union IO_APIC_reg_00 reg_00; 1253 union IO_APIC_reg_01 reg_01; 1254 union IO_APIC_reg_02 reg_02; 1255 union IO_APIC_reg_03 reg_03; 1256 unsigned long flags; 1257 1258 raw_spin_lock_irqsave(&ioapic_lock, flags); 1259 reg_00.raw = io_apic_read(ioapic_idx, 0); 1260 reg_01.raw = io_apic_read(ioapic_idx, 1); 1261 if (reg_01.bits.version >= 0x10) 1262 reg_02.raw = io_apic_read(ioapic_idx, 2); 1263 if (reg_01.bits.version >= 0x20) 1264 reg_03.raw = io_apic_read(ioapic_idx, 3); 1265 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1266 1267 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx)); 1268 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); 1269 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID); 1270 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type); 1271 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS); 1272 1273 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01); 1274 printk(KERN_DEBUG "....... : max redirection entries: %02X\n", 1275 reg_01.bits.entries); 1276 1277 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ); 1278 printk(KERN_DEBUG "....... : IO APIC version: %02X\n", 1279 reg_01.bits.version); 1280 1281 /* 1282 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02, 1283 * but the value of reg_02 is read as the previous read register 1284 * value, so ignore it if reg_02 == reg_01. 1285 */ 1286 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) { 1287 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw); 1288 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration); 1289 } 1290 1291 /* 1292 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02 1293 * or reg_03, but the value of reg_0[23] is read as the previous read 1294 * register value, so ignore it if reg_03 == reg_0[12]. 1295 */ 1296 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw && 1297 reg_03.raw != reg_01.raw) { 1298 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw); 1299 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT); 1300 } 1301 1302 printk(KERN_DEBUG ".... IRQ redirection table:\n"); 1303 io_apic_print_entries(ioapic_idx, reg_01.bits.entries); 1304 } 1305 1306 void __init print_IO_APICs(void) 1307 { 1308 int ioapic_idx; 1309 unsigned int irq; 1310 1311 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 1312 for_each_ioapic(ioapic_idx) 1313 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 1314 mpc_ioapic_id(ioapic_idx), 1315 ioapics[ioapic_idx].nr_registers); 1316 1317 /* 1318 * We are a bit conservative about what we expect. We have to 1319 * know about every hardware change ASAP. 1320 */ 1321 printk(KERN_INFO "testing the IO APIC.......................\n"); 1322 1323 for_each_ioapic(ioapic_idx) 1324 print_IO_APIC(ioapic_idx); 1325 1326 printk(KERN_DEBUG "IRQ to pin mappings:\n"); 1327 for_each_active_irq(irq) { 1328 struct irq_pin_list *entry; 1329 struct irq_chip *chip; 1330 struct mp_chip_data *data; 1331 1332 chip = irq_get_chip(irq); 1333 if (chip != &ioapic_chip && chip != &ioapic_ir_chip) 1334 continue; 1335 data = irq_get_chip_data(irq); 1336 if (!data) 1337 continue; 1338 if (list_empty(&data->irq_2_pin)) 1339 continue; 1340 1341 printk(KERN_DEBUG "IRQ%d ", irq); 1342 for_each_irq_pin(entry, data->irq_2_pin) 1343 pr_cont("-> %d:%d", entry->apic, entry->pin); 1344 pr_cont("\n"); 1345 } 1346 1347 printk(KERN_INFO ".................................... done.\n"); 1348 } 1349 1350 /* Where if anywhere is the i8259 connect in external int mode */ 1351 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 }; 1352 1353 void __init enable_IO_APIC(void) 1354 { 1355 int i8259_apic, i8259_pin; 1356 int apic, pin; 1357 1358 if (skip_ioapic_setup) 1359 nr_ioapics = 0; 1360 1361 if (!nr_legacy_irqs() || !nr_ioapics) 1362 return; 1363 1364 for_each_ioapic_pin(apic, pin) { 1365 /* See if any of the pins is in ExtINT mode */ 1366 struct IO_APIC_route_entry entry = ioapic_read_entry(apic, pin); 1367 1368 /* If the interrupt line is enabled and in ExtInt mode 1369 * I have found the pin where the i8259 is connected. 1370 */ 1371 if ((entry.mask == 0) && 1372 (entry.delivery_mode == APIC_DELIVERY_MODE_EXTINT)) { 1373 ioapic_i8259.apic = apic; 1374 ioapic_i8259.pin = pin; 1375 goto found_i8259; 1376 } 1377 } 1378 found_i8259: 1379 /* Look to see what if the MP table has reported the ExtINT */ 1380 /* If we could not find the appropriate pin by looking at the ioapic 1381 * the i8259 probably is not connected the ioapic but give the 1382 * mptable a chance anyway. 1383 */ 1384 i8259_pin = find_isa_irq_pin(0, mp_ExtINT); 1385 i8259_apic = find_isa_irq_apic(0, mp_ExtINT); 1386 /* Trust the MP table if nothing is setup in the hardware */ 1387 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) { 1388 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n"); 1389 ioapic_i8259.pin = i8259_pin; 1390 ioapic_i8259.apic = i8259_apic; 1391 } 1392 /* Complain if the MP table and the hardware disagree */ 1393 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) && 1394 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0)) 1395 { 1396 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n"); 1397 } 1398 1399 /* 1400 * Do not trust the IO-APIC being empty at bootup 1401 */ 1402 clear_IO_APIC(); 1403 } 1404 1405 void native_restore_boot_irq_mode(void) 1406 { 1407 /* 1408 * If the i8259 is routed through an IOAPIC 1409 * Put that IOAPIC in virtual wire mode 1410 * so legacy interrupts can be delivered. 1411 */ 1412 if (ioapic_i8259.pin != -1) { 1413 struct IO_APIC_route_entry entry; 1414 1415 memset(&entry, 0, sizeof(entry)); 1416 entry.mask = IOAPIC_UNMASKED; 1417 entry.trigger = IOAPIC_EDGE; 1418 entry.polarity = IOAPIC_POL_HIGH; 1419 entry.dest_mode = IOAPIC_DEST_MODE_PHYSICAL; 1420 entry.delivery_mode = APIC_DELIVERY_MODE_EXTINT; 1421 entry.dest = read_apic_id(); 1422 1423 /* 1424 * Add it to the IO-APIC irq-routing table: 1425 */ 1426 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry); 1427 } 1428 1429 if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config()) 1430 disconnect_bsp_APIC(ioapic_i8259.pin != -1); 1431 } 1432 1433 void restore_boot_irq_mode(void) 1434 { 1435 if (!nr_legacy_irqs()) 1436 return; 1437 1438 x86_apic_ops.restore(); 1439 } 1440 1441 #ifdef CONFIG_X86_32 1442 /* 1443 * function to set the IO-APIC physical IDs based on the 1444 * values stored in the MPC table. 1445 * 1446 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999 1447 */ 1448 void __init setup_ioapic_ids_from_mpc_nocheck(void) 1449 { 1450 union IO_APIC_reg_00 reg_00; 1451 physid_mask_t phys_id_present_map; 1452 int ioapic_idx; 1453 int i; 1454 unsigned char old_id; 1455 unsigned long flags; 1456 1457 /* 1458 * This is broken; anything with a real cpu count has to 1459 * circumvent this idiocy regardless. 1460 */ 1461 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map); 1462 1463 /* 1464 * Set the IOAPIC ID to the value stored in the MPC table. 1465 */ 1466 for_each_ioapic(ioapic_idx) { 1467 /* Read the register 0 value */ 1468 raw_spin_lock_irqsave(&ioapic_lock, flags); 1469 reg_00.raw = io_apic_read(ioapic_idx, 0); 1470 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1471 1472 old_id = mpc_ioapic_id(ioapic_idx); 1473 1474 if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) { 1475 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n", 1476 ioapic_idx, mpc_ioapic_id(ioapic_idx)); 1477 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1478 reg_00.bits.ID); 1479 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID; 1480 } 1481 1482 /* 1483 * Sanity check, is the ID really free? Every APIC in a 1484 * system must have a unique ID or we get lots of nice 1485 * 'stuck on smp_invalidate_needed IPI wait' messages. 1486 */ 1487 if (apic->check_apicid_used(&phys_id_present_map, 1488 mpc_ioapic_id(ioapic_idx))) { 1489 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n", 1490 ioapic_idx, mpc_ioapic_id(ioapic_idx)); 1491 for (i = 0; i < get_physical_broadcast(); i++) 1492 if (!physid_isset(i, phys_id_present_map)) 1493 break; 1494 if (i >= get_physical_broadcast()) 1495 panic("Max APIC ID exceeded!\n"); 1496 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1497 i); 1498 physid_set(i, phys_id_present_map); 1499 ioapics[ioapic_idx].mp_config.apicid = i; 1500 } else { 1501 physid_mask_t tmp; 1502 apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx), 1503 &tmp); 1504 apic_printk(APIC_VERBOSE, "Setting %d in the " 1505 "phys_id_present_map\n", 1506 mpc_ioapic_id(ioapic_idx)); 1507 physids_or(phys_id_present_map, phys_id_present_map, tmp); 1508 } 1509 1510 /* 1511 * We need to adjust the IRQ routing table 1512 * if the ID changed. 1513 */ 1514 if (old_id != mpc_ioapic_id(ioapic_idx)) 1515 for (i = 0; i < mp_irq_entries; i++) 1516 if (mp_irqs[i].dstapic == old_id) 1517 mp_irqs[i].dstapic 1518 = mpc_ioapic_id(ioapic_idx); 1519 1520 /* 1521 * Update the ID register according to the right value 1522 * from the MPC table if they are different. 1523 */ 1524 if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID) 1525 continue; 1526 1527 apic_printk(APIC_VERBOSE, KERN_INFO 1528 "...changing IO-APIC physical APIC ID to %d ...", 1529 mpc_ioapic_id(ioapic_idx)); 1530 1531 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx); 1532 raw_spin_lock_irqsave(&ioapic_lock, flags); 1533 io_apic_write(ioapic_idx, 0, reg_00.raw); 1534 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1535 1536 /* 1537 * Sanity check 1538 */ 1539 raw_spin_lock_irqsave(&ioapic_lock, flags); 1540 reg_00.raw = io_apic_read(ioapic_idx, 0); 1541 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1542 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) 1543 pr_cont("could not set ID!\n"); 1544 else 1545 apic_printk(APIC_VERBOSE, " ok.\n"); 1546 } 1547 } 1548 1549 void __init setup_ioapic_ids_from_mpc(void) 1550 { 1551 1552 if (acpi_ioapic) 1553 return; 1554 /* 1555 * Don't check I/O APIC IDs for xAPIC systems. They have 1556 * no meaning without the serial APIC bus. 1557 */ 1558 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1559 || APIC_XAPIC(boot_cpu_apic_version)) 1560 return; 1561 setup_ioapic_ids_from_mpc_nocheck(); 1562 } 1563 #endif 1564 1565 int no_timer_check __initdata; 1566 1567 static int __init notimercheck(char *s) 1568 { 1569 no_timer_check = 1; 1570 return 1; 1571 } 1572 __setup("no_timer_check", notimercheck); 1573 1574 static void __init delay_with_tsc(void) 1575 { 1576 unsigned long long start, now; 1577 unsigned long end = jiffies + 4; 1578 1579 start = rdtsc(); 1580 1581 /* 1582 * We don't know the TSC frequency yet, but waiting for 1583 * 40000000000/HZ TSC cycles is safe: 1584 * 4 GHz == 10 jiffies 1585 * 1 GHz == 40 jiffies 1586 */ 1587 do { 1588 rep_nop(); 1589 now = rdtsc(); 1590 } while ((now - start) < 40000000000ULL / HZ && 1591 time_before_eq(jiffies, end)); 1592 } 1593 1594 static void __init delay_without_tsc(void) 1595 { 1596 unsigned long end = jiffies + 4; 1597 int band = 1; 1598 1599 /* 1600 * We don't know any frequency yet, but waiting for 1601 * 40940000000/HZ cycles is safe: 1602 * 4 GHz == 10 jiffies 1603 * 1 GHz == 40 jiffies 1604 * 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094 1605 */ 1606 do { 1607 __delay(((1U << band++) * 10000000UL) / HZ); 1608 } while (band < 12 && time_before_eq(jiffies, end)); 1609 } 1610 1611 /* 1612 * There is a nasty bug in some older SMP boards, their mptable lies 1613 * about the timer IRQ. We do the following to work around the situation: 1614 * 1615 * - timer IRQ defaults to IO-APIC IRQ 1616 * - if this function detects that timer IRQs are defunct, then we fall 1617 * back to ISA timer IRQs 1618 */ 1619 static int __init timer_irq_works(void) 1620 { 1621 unsigned long t1 = jiffies; 1622 unsigned long flags; 1623 1624 if (no_timer_check) 1625 return 1; 1626 1627 local_save_flags(flags); 1628 local_irq_enable(); 1629 1630 if (boot_cpu_has(X86_FEATURE_TSC)) 1631 delay_with_tsc(); 1632 else 1633 delay_without_tsc(); 1634 1635 local_irq_restore(flags); 1636 1637 /* 1638 * Expect a few ticks at least, to be sure some possible 1639 * glue logic does not lock up after one or two first 1640 * ticks in a non-ExtINT mode. Also the local APIC 1641 * might have cached one ExtINT interrupt. Finally, at 1642 * least one tick may be lost due to delays. 1643 */ 1644 1645 /* jiffies wrap? */ 1646 if (time_after(jiffies, t1 + 4)) 1647 return 1; 1648 return 0; 1649 } 1650 1651 /* 1652 * In the SMP+IOAPIC case it might happen that there are an unspecified 1653 * number of pending IRQ events unhandled. These cases are very rare, 1654 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much 1655 * better to do it this way as thus we do not have to be aware of 1656 * 'pending' interrupts in the IRQ path, except at this point. 1657 */ 1658 /* 1659 * Edge triggered needs to resend any interrupt 1660 * that was delayed but this is now handled in the device 1661 * independent code. 1662 */ 1663 1664 /* 1665 * Starting up a edge-triggered IO-APIC interrupt is 1666 * nasty - we need to make sure that we get the edge. 1667 * If it is already asserted for some reason, we need 1668 * return 1 to indicate that is was pending. 1669 * 1670 * This is not complete - we should be able to fake 1671 * an edge even if it isn't on the 8259A... 1672 */ 1673 static unsigned int startup_ioapic_irq(struct irq_data *data) 1674 { 1675 int was_pending = 0, irq = data->irq; 1676 unsigned long flags; 1677 1678 raw_spin_lock_irqsave(&ioapic_lock, flags); 1679 if (irq < nr_legacy_irqs()) { 1680 legacy_pic->mask(irq); 1681 if (legacy_pic->irq_pending(irq)) 1682 was_pending = 1; 1683 } 1684 __unmask_ioapic(data->chip_data); 1685 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1686 1687 return was_pending; 1688 } 1689 1690 atomic_t irq_mis_count; 1691 1692 #ifdef CONFIG_GENERIC_PENDING_IRQ 1693 static bool io_apic_level_ack_pending(struct mp_chip_data *data) 1694 { 1695 struct irq_pin_list *entry; 1696 unsigned long flags; 1697 1698 raw_spin_lock_irqsave(&ioapic_lock, flags); 1699 for_each_irq_pin(entry, data->irq_2_pin) { 1700 unsigned int reg; 1701 int pin; 1702 1703 pin = entry->pin; 1704 reg = io_apic_read(entry->apic, 0x10 + pin*2); 1705 /* Is the remote IRR bit set? */ 1706 if (reg & IO_APIC_REDIR_REMOTE_IRR) { 1707 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1708 return true; 1709 } 1710 } 1711 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1712 1713 return false; 1714 } 1715 1716 static inline bool ioapic_prepare_move(struct irq_data *data) 1717 { 1718 /* If we are moving the IRQ we need to mask it */ 1719 if (unlikely(irqd_is_setaffinity_pending(data))) { 1720 if (!irqd_irq_masked(data)) 1721 mask_ioapic_irq(data); 1722 return true; 1723 } 1724 return false; 1725 } 1726 1727 static inline void ioapic_finish_move(struct irq_data *data, bool moveit) 1728 { 1729 if (unlikely(moveit)) { 1730 /* Only migrate the irq if the ack has been received. 1731 * 1732 * On rare occasions the broadcast level triggered ack gets 1733 * delayed going to ioapics, and if we reprogram the 1734 * vector while Remote IRR is still set the irq will never 1735 * fire again. 1736 * 1737 * To prevent this scenario we read the Remote IRR bit 1738 * of the ioapic. This has two effects. 1739 * - On any sane system the read of the ioapic will 1740 * flush writes (and acks) going to the ioapic from 1741 * this cpu. 1742 * - We get to see if the ACK has actually been delivered. 1743 * 1744 * Based on failed experiments of reprogramming the 1745 * ioapic entry from outside of irq context starting 1746 * with masking the ioapic entry and then polling until 1747 * Remote IRR was clear before reprogramming the 1748 * ioapic I don't trust the Remote IRR bit to be 1749 * completey accurate. 1750 * 1751 * However there appears to be no other way to plug 1752 * this race, so if the Remote IRR bit is not 1753 * accurate and is causing problems then it is a hardware bug 1754 * and you can go talk to the chipset vendor about it. 1755 */ 1756 if (!io_apic_level_ack_pending(data->chip_data)) 1757 irq_move_masked_irq(data); 1758 /* If the IRQ is masked in the core, leave it: */ 1759 if (!irqd_irq_masked(data)) 1760 unmask_ioapic_irq(data); 1761 } 1762 } 1763 #else 1764 static inline bool ioapic_prepare_move(struct irq_data *data) 1765 { 1766 return false; 1767 } 1768 static inline void ioapic_finish_move(struct irq_data *data, bool moveit) 1769 { 1770 } 1771 #endif 1772 1773 static void ioapic_ack_level(struct irq_data *irq_data) 1774 { 1775 struct irq_cfg *cfg = irqd_cfg(irq_data); 1776 unsigned long v; 1777 bool moveit; 1778 int i; 1779 1780 irq_complete_move(cfg); 1781 moveit = ioapic_prepare_move(irq_data); 1782 1783 /* 1784 * It appears there is an erratum which affects at least version 0x11 1785 * of I/O APIC (that's the 82093AA and cores integrated into various 1786 * chipsets). Under certain conditions a level-triggered interrupt is 1787 * erroneously delivered as edge-triggered one but the respective IRR 1788 * bit gets set nevertheless. As a result the I/O unit expects an EOI 1789 * message but it will never arrive and further interrupts are blocked 1790 * from the source. The exact reason is so far unknown, but the 1791 * phenomenon was observed when two consecutive interrupt requests 1792 * from a given source get delivered to the same CPU and the source is 1793 * temporarily disabled in between. 1794 * 1795 * A workaround is to simulate an EOI message manually. We achieve it 1796 * by setting the trigger mode to edge and then to level when the edge 1797 * trigger mode gets detected in the TMR of a local APIC for a 1798 * level-triggered interrupt. We mask the source for the time of the 1799 * operation to prevent an edge-triggered interrupt escaping meanwhile. 1800 * The idea is from Manfred Spraul. --macro 1801 * 1802 * Also in the case when cpu goes offline, fixup_irqs() will forward 1803 * any unhandled interrupt on the offlined cpu to the new cpu 1804 * destination that is handling the corresponding interrupt. This 1805 * interrupt forwarding is done via IPI's. Hence, in this case also 1806 * level-triggered io-apic interrupt will be seen as an edge 1807 * interrupt in the IRR. And we can't rely on the cpu's EOI 1808 * to be broadcasted to the IO-APIC's which will clear the remoteIRR 1809 * corresponding to the level-triggered interrupt. Hence on IO-APIC's 1810 * supporting EOI register, we do an explicit EOI to clear the 1811 * remote IRR and on IO-APIC's which don't have an EOI register, 1812 * we use the above logic (mask+edge followed by unmask+level) from 1813 * Manfred Spraul to clear the remote IRR. 1814 */ 1815 i = cfg->vector; 1816 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); 1817 1818 /* 1819 * We must acknowledge the irq before we move it or the acknowledge will 1820 * not propagate properly. 1821 */ 1822 ack_APIC_irq(); 1823 1824 /* 1825 * Tail end of clearing remote IRR bit (either by delivering the EOI 1826 * message via io-apic EOI register write or simulating it using 1827 * mask+edge followed by unnask+level logic) manually when the 1828 * level triggered interrupt is seen as the edge triggered interrupt 1829 * at the cpu. 1830 */ 1831 if (!(v & (1 << (i & 0x1f)))) { 1832 atomic_inc(&irq_mis_count); 1833 eoi_ioapic_pin(cfg->vector, irq_data->chip_data); 1834 } 1835 1836 ioapic_finish_move(irq_data, moveit); 1837 } 1838 1839 static void ioapic_ir_ack_level(struct irq_data *irq_data) 1840 { 1841 struct mp_chip_data *data = irq_data->chip_data; 1842 1843 /* 1844 * Intr-remapping uses pin number as the virtual vector 1845 * in the RTE. Actual vector is programmed in 1846 * intr-remapping table entry. Hence for the io-apic 1847 * EOI we use the pin number. 1848 */ 1849 apic_ack_irq(irq_data); 1850 eoi_ioapic_pin(data->entry.vector, data); 1851 } 1852 1853 static void ioapic_configure_entry(struct irq_data *irqd) 1854 { 1855 struct mp_chip_data *mpd = irqd->chip_data; 1856 struct irq_cfg *cfg = irqd_cfg(irqd); 1857 struct irq_pin_list *entry; 1858 1859 /* 1860 * Only update when the parent is the vector domain, don't touch it 1861 * if the parent is the remapping domain. Check the installed 1862 * ioapic chip to verify that. 1863 */ 1864 if (irqd->chip == &ioapic_chip) { 1865 mpd->entry.dest = cfg->dest_apicid; 1866 mpd->entry.vector = cfg->vector; 1867 } 1868 for_each_irq_pin(entry, mpd->irq_2_pin) 1869 __ioapic_write_entry(entry->apic, entry->pin, mpd->entry); 1870 } 1871 1872 static int ioapic_set_affinity(struct irq_data *irq_data, 1873 const struct cpumask *mask, bool force) 1874 { 1875 struct irq_data *parent = irq_data->parent_data; 1876 unsigned long flags; 1877 int ret; 1878 1879 ret = parent->chip->irq_set_affinity(parent, mask, force); 1880 raw_spin_lock_irqsave(&ioapic_lock, flags); 1881 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) 1882 ioapic_configure_entry(irq_data); 1883 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1884 1885 return ret; 1886 } 1887 1888 /* 1889 * Interrupt shutdown masks the ioapic pin, but the interrupt might already 1890 * be in flight, but not yet serviced by the target CPU. That means 1891 * __synchronize_hardirq() would return and claim that everything is calmed 1892 * down. So free_irq() would proceed and deactivate the interrupt and free 1893 * resources. 1894 * 1895 * Once the target CPU comes around to service it it will find a cleared 1896 * vector and complain. While the spurious interrupt is harmless, the full 1897 * release of resources might prevent the interrupt from being acknowledged 1898 * which keeps the hardware in a weird state. 1899 * 1900 * Verify that the corresponding Remote-IRR bits are clear. 1901 */ 1902 static int ioapic_irq_get_chip_state(struct irq_data *irqd, 1903 enum irqchip_irq_state which, 1904 bool *state) 1905 { 1906 struct mp_chip_data *mcd = irqd->chip_data; 1907 struct IO_APIC_route_entry rentry; 1908 struct irq_pin_list *p; 1909 1910 if (which != IRQCHIP_STATE_ACTIVE) 1911 return -EINVAL; 1912 1913 *state = false; 1914 raw_spin_lock(&ioapic_lock); 1915 for_each_irq_pin(p, mcd->irq_2_pin) { 1916 rentry = __ioapic_read_entry(p->apic, p->pin); 1917 /* 1918 * The remote IRR is only valid in level trigger mode. It's 1919 * meaning is undefined for edge triggered interrupts and 1920 * irrelevant because the IO-APIC treats them as fire and 1921 * forget. 1922 */ 1923 if (rentry.irr && rentry.trigger) { 1924 *state = true; 1925 break; 1926 } 1927 } 1928 raw_spin_unlock(&ioapic_lock); 1929 return 0; 1930 } 1931 1932 static struct irq_chip ioapic_chip __read_mostly = { 1933 .name = "IO-APIC", 1934 .irq_startup = startup_ioapic_irq, 1935 .irq_mask = mask_ioapic_irq, 1936 .irq_unmask = unmask_ioapic_irq, 1937 .irq_ack = irq_chip_ack_parent, 1938 .irq_eoi = ioapic_ack_level, 1939 .irq_set_affinity = ioapic_set_affinity, 1940 .irq_retrigger = irq_chip_retrigger_hierarchy, 1941 .irq_get_irqchip_state = ioapic_irq_get_chip_state, 1942 .flags = IRQCHIP_SKIP_SET_WAKE, 1943 }; 1944 1945 static struct irq_chip ioapic_ir_chip __read_mostly = { 1946 .name = "IR-IO-APIC", 1947 .irq_startup = startup_ioapic_irq, 1948 .irq_mask = mask_ioapic_irq, 1949 .irq_unmask = unmask_ioapic_irq, 1950 .irq_ack = irq_chip_ack_parent, 1951 .irq_eoi = ioapic_ir_ack_level, 1952 .irq_set_affinity = ioapic_set_affinity, 1953 .irq_retrigger = irq_chip_retrigger_hierarchy, 1954 .irq_get_irqchip_state = ioapic_irq_get_chip_state, 1955 .flags = IRQCHIP_SKIP_SET_WAKE, 1956 }; 1957 1958 static inline void init_IO_APIC_traps(void) 1959 { 1960 struct irq_cfg *cfg; 1961 unsigned int irq; 1962 1963 for_each_active_irq(irq) { 1964 cfg = irq_cfg(irq); 1965 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) { 1966 /* 1967 * Hmm.. We don't have an entry for this, 1968 * so default to an old-fashioned 8259 1969 * interrupt if we can.. 1970 */ 1971 if (irq < nr_legacy_irqs()) 1972 legacy_pic->make_irq(irq); 1973 else 1974 /* Strange. Oh, well.. */ 1975 irq_set_chip(irq, &no_irq_chip); 1976 } 1977 } 1978 } 1979 1980 /* 1981 * The local APIC irq-chip implementation: 1982 */ 1983 1984 static void mask_lapic_irq(struct irq_data *data) 1985 { 1986 unsigned long v; 1987 1988 v = apic_read(APIC_LVT0); 1989 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 1990 } 1991 1992 static void unmask_lapic_irq(struct irq_data *data) 1993 { 1994 unsigned long v; 1995 1996 v = apic_read(APIC_LVT0); 1997 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); 1998 } 1999 2000 static void ack_lapic_irq(struct irq_data *data) 2001 { 2002 ack_APIC_irq(); 2003 } 2004 2005 static struct irq_chip lapic_chip __read_mostly = { 2006 .name = "local-APIC", 2007 .irq_mask = mask_lapic_irq, 2008 .irq_unmask = unmask_lapic_irq, 2009 .irq_ack = ack_lapic_irq, 2010 }; 2011 2012 static void lapic_register_intr(int irq) 2013 { 2014 irq_clear_status_flags(irq, IRQ_LEVEL); 2015 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, 2016 "edge"); 2017 } 2018 2019 /* 2020 * This looks a bit hackish but it's about the only one way of sending 2021 * a few INTA cycles to 8259As and any associated glue logic. ICR does 2022 * not support the ExtINT mode, unfortunately. We need to send these 2023 * cycles as some i82489DX-based boards have glue logic that keeps the 2024 * 8259A interrupt line asserted until INTA. --macro 2025 */ 2026 static inline void __init unlock_ExtINT_logic(void) 2027 { 2028 int apic, pin, i; 2029 struct IO_APIC_route_entry entry0, entry1; 2030 unsigned char save_control, save_freq_select; 2031 2032 pin = find_isa_irq_pin(8, mp_INT); 2033 if (pin == -1) { 2034 WARN_ON_ONCE(1); 2035 return; 2036 } 2037 apic = find_isa_irq_apic(8, mp_INT); 2038 if (apic == -1) { 2039 WARN_ON_ONCE(1); 2040 return; 2041 } 2042 2043 entry0 = ioapic_read_entry(apic, pin); 2044 clear_IO_APIC_pin(apic, pin); 2045 2046 memset(&entry1, 0, sizeof(entry1)); 2047 2048 entry1.dest_mode = IOAPIC_DEST_MODE_PHYSICAL; 2049 entry1.mask = IOAPIC_UNMASKED; 2050 entry1.dest = hard_smp_processor_id(); 2051 entry1.delivery_mode = APIC_DELIVERY_MODE_EXTINT; 2052 entry1.polarity = entry0.polarity; 2053 entry1.trigger = IOAPIC_EDGE; 2054 entry1.vector = 0; 2055 2056 ioapic_write_entry(apic, pin, entry1); 2057 2058 save_control = CMOS_READ(RTC_CONTROL); 2059 save_freq_select = CMOS_READ(RTC_FREQ_SELECT); 2060 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6, 2061 RTC_FREQ_SELECT); 2062 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL); 2063 2064 i = 100; 2065 while (i-- > 0) { 2066 mdelay(10); 2067 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF) 2068 i -= 10; 2069 } 2070 2071 CMOS_WRITE(save_control, RTC_CONTROL); 2072 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); 2073 clear_IO_APIC_pin(apic, pin); 2074 2075 ioapic_write_entry(apic, pin, entry0); 2076 } 2077 2078 static int disable_timer_pin_1 __initdata; 2079 /* Actually the next is obsolete, but keep it for paranoid reasons -AK */ 2080 static int __init disable_timer_pin_setup(char *arg) 2081 { 2082 disable_timer_pin_1 = 1; 2083 return 0; 2084 } 2085 early_param("disable_timer_pin_1", disable_timer_pin_setup); 2086 2087 static int mp_alloc_timer_irq(int ioapic, int pin) 2088 { 2089 int irq = -1; 2090 struct irq_domain *domain = mp_ioapic_irqdomain(ioapic); 2091 2092 if (domain) { 2093 struct irq_alloc_info info; 2094 2095 ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0); 2096 info.devid = mpc_ioapic_id(ioapic); 2097 info.ioapic.pin = pin; 2098 mutex_lock(&ioapic_mutex); 2099 irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info); 2100 mutex_unlock(&ioapic_mutex); 2101 } 2102 2103 return irq; 2104 } 2105 2106 /* 2107 * This code may look a bit paranoid, but it's supposed to cooperate with 2108 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ 2109 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast 2110 * fanatically on his truly buggy board. 2111 * 2112 * FIXME: really need to revamp this for all platforms. 2113 */ 2114 static inline void __init check_timer(void) 2115 { 2116 struct irq_data *irq_data = irq_get_irq_data(0); 2117 struct mp_chip_data *data = irq_data->chip_data; 2118 struct irq_cfg *cfg = irqd_cfg(irq_data); 2119 int node = cpu_to_node(0); 2120 int apic1, pin1, apic2, pin2; 2121 unsigned long flags; 2122 int no_pin1 = 0; 2123 2124 if (!global_clock_event) 2125 return; 2126 2127 local_irq_save(flags); 2128 2129 /* 2130 * get/set the timer IRQ vector: 2131 */ 2132 legacy_pic->mask(0); 2133 2134 /* 2135 * As IRQ0 is to be enabled in the 8259A, the virtual 2136 * wire has to be disabled in the local APIC. Also 2137 * timer interrupts need to be acknowledged manually in 2138 * the 8259A for the i82489DX when using the NMI 2139 * watchdog as that APIC treats NMIs as level-triggered. 2140 * The AEOI mode will finish them in the 8259A 2141 * automatically. 2142 */ 2143 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT); 2144 legacy_pic->init(1); 2145 2146 pin1 = find_isa_irq_pin(0, mp_INT); 2147 apic1 = find_isa_irq_apic(0, mp_INT); 2148 pin2 = ioapic_i8259.pin; 2149 apic2 = ioapic_i8259.apic; 2150 2151 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X " 2152 "apic1=%d pin1=%d apic2=%d pin2=%d\n", 2153 cfg->vector, apic1, pin1, apic2, pin2); 2154 2155 /* 2156 * Some BIOS writers are clueless and report the ExtINTA 2157 * I/O APIC input from the cascaded 8259A as the timer 2158 * interrupt input. So just in case, if only one pin 2159 * was found above, try it both directly and through the 2160 * 8259A. 2161 */ 2162 if (pin1 == -1) { 2163 panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC"); 2164 pin1 = pin2; 2165 apic1 = apic2; 2166 no_pin1 = 1; 2167 } else if (pin2 == -1) { 2168 pin2 = pin1; 2169 apic2 = apic1; 2170 } 2171 2172 if (pin1 != -1) { 2173 /* Ok, does IRQ0 through the IOAPIC work? */ 2174 if (no_pin1) { 2175 mp_alloc_timer_irq(apic1, pin1); 2176 } else { 2177 /* 2178 * for edge trigger, it's already unmasked, 2179 * so only need to unmask if it is level-trigger 2180 * do we really have level trigger timer? 2181 */ 2182 int idx; 2183 idx = find_irq_entry(apic1, pin1, mp_INT); 2184 if (idx != -1 && irq_trigger(idx)) 2185 unmask_ioapic_irq(irq_get_irq_data(0)); 2186 } 2187 irq_domain_deactivate_irq(irq_data); 2188 irq_domain_activate_irq(irq_data, false); 2189 if (timer_irq_works()) { 2190 if (disable_timer_pin_1 > 0) 2191 clear_IO_APIC_pin(0, pin1); 2192 goto out; 2193 } 2194 panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC"); 2195 local_irq_disable(); 2196 clear_IO_APIC_pin(apic1, pin1); 2197 if (!no_pin1) 2198 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: " 2199 "8254 timer not connected to IO-APIC\n"); 2200 2201 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer " 2202 "(IRQ0) through the 8259A ...\n"); 2203 apic_printk(APIC_QUIET, KERN_INFO 2204 "..... (found apic %d pin %d) ...\n", apic2, pin2); 2205 /* 2206 * legacy devices should be connected to IO APIC #0 2207 */ 2208 replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2); 2209 irq_domain_deactivate_irq(irq_data); 2210 irq_domain_activate_irq(irq_data, false); 2211 legacy_pic->unmask(0); 2212 if (timer_irq_works()) { 2213 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); 2214 goto out; 2215 } 2216 /* 2217 * Cleanup, just in case ... 2218 */ 2219 local_irq_disable(); 2220 legacy_pic->mask(0); 2221 clear_IO_APIC_pin(apic2, pin2); 2222 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n"); 2223 } 2224 2225 apic_printk(APIC_QUIET, KERN_INFO 2226 "...trying to set up timer as Virtual Wire IRQ...\n"); 2227 2228 lapic_register_intr(0); 2229 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ 2230 legacy_pic->unmask(0); 2231 2232 if (timer_irq_works()) { 2233 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n"); 2234 goto out; 2235 } 2236 local_irq_disable(); 2237 legacy_pic->mask(0); 2238 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector); 2239 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n"); 2240 2241 apic_printk(APIC_QUIET, KERN_INFO 2242 "...trying to set up timer as ExtINT IRQ...\n"); 2243 2244 legacy_pic->init(0); 2245 legacy_pic->make_irq(0); 2246 apic_write(APIC_LVT0, APIC_DM_EXTINT); 2247 legacy_pic->unmask(0); 2248 2249 unlock_ExtINT_logic(); 2250 2251 if (timer_irq_works()) { 2252 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n"); 2253 goto out; 2254 } 2255 local_irq_disable(); 2256 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n"); 2257 if (apic_is_x2apic_enabled()) 2258 apic_printk(APIC_QUIET, KERN_INFO 2259 "Perhaps problem with the pre-enabled x2apic mode\n" 2260 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n"); 2261 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a " 2262 "report. Then try booting with the 'noapic' option.\n"); 2263 out: 2264 local_irq_restore(flags); 2265 } 2266 2267 /* 2268 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available 2269 * to devices. However there may be an I/O APIC pin available for 2270 * this interrupt regardless. The pin may be left unconnected, but 2271 * typically it will be reused as an ExtINT cascade interrupt for 2272 * the master 8259A. In the MPS case such a pin will normally be 2273 * reported as an ExtINT interrupt in the MP table. With ACPI 2274 * there is no provision for ExtINT interrupts, and in the absence 2275 * of an override it would be treated as an ordinary ISA I/O APIC 2276 * interrupt, that is edge-triggered and unmasked by default. We 2277 * used to do this, but it caused problems on some systems because 2278 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using 2279 * the same ExtINT cascade interrupt to drive the local APIC of the 2280 * bootstrap processor. Therefore we refrain from routing IRQ2 to 2281 * the I/O APIC in all cases now. No actual device should request 2282 * it anyway. --macro 2283 */ 2284 #define PIC_IRQS (1UL << PIC_CASCADE_IR) 2285 2286 static int mp_irqdomain_create(int ioapic) 2287 { 2288 struct irq_alloc_info info; 2289 struct irq_domain *parent; 2290 int hwirqs = mp_ioapic_pin_count(ioapic); 2291 struct ioapic *ip = &ioapics[ioapic]; 2292 struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg; 2293 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2294 struct fwnode_handle *fn; 2295 char *name = "IO-APIC"; 2296 2297 if (cfg->type == IOAPIC_DOMAIN_INVALID) 2298 return 0; 2299 2300 init_irq_alloc_info(&info, NULL); 2301 info.type = X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT; 2302 info.devid = mpc_ioapic_id(ioapic); 2303 parent = irq_remapping_get_irq_domain(&info); 2304 if (!parent) 2305 parent = x86_vector_domain; 2306 else 2307 name = "IO-APIC-IR"; 2308 2309 /* Handle device tree enumerated APICs proper */ 2310 if (cfg->dev) { 2311 fn = of_node_to_fwnode(cfg->dev); 2312 } else { 2313 fn = irq_domain_alloc_named_id_fwnode(name, ioapic); 2314 if (!fn) 2315 return -ENOMEM; 2316 } 2317 2318 ip->irqdomain = irq_domain_create_linear(fn, hwirqs, cfg->ops, 2319 (void *)(long)ioapic); 2320 2321 if (!ip->irqdomain) { 2322 /* Release fw handle if it was allocated above */ 2323 if (!cfg->dev) 2324 irq_domain_free_fwnode(fn); 2325 return -ENOMEM; 2326 } 2327 2328 ip->irqdomain->parent = parent; 2329 2330 if (cfg->type == IOAPIC_DOMAIN_LEGACY || 2331 cfg->type == IOAPIC_DOMAIN_STRICT) 2332 ioapic_dynirq_base = max(ioapic_dynirq_base, 2333 gsi_cfg->gsi_end + 1); 2334 2335 return 0; 2336 } 2337 2338 static void ioapic_destroy_irqdomain(int idx) 2339 { 2340 struct ioapic_domain_cfg *cfg = &ioapics[idx].irqdomain_cfg; 2341 struct fwnode_handle *fn = ioapics[idx].irqdomain->fwnode; 2342 2343 if (ioapics[idx].irqdomain) { 2344 irq_domain_remove(ioapics[idx].irqdomain); 2345 if (!cfg->dev) 2346 irq_domain_free_fwnode(fn); 2347 ioapics[idx].irqdomain = NULL; 2348 } 2349 } 2350 2351 void __init setup_IO_APIC(void) 2352 { 2353 int ioapic; 2354 2355 if (skip_ioapic_setup || !nr_ioapics) 2356 return; 2357 2358 io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL; 2359 2360 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n"); 2361 for_each_ioapic(ioapic) 2362 BUG_ON(mp_irqdomain_create(ioapic)); 2363 2364 /* 2365 * Set up IO-APIC IRQ routing. 2366 */ 2367 x86_init.mpparse.setup_ioapic_ids(); 2368 2369 sync_Arb_IDs(); 2370 setup_IO_APIC_irqs(); 2371 init_IO_APIC_traps(); 2372 if (nr_legacy_irqs()) 2373 check_timer(); 2374 2375 ioapic_initialized = 1; 2376 } 2377 2378 static void resume_ioapic_id(int ioapic_idx) 2379 { 2380 unsigned long flags; 2381 union IO_APIC_reg_00 reg_00; 2382 2383 raw_spin_lock_irqsave(&ioapic_lock, flags); 2384 reg_00.raw = io_apic_read(ioapic_idx, 0); 2385 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) { 2386 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx); 2387 io_apic_write(ioapic_idx, 0, reg_00.raw); 2388 } 2389 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2390 } 2391 2392 static void ioapic_resume(void) 2393 { 2394 int ioapic_idx; 2395 2396 for_each_ioapic_reverse(ioapic_idx) 2397 resume_ioapic_id(ioapic_idx); 2398 2399 restore_ioapic_entries(); 2400 } 2401 2402 static struct syscore_ops ioapic_syscore_ops = { 2403 .suspend = save_ioapic_entries, 2404 .resume = ioapic_resume, 2405 }; 2406 2407 static int __init ioapic_init_ops(void) 2408 { 2409 register_syscore_ops(&ioapic_syscore_ops); 2410 2411 return 0; 2412 } 2413 2414 device_initcall(ioapic_init_ops); 2415 2416 static int io_apic_get_redir_entries(int ioapic) 2417 { 2418 union IO_APIC_reg_01 reg_01; 2419 unsigned long flags; 2420 2421 raw_spin_lock_irqsave(&ioapic_lock, flags); 2422 reg_01.raw = io_apic_read(ioapic, 1); 2423 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2424 2425 /* The register returns the maximum index redir index 2426 * supported, which is one less than the total number of redir 2427 * entries. 2428 */ 2429 return reg_01.bits.entries + 1; 2430 } 2431 2432 unsigned int arch_dynirq_lower_bound(unsigned int from) 2433 { 2434 /* 2435 * dmar_alloc_hwirq() may be called before setup_IO_APIC(), so use 2436 * gsi_top if ioapic_dynirq_base hasn't been initialized yet. 2437 */ 2438 if (!ioapic_initialized) 2439 return gsi_top; 2440 /* 2441 * For DT enabled machines ioapic_dynirq_base is irrelevant and not 2442 * updated. So simply return @from if ioapic_dynirq_base == 0. 2443 */ 2444 return ioapic_dynirq_base ? : from; 2445 } 2446 2447 #ifdef CONFIG_X86_32 2448 static int io_apic_get_unique_id(int ioapic, int apic_id) 2449 { 2450 union IO_APIC_reg_00 reg_00; 2451 static physid_mask_t apic_id_map = PHYSID_MASK_NONE; 2452 physid_mask_t tmp; 2453 unsigned long flags; 2454 int i = 0; 2455 2456 /* 2457 * The P4 platform supports up to 256 APIC IDs on two separate APIC 2458 * buses (one for LAPICs, one for IOAPICs), where predecessors only 2459 * supports up to 16 on one shared APIC bus. 2460 * 2461 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full 2462 * advantage of new APIC bus architecture. 2463 */ 2464 2465 if (physids_empty(apic_id_map)) 2466 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map); 2467 2468 raw_spin_lock_irqsave(&ioapic_lock, flags); 2469 reg_00.raw = io_apic_read(ioapic, 0); 2470 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2471 2472 if (apic_id >= get_physical_broadcast()) { 2473 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying " 2474 "%d\n", ioapic, apic_id, reg_00.bits.ID); 2475 apic_id = reg_00.bits.ID; 2476 } 2477 2478 /* 2479 * Every APIC in a system must have a unique ID or we get lots of nice 2480 * 'stuck on smp_invalidate_needed IPI wait' messages. 2481 */ 2482 if (apic->check_apicid_used(&apic_id_map, apic_id)) { 2483 2484 for (i = 0; i < get_physical_broadcast(); i++) { 2485 if (!apic->check_apicid_used(&apic_id_map, i)) 2486 break; 2487 } 2488 2489 if (i == get_physical_broadcast()) 2490 panic("Max apic_id exceeded!\n"); 2491 2492 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, " 2493 "trying %d\n", ioapic, apic_id, i); 2494 2495 apic_id = i; 2496 } 2497 2498 apic->apicid_to_cpu_present(apic_id, &tmp); 2499 physids_or(apic_id_map, apic_id_map, tmp); 2500 2501 if (reg_00.bits.ID != apic_id) { 2502 reg_00.bits.ID = apic_id; 2503 2504 raw_spin_lock_irqsave(&ioapic_lock, flags); 2505 io_apic_write(ioapic, 0, reg_00.raw); 2506 reg_00.raw = io_apic_read(ioapic, 0); 2507 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2508 2509 /* Sanity check */ 2510 if (reg_00.bits.ID != apic_id) { 2511 pr_err("IOAPIC[%d]: Unable to change apic_id!\n", 2512 ioapic); 2513 return -1; 2514 } 2515 } 2516 2517 apic_printk(APIC_VERBOSE, KERN_INFO 2518 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id); 2519 2520 return apic_id; 2521 } 2522 2523 static u8 io_apic_unique_id(int idx, u8 id) 2524 { 2525 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 2526 !APIC_XAPIC(boot_cpu_apic_version)) 2527 return io_apic_get_unique_id(idx, id); 2528 else 2529 return id; 2530 } 2531 #else 2532 static u8 io_apic_unique_id(int idx, u8 id) 2533 { 2534 union IO_APIC_reg_00 reg_00; 2535 DECLARE_BITMAP(used, 256); 2536 unsigned long flags; 2537 u8 new_id; 2538 int i; 2539 2540 bitmap_zero(used, 256); 2541 for_each_ioapic(i) 2542 __set_bit(mpc_ioapic_id(i), used); 2543 2544 /* Hand out the requested id if available */ 2545 if (!test_bit(id, used)) 2546 return id; 2547 2548 /* 2549 * Read the current id from the ioapic and keep it if 2550 * available. 2551 */ 2552 raw_spin_lock_irqsave(&ioapic_lock, flags); 2553 reg_00.raw = io_apic_read(idx, 0); 2554 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2555 new_id = reg_00.bits.ID; 2556 if (!test_bit(new_id, used)) { 2557 apic_printk(APIC_VERBOSE, KERN_INFO 2558 "IOAPIC[%d]: Using reg apic_id %d instead of %d\n", 2559 idx, new_id, id); 2560 return new_id; 2561 } 2562 2563 /* 2564 * Get the next free id and write it to the ioapic. 2565 */ 2566 new_id = find_first_zero_bit(used, 256); 2567 reg_00.bits.ID = new_id; 2568 raw_spin_lock_irqsave(&ioapic_lock, flags); 2569 io_apic_write(idx, 0, reg_00.raw); 2570 reg_00.raw = io_apic_read(idx, 0); 2571 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2572 /* Sanity check */ 2573 BUG_ON(reg_00.bits.ID != new_id); 2574 2575 return new_id; 2576 } 2577 #endif 2578 2579 static int io_apic_get_version(int ioapic) 2580 { 2581 union IO_APIC_reg_01 reg_01; 2582 unsigned long flags; 2583 2584 raw_spin_lock_irqsave(&ioapic_lock, flags); 2585 reg_01.raw = io_apic_read(ioapic, 1); 2586 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2587 2588 return reg_01.bits.version; 2589 } 2590 2591 int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity) 2592 { 2593 int ioapic, pin, idx; 2594 2595 if (skip_ioapic_setup) 2596 return -1; 2597 2598 ioapic = mp_find_ioapic(gsi); 2599 if (ioapic < 0) 2600 return -1; 2601 2602 pin = mp_find_ioapic_pin(ioapic, gsi); 2603 if (pin < 0) 2604 return -1; 2605 2606 idx = find_irq_entry(ioapic, pin, mp_INT); 2607 if (idx < 0) 2608 return -1; 2609 2610 *trigger = irq_trigger(idx); 2611 *polarity = irq_polarity(idx); 2612 return 0; 2613 } 2614 2615 /* 2616 * This function updates target affinity of IOAPIC interrupts to include 2617 * the CPUs which came online during SMP bringup. 2618 */ 2619 #define IOAPIC_RESOURCE_NAME_SIZE 11 2620 2621 static struct resource *ioapic_resources; 2622 2623 static struct resource * __init ioapic_setup_resources(void) 2624 { 2625 unsigned long n; 2626 struct resource *res; 2627 char *mem; 2628 int i; 2629 2630 if (nr_ioapics == 0) 2631 return NULL; 2632 2633 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource); 2634 n *= nr_ioapics; 2635 2636 mem = memblock_alloc(n, SMP_CACHE_BYTES); 2637 if (!mem) 2638 panic("%s: Failed to allocate %lu bytes\n", __func__, n); 2639 res = (void *)mem; 2640 2641 mem += sizeof(struct resource) * nr_ioapics; 2642 2643 for_each_ioapic(i) { 2644 res[i].name = mem; 2645 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY; 2646 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i); 2647 mem += IOAPIC_RESOURCE_NAME_SIZE; 2648 ioapics[i].iomem_res = &res[i]; 2649 } 2650 2651 ioapic_resources = res; 2652 2653 return res; 2654 } 2655 2656 void __init io_apic_init_mappings(void) 2657 { 2658 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0; 2659 struct resource *ioapic_res; 2660 int i; 2661 2662 ioapic_res = ioapic_setup_resources(); 2663 for_each_ioapic(i) { 2664 if (smp_found_config) { 2665 ioapic_phys = mpc_ioapic_addr(i); 2666 #ifdef CONFIG_X86_32 2667 if (!ioapic_phys) { 2668 printk(KERN_ERR 2669 "WARNING: bogus zero IO-APIC " 2670 "address found in MPTABLE, " 2671 "disabling IO/APIC support!\n"); 2672 smp_found_config = 0; 2673 skip_ioapic_setup = 1; 2674 goto fake_ioapic_page; 2675 } 2676 #endif 2677 } else { 2678 #ifdef CONFIG_X86_32 2679 fake_ioapic_page: 2680 #endif 2681 ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE, 2682 PAGE_SIZE); 2683 if (!ioapic_phys) 2684 panic("%s: Failed to allocate %lu bytes align=0x%lx\n", 2685 __func__, PAGE_SIZE, PAGE_SIZE); 2686 ioapic_phys = __pa(ioapic_phys); 2687 } 2688 set_fixmap_nocache(idx, ioapic_phys); 2689 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n", 2690 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK), 2691 ioapic_phys); 2692 idx++; 2693 2694 ioapic_res->start = ioapic_phys; 2695 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1; 2696 ioapic_res++; 2697 } 2698 } 2699 2700 void __init ioapic_insert_resources(void) 2701 { 2702 int i; 2703 struct resource *r = ioapic_resources; 2704 2705 if (!r) { 2706 if (nr_ioapics > 0) 2707 printk(KERN_ERR 2708 "IO APIC resources couldn't be allocated.\n"); 2709 return; 2710 } 2711 2712 for_each_ioapic(i) { 2713 insert_resource(&iomem_resource, r); 2714 r++; 2715 } 2716 } 2717 2718 int mp_find_ioapic(u32 gsi) 2719 { 2720 int i; 2721 2722 if (nr_ioapics == 0) 2723 return -1; 2724 2725 /* Find the IOAPIC that manages this GSI. */ 2726 for_each_ioapic(i) { 2727 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i); 2728 if (gsi >= gsi_cfg->gsi_base && gsi <= gsi_cfg->gsi_end) 2729 return i; 2730 } 2731 2732 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi); 2733 return -1; 2734 } 2735 2736 int mp_find_ioapic_pin(int ioapic, u32 gsi) 2737 { 2738 struct mp_ioapic_gsi *gsi_cfg; 2739 2740 if (WARN_ON(ioapic < 0)) 2741 return -1; 2742 2743 gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2744 if (WARN_ON(gsi > gsi_cfg->gsi_end)) 2745 return -1; 2746 2747 return gsi - gsi_cfg->gsi_base; 2748 } 2749 2750 static int bad_ioapic_register(int idx) 2751 { 2752 union IO_APIC_reg_00 reg_00; 2753 union IO_APIC_reg_01 reg_01; 2754 union IO_APIC_reg_02 reg_02; 2755 2756 reg_00.raw = io_apic_read(idx, 0); 2757 reg_01.raw = io_apic_read(idx, 1); 2758 reg_02.raw = io_apic_read(idx, 2); 2759 2760 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) { 2761 pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n", 2762 mpc_ioapic_addr(idx)); 2763 return 1; 2764 } 2765 2766 return 0; 2767 } 2768 2769 static int find_free_ioapic_entry(void) 2770 { 2771 int idx; 2772 2773 for (idx = 0; idx < MAX_IO_APICS; idx++) 2774 if (ioapics[idx].nr_registers == 0) 2775 return idx; 2776 2777 return MAX_IO_APICS; 2778 } 2779 2780 /** 2781 * mp_register_ioapic - Register an IOAPIC device 2782 * @id: hardware IOAPIC ID 2783 * @address: physical address of IOAPIC register area 2784 * @gsi_base: base of GSI associated with the IOAPIC 2785 * @cfg: configuration information for the IOAPIC 2786 */ 2787 int mp_register_ioapic(int id, u32 address, u32 gsi_base, 2788 struct ioapic_domain_cfg *cfg) 2789 { 2790 bool hotplug = !!ioapic_initialized; 2791 struct mp_ioapic_gsi *gsi_cfg; 2792 int idx, ioapic, entries; 2793 u32 gsi_end; 2794 2795 if (!address) { 2796 pr_warn("Bogus (zero) I/O APIC address found, skipping!\n"); 2797 return -EINVAL; 2798 } 2799 for_each_ioapic(ioapic) 2800 if (ioapics[ioapic].mp_config.apicaddr == address) { 2801 pr_warn("address 0x%x conflicts with IOAPIC%d\n", 2802 address, ioapic); 2803 return -EEXIST; 2804 } 2805 2806 idx = find_free_ioapic_entry(); 2807 if (idx >= MAX_IO_APICS) { 2808 pr_warn("Max # of I/O APICs (%d) exceeded (found %d), skipping\n", 2809 MAX_IO_APICS, idx); 2810 return -ENOSPC; 2811 } 2812 2813 ioapics[idx].mp_config.type = MP_IOAPIC; 2814 ioapics[idx].mp_config.flags = MPC_APIC_USABLE; 2815 ioapics[idx].mp_config.apicaddr = address; 2816 2817 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address); 2818 if (bad_ioapic_register(idx)) { 2819 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2820 return -ENODEV; 2821 } 2822 2823 ioapics[idx].mp_config.apicid = io_apic_unique_id(idx, id); 2824 ioapics[idx].mp_config.apicver = io_apic_get_version(idx); 2825 2826 /* 2827 * Build basic GSI lookup table to facilitate gsi->io_apic lookups 2828 * and to prevent reprogramming of IOAPIC pins (PCI GSIs). 2829 */ 2830 entries = io_apic_get_redir_entries(idx); 2831 gsi_end = gsi_base + entries - 1; 2832 for_each_ioapic(ioapic) { 2833 gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2834 if ((gsi_base >= gsi_cfg->gsi_base && 2835 gsi_base <= gsi_cfg->gsi_end) || 2836 (gsi_end >= gsi_cfg->gsi_base && 2837 gsi_end <= gsi_cfg->gsi_end)) { 2838 pr_warn("GSI range [%u-%u] for new IOAPIC conflicts with GSI[%u-%u]\n", 2839 gsi_base, gsi_end, 2840 gsi_cfg->gsi_base, gsi_cfg->gsi_end); 2841 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2842 return -ENOSPC; 2843 } 2844 } 2845 gsi_cfg = mp_ioapic_gsi_routing(idx); 2846 gsi_cfg->gsi_base = gsi_base; 2847 gsi_cfg->gsi_end = gsi_end; 2848 2849 ioapics[idx].irqdomain = NULL; 2850 ioapics[idx].irqdomain_cfg = *cfg; 2851 2852 /* 2853 * If mp_register_ioapic() is called during early boot stage when 2854 * walking ACPI/SFI/DT tables, it's too early to create irqdomain, 2855 * we are still using bootmem allocator. So delay it to setup_IO_APIC(). 2856 */ 2857 if (hotplug) { 2858 if (mp_irqdomain_create(idx)) { 2859 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2860 return -ENOMEM; 2861 } 2862 alloc_ioapic_saved_registers(idx); 2863 } 2864 2865 if (gsi_cfg->gsi_end >= gsi_top) 2866 gsi_top = gsi_cfg->gsi_end + 1; 2867 if (nr_ioapics <= idx) 2868 nr_ioapics = idx + 1; 2869 2870 /* Set nr_registers to mark entry present */ 2871 ioapics[idx].nr_registers = entries; 2872 2873 pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n", 2874 idx, mpc_ioapic_id(idx), 2875 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx), 2876 gsi_cfg->gsi_base, gsi_cfg->gsi_end); 2877 2878 return 0; 2879 } 2880 2881 int mp_unregister_ioapic(u32 gsi_base) 2882 { 2883 int ioapic, pin; 2884 int found = 0; 2885 2886 for_each_ioapic(ioapic) 2887 if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) { 2888 found = 1; 2889 break; 2890 } 2891 if (!found) { 2892 pr_warn("can't find IOAPIC for GSI %d\n", gsi_base); 2893 return -ENODEV; 2894 } 2895 2896 for_each_pin(ioapic, pin) { 2897 u32 gsi = mp_pin_to_gsi(ioapic, pin); 2898 int irq = mp_map_gsi_to_irq(gsi, 0, NULL); 2899 struct mp_chip_data *data; 2900 2901 if (irq >= 0) { 2902 data = irq_get_chip_data(irq); 2903 if (data && data->count) { 2904 pr_warn("pin%d on IOAPIC%d is still in use.\n", 2905 pin, ioapic); 2906 return -EBUSY; 2907 } 2908 } 2909 } 2910 2911 /* Mark entry not present */ 2912 ioapics[ioapic].nr_registers = 0; 2913 ioapic_destroy_irqdomain(ioapic); 2914 free_ioapic_saved_registers(ioapic); 2915 if (ioapics[ioapic].iomem_res) 2916 release_resource(ioapics[ioapic].iomem_res); 2917 clear_fixmap(FIX_IO_APIC_BASE_0 + ioapic); 2918 memset(&ioapics[ioapic], 0, sizeof(ioapics[ioapic])); 2919 2920 return 0; 2921 } 2922 2923 int mp_ioapic_registered(u32 gsi_base) 2924 { 2925 int ioapic; 2926 2927 for_each_ioapic(ioapic) 2928 if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) 2929 return 1; 2930 2931 return 0; 2932 } 2933 2934 static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data, 2935 struct irq_alloc_info *info) 2936 { 2937 if (info && info->ioapic.valid) { 2938 data->trigger = info->ioapic.trigger; 2939 data->polarity = info->ioapic.polarity; 2940 } else if (acpi_get_override_irq(gsi, &data->trigger, 2941 &data->polarity) < 0) { 2942 /* PCI interrupts are always active low level triggered. */ 2943 data->trigger = IOAPIC_LEVEL; 2944 data->polarity = IOAPIC_POL_LOW; 2945 } 2946 } 2947 2948 static void mp_setup_entry(struct irq_cfg *cfg, struct mp_chip_data *data, 2949 struct IO_APIC_route_entry *entry) 2950 { 2951 memset(entry, 0, sizeof(*entry)); 2952 entry->delivery_mode = apic->delivery_mode; 2953 entry->dest_mode = apic->irq_dest_mode; 2954 entry->dest = cfg->dest_apicid; 2955 entry->vector = cfg->vector; 2956 entry->trigger = data->trigger; 2957 entry->polarity = data->polarity; 2958 /* 2959 * Mask level triggered irqs. Edge triggered irqs are masked 2960 * by the irq core code in case they fire. 2961 */ 2962 if (data->trigger == IOAPIC_LEVEL) 2963 entry->mask = IOAPIC_MASKED; 2964 else 2965 entry->mask = IOAPIC_UNMASKED; 2966 } 2967 2968 int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, 2969 unsigned int nr_irqs, void *arg) 2970 { 2971 int ret, ioapic, pin; 2972 struct irq_cfg *cfg; 2973 struct irq_data *irq_data; 2974 struct mp_chip_data *data; 2975 struct irq_alloc_info *info = arg; 2976 unsigned long flags; 2977 2978 if (!info || nr_irqs > 1) 2979 return -EINVAL; 2980 irq_data = irq_domain_get_irq_data(domain, virq); 2981 if (!irq_data) 2982 return -EINVAL; 2983 2984 ioapic = mp_irqdomain_ioapic_idx(domain); 2985 pin = info->ioapic.pin; 2986 if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0) 2987 return -EEXIST; 2988 2989 data = kzalloc(sizeof(*data), GFP_KERNEL); 2990 if (!data) 2991 return -ENOMEM; 2992 2993 info->ioapic.entry = &data->entry; 2994 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info); 2995 if (ret < 0) { 2996 kfree(data); 2997 return ret; 2998 } 2999 3000 INIT_LIST_HEAD(&data->irq_2_pin); 3001 irq_data->hwirq = info->ioapic.pin; 3002 irq_data->chip = (domain->parent == x86_vector_domain) ? 3003 &ioapic_chip : &ioapic_ir_chip; 3004 irq_data->chip_data = data; 3005 mp_irqdomain_get_attr(mp_pin_to_gsi(ioapic, pin), data, info); 3006 3007 cfg = irqd_cfg(irq_data); 3008 add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin); 3009 3010 local_irq_save(flags); 3011 if (info->ioapic.entry) 3012 mp_setup_entry(cfg, data, info->ioapic.entry); 3013 mp_register_handler(virq, data->trigger); 3014 if (virq < nr_legacy_irqs()) 3015 legacy_pic->mask(virq); 3016 local_irq_restore(flags); 3017 3018 apic_printk(APIC_VERBOSE, KERN_DEBUG 3019 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i Dest:%d)\n", 3020 ioapic, mpc_ioapic_id(ioapic), pin, cfg->vector, 3021 virq, data->trigger, data->polarity, cfg->dest_apicid); 3022 3023 return 0; 3024 } 3025 3026 void mp_irqdomain_free(struct irq_domain *domain, unsigned int virq, 3027 unsigned int nr_irqs) 3028 { 3029 struct irq_data *irq_data; 3030 struct mp_chip_data *data; 3031 3032 BUG_ON(nr_irqs != 1); 3033 irq_data = irq_domain_get_irq_data(domain, virq); 3034 if (irq_data && irq_data->chip_data) { 3035 data = irq_data->chip_data; 3036 __remove_pin_from_irq(data, mp_irqdomain_ioapic_idx(domain), 3037 (int)irq_data->hwirq); 3038 WARN_ON(!list_empty(&data->irq_2_pin)); 3039 kfree(irq_data->chip_data); 3040 } 3041 irq_domain_free_irqs_top(domain, virq, nr_irqs); 3042 } 3043 3044 int mp_irqdomain_activate(struct irq_domain *domain, 3045 struct irq_data *irq_data, bool reserve) 3046 { 3047 unsigned long flags; 3048 3049 raw_spin_lock_irqsave(&ioapic_lock, flags); 3050 ioapic_configure_entry(irq_data); 3051 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 3052 return 0; 3053 } 3054 3055 void mp_irqdomain_deactivate(struct irq_domain *domain, 3056 struct irq_data *irq_data) 3057 { 3058 /* It won't be called for IRQ with multiple IOAPIC pins associated */ 3059 ioapic_mask_entry(mp_irqdomain_ioapic_idx(domain), 3060 (int)irq_data->hwirq); 3061 } 3062 3063 int mp_irqdomain_ioapic_idx(struct irq_domain *domain) 3064 { 3065 return (int)(long)domain->host_data; 3066 } 3067 3068 const struct irq_domain_ops mp_ioapic_irqdomain_ops = { 3069 .alloc = mp_irqdomain_alloc, 3070 .free = mp_irqdomain_free, 3071 .activate = mp_irqdomain_activate, 3072 .deactivate = mp_irqdomain_deactivate, 3073 }; 3074