1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel IO-APIC support for multi-Pentium hosts. 4 * 5 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo 6 * 7 * Many thanks to Stig Venaas for trying out countless experimental 8 * patches and reporting/debugging problems patiently! 9 * 10 * (c) 1999, Multiple IO-APIC support, developed by 11 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and 12 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>, 13 * further tested and cleaned up by Zach Brown <zab@redhat.com> 14 * and Ingo Molnar <mingo@redhat.com> 15 * 16 * Fixes 17 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 18 * thanks to Eric Gilmore 19 * and Rolf G. Tews 20 * for testing these extensively 21 * Paul Diefenbaugh : Added full ACPI support 22 * 23 * Historical information which is worth to be preserved: 24 * 25 * - SiS APIC rmw bug: 26 * 27 * We used to have a workaround for a bug in SiS chips which 28 * required to rewrite the index register for a read-modify-write 29 * operation as the chip lost the index information which was 30 * setup for the read already. We cache the data now, so that 31 * workaround has been removed. 32 */ 33 34 #include <linux/mm.h> 35 #include <linux/interrupt.h> 36 #include <linux/irq.h> 37 #include <linux/init.h> 38 #include <linux/delay.h> 39 #include <linux/sched.h> 40 #include <linux/pci.h> 41 #include <linux/mc146818rtc.h> 42 #include <linux/compiler.h> 43 #include <linux/acpi.h> 44 #include <linux/export.h> 45 #include <linux/syscore_ops.h> 46 #include <linux/freezer.h> 47 #include <linux/kthread.h> 48 #include <linux/jiffies.h> /* time_after() */ 49 #include <linux/slab.h> 50 #include <linux/memblock.h> 51 52 #include <asm/irqdomain.h> 53 #include <asm/io.h> 54 #include <asm/smp.h> 55 #include <asm/cpu.h> 56 #include <asm/desc.h> 57 #include <asm/proto.h> 58 #include <asm/acpi.h> 59 #include <asm/dma.h> 60 #include <asm/timer.h> 61 #include <asm/time.h> 62 #include <asm/i8259.h> 63 #include <asm/setup.h> 64 #include <asm/irq_remapping.h> 65 #include <asm/hw_irq.h> 66 67 #include <asm/apic.h> 68 69 #define for_each_ioapic(idx) \ 70 for ((idx) = 0; (idx) < nr_ioapics; (idx)++) 71 #define for_each_ioapic_reverse(idx) \ 72 for ((idx) = nr_ioapics - 1; (idx) >= 0; (idx)--) 73 #define for_each_pin(idx, pin) \ 74 for ((pin) = 0; (pin) < ioapics[(idx)].nr_registers; (pin)++) 75 #define for_each_ioapic_pin(idx, pin) \ 76 for_each_ioapic((idx)) \ 77 for_each_pin((idx), (pin)) 78 #define for_each_irq_pin(entry, head) \ 79 list_for_each_entry(entry, &head, list) 80 81 static DEFINE_RAW_SPINLOCK(ioapic_lock); 82 static DEFINE_MUTEX(ioapic_mutex); 83 static unsigned int ioapic_dynirq_base; 84 static int ioapic_initialized; 85 86 struct irq_pin_list { 87 struct list_head list; 88 int apic, pin; 89 }; 90 91 struct mp_chip_data { 92 struct list_head irq_2_pin; 93 struct IO_APIC_route_entry entry; 94 bool is_level; 95 bool active_low; 96 bool isa_irq; 97 u32 count; 98 }; 99 100 struct mp_ioapic_gsi { 101 u32 gsi_base; 102 u32 gsi_end; 103 }; 104 105 static struct ioapic { 106 /* 107 * # of IRQ routing registers 108 */ 109 int nr_registers; 110 /* 111 * Saved state during suspend/resume, or while enabling intr-remap. 112 */ 113 struct IO_APIC_route_entry *saved_registers; 114 /* I/O APIC config */ 115 struct mpc_ioapic mp_config; 116 /* IO APIC gsi routing info */ 117 struct mp_ioapic_gsi gsi_config; 118 struct ioapic_domain_cfg irqdomain_cfg; 119 struct irq_domain *irqdomain; 120 struct resource *iomem_res; 121 } ioapics[MAX_IO_APICS]; 122 123 #define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver 124 125 int mpc_ioapic_id(int ioapic_idx) 126 { 127 return ioapics[ioapic_idx].mp_config.apicid; 128 } 129 130 unsigned int mpc_ioapic_addr(int ioapic_idx) 131 { 132 return ioapics[ioapic_idx].mp_config.apicaddr; 133 } 134 135 static inline struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx) 136 { 137 return &ioapics[ioapic_idx].gsi_config; 138 } 139 140 static inline int mp_ioapic_pin_count(int ioapic) 141 { 142 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic); 143 144 return gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1; 145 } 146 147 static inline u32 mp_pin_to_gsi(int ioapic, int pin) 148 { 149 return mp_ioapic_gsi_routing(ioapic)->gsi_base + pin; 150 } 151 152 static inline bool mp_is_legacy_irq(int irq) 153 { 154 return irq >= 0 && irq < nr_legacy_irqs(); 155 } 156 157 static inline struct irq_domain *mp_ioapic_irqdomain(int ioapic) 158 { 159 return ioapics[ioapic].irqdomain; 160 } 161 162 int nr_ioapics; 163 164 /* The one past the highest gsi number used */ 165 u32 gsi_top; 166 167 /* MP IRQ source entries */ 168 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES]; 169 170 /* # of MP IRQ source entries */ 171 int mp_irq_entries; 172 173 #ifdef CONFIG_EISA 174 int mp_bus_id_to_type[MAX_MP_BUSSES]; 175 #endif 176 177 DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); 178 179 int skip_ioapic_setup; 180 181 /** 182 * disable_ioapic_support() - disables ioapic support at runtime 183 */ 184 void disable_ioapic_support(void) 185 { 186 #ifdef CONFIG_PCI 187 noioapicquirk = 1; 188 noioapicreroute = -1; 189 #endif 190 skip_ioapic_setup = 1; 191 } 192 193 static int __init parse_noapic(char *str) 194 { 195 /* disable IO-APIC */ 196 disable_ioapic_support(); 197 return 0; 198 } 199 early_param("noapic", parse_noapic); 200 201 /* Will be called in mpparse/acpi/sfi codes for saving IRQ info */ 202 void mp_save_irq(struct mpc_intsrc *m) 203 { 204 int i; 205 206 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x," 207 " IRQ %02x, APIC ID %x, APIC INT %02x\n", 208 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus, 209 m->srcbusirq, m->dstapic, m->dstirq); 210 211 for (i = 0; i < mp_irq_entries; i++) { 212 if (!memcmp(&mp_irqs[i], m, sizeof(*m))) 213 return; 214 } 215 216 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m)); 217 if (++mp_irq_entries == MAX_IRQ_SOURCES) 218 panic("Max # of irq sources exceeded!!\n"); 219 } 220 221 static void alloc_ioapic_saved_registers(int idx) 222 { 223 size_t size; 224 225 if (ioapics[idx].saved_registers) 226 return; 227 228 size = sizeof(struct IO_APIC_route_entry) * ioapics[idx].nr_registers; 229 ioapics[idx].saved_registers = kzalloc(size, GFP_KERNEL); 230 if (!ioapics[idx].saved_registers) 231 pr_err("IOAPIC %d: suspend/resume impossible!\n", idx); 232 } 233 234 static void free_ioapic_saved_registers(int idx) 235 { 236 kfree(ioapics[idx].saved_registers); 237 ioapics[idx].saved_registers = NULL; 238 } 239 240 int __init arch_early_ioapic_init(void) 241 { 242 int i; 243 244 if (!nr_legacy_irqs()) 245 io_apic_irqs = ~0UL; 246 247 for_each_ioapic(i) 248 alloc_ioapic_saved_registers(i); 249 250 return 0; 251 } 252 253 struct io_apic { 254 unsigned int index; 255 unsigned int unused[3]; 256 unsigned int data; 257 unsigned int unused2[11]; 258 unsigned int eoi; 259 }; 260 261 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) 262 { 263 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx) 264 + (mpc_ioapic_addr(idx) & ~PAGE_MASK); 265 } 266 267 static inline void io_apic_eoi(unsigned int apic, unsigned int vector) 268 { 269 struct io_apic __iomem *io_apic = io_apic_base(apic); 270 writel(vector, &io_apic->eoi); 271 } 272 273 unsigned int native_io_apic_read(unsigned int apic, unsigned int reg) 274 { 275 struct io_apic __iomem *io_apic = io_apic_base(apic); 276 writel(reg, &io_apic->index); 277 return readl(&io_apic->data); 278 } 279 280 static void io_apic_write(unsigned int apic, unsigned int reg, 281 unsigned int value) 282 { 283 struct io_apic __iomem *io_apic = io_apic_base(apic); 284 285 writel(reg, &io_apic->index); 286 writel(value, &io_apic->data); 287 } 288 289 static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin) 290 { 291 struct IO_APIC_route_entry entry; 292 293 entry.w1 = io_apic_read(apic, 0x10 + 2 * pin); 294 entry.w2 = io_apic_read(apic, 0x11 + 2 * pin); 295 296 return entry; 297 } 298 299 static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin) 300 { 301 struct IO_APIC_route_entry entry; 302 unsigned long flags; 303 304 raw_spin_lock_irqsave(&ioapic_lock, flags); 305 entry = __ioapic_read_entry(apic, pin); 306 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 307 308 return entry; 309 } 310 311 /* 312 * When we write a new IO APIC routing entry, we need to write the high 313 * word first! If the mask bit in the low word is clear, we will enable 314 * the interrupt, and we need to make sure the entry is fully populated 315 * before that happens. 316 */ 317 static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 318 { 319 io_apic_write(apic, 0x11 + 2*pin, e.w2); 320 io_apic_write(apic, 0x10 + 2*pin, e.w1); 321 } 322 323 static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 324 { 325 unsigned long flags; 326 327 raw_spin_lock_irqsave(&ioapic_lock, flags); 328 __ioapic_write_entry(apic, pin, e); 329 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 330 } 331 332 /* 333 * When we mask an IO APIC routing entry, we need to write the low 334 * word first, in order to set the mask bit before we change the 335 * high bits! 336 */ 337 static void ioapic_mask_entry(int apic, int pin) 338 { 339 struct IO_APIC_route_entry e = { .masked = true }; 340 unsigned long flags; 341 342 raw_spin_lock_irqsave(&ioapic_lock, flags); 343 io_apic_write(apic, 0x10 + 2*pin, e.w1); 344 io_apic_write(apic, 0x11 + 2*pin, e.w2); 345 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 346 } 347 348 /* 349 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are 350 * shared ISA-space IRQs, so we have to support them. We are super 351 * fast in the common case, and fast for shared ISA-space IRQs. 352 */ 353 static int __add_pin_to_irq_node(struct mp_chip_data *data, 354 int node, int apic, int pin) 355 { 356 struct irq_pin_list *entry; 357 358 /* don't allow duplicates */ 359 for_each_irq_pin(entry, data->irq_2_pin) 360 if (entry->apic == apic && entry->pin == pin) 361 return 0; 362 363 entry = kzalloc_node(sizeof(struct irq_pin_list), GFP_ATOMIC, node); 364 if (!entry) { 365 pr_err("can not alloc irq_pin_list (%d,%d,%d)\n", 366 node, apic, pin); 367 return -ENOMEM; 368 } 369 entry->apic = apic; 370 entry->pin = pin; 371 list_add_tail(&entry->list, &data->irq_2_pin); 372 373 return 0; 374 } 375 376 static void __remove_pin_from_irq(struct mp_chip_data *data, int apic, int pin) 377 { 378 struct irq_pin_list *tmp, *entry; 379 380 list_for_each_entry_safe(entry, tmp, &data->irq_2_pin, list) 381 if (entry->apic == apic && entry->pin == pin) { 382 list_del(&entry->list); 383 kfree(entry); 384 return; 385 } 386 } 387 388 static void add_pin_to_irq_node(struct mp_chip_data *data, 389 int node, int apic, int pin) 390 { 391 if (__add_pin_to_irq_node(data, node, apic, pin)) 392 panic("IO-APIC: failed to add irq-pin. Can not proceed\n"); 393 } 394 395 /* 396 * Reroute an IRQ to a different pin. 397 */ 398 static void __init replace_pin_at_irq_node(struct mp_chip_data *data, int node, 399 int oldapic, int oldpin, 400 int newapic, int newpin) 401 { 402 struct irq_pin_list *entry; 403 404 for_each_irq_pin(entry, data->irq_2_pin) { 405 if (entry->apic == oldapic && entry->pin == oldpin) { 406 entry->apic = newapic; 407 entry->pin = newpin; 408 /* every one is different, right? */ 409 return; 410 } 411 } 412 413 /* old apic/pin didn't exist, so just add new ones */ 414 add_pin_to_irq_node(data, node, newapic, newpin); 415 } 416 417 static void io_apic_modify_irq(struct mp_chip_data *data, bool masked, 418 void (*final)(struct irq_pin_list *entry)) 419 { 420 struct irq_pin_list *entry; 421 422 data->entry.masked = masked; 423 424 for_each_irq_pin(entry, data->irq_2_pin) { 425 io_apic_write(entry->apic, 0x10 + 2 * entry->pin, data->entry.w1); 426 if (final) 427 final(entry); 428 } 429 } 430 431 static void io_apic_sync(struct irq_pin_list *entry) 432 { 433 /* 434 * Synchronize the IO-APIC and the CPU by doing 435 * a dummy read from the IO-APIC 436 */ 437 struct io_apic __iomem *io_apic; 438 439 io_apic = io_apic_base(entry->apic); 440 readl(&io_apic->data); 441 } 442 443 static void mask_ioapic_irq(struct irq_data *irq_data) 444 { 445 struct mp_chip_data *data = irq_data->chip_data; 446 unsigned long flags; 447 448 raw_spin_lock_irqsave(&ioapic_lock, flags); 449 io_apic_modify_irq(data, true, &io_apic_sync); 450 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 451 } 452 453 static void __unmask_ioapic(struct mp_chip_data *data) 454 { 455 io_apic_modify_irq(data, false, NULL); 456 } 457 458 static void unmask_ioapic_irq(struct irq_data *irq_data) 459 { 460 struct mp_chip_data *data = irq_data->chip_data; 461 unsigned long flags; 462 463 raw_spin_lock_irqsave(&ioapic_lock, flags); 464 __unmask_ioapic(data); 465 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 466 } 467 468 /* 469 * IO-APIC versions below 0x20 don't support EOI register. 470 * For the record, here is the information about various versions: 471 * 0Xh 82489DX 472 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant 473 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant 474 * 30h-FFh Reserved 475 * 476 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic 477 * version as 0x2. This is an error with documentation and these ICH chips 478 * use io-apic's of version 0x20. 479 * 480 * For IO-APIC's with EOI register, we use that to do an explicit EOI. 481 * Otherwise, we simulate the EOI message manually by changing the trigger 482 * mode to edge and then back to level, with RTE being masked during this. 483 */ 484 static void __eoi_ioapic_pin(int apic, int pin, int vector) 485 { 486 if (mpc_ioapic_ver(apic) >= 0x20) { 487 io_apic_eoi(apic, vector); 488 } else { 489 struct IO_APIC_route_entry entry, entry1; 490 491 entry = entry1 = __ioapic_read_entry(apic, pin); 492 493 /* 494 * Mask the entry and change the trigger mode to edge. 495 */ 496 entry1.masked = true; 497 entry1.is_level = false; 498 499 __ioapic_write_entry(apic, pin, entry1); 500 501 /* 502 * Restore the previous level triggered entry. 503 */ 504 __ioapic_write_entry(apic, pin, entry); 505 } 506 } 507 508 static void eoi_ioapic_pin(int vector, struct mp_chip_data *data) 509 { 510 unsigned long flags; 511 struct irq_pin_list *entry; 512 513 raw_spin_lock_irqsave(&ioapic_lock, flags); 514 for_each_irq_pin(entry, data->irq_2_pin) 515 __eoi_ioapic_pin(entry->apic, entry->pin, vector); 516 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 517 } 518 519 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) 520 { 521 struct IO_APIC_route_entry entry; 522 523 /* Check delivery_mode to be sure we're not clearing an SMI pin */ 524 entry = ioapic_read_entry(apic, pin); 525 if (entry.delivery_mode == APIC_DELIVERY_MODE_SMI) 526 return; 527 528 /* 529 * Make sure the entry is masked and re-read the contents to check 530 * if it is a level triggered pin and if the remote-IRR is set. 531 */ 532 if (!entry.masked) { 533 entry.masked = true; 534 ioapic_write_entry(apic, pin, entry); 535 entry = ioapic_read_entry(apic, pin); 536 } 537 538 if (entry.irr) { 539 unsigned long flags; 540 541 /* 542 * Make sure the trigger mode is set to level. Explicit EOI 543 * doesn't clear the remote-IRR if the trigger mode is not 544 * set to level. 545 */ 546 if (!entry.is_level) { 547 entry.is_level = true; 548 ioapic_write_entry(apic, pin, entry); 549 } 550 raw_spin_lock_irqsave(&ioapic_lock, flags); 551 __eoi_ioapic_pin(apic, pin, entry.vector); 552 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 553 } 554 555 /* 556 * Clear the rest of the bits in the IO-APIC RTE except for the mask 557 * bit. 558 */ 559 ioapic_mask_entry(apic, pin); 560 entry = ioapic_read_entry(apic, pin); 561 if (entry.irr) 562 pr_err("Unable to reset IRR for apic: %d, pin :%d\n", 563 mpc_ioapic_id(apic), pin); 564 } 565 566 void clear_IO_APIC (void) 567 { 568 int apic, pin; 569 570 for_each_ioapic_pin(apic, pin) 571 clear_IO_APIC_pin(apic, pin); 572 } 573 574 #ifdef CONFIG_X86_32 575 /* 576 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to 577 * specific CPU-side IRQs. 578 */ 579 580 #define MAX_PIRQS 8 581 static int pirq_entries[MAX_PIRQS] = { 582 [0 ... MAX_PIRQS - 1] = -1 583 }; 584 585 static int __init ioapic_pirq_setup(char *str) 586 { 587 int i, max; 588 int ints[MAX_PIRQS+1]; 589 590 get_options(str, ARRAY_SIZE(ints), ints); 591 592 apic_printk(APIC_VERBOSE, KERN_INFO 593 "PIRQ redirection, working around broken MP-BIOS.\n"); 594 max = MAX_PIRQS; 595 if (ints[0] < MAX_PIRQS) 596 max = ints[0]; 597 598 for (i = 0; i < max; i++) { 599 apic_printk(APIC_VERBOSE, KERN_DEBUG 600 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]); 601 /* 602 * PIRQs are mapped upside down, usually. 603 */ 604 pirq_entries[MAX_PIRQS-i-1] = ints[i+1]; 605 } 606 return 1; 607 } 608 609 __setup("pirq=", ioapic_pirq_setup); 610 #endif /* CONFIG_X86_32 */ 611 612 /* 613 * Saves all the IO-APIC RTE's 614 */ 615 int save_ioapic_entries(void) 616 { 617 int apic, pin; 618 int err = 0; 619 620 for_each_ioapic(apic) { 621 if (!ioapics[apic].saved_registers) { 622 err = -ENOMEM; 623 continue; 624 } 625 626 for_each_pin(apic, pin) 627 ioapics[apic].saved_registers[pin] = 628 ioapic_read_entry(apic, pin); 629 } 630 631 return err; 632 } 633 634 /* 635 * Mask all IO APIC entries. 636 */ 637 void mask_ioapic_entries(void) 638 { 639 int apic, pin; 640 641 for_each_ioapic(apic) { 642 if (!ioapics[apic].saved_registers) 643 continue; 644 645 for_each_pin(apic, pin) { 646 struct IO_APIC_route_entry entry; 647 648 entry = ioapics[apic].saved_registers[pin]; 649 if (!entry.masked) { 650 entry.masked = true; 651 ioapic_write_entry(apic, pin, entry); 652 } 653 } 654 } 655 } 656 657 /* 658 * Restore IO APIC entries which was saved in the ioapic structure. 659 */ 660 int restore_ioapic_entries(void) 661 { 662 int apic, pin; 663 664 for_each_ioapic(apic) { 665 if (!ioapics[apic].saved_registers) 666 continue; 667 668 for_each_pin(apic, pin) 669 ioapic_write_entry(apic, pin, 670 ioapics[apic].saved_registers[pin]); 671 } 672 return 0; 673 } 674 675 /* 676 * Find the IRQ entry number of a certain pin. 677 */ 678 static int find_irq_entry(int ioapic_idx, int pin, int type) 679 { 680 int i; 681 682 for (i = 0; i < mp_irq_entries; i++) 683 if (mp_irqs[i].irqtype == type && 684 (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) || 685 mp_irqs[i].dstapic == MP_APIC_ALL) && 686 mp_irqs[i].dstirq == pin) 687 return i; 688 689 return -1; 690 } 691 692 /* 693 * Find the pin to which IRQ[irq] (ISA) is connected 694 */ 695 static int __init find_isa_irq_pin(int irq, int type) 696 { 697 int i; 698 699 for (i = 0; i < mp_irq_entries; i++) { 700 int lbus = mp_irqs[i].srcbus; 701 702 if (test_bit(lbus, mp_bus_not_pci) && 703 (mp_irqs[i].irqtype == type) && 704 (mp_irqs[i].srcbusirq == irq)) 705 706 return mp_irqs[i].dstirq; 707 } 708 return -1; 709 } 710 711 static int __init find_isa_irq_apic(int irq, int type) 712 { 713 int i; 714 715 for (i = 0; i < mp_irq_entries; i++) { 716 int lbus = mp_irqs[i].srcbus; 717 718 if (test_bit(lbus, mp_bus_not_pci) && 719 (mp_irqs[i].irqtype == type) && 720 (mp_irqs[i].srcbusirq == irq)) 721 break; 722 } 723 724 if (i < mp_irq_entries) { 725 int ioapic_idx; 726 727 for_each_ioapic(ioapic_idx) 728 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic) 729 return ioapic_idx; 730 } 731 732 return -1; 733 } 734 735 static bool irq_active_low(int idx) 736 { 737 int bus = mp_irqs[idx].srcbus; 738 739 /* 740 * Determine IRQ line polarity (high active or low active): 741 */ 742 switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) { 743 case MP_IRQPOL_DEFAULT: 744 /* 745 * Conforms to spec, ie. bus-type dependent polarity. PCI 746 * defaults to low active. [E]ISA defaults to high active. 747 */ 748 return !test_bit(bus, mp_bus_not_pci); 749 case MP_IRQPOL_ACTIVE_HIGH: 750 return false; 751 case MP_IRQPOL_RESERVED: 752 pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n"); 753 fallthrough; 754 case MP_IRQPOL_ACTIVE_LOW: 755 default: /* Pointless default required due to do gcc stupidity */ 756 return true; 757 } 758 } 759 760 #ifdef CONFIG_EISA 761 /* 762 * EISA Edge/Level control register, ELCR 763 */ 764 static bool EISA_ELCR(unsigned int irq) 765 { 766 if (irq < nr_legacy_irqs()) { 767 unsigned int port = 0x4d0 + (irq >> 3); 768 return (inb(port) >> (irq & 7)) & 1; 769 } 770 apic_printk(APIC_VERBOSE, KERN_INFO 771 "Broken MPtable reports ISA irq %d\n", irq); 772 return false; 773 } 774 775 /* 776 * EISA interrupts are always active high and can be edge or level 777 * triggered depending on the ELCR value. If an interrupt is listed as 778 * EISA conforming in the MP table, that means its trigger type must be 779 * read in from the ELCR. 780 */ 781 static bool eisa_irq_is_level(int idx, int bus, bool level) 782 { 783 switch (mp_bus_id_to_type[bus]) { 784 case MP_BUS_PCI: 785 case MP_BUS_ISA: 786 return level; 787 case MP_BUS_EISA: 788 return EISA_ELCR(mp_irqs[idx].srcbusirq); 789 } 790 pr_warn("IOAPIC: Invalid srcbus: %d defaulting to level\n", bus); 791 return true; 792 } 793 #else 794 static inline int eisa_irq_is_level(int idx, int bus, bool level) 795 { 796 return level; 797 } 798 #endif 799 800 static bool irq_is_level(int idx) 801 { 802 int bus = mp_irqs[idx].srcbus; 803 bool level; 804 805 /* 806 * Determine IRQ trigger mode (edge or level sensitive): 807 */ 808 switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) { 809 case MP_IRQTRIG_DEFAULT: 810 /* 811 * Conforms to spec, ie. bus-type dependent trigger 812 * mode. PCI defaults to egde, ISA to level. 813 */ 814 level = test_bit(bus, mp_bus_not_pci); 815 /* Take EISA into account */ 816 return eisa_irq_is_level(idx, bus, level); 817 case MP_IRQTRIG_EDGE: 818 return false; 819 case MP_IRQTRIG_RESERVED: 820 pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n"); 821 fallthrough; 822 case MP_IRQTRIG_LEVEL: 823 default: /* Pointless default required due to do gcc stupidity */ 824 return true; 825 } 826 } 827 828 static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity) 829 { 830 int ioapic, pin, idx; 831 832 if (skip_ioapic_setup) 833 return -1; 834 835 ioapic = mp_find_ioapic(gsi); 836 if (ioapic < 0) 837 return -1; 838 839 pin = mp_find_ioapic_pin(ioapic, gsi); 840 if (pin < 0) 841 return -1; 842 843 idx = find_irq_entry(ioapic, pin, mp_INT); 844 if (idx < 0) 845 return -1; 846 847 *trigger = irq_is_level(idx); 848 *polarity = irq_active_low(idx); 849 return 0; 850 } 851 852 #ifdef CONFIG_ACPI 853 int acpi_get_override_irq(u32 gsi, int *is_level, int *active_low) 854 { 855 *is_level = *active_low = 0; 856 return __acpi_get_override_irq(gsi, (bool *)is_level, 857 (bool *)active_low); 858 } 859 #endif 860 861 void ioapic_set_alloc_attr(struct irq_alloc_info *info, int node, 862 int trigger, int polarity) 863 { 864 init_irq_alloc_info(info, NULL); 865 info->type = X86_IRQ_ALLOC_TYPE_IOAPIC; 866 info->ioapic.node = node; 867 info->ioapic.is_level = trigger; 868 info->ioapic.active_low = polarity; 869 info->ioapic.valid = 1; 870 } 871 872 static void ioapic_copy_alloc_attr(struct irq_alloc_info *dst, 873 struct irq_alloc_info *src, 874 u32 gsi, int ioapic_idx, int pin) 875 { 876 bool level, pol_low; 877 878 copy_irq_alloc_info(dst, src); 879 dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC; 880 dst->devid = mpc_ioapic_id(ioapic_idx); 881 dst->ioapic.pin = pin; 882 dst->ioapic.valid = 1; 883 if (src && src->ioapic.valid) { 884 dst->ioapic.node = src->ioapic.node; 885 dst->ioapic.is_level = src->ioapic.is_level; 886 dst->ioapic.active_low = src->ioapic.active_low; 887 } else { 888 dst->ioapic.node = NUMA_NO_NODE; 889 if (__acpi_get_override_irq(gsi, &level, &pol_low) >= 0) { 890 dst->ioapic.is_level = level; 891 dst->ioapic.active_low = pol_low; 892 } else { 893 /* 894 * PCI interrupts are always active low level 895 * triggered. 896 */ 897 dst->ioapic.is_level = true; 898 dst->ioapic.active_low = true; 899 } 900 } 901 } 902 903 static int ioapic_alloc_attr_node(struct irq_alloc_info *info) 904 { 905 return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE; 906 } 907 908 static void mp_register_handler(unsigned int irq, bool level) 909 { 910 irq_flow_handler_t hdl; 911 bool fasteoi; 912 913 if (level) { 914 irq_set_status_flags(irq, IRQ_LEVEL); 915 fasteoi = true; 916 } else { 917 irq_clear_status_flags(irq, IRQ_LEVEL); 918 fasteoi = false; 919 } 920 921 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq; 922 __irq_set_handler(irq, hdl, 0, fasteoi ? "fasteoi" : "edge"); 923 } 924 925 static bool mp_check_pin_attr(int irq, struct irq_alloc_info *info) 926 { 927 struct mp_chip_data *data = irq_get_chip_data(irq); 928 929 /* 930 * setup_IO_APIC_irqs() programs all legacy IRQs with default trigger 931 * and polarity attirbutes. So allow the first user to reprogram the 932 * pin with real trigger and polarity attributes. 933 */ 934 if (irq < nr_legacy_irqs() && data->count == 1) { 935 if (info->ioapic.is_level != data->is_level) 936 mp_register_handler(irq, info->ioapic.is_level); 937 data->entry.is_level = data->is_level = info->ioapic.is_level; 938 data->entry.active_low = data->active_low = info->ioapic.active_low; 939 } 940 941 return data->is_level == info->ioapic.is_level && 942 data->active_low == info->ioapic.active_low; 943 } 944 945 static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi, 946 struct irq_alloc_info *info) 947 { 948 bool legacy = false; 949 int irq = -1; 950 int type = ioapics[ioapic].irqdomain_cfg.type; 951 952 switch (type) { 953 case IOAPIC_DOMAIN_LEGACY: 954 /* 955 * Dynamically allocate IRQ number for non-ISA IRQs in the first 956 * 16 GSIs on some weird platforms. 957 */ 958 if (!ioapic_initialized || gsi >= nr_legacy_irqs()) 959 irq = gsi; 960 legacy = mp_is_legacy_irq(irq); 961 break; 962 case IOAPIC_DOMAIN_STRICT: 963 irq = gsi; 964 break; 965 case IOAPIC_DOMAIN_DYNAMIC: 966 break; 967 default: 968 WARN(1, "ioapic: unknown irqdomain type %d\n", type); 969 return -1; 970 } 971 972 return __irq_domain_alloc_irqs(domain, irq, 1, 973 ioapic_alloc_attr_node(info), 974 info, legacy, NULL); 975 } 976 977 /* 978 * Need special handling for ISA IRQs because there may be multiple IOAPIC pins 979 * sharing the same ISA IRQ number and irqdomain only supports 1:1 mapping 980 * between IOAPIC pin and IRQ number. A typical IOAPIC has 24 pins, pin 0-15 are 981 * used for legacy IRQs and pin 16-23 are used for PCI IRQs (PIRQ A-H). 982 * When ACPI is disabled, only legacy IRQ numbers (IRQ0-15) are available, and 983 * some BIOSes may use MP Interrupt Source records to override IRQ numbers for 984 * PIRQs instead of reprogramming the interrupt routing logic. Thus there may be 985 * multiple pins sharing the same legacy IRQ number when ACPI is disabled. 986 */ 987 static int alloc_isa_irq_from_domain(struct irq_domain *domain, 988 int irq, int ioapic, int pin, 989 struct irq_alloc_info *info) 990 { 991 struct mp_chip_data *data; 992 struct irq_data *irq_data = irq_get_irq_data(irq); 993 int node = ioapic_alloc_attr_node(info); 994 995 /* 996 * Legacy ISA IRQ has already been allocated, just add pin to 997 * the pin list assoicated with this IRQ and program the IOAPIC 998 * entry. The IOAPIC entry 999 */ 1000 if (irq_data && irq_data->parent_data) { 1001 if (!mp_check_pin_attr(irq, info)) 1002 return -EBUSY; 1003 if (__add_pin_to_irq_node(irq_data->chip_data, node, ioapic, 1004 info->ioapic.pin)) 1005 return -ENOMEM; 1006 } else { 1007 info->flags |= X86_IRQ_ALLOC_LEGACY; 1008 irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true, 1009 NULL); 1010 if (irq >= 0) { 1011 irq_data = irq_domain_get_irq_data(domain, irq); 1012 data = irq_data->chip_data; 1013 data->isa_irq = true; 1014 } 1015 } 1016 1017 return irq; 1018 } 1019 1020 static int mp_map_pin_to_irq(u32 gsi, int idx, int ioapic, int pin, 1021 unsigned int flags, struct irq_alloc_info *info) 1022 { 1023 int irq; 1024 bool legacy = false; 1025 struct irq_alloc_info tmp; 1026 struct mp_chip_data *data; 1027 struct irq_domain *domain = mp_ioapic_irqdomain(ioapic); 1028 1029 if (!domain) 1030 return -ENOSYS; 1031 1032 if (idx >= 0 && test_bit(mp_irqs[idx].srcbus, mp_bus_not_pci)) { 1033 irq = mp_irqs[idx].srcbusirq; 1034 legacy = mp_is_legacy_irq(irq); 1035 } 1036 1037 mutex_lock(&ioapic_mutex); 1038 if (!(flags & IOAPIC_MAP_ALLOC)) { 1039 if (!legacy) { 1040 irq = irq_find_mapping(domain, pin); 1041 if (irq == 0) 1042 irq = -ENOENT; 1043 } 1044 } else { 1045 ioapic_copy_alloc_attr(&tmp, info, gsi, ioapic, pin); 1046 if (legacy) 1047 irq = alloc_isa_irq_from_domain(domain, irq, 1048 ioapic, pin, &tmp); 1049 else if ((irq = irq_find_mapping(domain, pin)) == 0) 1050 irq = alloc_irq_from_domain(domain, ioapic, gsi, &tmp); 1051 else if (!mp_check_pin_attr(irq, &tmp)) 1052 irq = -EBUSY; 1053 if (irq >= 0) { 1054 data = irq_get_chip_data(irq); 1055 data->count++; 1056 } 1057 } 1058 mutex_unlock(&ioapic_mutex); 1059 1060 return irq; 1061 } 1062 1063 static int pin_2_irq(int idx, int ioapic, int pin, unsigned int flags) 1064 { 1065 u32 gsi = mp_pin_to_gsi(ioapic, pin); 1066 1067 /* 1068 * Debugging check, we are in big trouble if this message pops up! 1069 */ 1070 if (mp_irqs[idx].dstirq != pin) 1071 pr_err("broken BIOS or MPTABLE parser, ayiee!!\n"); 1072 1073 #ifdef CONFIG_X86_32 1074 /* 1075 * PCI IRQ command line redirection. Yes, limits are hardcoded. 1076 */ 1077 if ((pin >= 16) && (pin <= 23)) { 1078 if (pirq_entries[pin-16] != -1) { 1079 if (!pirq_entries[pin-16]) { 1080 apic_printk(APIC_VERBOSE, KERN_DEBUG 1081 "disabling PIRQ%d\n", pin-16); 1082 } else { 1083 int irq = pirq_entries[pin-16]; 1084 apic_printk(APIC_VERBOSE, KERN_DEBUG 1085 "using PIRQ%d -> IRQ %d\n", 1086 pin-16, irq); 1087 return irq; 1088 } 1089 } 1090 } 1091 #endif 1092 1093 return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, NULL); 1094 } 1095 1096 int mp_map_gsi_to_irq(u32 gsi, unsigned int flags, struct irq_alloc_info *info) 1097 { 1098 int ioapic, pin, idx; 1099 1100 ioapic = mp_find_ioapic(gsi); 1101 if (ioapic < 0) 1102 return -ENODEV; 1103 1104 pin = mp_find_ioapic_pin(ioapic, gsi); 1105 idx = find_irq_entry(ioapic, pin, mp_INT); 1106 if ((flags & IOAPIC_MAP_CHECK) && idx < 0) 1107 return -ENODEV; 1108 1109 return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, info); 1110 } 1111 1112 void mp_unmap_irq(int irq) 1113 { 1114 struct irq_data *irq_data = irq_get_irq_data(irq); 1115 struct mp_chip_data *data; 1116 1117 if (!irq_data || !irq_data->domain) 1118 return; 1119 1120 data = irq_data->chip_data; 1121 if (!data || data->isa_irq) 1122 return; 1123 1124 mutex_lock(&ioapic_mutex); 1125 if (--data->count == 0) 1126 irq_domain_free_irqs(irq, 1); 1127 mutex_unlock(&ioapic_mutex); 1128 } 1129 1130 /* 1131 * Find a specific PCI IRQ entry. 1132 * Not an __init, possibly needed by modules 1133 */ 1134 int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin) 1135 { 1136 int irq, i, best_ioapic = -1, best_idx = -1; 1137 1138 apic_printk(APIC_DEBUG, 1139 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n", 1140 bus, slot, pin); 1141 if (test_bit(bus, mp_bus_not_pci)) { 1142 apic_printk(APIC_VERBOSE, 1143 "PCI BIOS passed nonexistent PCI bus %d!\n", bus); 1144 return -1; 1145 } 1146 1147 for (i = 0; i < mp_irq_entries; i++) { 1148 int lbus = mp_irqs[i].srcbus; 1149 int ioapic_idx, found = 0; 1150 1151 if (bus != lbus || mp_irqs[i].irqtype != mp_INT || 1152 slot != ((mp_irqs[i].srcbusirq >> 2) & 0x1f)) 1153 continue; 1154 1155 for_each_ioapic(ioapic_idx) 1156 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic || 1157 mp_irqs[i].dstapic == MP_APIC_ALL) { 1158 found = 1; 1159 break; 1160 } 1161 if (!found) 1162 continue; 1163 1164 /* Skip ISA IRQs */ 1165 irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq, 0); 1166 if (irq > 0 && !IO_APIC_IRQ(irq)) 1167 continue; 1168 1169 if (pin == (mp_irqs[i].srcbusirq & 3)) { 1170 best_idx = i; 1171 best_ioapic = ioapic_idx; 1172 goto out; 1173 } 1174 1175 /* 1176 * Use the first all-but-pin matching entry as a 1177 * best-guess fuzzy result for broken mptables. 1178 */ 1179 if (best_idx < 0) { 1180 best_idx = i; 1181 best_ioapic = ioapic_idx; 1182 } 1183 } 1184 if (best_idx < 0) 1185 return -1; 1186 1187 out: 1188 return pin_2_irq(best_idx, best_ioapic, mp_irqs[best_idx].dstirq, 1189 IOAPIC_MAP_ALLOC); 1190 } 1191 EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector); 1192 1193 static struct irq_chip ioapic_chip, ioapic_ir_chip; 1194 1195 static void __init setup_IO_APIC_irqs(void) 1196 { 1197 unsigned int ioapic, pin; 1198 int idx; 1199 1200 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1201 1202 for_each_ioapic_pin(ioapic, pin) { 1203 idx = find_irq_entry(ioapic, pin, mp_INT); 1204 if (idx < 0) 1205 apic_printk(APIC_VERBOSE, 1206 KERN_DEBUG " apic %d pin %d not connected\n", 1207 mpc_ioapic_id(ioapic), pin); 1208 else 1209 pin_2_irq(idx, ioapic, pin, 1210 ioapic ? 0 : IOAPIC_MAP_ALLOC); 1211 } 1212 } 1213 1214 void ioapic_zap_locks(void) 1215 { 1216 raw_spin_lock_init(&ioapic_lock); 1217 } 1218 1219 static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries) 1220 { 1221 struct IO_APIC_route_entry entry; 1222 char buf[256]; 1223 int i; 1224 1225 printk(KERN_DEBUG "IOAPIC %d:\n", apic); 1226 for (i = 0; i <= nr_entries; i++) { 1227 entry = ioapic_read_entry(apic, i); 1228 snprintf(buf, sizeof(buf), 1229 " pin%02x, %s, %s, %s, V(%02X), IRR(%1d), S(%1d)", 1230 i, 1231 entry.masked ? "disabled" : "enabled ", 1232 entry.is_level ? "level" : "edge ", 1233 entry.active_low ? "low " : "high", 1234 entry.vector, entry.irr, entry.delivery_status); 1235 if (entry.ir_format) { 1236 printk(KERN_DEBUG "%s, remapped, I(%04X), Z(%X)\n", 1237 buf, 1238 (entry.ir_index_15 << 15) | entry.ir_index_0_14, 1239 entry.ir_zero); 1240 } else { 1241 printk(KERN_DEBUG "%s, %s, D(%02X), M(%1d)\n", buf, 1242 entry.dest_mode_logical ? "logical " : "physical", 1243 entry.destid_0_7, entry.delivery_mode); 1244 } 1245 } 1246 } 1247 1248 static void __init print_IO_APIC(int ioapic_idx) 1249 { 1250 union IO_APIC_reg_00 reg_00; 1251 union IO_APIC_reg_01 reg_01; 1252 union IO_APIC_reg_02 reg_02; 1253 union IO_APIC_reg_03 reg_03; 1254 unsigned long flags; 1255 1256 raw_spin_lock_irqsave(&ioapic_lock, flags); 1257 reg_00.raw = io_apic_read(ioapic_idx, 0); 1258 reg_01.raw = io_apic_read(ioapic_idx, 1); 1259 if (reg_01.bits.version >= 0x10) 1260 reg_02.raw = io_apic_read(ioapic_idx, 2); 1261 if (reg_01.bits.version >= 0x20) 1262 reg_03.raw = io_apic_read(ioapic_idx, 3); 1263 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1264 1265 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx)); 1266 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); 1267 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID); 1268 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type); 1269 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS); 1270 1271 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01); 1272 printk(KERN_DEBUG "....... : max redirection entries: %02X\n", 1273 reg_01.bits.entries); 1274 1275 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ); 1276 printk(KERN_DEBUG "....... : IO APIC version: %02X\n", 1277 reg_01.bits.version); 1278 1279 /* 1280 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02, 1281 * but the value of reg_02 is read as the previous read register 1282 * value, so ignore it if reg_02 == reg_01. 1283 */ 1284 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) { 1285 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw); 1286 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration); 1287 } 1288 1289 /* 1290 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02 1291 * or reg_03, but the value of reg_0[23] is read as the previous read 1292 * register value, so ignore it if reg_03 == reg_0[12]. 1293 */ 1294 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw && 1295 reg_03.raw != reg_01.raw) { 1296 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw); 1297 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT); 1298 } 1299 1300 printk(KERN_DEBUG ".... IRQ redirection table:\n"); 1301 io_apic_print_entries(ioapic_idx, reg_01.bits.entries); 1302 } 1303 1304 void __init print_IO_APICs(void) 1305 { 1306 int ioapic_idx; 1307 unsigned int irq; 1308 1309 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 1310 for_each_ioapic(ioapic_idx) 1311 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 1312 mpc_ioapic_id(ioapic_idx), 1313 ioapics[ioapic_idx].nr_registers); 1314 1315 /* 1316 * We are a bit conservative about what we expect. We have to 1317 * know about every hardware change ASAP. 1318 */ 1319 printk(KERN_INFO "testing the IO APIC.......................\n"); 1320 1321 for_each_ioapic(ioapic_idx) 1322 print_IO_APIC(ioapic_idx); 1323 1324 printk(KERN_DEBUG "IRQ to pin mappings:\n"); 1325 for_each_active_irq(irq) { 1326 struct irq_pin_list *entry; 1327 struct irq_chip *chip; 1328 struct mp_chip_data *data; 1329 1330 chip = irq_get_chip(irq); 1331 if (chip != &ioapic_chip && chip != &ioapic_ir_chip) 1332 continue; 1333 data = irq_get_chip_data(irq); 1334 if (!data) 1335 continue; 1336 if (list_empty(&data->irq_2_pin)) 1337 continue; 1338 1339 printk(KERN_DEBUG "IRQ%d ", irq); 1340 for_each_irq_pin(entry, data->irq_2_pin) 1341 pr_cont("-> %d:%d", entry->apic, entry->pin); 1342 pr_cont("\n"); 1343 } 1344 1345 printk(KERN_INFO ".................................... done.\n"); 1346 } 1347 1348 /* Where if anywhere is the i8259 connect in external int mode */ 1349 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 }; 1350 1351 void __init enable_IO_APIC(void) 1352 { 1353 int i8259_apic, i8259_pin; 1354 int apic, pin; 1355 1356 if (skip_ioapic_setup) 1357 nr_ioapics = 0; 1358 1359 if (!nr_legacy_irqs() || !nr_ioapics) 1360 return; 1361 1362 for_each_ioapic_pin(apic, pin) { 1363 /* See if any of the pins is in ExtINT mode */ 1364 struct IO_APIC_route_entry entry = ioapic_read_entry(apic, pin); 1365 1366 /* If the interrupt line is enabled and in ExtInt mode 1367 * I have found the pin where the i8259 is connected. 1368 */ 1369 if (!entry.masked && 1370 entry.delivery_mode == APIC_DELIVERY_MODE_EXTINT) { 1371 ioapic_i8259.apic = apic; 1372 ioapic_i8259.pin = pin; 1373 goto found_i8259; 1374 } 1375 } 1376 found_i8259: 1377 /* Look to see what if the MP table has reported the ExtINT */ 1378 /* If we could not find the appropriate pin by looking at the ioapic 1379 * the i8259 probably is not connected the ioapic but give the 1380 * mptable a chance anyway. 1381 */ 1382 i8259_pin = find_isa_irq_pin(0, mp_ExtINT); 1383 i8259_apic = find_isa_irq_apic(0, mp_ExtINT); 1384 /* Trust the MP table if nothing is setup in the hardware */ 1385 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) { 1386 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n"); 1387 ioapic_i8259.pin = i8259_pin; 1388 ioapic_i8259.apic = i8259_apic; 1389 } 1390 /* Complain if the MP table and the hardware disagree */ 1391 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) && 1392 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0)) 1393 { 1394 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n"); 1395 } 1396 1397 /* 1398 * Do not trust the IO-APIC being empty at bootup 1399 */ 1400 clear_IO_APIC(); 1401 } 1402 1403 void native_restore_boot_irq_mode(void) 1404 { 1405 /* 1406 * If the i8259 is routed through an IOAPIC 1407 * Put that IOAPIC in virtual wire mode 1408 * so legacy interrupts can be delivered. 1409 */ 1410 if (ioapic_i8259.pin != -1) { 1411 struct IO_APIC_route_entry entry; 1412 1413 memset(&entry, 0, sizeof(entry)); 1414 entry.masked = false; 1415 entry.is_level = false; 1416 entry.active_low = false; 1417 entry.dest_mode_logical = false; 1418 entry.delivery_mode = APIC_DELIVERY_MODE_EXTINT; 1419 entry.destid_0_7 = read_apic_id(); 1420 1421 /* 1422 * Add it to the IO-APIC irq-routing table: 1423 */ 1424 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry); 1425 } 1426 1427 if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config()) 1428 disconnect_bsp_APIC(ioapic_i8259.pin != -1); 1429 } 1430 1431 void restore_boot_irq_mode(void) 1432 { 1433 if (!nr_legacy_irqs()) 1434 return; 1435 1436 x86_apic_ops.restore(); 1437 } 1438 1439 #ifdef CONFIG_X86_32 1440 /* 1441 * function to set the IO-APIC physical IDs based on the 1442 * values stored in the MPC table. 1443 * 1444 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999 1445 */ 1446 void __init setup_ioapic_ids_from_mpc_nocheck(void) 1447 { 1448 union IO_APIC_reg_00 reg_00; 1449 physid_mask_t phys_id_present_map; 1450 int ioapic_idx; 1451 int i; 1452 unsigned char old_id; 1453 unsigned long flags; 1454 1455 /* 1456 * This is broken; anything with a real cpu count has to 1457 * circumvent this idiocy regardless. 1458 */ 1459 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map); 1460 1461 /* 1462 * Set the IOAPIC ID to the value stored in the MPC table. 1463 */ 1464 for_each_ioapic(ioapic_idx) { 1465 /* Read the register 0 value */ 1466 raw_spin_lock_irqsave(&ioapic_lock, flags); 1467 reg_00.raw = io_apic_read(ioapic_idx, 0); 1468 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1469 1470 old_id = mpc_ioapic_id(ioapic_idx); 1471 1472 if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) { 1473 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n", 1474 ioapic_idx, mpc_ioapic_id(ioapic_idx)); 1475 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1476 reg_00.bits.ID); 1477 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID; 1478 } 1479 1480 /* 1481 * Sanity check, is the ID really free? Every APIC in a 1482 * system must have a unique ID or we get lots of nice 1483 * 'stuck on smp_invalidate_needed IPI wait' messages. 1484 */ 1485 if (apic->check_apicid_used(&phys_id_present_map, 1486 mpc_ioapic_id(ioapic_idx))) { 1487 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n", 1488 ioapic_idx, mpc_ioapic_id(ioapic_idx)); 1489 for (i = 0; i < get_physical_broadcast(); i++) 1490 if (!physid_isset(i, phys_id_present_map)) 1491 break; 1492 if (i >= get_physical_broadcast()) 1493 panic("Max APIC ID exceeded!\n"); 1494 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n", 1495 i); 1496 physid_set(i, phys_id_present_map); 1497 ioapics[ioapic_idx].mp_config.apicid = i; 1498 } else { 1499 physid_mask_t tmp; 1500 apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx), 1501 &tmp); 1502 apic_printk(APIC_VERBOSE, "Setting %d in the " 1503 "phys_id_present_map\n", 1504 mpc_ioapic_id(ioapic_idx)); 1505 physids_or(phys_id_present_map, phys_id_present_map, tmp); 1506 } 1507 1508 /* 1509 * We need to adjust the IRQ routing table 1510 * if the ID changed. 1511 */ 1512 if (old_id != mpc_ioapic_id(ioapic_idx)) 1513 for (i = 0; i < mp_irq_entries; i++) 1514 if (mp_irqs[i].dstapic == old_id) 1515 mp_irqs[i].dstapic 1516 = mpc_ioapic_id(ioapic_idx); 1517 1518 /* 1519 * Update the ID register according to the right value 1520 * from the MPC table if they are different. 1521 */ 1522 if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID) 1523 continue; 1524 1525 apic_printk(APIC_VERBOSE, KERN_INFO 1526 "...changing IO-APIC physical APIC ID to %d ...", 1527 mpc_ioapic_id(ioapic_idx)); 1528 1529 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx); 1530 raw_spin_lock_irqsave(&ioapic_lock, flags); 1531 io_apic_write(ioapic_idx, 0, reg_00.raw); 1532 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1533 1534 /* 1535 * Sanity check 1536 */ 1537 raw_spin_lock_irqsave(&ioapic_lock, flags); 1538 reg_00.raw = io_apic_read(ioapic_idx, 0); 1539 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1540 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) 1541 pr_cont("could not set ID!\n"); 1542 else 1543 apic_printk(APIC_VERBOSE, " ok.\n"); 1544 } 1545 } 1546 1547 void __init setup_ioapic_ids_from_mpc(void) 1548 { 1549 1550 if (acpi_ioapic) 1551 return; 1552 /* 1553 * Don't check I/O APIC IDs for xAPIC systems. They have 1554 * no meaning without the serial APIC bus. 1555 */ 1556 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1557 || APIC_XAPIC(boot_cpu_apic_version)) 1558 return; 1559 setup_ioapic_ids_from_mpc_nocheck(); 1560 } 1561 #endif 1562 1563 int no_timer_check __initdata; 1564 1565 static int __init notimercheck(char *s) 1566 { 1567 no_timer_check = 1; 1568 return 1; 1569 } 1570 __setup("no_timer_check", notimercheck); 1571 1572 static void __init delay_with_tsc(void) 1573 { 1574 unsigned long long start, now; 1575 unsigned long end = jiffies + 4; 1576 1577 start = rdtsc(); 1578 1579 /* 1580 * We don't know the TSC frequency yet, but waiting for 1581 * 40000000000/HZ TSC cycles is safe: 1582 * 4 GHz == 10 jiffies 1583 * 1 GHz == 40 jiffies 1584 */ 1585 do { 1586 rep_nop(); 1587 now = rdtsc(); 1588 } while ((now - start) < 40000000000ULL / HZ && 1589 time_before_eq(jiffies, end)); 1590 } 1591 1592 static void __init delay_without_tsc(void) 1593 { 1594 unsigned long end = jiffies + 4; 1595 int band = 1; 1596 1597 /* 1598 * We don't know any frequency yet, but waiting for 1599 * 40940000000/HZ cycles is safe: 1600 * 4 GHz == 10 jiffies 1601 * 1 GHz == 40 jiffies 1602 * 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094 1603 */ 1604 do { 1605 __delay(((1U << band++) * 10000000UL) / HZ); 1606 } while (band < 12 && time_before_eq(jiffies, end)); 1607 } 1608 1609 /* 1610 * There is a nasty bug in some older SMP boards, their mptable lies 1611 * about the timer IRQ. We do the following to work around the situation: 1612 * 1613 * - timer IRQ defaults to IO-APIC IRQ 1614 * - if this function detects that timer IRQs are defunct, then we fall 1615 * back to ISA timer IRQs 1616 */ 1617 static int __init timer_irq_works(void) 1618 { 1619 unsigned long t1 = jiffies; 1620 unsigned long flags; 1621 1622 if (no_timer_check) 1623 return 1; 1624 1625 local_save_flags(flags); 1626 local_irq_enable(); 1627 1628 if (boot_cpu_has(X86_FEATURE_TSC)) 1629 delay_with_tsc(); 1630 else 1631 delay_without_tsc(); 1632 1633 local_irq_restore(flags); 1634 1635 /* 1636 * Expect a few ticks at least, to be sure some possible 1637 * glue logic does not lock up after one or two first 1638 * ticks in a non-ExtINT mode. Also the local APIC 1639 * might have cached one ExtINT interrupt. Finally, at 1640 * least one tick may be lost due to delays. 1641 */ 1642 1643 /* jiffies wrap? */ 1644 if (time_after(jiffies, t1 + 4)) 1645 return 1; 1646 return 0; 1647 } 1648 1649 /* 1650 * In the SMP+IOAPIC case it might happen that there are an unspecified 1651 * number of pending IRQ events unhandled. These cases are very rare, 1652 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much 1653 * better to do it this way as thus we do not have to be aware of 1654 * 'pending' interrupts in the IRQ path, except at this point. 1655 */ 1656 /* 1657 * Edge triggered needs to resend any interrupt 1658 * that was delayed but this is now handled in the device 1659 * independent code. 1660 */ 1661 1662 /* 1663 * Starting up a edge-triggered IO-APIC interrupt is 1664 * nasty - we need to make sure that we get the edge. 1665 * If it is already asserted for some reason, we need 1666 * return 1 to indicate that is was pending. 1667 * 1668 * This is not complete - we should be able to fake 1669 * an edge even if it isn't on the 8259A... 1670 */ 1671 static unsigned int startup_ioapic_irq(struct irq_data *data) 1672 { 1673 int was_pending = 0, irq = data->irq; 1674 unsigned long flags; 1675 1676 raw_spin_lock_irqsave(&ioapic_lock, flags); 1677 if (irq < nr_legacy_irqs()) { 1678 legacy_pic->mask(irq); 1679 if (legacy_pic->irq_pending(irq)) 1680 was_pending = 1; 1681 } 1682 __unmask_ioapic(data->chip_data); 1683 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1684 1685 return was_pending; 1686 } 1687 1688 atomic_t irq_mis_count; 1689 1690 #ifdef CONFIG_GENERIC_PENDING_IRQ 1691 static bool io_apic_level_ack_pending(struct mp_chip_data *data) 1692 { 1693 struct irq_pin_list *entry; 1694 unsigned long flags; 1695 1696 raw_spin_lock_irqsave(&ioapic_lock, flags); 1697 for_each_irq_pin(entry, data->irq_2_pin) { 1698 struct IO_APIC_route_entry e; 1699 int pin; 1700 1701 pin = entry->pin; 1702 e.w1 = io_apic_read(entry->apic, 0x10 + pin*2); 1703 /* Is the remote IRR bit set? */ 1704 if (e.irr) { 1705 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1706 return true; 1707 } 1708 } 1709 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1710 1711 return false; 1712 } 1713 1714 static inline bool ioapic_prepare_move(struct irq_data *data) 1715 { 1716 /* If we are moving the IRQ we need to mask it */ 1717 if (unlikely(irqd_is_setaffinity_pending(data))) { 1718 if (!irqd_irq_masked(data)) 1719 mask_ioapic_irq(data); 1720 return true; 1721 } 1722 return false; 1723 } 1724 1725 static inline void ioapic_finish_move(struct irq_data *data, bool moveit) 1726 { 1727 if (unlikely(moveit)) { 1728 /* Only migrate the irq if the ack has been received. 1729 * 1730 * On rare occasions the broadcast level triggered ack gets 1731 * delayed going to ioapics, and if we reprogram the 1732 * vector while Remote IRR is still set the irq will never 1733 * fire again. 1734 * 1735 * To prevent this scenario we read the Remote IRR bit 1736 * of the ioapic. This has two effects. 1737 * - On any sane system the read of the ioapic will 1738 * flush writes (and acks) going to the ioapic from 1739 * this cpu. 1740 * - We get to see if the ACK has actually been delivered. 1741 * 1742 * Based on failed experiments of reprogramming the 1743 * ioapic entry from outside of irq context starting 1744 * with masking the ioapic entry and then polling until 1745 * Remote IRR was clear before reprogramming the 1746 * ioapic I don't trust the Remote IRR bit to be 1747 * completey accurate. 1748 * 1749 * However there appears to be no other way to plug 1750 * this race, so if the Remote IRR bit is not 1751 * accurate and is causing problems then it is a hardware bug 1752 * and you can go talk to the chipset vendor about it. 1753 */ 1754 if (!io_apic_level_ack_pending(data->chip_data)) 1755 irq_move_masked_irq(data); 1756 /* If the IRQ is masked in the core, leave it: */ 1757 if (!irqd_irq_masked(data)) 1758 unmask_ioapic_irq(data); 1759 } 1760 } 1761 #else 1762 static inline bool ioapic_prepare_move(struct irq_data *data) 1763 { 1764 return false; 1765 } 1766 static inline void ioapic_finish_move(struct irq_data *data, bool moveit) 1767 { 1768 } 1769 #endif 1770 1771 static void ioapic_ack_level(struct irq_data *irq_data) 1772 { 1773 struct irq_cfg *cfg = irqd_cfg(irq_data); 1774 unsigned long v; 1775 bool moveit; 1776 int i; 1777 1778 irq_complete_move(cfg); 1779 moveit = ioapic_prepare_move(irq_data); 1780 1781 /* 1782 * It appears there is an erratum which affects at least version 0x11 1783 * of I/O APIC (that's the 82093AA and cores integrated into various 1784 * chipsets). Under certain conditions a level-triggered interrupt is 1785 * erroneously delivered as edge-triggered one but the respective IRR 1786 * bit gets set nevertheless. As a result the I/O unit expects an EOI 1787 * message but it will never arrive and further interrupts are blocked 1788 * from the source. The exact reason is so far unknown, but the 1789 * phenomenon was observed when two consecutive interrupt requests 1790 * from a given source get delivered to the same CPU and the source is 1791 * temporarily disabled in between. 1792 * 1793 * A workaround is to simulate an EOI message manually. We achieve it 1794 * by setting the trigger mode to edge and then to level when the edge 1795 * trigger mode gets detected in the TMR of a local APIC for a 1796 * level-triggered interrupt. We mask the source for the time of the 1797 * operation to prevent an edge-triggered interrupt escaping meanwhile. 1798 * The idea is from Manfred Spraul. --macro 1799 * 1800 * Also in the case when cpu goes offline, fixup_irqs() will forward 1801 * any unhandled interrupt on the offlined cpu to the new cpu 1802 * destination that is handling the corresponding interrupt. This 1803 * interrupt forwarding is done via IPI's. Hence, in this case also 1804 * level-triggered io-apic interrupt will be seen as an edge 1805 * interrupt in the IRR. And we can't rely on the cpu's EOI 1806 * to be broadcasted to the IO-APIC's which will clear the remoteIRR 1807 * corresponding to the level-triggered interrupt. Hence on IO-APIC's 1808 * supporting EOI register, we do an explicit EOI to clear the 1809 * remote IRR and on IO-APIC's which don't have an EOI register, 1810 * we use the above logic (mask+edge followed by unmask+level) from 1811 * Manfred Spraul to clear the remote IRR. 1812 */ 1813 i = cfg->vector; 1814 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); 1815 1816 /* 1817 * We must acknowledge the irq before we move it or the acknowledge will 1818 * not propagate properly. 1819 */ 1820 ack_APIC_irq(); 1821 1822 /* 1823 * Tail end of clearing remote IRR bit (either by delivering the EOI 1824 * message via io-apic EOI register write or simulating it using 1825 * mask+edge followed by unnask+level logic) manually when the 1826 * level triggered interrupt is seen as the edge triggered interrupt 1827 * at the cpu. 1828 */ 1829 if (!(v & (1 << (i & 0x1f)))) { 1830 atomic_inc(&irq_mis_count); 1831 eoi_ioapic_pin(cfg->vector, irq_data->chip_data); 1832 } 1833 1834 ioapic_finish_move(irq_data, moveit); 1835 } 1836 1837 static void ioapic_ir_ack_level(struct irq_data *irq_data) 1838 { 1839 struct mp_chip_data *data = irq_data->chip_data; 1840 1841 /* 1842 * Intr-remapping uses pin number as the virtual vector 1843 * in the RTE. Actual vector is programmed in 1844 * intr-remapping table entry. Hence for the io-apic 1845 * EOI we use the pin number. 1846 */ 1847 apic_ack_irq(irq_data); 1848 eoi_ioapic_pin(data->entry.vector, data); 1849 } 1850 1851 static void ioapic_configure_entry(struct irq_data *irqd) 1852 { 1853 struct mp_chip_data *mpd = irqd->chip_data; 1854 struct irq_cfg *cfg = irqd_cfg(irqd); 1855 struct irq_pin_list *entry; 1856 1857 /* 1858 * Only update when the parent is the vector domain, don't touch it 1859 * if the parent is the remapping domain. Check the installed 1860 * ioapic chip to verify that. 1861 */ 1862 if (irqd->chip == &ioapic_chip) { 1863 mpd->entry.destid_0_7 = cfg->dest_apicid; 1864 mpd->entry.vector = cfg->vector; 1865 } 1866 for_each_irq_pin(entry, mpd->irq_2_pin) 1867 __ioapic_write_entry(entry->apic, entry->pin, mpd->entry); 1868 } 1869 1870 static int ioapic_set_affinity(struct irq_data *irq_data, 1871 const struct cpumask *mask, bool force) 1872 { 1873 struct irq_data *parent = irq_data->parent_data; 1874 unsigned long flags; 1875 int ret; 1876 1877 ret = parent->chip->irq_set_affinity(parent, mask, force); 1878 raw_spin_lock_irqsave(&ioapic_lock, flags); 1879 if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) 1880 ioapic_configure_entry(irq_data); 1881 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 1882 1883 return ret; 1884 } 1885 1886 /* 1887 * Interrupt shutdown masks the ioapic pin, but the interrupt might already 1888 * be in flight, but not yet serviced by the target CPU. That means 1889 * __synchronize_hardirq() would return and claim that everything is calmed 1890 * down. So free_irq() would proceed and deactivate the interrupt and free 1891 * resources. 1892 * 1893 * Once the target CPU comes around to service it it will find a cleared 1894 * vector and complain. While the spurious interrupt is harmless, the full 1895 * release of resources might prevent the interrupt from being acknowledged 1896 * which keeps the hardware in a weird state. 1897 * 1898 * Verify that the corresponding Remote-IRR bits are clear. 1899 */ 1900 static int ioapic_irq_get_chip_state(struct irq_data *irqd, 1901 enum irqchip_irq_state which, 1902 bool *state) 1903 { 1904 struct mp_chip_data *mcd = irqd->chip_data; 1905 struct IO_APIC_route_entry rentry; 1906 struct irq_pin_list *p; 1907 1908 if (which != IRQCHIP_STATE_ACTIVE) 1909 return -EINVAL; 1910 1911 *state = false; 1912 raw_spin_lock(&ioapic_lock); 1913 for_each_irq_pin(p, mcd->irq_2_pin) { 1914 rentry = __ioapic_read_entry(p->apic, p->pin); 1915 /* 1916 * The remote IRR is only valid in level trigger mode. It's 1917 * meaning is undefined for edge triggered interrupts and 1918 * irrelevant because the IO-APIC treats them as fire and 1919 * forget. 1920 */ 1921 if (rentry.irr && rentry.is_level) { 1922 *state = true; 1923 break; 1924 } 1925 } 1926 raw_spin_unlock(&ioapic_lock); 1927 return 0; 1928 } 1929 1930 static struct irq_chip ioapic_chip __read_mostly = { 1931 .name = "IO-APIC", 1932 .irq_startup = startup_ioapic_irq, 1933 .irq_mask = mask_ioapic_irq, 1934 .irq_unmask = unmask_ioapic_irq, 1935 .irq_ack = irq_chip_ack_parent, 1936 .irq_eoi = ioapic_ack_level, 1937 .irq_set_affinity = ioapic_set_affinity, 1938 .irq_retrigger = irq_chip_retrigger_hierarchy, 1939 .irq_get_irqchip_state = ioapic_irq_get_chip_state, 1940 .flags = IRQCHIP_SKIP_SET_WAKE, 1941 }; 1942 1943 static struct irq_chip ioapic_ir_chip __read_mostly = { 1944 .name = "IR-IO-APIC", 1945 .irq_startup = startup_ioapic_irq, 1946 .irq_mask = mask_ioapic_irq, 1947 .irq_unmask = unmask_ioapic_irq, 1948 .irq_ack = irq_chip_ack_parent, 1949 .irq_eoi = ioapic_ir_ack_level, 1950 .irq_set_affinity = ioapic_set_affinity, 1951 .irq_retrigger = irq_chip_retrigger_hierarchy, 1952 .irq_get_irqchip_state = ioapic_irq_get_chip_state, 1953 .flags = IRQCHIP_SKIP_SET_WAKE, 1954 }; 1955 1956 static inline void init_IO_APIC_traps(void) 1957 { 1958 struct irq_cfg *cfg; 1959 unsigned int irq; 1960 1961 for_each_active_irq(irq) { 1962 cfg = irq_cfg(irq); 1963 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) { 1964 /* 1965 * Hmm.. We don't have an entry for this, 1966 * so default to an old-fashioned 8259 1967 * interrupt if we can.. 1968 */ 1969 if (irq < nr_legacy_irqs()) 1970 legacy_pic->make_irq(irq); 1971 else 1972 /* Strange. Oh, well.. */ 1973 irq_set_chip(irq, &no_irq_chip); 1974 } 1975 } 1976 } 1977 1978 /* 1979 * The local APIC irq-chip implementation: 1980 */ 1981 1982 static void mask_lapic_irq(struct irq_data *data) 1983 { 1984 unsigned long v; 1985 1986 v = apic_read(APIC_LVT0); 1987 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 1988 } 1989 1990 static void unmask_lapic_irq(struct irq_data *data) 1991 { 1992 unsigned long v; 1993 1994 v = apic_read(APIC_LVT0); 1995 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); 1996 } 1997 1998 static void ack_lapic_irq(struct irq_data *data) 1999 { 2000 ack_APIC_irq(); 2001 } 2002 2003 static struct irq_chip lapic_chip __read_mostly = { 2004 .name = "local-APIC", 2005 .irq_mask = mask_lapic_irq, 2006 .irq_unmask = unmask_lapic_irq, 2007 .irq_ack = ack_lapic_irq, 2008 }; 2009 2010 static void lapic_register_intr(int irq) 2011 { 2012 irq_clear_status_flags(irq, IRQ_LEVEL); 2013 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, 2014 "edge"); 2015 } 2016 2017 /* 2018 * This looks a bit hackish but it's about the only one way of sending 2019 * a few INTA cycles to 8259As and any associated glue logic. ICR does 2020 * not support the ExtINT mode, unfortunately. We need to send these 2021 * cycles as some i82489DX-based boards have glue logic that keeps the 2022 * 8259A interrupt line asserted until INTA. --macro 2023 */ 2024 static inline void __init unlock_ExtINT_logic(void) 2025 { 2026 int apic, pin, i; 2027 struct IO_APIC_route_entry entry0, entry1; 2028 unsigned char save_control, save_freq_select; 2029 2030 pin = find_isa_irq_pin(8, mp_INT); 2031 if (pin == -1) { 2032 WARN_ON_ONCE(1); 2033 return; 2034 } 2035 apic = find_isa_irq_apic(8, mp_INT); 2036 if (apic == -1) { 2037 WARN_ON_ONCE(1); 2038 return; 2039 } 2040 2041 entry0 = ioapic_read_entry(apic, pin); 2042 clear_IO_APIC_pin(apic, pin); 2043 2044 memset(&entry1, 0, sizeof(entry1)); 2045 2046 entry1.dest_mode_logical = true; 2047 entry1.masked = false; 2048 entry1.destid_0_7 = hard_smp_processor_id(); 2049 entry1.delivery_mode = APIC_DELIVERY_MODE_EXTINT; 2050 entry1.active_low = entry0.active_low; 2051 entry1.is_level = false; 2052 entry1.vector = 0; 2053 2054 ioapic_write_entry(apic, pin, entry1); 2055 2056 save_control = CMOS_READ(RTC_CONTROL); 2057 save_freq_select = CMOS_READ(RTC_FREQ_SELECT); 2058 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6, 2059 RTC_FREQ_SELECT); 2060 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL); 2061 2062 i = 100; 2063 while (i-- > 0) { 2064 mdelay(10); 2065 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF) 2066 i -= 10; 2067 } 2068 2069 CMOS_WRITE(save_control, RTC_CONTROL); 2070 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); 2071 clear_IO_APIC_pin(apic, pin); 2072 2073 ioapic_write_entry(apic, pin, entry0); 2074 } 2075 2076 static int disable_timer_pin_1 __initdata; 2077 /* Actually the next is obsolete, but keep it for paranoid reasons -AK */ 2078 static int __init disable_timer_pin_setup(char *arg) 2079 { 2080 disable_timer_pin_1 = 1; 2081 return 0; 2082 } 2083 early_param("disable_timer_pin_1", disable_timer_pin_setup); 2084 2085 static int mp_alloc_timer_irq(int ioapic, int pin) 2086 { 2087 int irq = -1; 2088 struct irq_domain *domain = mp_ioapic_irqdomain(ioapic); 2089 2090 if (domain) { 2091 struct irq_alloc_info info; 2092 2093 ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0); 2094 info.devid = mpc_ioapic_id(ioapic); 2095 info.ioapic.pin = pin; 2096 mutex_lock(&ioapic_mutex); 2097 irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info); 2098 mutex_unlock(&ioapic_mutex); 2099 } 2100 2101 return irq; 2102 } 2103 2104 /* 2105 * This code may look a bit paranoid, but it's supposed to cooperate with 2106 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ 2107 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast 2108 * fanatically on his truly buggy board. 2109 * 2110 * FIXME: really need to revamp this for all platforms. 2111 */ 2112 static inline void __init check_timer(void) 2113 { 2114 struct irq_data *irq_data = irq_get_irq_data(0); 2115 struct mp_chip_data *data = irq_data->chip_data; 2116 struct irq_cfg *cfg = irqd_cfg(irq_data); 2117 int node = cpu_to_node(0); 2118 int apic1, pin1, apic2, pin2; 2119 unsigned long flags; 2120 int no_pin1 = 0; 2121 2122 if (!global_clock_event) 2123 return; 2124 2125 local_irq_save(flags); 2126 2127 /* 2128 * get/set the timer IRQ vector: 2129 */ 2130 legacy_pic->mask(0); 2131 2132 /* 2133 * As IRQ0 is to be enabled in the 8259A, the virtual 2134 * wire has to be disabled in the local APIC. Also 2135 * timer interrupts need to be acknowledged manually in 2136 * the 8259A for the i82489DX when using the NMI 2137 * watchdog as that APIC treats NMIs as level-triggered. 2138 * The AEOI mode will finish them in the 8259A 2139 * automatically. 2140 */ 2141 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT); 2142 legacy_pic->init(1); 2143 2144 pin1 = find_isa_irq_pin(0, mp_INT); 2145 apic1 = find_isa_irq_apic(0, mp_INT); 2146 pin2 = ioapic_i8259.pin; 2147 apic2 = ioapic_i8259.apic; 2148 2149 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X " 2150 "apic1=%d pin1=%d apic2=%d pin2=%d\n", 2151 cfg->vector, apic1, pin1, apic2, pin2); 2152 2153 /* 2154 * Some BIOS writers are clueless and report the ExtINTA 2155 * I/O APIC input from the cascaded 8259A as the timer 2156 * interrupt input. So just in case, if only one pin 2157 * was found above, try it both directly and through the 2158 * 8259A. 2159 */ 2160 if (pin1 == -1) { 2161 panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC"); 2162 pin1 = pin2; 2163 apic1 = apic2; 2164 no_pin1 = 1; 2165 } else if (pin2 == -1) { 2166 pin2 = pin1; 2167 apic2 = apic1; 2168 } 2169 2170 if (pin1 != -1) { 2171 /* Ok, does IRQ0 through the IOAPIC work? */ 2172 if (no_pin1) { 2173 mp_alloc_timer_irq(apic1, pin1); 2174 } else { 2175 /* 2176 * for edge trigger, it's already unmasked, 2177 * so only need to unmask if it is level-trigger 2178 * do we really have level trigger timer? 2179 */ 2180 int idx = find_irq_entry(apic1, pin1, mp_INT); 2181 2182 if (idx != -1 && irq_is_level(idx)) 2183 unmask_ioapic_irq(irq_get_irq_data(0)); 2184 } 2185 irq_domain_deactivate_irq(irq_data); 2186 irq_domain_activate_irq(irq_data, false); 2187 if (timer_irq_works()) { 2188 if (disable_timer_pin_1 > 0) 2189 clear_IO_APIC_pin(0, pin1); 2190 goto out; 2191 } 2192 panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC"); 2193 local_irq_disable(); 2194 clear_IO_APIC_pin(apic1, pin1); 2195 if (!no_pin1) 2196 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: " 2197 "8254 timer not connected to IO-APIC\n"); 2198 2199 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer " 2200 "(IRQ0) through the 8259A ...\n"); 2201 apic_printk(APIC_QUIET, KERN_INFO 2202 "..... (found apic %d pin %d) ...\n", apic2, pin2); 2203 /* 2204 * legacy devices should be connected to IO APIC #0 2205 */ 2206 replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2); 2207 irq_domain_deactivate_irq(irq_data); 2208 irq_domain_activate_irq(irq_data, false); 2209 legacy_pic->unmask(0); 2210 if (timer_irq_works()) { 2211 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); 2212 goto out; 2213 } 2214 /* 2215 * Cleanup, just in case ... 2216 */ 2217 local_irq_disable(); 2218 legacy_pic->mask(0); 2219 clear_IO_APIC_pin(apic2, pin2); 2220 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n"); 2221 } 2222 2223 apic_printk(APIC_QUIET, KERN_INFO 2224 "...trying to set up timer as Virtual Wire IRQ...\n"); 2225 2226 lapic_register_intr(0); 2227 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ 2228 legacy_pic->unmask(0); 2229 2230 if (timer_irq_works()) { 2231 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n"); 2232 goto out; 2233 } 2234 local_irq_disable(); 2235 legacy_pic->mask(0); 2236 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector); 2237 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n"); 2238 2239 apic_printk(APIC_QUIET, KERN_INFO 2240 "...trying to set up timer as ExtINT IRQ...\n"); 2241 2242 legacy_pic->init(0); 2243 legacy_pic->make_irq(0); 2244 apic_write(APIC_LVT0, APIC_DM_EXTINT); 2245 legacy_pic->unmask(0); 2246 2247 unlock_ExtINT_logic(); 2248 2249 if (timer_irq_works()) { 2250 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n"); 2251 goto out; 2252 } 2253 local_irq_disable(); 2254 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n"); 2255 if (apic_is_x2apic_enabled()) 2256 apic_printk(APIC_QUIET, KERN_INFO 2257 "Perhaps problem with the pre-enabled x2apic mode\n" 2258 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n"); 2259 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a " 2260 "report. Then try booting with the 'noapic' option.\n"); 2261 out: 2262 local_irq_restore(flags); 2263 } 2264 2265 /* 2266 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available 2267 * to devices. However there may be an I/O APIC pin available for 2268 * this interrupt regardless. The pin may be left unconnected, but 2269 * typically it will be reused as an ExtINT cascade interrupt for 2270 * the master 8259A. In the MPS case such a pin will normally be 2271 * reported as an ExtINT interrupt in the MP table. With ACPI 2272 * there is no provision for ExtINT interrupts, and in the absence 2273 * of an override it would be treated as an ordinary ISA I/O APIC 2274 * interrupt, that is edge-triggered and unmasked by default. We 2275 * used to do this, but it caused problems on some systems because 2276 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using 2277 * the same ExtINT cascade interrupt to drive the local APIC of the 2278 * bootstrap processor. Therefore we refrain from routing IRQ2 to 2279 * the I/O APIC in all cases now. No actual device should request 2280 * it anyway. --macro 2281 */ 2282 #define PIC_IRQS (1UL << PIC_CASCADE_IR) 2283 2284 static int mp_irqdomain_create(int ioapic) 2285 { 2286 struct irq_alloc_info info; 2287 struct irq_domain *parent; 2288 int hwirqs = mp_ioapic_pin_count(ioapic); 2289 struct ioapic *ip = &ioapics[ioapic]; 2290 struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg; 2291 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2292 struct fwnode_handle *fn; 2293 char *name = "IO-APIC"; 2294 2295 if (cfg->type == IOAPIC_DOMAIN_INVALID) 2296 return 0; 2297 2298 init_irq_alloc_info(&info, NULL); 2299 info.type = X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT; 2300 info.devid = mpc_ioapic_id(ioapic); 2301 parent = irq_remapping_get_irq_domain(&info); 2302 if (!parent) 2303 parent = x86_vector_domain; 2304 else 2305 name = "IO-APIC-IR"; 2306 2307 /* Handle device tree enumerated APICs proper */ 2308 if (cfg->dev) { 2309 fn = of_node_to_fwnode(cfg->dev); 2310 } else { 2311 fn = irq_domain_alloc_named_id_fwnode(name, ioapic); 2312 if (!fn) 2313 return -ENOMEM; 2314 } 2315 2316 ip->irqdomain = irq_domain_create_linear(fn, hwirqs, cfg->ops, 2317 (void *)(long)ioapic); 2318 2319 if (!ip->irqdomain) { 2320 /* Release fw handle if it was allocated above */ 2321 if (!cfg->dev) 2322 irq_domain_free_fwnode(fn); 2323 return -ENOMEM; 2324 } 2325 2326 ip->irqdomain->parent = parent; 2327 2328 if (cfg->type == IOAPIC_DOMAIN_LEGACY || 2329 cfg->type == IOAPIC_DOMAIN_STRICT) 2330 ioapic_dynirq_base = max(ioapic_dynirq_base, 2331 gsi_cfg->gsi_end + 1); 2332 2333 return 0; 2334 } 2335 2336 static void ioapic_destroy_irqdomain(int idx) 2337 { 2338 struct ioapic_domain_cfg *cfg = &ioapics[idx].irqdomain_cfg; 2339 struct fwnode_handle *fn = ioapics[idx].irqdomain->fwnode; 2340 2341 if (ioapics[idx].irqdomain) { 2342 irq_domain_remove(ioapics[idx].irqdomain); 2343 if (!cfg->dev) 2344 irq_domain_free_fwnode(fn); 2345 ioapics[idx].irqdomain = NULL; 2346 } 2347 } 2348 2349 void __init setup_IO_APIC(void) 2350 { 2351 int ioapic; 2352 2353 if (skip_ioapic_setup || !nr_ioapics) 2354 return; 2355 2356 io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL; 2357 2358 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n"); 2359 for_each_ioapic(ioapic) 2360 BUG_ON(mp_irqdomain_create(ioapic)); 2361 2362 /* 2363 * Set up IO-APIC IRQ routing. 2364 */ 2365 x86_init.mpparse.setup_ioapic_ids(); 2366 2367 sync_Arb_IDs(); 2368 setup_IO_APIC_irqs(); 2369 init_IO_APIC_traps(); 2370 if (nr_legacy_irqs()) 2371 check_timer(); 2372 2373 ioapic_initialized = 1; 2374 } 2375 2376 static void resume_ioapic_id(int ioapic_idx) 2377 { 2378 unsigned long flags; 2379 union IO_APIC_reg_00 reg_00; 2380 2381 raw_spin_lock_irqsave(&ioapic_lock, flags); 2382 reg_00.raw = io_apic_read(ioapic_idx, 0); 2383 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) { 2384 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx); 2385 io_apic_write(ioapic_idx, 0, reg_00.raw); 2386 } 2387 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2388 } 2389 2390 static void ioapic_resume(void) 2391 { 2392 int ioapic_idx; 2393 2394 for_each_ioapic_reverse(ioapic_idx) 2395 resume_ioapic_id(ioapic_idx); 2396 2397 restore_ioapic_entries(); 2398 } 2399 2400 static struct syscore_ops ioapic_syscore_ops = { 2401 .suspend = save_ioapic_entries, 2402 .resume = ioapic_resume, 2403 }; 2404 2405 static int __init ioapic_init_ops(void) 2406 { 2407 register_syscore_ops(&ioapic_syscore_ops); 2408 2409 return 0; 2410 } 2411 2412 device_initcall(ioapic_init_ops); 2413 2414 static int io_apic_get_redir_entries(int ioapic) 2415 { 2416 union IO_APIC_reg_01 reg_01; 2417 unsigned long flags; 2418 2419 raw_spin_lock_irqsave(&ioapic_lock, flags); 2420 reg_01.raw = io_apic_read(ioapic, 1); 2421 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2422 2423 /* The register returns the maximum index redir index 2424 * supported, which is one less than the total number of redir 2425 * entries. 2426 */ 2427 return reg_01.bits.entries + 1; 2428 } 2429 2430 unsigned int arch_dynirq_lower_bound(unsigned int from) 2431 { 2432 /* 2433 * dmar_alloc_hwirq() may be called before setup_IO_APIC(), so use 2434 * gsi_top if ioapic_dynirq_base hasn't been initialized yet. 2435 */ 2436 if (!ioapic_initialized) 2437 return gsi_top; 2438 /* 2439 * For DT enabled machines ioapic_dynirq_base is irrelevant and not 2440 * updated. So simply return @from if ioapic_dynirq_base == 0. 2441 */ 2442 return ioapic_dynirq_base ? : from; 2443 } 2444 2445 #ifdef CONFIG_X86_32 2446 static int io_apic_get_unique_id(int ioapic, int apic_id) 2447 { 2448 union IO_APIC_reg_00 reg_00; 2449 static physid_mask_t apic_id_map = PHYSID_MASK_NONE; 2450 physid_mask_t tmp; 2451 unsigned long flags; 2452 int i = 0; 2453 2454 /* 2455 * The P4 platform supports up to 256 APIC IDs on two separate APIC 2456 * buses (one for LAPICs, one for IOAPICs), where predecessors only 2457 * supports up to 16 on one shared APIC bus. 2458 * 2459 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full 2460 * advantage of new APIC bus architecture. 2461 */ 2462 2463 if (physids_empty(apic_id_map)) 2464 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map); 2465 2466 raw_spin_lock_irqsave(&ioapic_lock, flags); 2467 reg_00.raw = io_apic_read(ioapic, 0); 2468 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2469 2470 if (apic_id >= get_physical_broadcast()) { 2471 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying " 2472 "%d\n", ioapic, apic_id, reg_00.bits.ID); 2473 apic_id = reg_00.bits.ID; 2474 } 2475 2476 /* 2477 * Every APIC in a system must have a unique ID or we get lots of nice 2478 * 'stuck on smp_invalidate_needed IPI wait' messages. 2479 */ 2480 if (apic->check_apicid_used(&apic_id_map, apic_id)) { 2481 2482 for (i = 0; i < get_physical_broadcast(); i++) { 2483 if (!apic->check_apicid_used(&apic_id_map, i)) 2484 break; 2485 } 2486 2487 if (i == get_physical_broadcast()) 2488 panic("Max apic_id exceeded!\n"); 2489 2490 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, " 2491 "trying %d\n", ioapic, apic_id, i); 2492 2493 apic_id = i; 2494 } 2495 2496 apic->apicid_to_cpu_present(apic_id, &tmp); 2497 physids_or(apic_id_map, apic_id_map, tmp); 2498 2499 if (reg_00.bits.ID != apic_id) { 2500 reg_00.bits.ID = apic_id; 2501 2502 raw_spin_lock_irqsave(&ioapic_lock, flags); 2503 io_apic_write(ioapic, 0, reg_00.raw); 2504 reg_00.raw = io_apic_read(ioapic, 0); 2505 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2506 2507 /* Sanity check */ 2508 if (reg_00.bits.ID != apic_id) { 2509 pr_err("IOAPIC[%d]: Unable to change apic_id!\n", 2510 ioapic); 2511 return -1; 2512 } 2513 } 2514 2515 apic_printk(APIC_VERBOSE, KERN_INFO 2516 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id); 2517 2518 return apic_id; 2519 } 2520 2521 static u8 io_apic_unique_id(int idx, u8 id) 2522 { 2523 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && 2524 !APIC_XAPIC(boot_cpu_apic_version)) 2525 return io_apic_get_unique_id(idx, id); 2526 else 2527 return id; 2528 } 2529 #else 2530 static u8 io_apic_unique_id(int idx, u8 id) 2531 { 2532 union IO_APIC_reg_00 reg_00; 2533 DECLARE_BITMAP(used, 256); 2534 unsigned long flags; 2535 u8 new_id; 2536 int i; 2537 2538 bitmap_zero(used, 256); 2539 for_each_ioapic(i) 2540 __set_bit(mpc_ioapic_id(i), used); 2541 2542 /* Hand out the requested id if available */ 2543 if (!test_bit(id, used)) 2544 return id; 2545 2546 /* 2547 * Read the current id from the ioapic and keep it if 2548 * available. 2549 */ 2550 raw_spin_lock_irqsave(&ioapic_lock, flags); 2551 reg_00.raw = io_apic_read(idx, 0); 2552 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2553 new_id = reg_00.bits.ID; 2554 if (!test_bit(new_id, used)) { 2555 apic_printk(APIC_VERBOSE, KERN_INFO 2556 "IOAPIC[%d]: Using reg apic_id %d instead of %d\n", 2557 idx, new_id, id); 2558 return new_id; 2559 } 2560 2561 /* 2562 * Get the next free id and write it to the ioapic. 2563 */ 2564 new_id = find_first_zero_bit(used, 256); 2565 reg_00.bits.ID = new_id; 2566 raw_spin_lock_irqsave(&ioapic_lock, flags); 2567 io_apic_write(idx, 0, reg_00.raw); 2568 reg_00.raw = io_apic_read(idx, 0); 2569 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2570 /* Sanity check */ 2571 BUG_ON(reg_00.bits.ID != new_id); 2572 2573 return new_id; 2574 } 2575 #endif 2576 2577 static int io_apic_get_version(int ioapic) 2578 { 2579 union IO_APIC_reg_01 reg_01; 2580 unsigned long flags; 2581 2582 raw_spin_lock_irqsave(&ioapic_lock, flags); 2583 reg_01.raw = io_apic_read(ioapic, 1); 2584 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 2585 2586 return reg_01.bits.version; 2587 } 2588 2589 /* 2590 * This function updates target affinity of IOAPIC interrupts to include 2591 * the CPUs which came online during SMP bringup. 2592 */ 2593 #define IOAPIC_RESOURCE_NAME_SIZE 11 2594 2595 static struct resource *ioapic_resources; 2596 2597 static struct resource * __init ioapic_setup_resources(void) 2598 { 2599 unsigned long n; 2600 struct resource *res; 2601 char *mem; 2602 int i; 2603 2604 if (nr_ioapics == 0) 2605 return NULL; 2606 2607 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource); 2608 n *= nr_ioapics; 2609 2610 mem = memblock_alloc(n, SMP_CACHE_BYTES); 2611 if (!mem) 2612 panic("%s: Failed to allocate %lu bytes\n", __func__, n); 2613 res = (void *)mem; 2614 2615 mem += sizeof(struct resource) * nr_ioapics; 2616 2617 for_each_ioapic(i) { 2618 res[i].name = mem; 2619 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY; 2620 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i); 2621 mem += IOAPIC_RESOURCE_NAME_SIZE; 2622 ioapics[i].iomem_res = &res[i]; 2623 } 2624 2625 ioapic_resources = res; 2626 2627 return res; 2628 } 2629 2630 void __init io_apic_init_mappings(void) 2631 { 2632 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0; 2633 struct resource *ioapic_res; 2634 int i; 2635 2636 ioapic_res = ioapic_setup_resources(); 2637 for_each_ioapic(i) { 2638 if (smp_found_config) { 2639 ioapic_phys = mpc_ioapic_addr(i); 2640 #ifdef CONFIG_X86_32 2641 if (!ioapic_phys) { 2642 printk(KERN_ERR 2643 "WARNING: bogus zero IO-APIC " 2644 "address found in MPTABLE, " 2645 "disabling IO/APIC support!\n"); 2646 smp_found_config = 0; 2647 skip_ioapic_setup = 1; 2648 goto fake_ioapic_page; 2649 } 2650 #endif 2651 } else { 2652 #ifdef CONFIG_X86_32 2653 fake_ioapic_page: 2654 #endif 2655 ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE, 2656 PAGE_SIZE); 2657 if (!ioapic_phys) 2658 panic("%s: Failed to allocate %lu bytes align=0x%lx\n", 2659 __func__, PAGE_SIZE, PAGE_SIZE); 2660 ioapic_phys = __pa(ioapic_phys); 2661 } 2662 set_fixmap_nocache(idx, ioapic_phys); 2663 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n", 2664 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK), 2665 ioapic_phys); 2666 idx++; 2667 2668 ioapic_res->start = ioapic_phys; 2669 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1; 2670 ioapic_res++; 2671 } 2672 } 2673 2674 void __init ioapic_insert_resources(void) 2675 { 2676 int i; 2677 struct resource *r = ioapic_resources; 2678 2679 if (!r) { 2680 if (nr_ioapics > 0) 2681 printk(KERN_ERR 2682 "IO APIC resources couldn't be allocated.\n"); 2683 return; 2684 } 2685 2686 for_each_ioapic(i) { 2687 insert_resource(&iomem_resource, r); 2688 r++; 2689 } 2690 } 2691 2692 int mp_find_ioapic(u32 gsi) 2693 { 2694 int i; 2695 2696 if (nr_ioapics == 0) 2697 return -1; 2698 2699 /* Find the IOAPIC that manages this GSI. */ 2700 for_each_ioapic(i) { 2701 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i); 2702 if (gsi >= gsi_cfg->gsi_base && gsi <= gsi_cfg->gsi_end) 2703 return i; 2704 } 2705 2706 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi); 2707 return -1; 2708 } 2709 2710 int mp_find_ioapic_pin(int ioapic, u32 gsi) 2711 { 2712 struct mp_ioapic_gsi *gsi_cfg; 2713 2714 if (WARN_ON(ioapic < 0)) 2715 return -1; 2716 2717 gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2718 if (WARN_ON(gsi > gsi_cfg->gsi_end)) 2719 return -1; 2720 2721 return gsi - gsi_cfg->gsi_base; 2722 } 2723 2724 static int bad_ioapic_register(int idx) 2725 { 2726 union IO_APIC_reg_00 reg_00; 2727 union IO_APIC_reg_01 reg_01; 2728 union IO_APIC_reg_02 reg_02; 2729 2730 reg_00.raw = io_apic_read(idx, 0); 2731 reg_01.raw = io_apic_read(idx, 1); 2732 reg_02.raw = io_apic_read(idx, 2); 2733 2734 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) { 2735 pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n", 2736 mpc_ioapic_addr(idx)); 2737 return 1; 2738 } 2739 2740 return 0; 2741 } 2742 2743 static int find_free_ioapic_entry(void) 2744 { 2745 int idx; 2746 2747 for (idx = 0; idx < MAX_IO_APICS; idx++) 2748 if (ioapics[idx].nr_registers == 0) 2749 return idx; 2750 2751 return MAX_IO_APICS; 2752 } 2753 2754 /** 2755 * mp_register_ioapic - Register an IOAPIC device 2756 * @id: hardware IOAPIC ID 2757 * @address: physical address of IOAPIC register area 2758 * @gsi_base: base of GSI associated with the IOAPIC 2759 * @cfg: configuration information for the IOAPIC 2760 */ 2761 int mp_register_ioapic(int id, u32 address, u32 gsi_base, 2762 struct ioapic_domain_cfg *cfg) 2763 { 2764 bool hotplug = !!ioapic_initialized; 2765 struct mp_ioapic_gsi *gsi_cfg; 2766 int idx, ioapic, entries; 2767 u32 gsi_end; 2768 2769 if (!address) { 2770 pr_warn("Bogus (zero) I/O APIC address found, skipping!\n"); 2771 return -EINVAL; 2772 } 2773 for_each_ioapic(ioapic) 2774 if (ioapics[ioapic].mp_config.apicaddr == address) { 2775 pr_warn("address 0x%x conflicts with IOAPIC%d\n", 2776 address, ioapic); 2777 return -EEXIST; 2778 } 2779 2780 idx = find_free_ioapic_entry(); 2781 if (idx >= MAX_IO_APICS) { 2782 pr_warn("Max # of I/O APICs (%d) exceeded (found %d), skipping\n", 2783 MAX_IO_APICS, idx); 2784 return -ENOSPC; 2785 } 2786 2787 ioapics[idx].mp_config.type = MP_IOAPIC; 2788 ioapics[idx].mp_config.flags = MPC_APIC_USABLE; 2789 ioapics[idx].mp_config.apicaddr = address; 2790 2791 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address); 2792 if (bad_ioapic_register(idx)) { 2793 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2794 return -ENODEV; 2795 } 2796 2797 ioapics[idx].mp_config.apicid = io_apic_unique_id(idx, id); 2798 ioapics[idx].mp_config.apicver = io_apic_get_version(idx); 2799 2800 /* 2801 * Build basic GSI lookup table to facilitate gsi->io_apic lookups 2802 * and to prevent reprogramming of IOAPIC pins (PCI GSIs). 2803 */ 2804 entries = io_apic_get_redir_entries(idx); 2805 gsi_end = gsi_base + entries - 1; 2806 for_each_ioapic(ioapic) { 2807 gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2808 if ((gsi_base >= gsi_cfg->gsi_base && 2809 gsi_base <= gsi_cfg->gsi_end) || 2810 (gsi_end >= gsi_cfg->gsi_base && 2811 gsi_end <= gsi_cfg->gsi_end)) { 2812 pr_warn("GSI range [%u-%u] for new IOAPIC conflicts with GSI[%u-%u]\n", 2813 gsi_base, gsi_end, 2814 gsi_cfg->gsi_base, gsi_cfg->gsi_end); 2815 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2816 return -ENOSPC; 2817 } 2818 } 2819 gsi_cfg = mp_ioapic_gsi_routing(idx); 2820 gsi_cfg->gsi_base = gsi_base; 2821 gsi_cfg->gsi_end = gsi_end; 2822 2823 ioapics[idx].irqdomain = NULL; 2824 ioapics[idx].irqdomain_cfg = *cfg; 2825 2826 /* 2827 * If mp_register_ioapic() is called during early boot stage when 2828 * walking ACPI/SFI/DT tables, it's too early to create irqdomain, 2829 * we are still using bootmem allocator. So delay it to setup_IO_APIC(). 2830 */ 2831 if (hotplug) { 2832 if (mp_irqdomain_create(idx)) { 2833 clear_fixmap(FIX_IO_APIC_BASE_0 + idx); 2834 return -ENOMEM; 2835 } 2836 alloc_ioapic_saved_registers(idx); 2837 } 2838 2839 if (gsi_cfg->gsi_end >= gsi_top) 2840 gsi_top = gsi_cfg->gsi_end + 1; 2841 if (nr_ioapics <= idx) 2842 nr_ioapics = idx + 1; 2843 2844 /* Set nr_registers to mark entry present */ 2845 ioapics[idx].nr_registers = entries; 2846 2847 pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n", 2848 idx, mpc_ioapic_id(idx), 2849 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx), 2850 gsi_cfg->gsi_base, gsi_cfg->gsi_end); 2851 2852 return 0; 2853 } 2854 2855 int mp_unregister_ioapic(u32 gsi_base) 2856 { 2857 int ioapic, pin; 2858 int found = 0; 2859 2860 for_each_ioapic(ioapic) 2861 if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) { 2862 found = 1; 2863 break; 2864 } 2865 if (!found) { 2866 pr_warn("can't find IOAPIC for GSI %d\n", gsi_base); 2867 return -ENODEV; 2868 } 2869 2870 for_each_pin(ioapic, pin) { 2871 u32 gsi = mp_pin_to_gsi(ioapic, pin); 2872 int irq = mp_map_gsi_to_irq(gsi, 0, NULL); 2873 struct mp_chip_data *data; 2874 2875 if (irq >= 0) { 2876 data = irq_get_chip_data(irq); 2877 if (data && data->count) { 2878 pr_warn("pin%d on IOAPIC%d is still in use.\n", 2879 pin, ioapic); 2880 return -EBUSY; 2881 } 2882 } 2883 } 2884 2885 /* Mark entry not present */ 2886 ioapics[ioapic].nr_registers = 0; 2887 ioapic_destroy_irqdomain(ioapic); 2888 free_ioapic_saved_registers(ioapic); 2889 if (ioapics[ioapic].iomem_res) 2890 release_resource(ioapics[ioapic].iomem_res); 2891 clear_fixmap(FIX_IO_APIC_BASE_0 + ioapic); 2892 memset(&ioapics[ioapic], 0, sizeof(ioapics[ioapic])); 2893 2894 return 0; 2895 } 2896 2897 int mp_ioapic_registered(u32 gsi_base) 2898 { 2899 int ioapic; 2900 2901 for_each_ioapic(ioapic) 2902 if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) 2903 return 1; 2904 2905 return 0; 2906 } 2907 2908 static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data, 2909 struct irq_alloc_info *info) 2910 { 2911 if (info && info->ioapic.valid) { 2912 data->is_level = info->ioapic.is_level; 2913 data->active_low = info->ioapic.active_low; 2914 } else if (__acpi_get_override_irq(gsi, &data->is_level, 2915 &data->active_low) < 0) { 2916 /* PCI interrupts are always active low level triggered. */ 2917 data->is_level = true; 2918 data->active_low = true; 2919 } 2920 } 2921 2922 static void mp_setup_entry(struct irq_cfg *cfg, struct mp_chip_data *data, 2923 struct IO_APIC_route_entry *entry) 2924 { 2925 memset(entry, 0, sizeof(*entry)); 2926 entry->delivery_mode = apic->delivery_mode; 2927 entry->dest_mode_logical = apic->dest_mode_logical; 2928 entry->destid_0_7 = cfg->dest_apicid; 2929 entry->vector = cfg->vector; 2930 entry->is_level = data->is_level; 2931 entry->active_low = data->active_low; 2932 /* 2933 * Mask level triggered irqs. Edge triggered irqs are masked 2934 * by the irq core code in case they fire. 2935 */ 2936 entry->masked = data->is_level; 2937 } 2938 2939 int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, 2940 unsigned int nr_irqs, void *arg) 2941 { 2942 int ret, ioapic, pin; 2943 struct irq_cfg *cfg; 2944 struct irq_data *irq_data; 2945 struct mp_chip_data *data; 2946 struct irq_alloc_info *info = arg; 2947 unsigned long flags; 2948 2949 if (!info || nr_irqs > 1) 2950 return -EINVAL; 2951 irq_data = irq_domain_get_irq_data(domain, virq); 2952 if (!irq_data) 2953 return -EINVAL; 2954 2955 ioapic = mp_irqdomain_ioapic_idx(domain); 2956 pin = info->ioapic.pin; 2957 if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0) 2958 return -EEXIST; 2959 2960 data = kzalloc(sizeof(*data), GFP_KERNEL); 2961 if (!data) 2962 return -ENOMEM; 2963 2964 info->ioapic.entry = &data->entry; 2965 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info); 2966 if (ret < 0) { 2967 kfree(data); 2968 return ret; 2969 } 2970 2971 INIT_LIST_HEAD(&data->irq_2_pin); 2972 irq_data->hwirq = info->ioapic.pin; 2973 irq_data->chip = (domain->parent == x86_vector_domain) ? 2974 &ioapic_chip : &ioapic_ir_chip; 2975 irq_data->chip_data = data; 2976 mp_irqdomain_get_attr(mp_pin_to_gsi(ioapic, pin), data, info); 2977 2978 cfg = irqd_cfg(irq_data); 2979 add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin); 2980 2981 local_irq_save(flags); 2982 if (info->ioapic.entry) 2983 mp_setup_entry(cfg, data, info->ioapic.entry); 2984 mp_register_handler(virq, data->is_level); 2985 if (virq < nr_legacy_irqs()) 2986 legacy_pic->mask(virq); 2987 local_irq_restore(flags); 2988 2989 apic_printk(APIC_VERBOSE, KERN_DEBUG 2990 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i Dest:%d)\n", 2991 ioapic, mpc_ioapic_id(ioapic), pin, cfg->vector, 2992 virq, data->is_level, data->active_low, 2993 cfg->dest_apicid); 2994 2995 return 0; 2996 } 2997 2998 void mp_irqdomain_free(struct irq_domain *domain, unsigned int virq, 2999 unsigned int nr_irqs) 3000 { 3001 struct irq_data *irq_data; 3002 struct mp_chip_data *data; 3003 3004 BUG_ON(nr_irqs != 1); 3005 irq_data = irq_domain_get_irq_data(domain, virq); 3006 if (irq_data && irq_data->chip_data) { 3007 data = irq_data->chip_data; 3008 __remove_pin_from_irq(data, mp_irqdomain_ioapic_idx(domain), 3009 (int)irq_data->hwirq); 3010 WARN_ON(!list_empty(&data->irq_2_pin)); 3011 kfree(irq_data->chip_data); 3012 } 3013 irq_domain_free_irqs_top(domain, virq, nr_irqs); 3014 } 3015 3016 int mp_irqdomain_activate(struct irq_domain *domain, 3017 struct irq_data *irq_data, bool reserve) 3018 { 3019 unsigned long flags; 3020 3021 raw_spin_lock_irqsave(&ioapic_lock, flags); 3022 ioapic_configure_entry(irq_data); 3023 raw_spin_unlock_irqrestore(&ioapic_lock, flags); 3024 return 0; 3025 } 3026 3027 void mp_irqdomain_deactivate(struct irq_domain *domain, 3028 struct irq_data *irq_data) 3029 { 3030 /* It won't be called for IRQ with multiple IOAPIC pins associated */ 3031 ioapic_mask_entry(mp_irqdomain_ioapic_idx(domain), 3032 (int)irq_data->hwirq); 3033 } 3034 3035 int mp_irqdomain_ioapic_idx(struct irq_domain *domain) 3036 { 3037 return (int)(long)domain->host_data; 3038 } 3039 3040 const struct irq_domain_ops mp_ioapic_irqdomain_ops = { 3041 .alloc = mp_irqdomain_alloc, 3042 .free = mp_irqdomain_free, 3043 .activate = mp_irqdomain_activate, 3044 .deactivate = mp_irqdomain_deactivate, 3045 }; 3046