1 // SPDX-License-Identifier: GPL-2.0 2 3 #define pr_fmt(fmt) "DMAR-IR: " fmt 4 5 #include <linux/interrupt.h> 6 #include <linux/dmar.h> 7 #include <linux/spinlock.h> 8 #include <linux/slab.h> 9 #include <linux/jiffies.h> 10 #include <linux/hpet.h> 11 #include <linux/pci.h> 12 #include <linux/irq.h> 13 #include <linux/intel-iommu.h> 14 #include <linux/acpi.h> 15 #include <linux/irqdomain.h> 16 #include <linux/crash_dump.h> 17 #include <asm/io_apic.h> 18 #include <asm/smp.h> 19 #include <asm/cpu.h> 20 #include <asm/irq_remapping.h> 21 #include <asm/pci-direct.h> 22 #include <asm/msidef.h> 23 24 #include "../irq_remapping.h" 25 26 enum irq_mode { 27 IRQ_REMAPPING, 28 IRQ_POSTING, 29 }; 30 31 struct ioapic_scope { 32 struct intel_iommu *iommu; 33 unsigned int id; 34 unsigned int bus; /* PCI bus number */ 35 unsigned int devfn; /* PCI devfn number */ 36 }; 37 38 struct hpet_scope { 39 struct intel_iommu *iommu; 40 u8 id; 41 unsigned int bus; 42 unsigned int devfn; 43 }; 44 45 struct irq_2_iommu { 46 struct intel_iommu *iommu; 47 u16 irte_index; 48 u16 sub_handle; 49 u8 irte_mask; 50 enum irq_mode mode; 51 }; 52 53 struct intel_ir_data { 54 struct irq_2_iommu irq_2_iommu; 55 struct irte irte_entry; 56 union { 57 struct msi_msg msi_entry; 58 }; 59 }; 60 61 #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) 62 #define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8) 63 64 static int __read_mostly eim_mode; 65 static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; 66 static struct hpet_scope ir_hpet[MAX_HPET_TBS]; 67 68 /* 69 * Lock ordering: 70 * ->dmar_global_lock 71 * ->irq_2_ir_lock 72 * ->qi->q_lock 73 * ->iommu->register_lock 74 * Note: 75 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called 76 * in single-threaded environment with interrupt disabled, so no need to tabke 77 * the dmar_global_lock. 78 */ 79 DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 80 static const struct irq_domain_ops intel_ir_domain_ops; 81 82 static void iommu_disable_irq_remapping(struct intel_iommu *iommu); 83 static int __init parse_ioapics_under_ir(void); 84 85 static bool ir_pre_enabled(struct intel_iommu *iommu) 86 { 87 return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); 88 } 89 90 static void clear_ir_pre_enabled(struct intel_iommu *iommu) 91 { 92 iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 93 } 94 95 static void init_ir_status(struct intel_iommu *iommu) 96 { 97 u32 gsts; 98 99 gsts = readl(iommu->reg + DMAR_GSTS_REG); 100 if (gsts & DMA_GSTS_IRES) 101 iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 102 } 103 104 static int alloc_irte(struct intel_iommu *iommu, 105 struct irq_2_iommu *irq_iommu, u16 count) 106 { 107 struct ir_table *table = iommu->ir_table; 108 unsigned int mask = 0; 109 unsigned long flags; 110 int index; 111 112 if (!count || !irq_iommu) 113 return -1; 114 115 if (count > 1) { 116 count = __roundup_pow_of_two(count); 117 mask = ilog2(count); 118 } 119 120 if (mask > ecap_max_handle_mask(iommu->ecap)) { 121 pr_err("Requested mask %x exceeds the max invalidation handle" 122 " mask value %Lx\n", mask, 123 ecap_max_handle_mask(iommu->ecap)); 124 return -1; 125 } 126 127 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 128 index = bitmap_find_free_region(table->bitmap, 129 INTR_REMAP_TABLE_ENTRIES, mask); 130 if (index < 0) { 131 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); 132 } else { 133 irq_iommu->iommu = iommu; 134 irq_iommu->irte_index = index; 135 irq_iommu->sub_handle = 0; 136 irq_iommu->irte_mask = mask; 137 irq_iommu->mode = IRQ_REMAPPING; 138 } 139 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 140 141 return index; 142 } 143 144 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) 145 { 146 struct qi_desc desc; 147 148 desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) 149 | QI_IEC_SELECTIVE; 150 desc.qw1 = 0; 151 desc.qw2 = 0; 152 desc.qw3 = 0; 153 154 return qi_submit_sync(iommu, &desc, 1, 0); 155 } 156 157 static int modify_irte(struct irq_2_iommu *irq_iommu, 158 struct irte *irte_modified) 159 { 160 struct intel_iommu *iommu; 161 unsigned long flags; 162 struct irte *irte; 163 int rc, index; 164 165 if (!irq_iommu) 166 return -1; 167 168 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 169 170 iommu = irq_iommu->iommu; 171 172 index = irq_iommu->irte_index + irq_iommu->sub_handle; 173 irte = &iommu->ir_table->base[index]; 174 175 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) 176 if ((irte->pst == 1) || (irte_modified->pst == 1)) { 177 bool ret; 178 179 ret = cmpxchg_double(&irte->low, &irte->high, 180 irte->low, irte->high, 181 irte_modified->low, irte_modified->high); 182 /* 183 * We use cmpxchg16 to atomically update the 128-bit IRTE, 184 * and it cannot be updated by the hardware or other processors 185 * behind us, so the return value of cmpxchg16 should be the 186 * same as the old value. 187 */ 188 WARN_ON(!ret); 189 } else 190 #endif 191 { 192 set_64bit(&irte->low, irte_modified->low); 193 set_64bit(&irte->high, irte_modified->high); 194 } 195 __iommu_flush_cache(iommu, irte, sizeof(*irte)); 196 197 rc = qi_flush_iec(iommu, index, 0); 198 199 /* Update iommu mode according to the IRTE mode */ 200 irq_iommu->mode = irte->pst ? IRQ_POSTING : IRQ_REMAPPING; 201 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 202 203 return rc; 204 } 205 206 static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) 207 { 208 int i; 209 210 for (i = 0; i < MAX_HPET_TBS; i++) 211 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) 212 return ir_hpet[i].iommu; 213 return NULL; 214 } 215 216 static struct intel_iommu *map_ioapic_to_ir(int apic) 217 { 218 int i; 219 220 for (i = 0; i < MAX_IO_APICS; i++) 221 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) 222 return ir_ioapic[i].iommu; 223 return NULL; 224 } 225 226 static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) 227 { 228 struct dmar_drhd_unit *drhd; 229 230 drhd = dmar_find_matched_drhd_unit(dev); 231 if (!drhd) 232 return NULL; 233 234 return drhd->iommu; 235 } 236 237 static int clear_entries(struct irq_2_iommu *irq_iommu) 238 { 239 struct irte *start, *entry, *end; 240 struct intel_iommu *iommu; 241 int index; 242 243 if (irq_iommu->sub_handle) 244 return 0; 245 246 iommu = irq_iommu->iommu; 247 index = irq_iommu->irte_index; 248 249 start = iommu->ir_table->base + index; 250 end = start + (1 << irq_iommu->irte_mask); 251 252 for (entry = start; entry < end; entry++) { 253 set_64bit(&entry->low, 0); 254 set_64bit(&entry->high, 0); 255 } 256 bitmap_release_region(iommu->ir_table->bitmap, index, 257 irq_iommu->irte_mask); 258 259 return qi_flush_iec(iommu, index, irq_iommu->irte_mask); 260 } 261 262 /* 263 * source validation type 264 */ 265 #define SVT_NO_VERIFY 0x0 /* no verification is required */ 266 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ 267 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ 268 269 /* 270 * source-id qualifier 271 */ 272 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ 273 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore 274 * the third least significant bit 275 */ 276 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore 277 * the second and third least significant bits 278 */ 279 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore 280 * the least three significant bits 281 */ 282 283 /* 284 * set SVT, SQ and SID fields of irte to verify 285 * source ids of interrupt requests 286 */ 287 static void set_irte_sid(struct irte *irte, unsigned int svt, 288 unsigned int sq, unsigned int sid) 289 { 290 if (disable_sourceid_checking) 291 svt = SVT_NO_VERIFY; 292 irte->svt = svt; 293 irte->sq = sq; 294 irte->sid = sid; 295 } 296 297 /* 298 * Set an IRTE to match only the bus number. Interrupt requests that reference 299 * this IRTE must have a requester-id whose bus number is between or equal 300 * to the start_bus and end_bus arguments. 301 */ 302 static void set_irte_verify_bus(struct irte *irte, unsigned int start_bus, 303 unsigned int end_bus) 304 { 305 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, 306 (start_bus << 8) | end_bus); 307 } 308 309 static int set_ioapic_sid(struct irte *irte, int apic) 310 { 311 int i; 312 u16 sid = 0; 313 314 if (!irte) 315 return -1; 316 317 down_read(&dmar_global_lock); 318 for (i = 0; i < MAX_IO_APICS; i++) { 319 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { 320 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; 321 break; 322 } 323 } 324 up_read(&dmar_global_lock); 325 326 if (sid == 0) { 327 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); 328 return -1; 329 } 330 331 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid); 332 333 return 0; 334 } 335 336 static int set_hpet_sid(struct irte *irte, u8 id) 337 { 338 int i; 339 u16 sid = 0; 340 341 if (!irte) 342 return -1; 343 344 down_read(&dmar_global_lock); 345 for (i = 0; i < MAX_HPET_TBS; i++) { 346 if (ir_hpet[i].iommu && ir_hpet[i].id == id) { 347 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; 348 break; 349 } 350 } 351 up_read(&dmar_global_lock); 352 353 if (sid == 0) { 354 pr_warn("Failed to set source-id of HPET block (%d)\n", id); 355 return -1; 356 } 357 358 /* 359 * Should really use SQ_ALL_16. Some platforms are broken. 360 * While we figure out the right quirks for these broken platforms, use 361 * SQ_13_IGNORE_3 for now. 362 */ 363 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); 364 365 return 0; 366 } 367 368 struct set_msi_sid_data { 369 struct pci_dev *pdev; 370 u16 alias; 371 int count; 372 int busmatch_count; 373 }; 374 375 static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque) 376 { 377 struct set_msi_sid_data *data = opaque; 378 379 if (data->count == 0 || PCI_BUS_NUM(alias) == PCI_BUS_NUM(data->alias)) 380 data->busmatch_count++; 381 382 data->pdev = pdev; 383 data->alias = alias; 384 data->count++; 385 386 return 0; 387 } 388 389 static int set_msi_sid(struct irte *irte, struct pci_dev *dev) 390 { 391 struct set_msi_sid_data data; 392 393 if (!irte || !dev) 394 return -1; 395 396 data.count = 0; 397 data.busmatch_count = 0; 398 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data); 399 400 /* 401 * DMA alias provides us with a PCI device and alias. The only case 402 * where the it will return an alias on a different bus than the 403 * device is the case of a PCIe-to-PCI bridge, where the alias is for 404 * the subordinate bus. In this case we can only verify the bus. 405 * 406 * If there are multiple aliases, all with the same bus number, 407 * then all we can do is verify the bus. This is typical in NTB 408 * hardware which use proxy IDs where the device will generate traffic 409 * from multiple devfn numbers on the same bus. 410 * 411 * If the alias device is on a different bus than our source device 412 * then we have a topology based alias, use it. 413 * 414 * Otherwise, the alias is for a device DMA quirk and we cannot 415 * assume that MSI uses the same requester ID. Therefore use the 416 * original device. 417 */ 418 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number) 419 set_irte_verify_bus(irte, PCI_BUS_NUM(data.alias), 420 dev->bus->number); 421 else if (data.count >= 2 && data.busmatch_count == data.count) 422 set_irte_verify_bus(irte, dev->bus->number, dev->bus->number); 423 else if (data.pdev->bus->number != dev->bus->number) 424 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias); 425 else 426 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, 427 pci_dev_id(dev)); 428 429 return 0; 430 } 431 432 static int iommu_load_old_irte(struct intel_iommu *iommu) 433 { 434 struct irte *old_ir_table; 435 phys_addr_t irt_phys; 436 unsigned int i; 437 size_t size; 438 u64 irta; 439 440 /* Check whether the old ir-table has the same size as ours */ 441 irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); 442 if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) 443 != INTR_REMAP_TABLE_REG_SIZE) 444 return -EINVAL; 445 446 irt_phys = irta & VTD_PAGE_MASK; 447 size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte); 448 449 /* Map the old IR table */ 450 old_ir_table = memremap(irt_phys, size, MEMREMAP_WB); 451 if (!old_ir_table) 452 return -ENOMEM; 453 454 /* Copy data over */ 455 memcpy(iommu->ir_table->base, old_ir_table, size); 456 457 __iommu_flush_cache(iommu, iommu->ir_table->base, size); 458 459 /* 460 * Now check the table for used entries and mark those as 461 * allocated in the bitmap 462 */ 463 for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) { 464 if (iommu->ir_table->base[i].present) 465 bitmap_set(iommu->ir_table->bitmap, i, 1); 466 } 467 468 memunmap(old_ir_table); 469 470 return 0; 471 } 472 473 474 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) 475 { 476 unsigned long flags; 477 u64 addr; 478 u32 sts; 479 480 addr = virt_to_phys((void *)iommu->ir_table->base); 481 482 raw_spin_lock_irqsave(&iommu->register_lock, flags); 483 484 dmar_writeq(iommu->reg + DMAR_IRTA_REG, 485 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); 486 487 /* Set interrupt-remapping table pointer */ 488 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); 489 490 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 491 readl, (sts & DMA_GSTS_IRTPS), sts); 492 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 493 494 /* 495 * Global invalidation of interrupt entry cache to make sure the 496 * hardware uses the new irq remapping table. 497 */ 498 qi_global_iec(iommu); 499 } 500 501 static void iommu_enable_irq_remapping(struct intel_iommu *iommu) 502 { 503 unsigned long flags; 504 u32 sts; 505 506 raw_spin_lock_irqsave(&iommu->register_lock, flags); 507 508 /* Enable interrupt-remapping */ 509 iommu->gcmd |= DMA_GCMD_IRE; 510 iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ 511 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 512 513 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 514 readl, (sts & DMA_GSTS_IRES), sts); 515 516 /* 517 * With CFI clear in the Global Command register, we should be 518 * protected from dangerous (i.e. compatibility) interrupts 519 * regardless of x2apic status. Check just to be sure. 520 */ 521 if (sts & DMA_GSTS_CFIS) 522 WARN(1, KERN_WARNING 523 "Compatibility-format IRQs enabled despite intr remapping;\n" 524 "you are vulnerable to IRQ injection.\n"); 525 526 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 527 } 528 529 static int intel_setup_irq_remapping(struct intel_iommu *iommu) 530 { 531 struct ir_table *ir_table; 532 struct fwnode_handle *fn; 533 unsigned long *bitmap; 534 struct page *pages; 535 536 if (iommu->ir_table) 537 return 0; 538 539 ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL); 540 if (!ir_table) 541 return -ENOMEM; 542 543 pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 544 INTR_REMAP_PAGE_ORDER); 545 if (!pages) { 546 pr_err("IR%d: failed to allocate pages of order %d\n", 547 iommu->seq_id, INTR_REMAP_PAGE_ORDER); 548 goto out_free_table; 549 } 550 551 bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_ATOMIC); 552 if (bitmap == NULL) { 553 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); 554 goto out_free_pages; 555 } 556 557 fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id); 558 if (!fn) 559 goto out_free_bitmap; 560 561 iommu->ir_domain = 562 irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 563 0, INTR_REMAP_TABLE_ENTRIES, 564 fn, &intel_ir_domain_ops, 565 iommu); 566 if (!iommu->ir_domain) { 567 irq_domain_free_fwnode(fn); 568 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id); 569 goto out_free_bitmap; 570 } 571 iommu->ir_msi_domain = 572 arch_create_remap_msi_irq_domain(iommu->ir_domain, 573 "INTEL-IR-MSI", 574 iommu->seq_id); 575 576 ir_table->base = page_address(pages); 577 ir_table->bitmap = bitmap; 578 iommu->ir_table = ir_table; 579 580 /* 581 * If the queued invalidation is already initialized, 582 * shouldn't disable it. 583 */ 584 if (!iommu->qi) { 585 /* 586 * Clear previous faults. 587 */ 588 dmar_fault(-1, iommu); 589 dmar_disable_qi(iommu); 590 591 if (dmar_enable_qi(iommu)) { 592 pr_err("Failed to enable queued invalidation\n"); 593 goto out_free_bitmap; 594 } 595 } 596 597 init_ir_status(iommu); 598 599 if (ir_pre_enabled(iommu)) { 600 if (!is_kdump_kernel()) { 601 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", 602 iommu->name); 603 clear_ir_pre_enabled(iommu); 604 iommu_disable_irq_remapping(iommu); 605 } else if (iommu_load_old_irte(iommu)) 606 pr_err("Failed to copy IR table for %s from previous kernel\n", 607 iommu->name); 608 else 609 pr_info("Copied IR table for %s from previous kernel\n", 610 iommu->name); 611 } 612 613 iommu_set_irq_remapping(iommu, eim_mode); 614 615 return 0; 616 617 out_free_bitmap: 618 bitmap_free(bitmap); 619 out_free_pages: 620 __free_pages(pages, INTR_REMAP_PAGE_ORDER); 621 out_free_table: 622 kfree(ir_table); 623 624 iommu->ir_table = NULL; 625 626 return -ENOMEM; 627 } 628 629 static void intel_teardown_irq_remapping(struct intel_iommu *iommu) 630 { 631 if (iommu && iommu->ir_table) { 632 if (iommu->ir_msi_domain) { 633 irq_domain_remove(iommu->ir_msi_domain); 634 iommu->ir_msi_domain = NULL; 635 } 636 if (iommu->ir_domain) { 637 irq_domain_remove(iommu->ir_domain); 638 iommu->ir_domain = NULL; 639 } 640 free_pages((unsigned long)iommu->ir_table->base, 641 INTR_REMAP_PAGE_ORDER); 642 bitmap_free(iommu->ir_table->bitmap); 643 kfree(iommu->ir_table); 644 iommu->ir_table = NULL; 645 } 646 } 647 648 /* 649 * Disable Interrupt Remapping. 650 */ 651 static void iommu_disable_irq_remapping(struct intel_iommu *iommu) 652 { 653 unsigned long flags; 654 u32 sts; 655 656 if (!ecap_ir_support(iommu->ecap)) 657 return; 658 659 /* 660 * global invalidation of interrupt entry cache before disabling 661 * interrupt-remapping. 662 */ 663 qi_global_iec(iommu); 664 665 raw_spin_lock_irqsave(&iommu->register_lock, flags); 666 667 sts = readl(iommu->reg + DMAR_GSTS_REG); 668 if (!(sts & DMA_GSTS_IRES)) 669 goto end; 670 671 iommu->gcmd &= ~DMA_GCMD_IRE; 672 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 673 674 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 675 readl, !(sts & DMA_GSTS_IRES), sts); 676 677 end: 678 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 679 } 680 681 static int __init dmar_x2apic_optout(void) 682 { 683 struct acpi_table_dmar *dmar; 684 dmar = (struct acpi_table_dmar *)dmar_tbl; 685 if (!dmar || no_x2apic_optout) 686 return 0; 687 return dmar->flags & DMAR_X2APIC_OPT_OUT; 688 } 689 690 static void __init intel_cleanup_irq_remapping(void) 691 { 692 struct dmar_drhd_unit *drhd; 693 struct intel_iommu *iommu; 694 695 for_each_iommu(iommu, drhd) { 696 if (ecap_ir_support(iommu->ecap)) { 697 iommu_disable_irq_remapping(iommu); 698 intel_teardown_irq_remapping(iommu); 699 } 700 } 701 702 if (x2apic_supported()) 703 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); 704 } 705 706 static int __init intel_prepare_irq_remapping(void) 707 { 708 struct dmar_drhd_unit *drhd; 709 struct intel_iommu *iommu; 710 int eim = 0; 711 712 if (irq_remap_broken) { 713 pr_warn("This system BIOS has enabled interrupt remapping\n" 714 "on a chipset that contains an erratum making that\n" 715 "feature unstable. To maintain system stability\n" 716 "interrupt remapping is being disabled. Please\n" 717 "contact your BIOS vendor for an update\n"); 718 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 719 return -ENODEV; 720 } 721 722 if (dmar_table_init() < 0) 723 return -ENODEV; 724 725 if (!dmar_ir_support()) 726 return -ENODEV; 727 728 if (parse_ioapics_under_ir()) { 729 pr_info("Not enabling interrupt remapping\n"); 730 goto error; 731 } 732 733 /* First make sure all IOMMUs support IRQ remapping */ 734 for_each_iommu(iommu, drhd) 735 if (!ecap_ir_support(iommu->ecap)) 736 goto error; 737 738 /* Detect remapping mode: lapic or x2apic */ 739 if (x2apic_supported()) { 740 eim = !dmar_x2apic_optout(); 741 if (!eim) { 742 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); 743 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); 744 } 745 } 746 747 for_each_iommu(iommu, drhd) { 748 if (eim && !ecap_eim_support(iommu->ecap)) { 749 pr_info("%s does not support EIM\n", iommu->name); 750 eim = 0; 751 } 752 } 753 754 eim_mode = eim; 755 if (eim) 756 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); 757 758 /* Do the initializations early */ 759 for_each_iommu(iommu, drhd) { 760 if (intel_setup_irq_remapping(iommu)) { 761 pr_err("Failed to setup irq remapping for %s\n", 762 iommu->name); 763 goto error; 764 } 765 } 766 767 return 0; 768 769 error: 770 intel_cleanup_irq_remapping(); 771 return -ENODEV; 772 } 773 774 /* 775 * Set Posted-Interrupts capability. 776 */ 777 static inline void set_irq_posting_cap(void) 778 { 779 struct dmar_drhd_unit *drhd; 780 struct intel_iommu *iommu; 781 782 if (!disable_irq_post) { 783 /* 784 * If IRTE is in posted format, the 'pda' field goes across the 785 * 64-bit boundary, we need use cmpxchg16b to atomically update 786 * it. We only expose posted-interrupt when X86_FEATURE_CX16 787 * is supported. Actually, hardware platforms supporting PI 788 * should have X86_FEATURE_CX16 support, this has been confirmed 789 * with Intel hardware guys. 790 */ 791 if (boot_cpu_has(X86_FEATURE_CX16)) 792 intel_irq_remap_ops.capability |= 1 << IRQ_POSTING_CAP; 793 794 for_each_iommu(iommu, drhd) 795 if (!cap_pi_support(iommu->cap)) { 796 intel_irq_remap_ops.capability &= 797 ~(1 << IRQ_POSTING_CAP); 798 break; 799 } 800 } 801 } 802 803 static int __init intel_enable_irq_remapping(void) 804 { 805 struct dmar_drhd_unit *drhd; 806 struct intel_iommu *iommu; 807 bool setup = false; 808 809 /* 810 * Setup Interrupt-remapping for all the DRHD's now. 811 */ 812 for_each_iommu(iommu, drhd) { 813 if (!ir_pre_enabled(iommu)) 814 iommu_enable_irq_remapping(iommu); 815 setup = true; 816 } 817 818 if (!setup) 819 goto error; 820 821 irq_remapping_enabled = 1; 822 823 set_irq_posting_cap(); 824 825 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); 826 827 return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; 828 829 error: 830 intel_cleanup_irq_remapping(); 831 return -1; 832 } 833 834 static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, 835 struct intel_iommu *iommu, 836 struct acpi_dmar_hardware_unit *drhd) 837 { 838 struct acpi_dmar_pci_path *path; 839 u8 bus; 840 int count, free = -1; 841 842 bus = scope->bus; 843 path = (struct acpi_dmar_pci_path *)(scope + 1); 844 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 845 / sizeof(struct acpi_dmar_pci_path); 846 847 while (--count > 0) { 848 /* 849 * Access PCI directly due to the PCI 850 * subsystem isn't initialized yet. 851 */ 852 bus = read_pci_config_byte(bus, path->device, path->function, 853 PCI_SECONDARY_BUS); 854 path++; 855 } 856 857 for (count = 0; count < MAX_HPET_TBS; count++) { 858 if (ir_hpet[count].iommu == iommu && 859 ir_hpet[count].id == scope->enumeration_id) 860 return 0; 861 else if (ir_hpet[count].iommu == NULL && free == -1) 862 free = count; 863 } 864 if (free == -1) { 865 pr_warn("Exceeded Max HPET blocks\n"); 866 return -ENOSPC; 867 } 868 869 ir_hpet[free].iommu = iommu; 870 ir_hpet[free].id = scope->enumeration_id; 871 ir_hpet[free].bus = bus; 872 ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function); 873 pr_info("HPET id %d under DRHD base 0x%Lx\n", 874 scope->enumeration_id, drhd->address); 875 876 return 0; 877 } 878 879 static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, 880 struct intel_iommu *iommu, 881 struct acpi_dmar_hardware_unit *drhd) 882 { 883 struct acpi_dmar_pci_path *path; 884 u8 bus; 885 int count, free = -1; 886 887 bus = scope->bus; 888 path = (struct acpi_dmar_pci_path *)(scope + 1); 889 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 890 / sizeof(struct acpi_dmar_pci_path); 891 892 while (--count > 0) { 893 /* 894 * Access PCI directly due to the PCI 895 * subsystem isn't initialized yet. 896 */ 897 bus = read_pci_config_byte(bus, path->device, path->function, 898 PCI_SECONDARY_BUS); 899 path++; 900 } 901 902 for (count = 0; count < MAX_IO_APICS; count++) { 903 if (ir_ioapic[count].iommu == iommu && 904 ir_ioapic[count].id == scope->enumeration_id) 905 return 0; 906 else if (ir_ioapic[count].iommu == NULL && free == -1) 907 free = count; 908 } 909 if (free == -1) { 910 pr_warn("Exceeded Max IO APICS\n"); 911 return -ENOSPC; 912 } 913 914 ir_ioapic[free].bus = bus; 915 ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function); 916 ir_ioapic[free].iommu = iommu; 917 ir_ioapic[free].id = scope->enumeration_id; 918 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", 919 scope->enumeration_id, drhd->address, iommu->seq_id); 920 921 return 0; 922 } 923 924 static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, 925 struct intel_iommu *iommu) 926 { 927 int ret = 0; 928 struct acpi_dmar_hardware_unit *drhd; 929 struct acpi_dmar_device_scope *scope; 930 void *start, *end; 931 932 drhd = (struct acpi_dmar_hardware_unit *)header; 933 start = (void *)(drhd + 1); 934 end = ((void *)drhd) + header->length; 935 936 while (start < end && ret == 0) { 937 scope = start; 938 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) 939 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); 940 else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) 941 ret = ir_parse_one_hpet_scope(scope, iommu, drhd); 942 start += scope->length; 943 } 944 945 return ret; 946 } 947 948 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) 949 { 950 int i; 951 952 for (i = 0; i < MAX_HPET_TBS; i++) 953 if (ir_hpet[i].iommu == iommu) 954 ir_hpet[i].iommu = NULL; 955 956 for (i = 0; i < MAX_IO_APICS; i++) 957 if (ir_ioapic[i].iommu == iommu) 958 ir_ioapic[i].iommu = NULL; 959 } 960 961 /* 962 * Finds the assocaition between IOAPIC's and its Interrupt-remapping 963 * hardware unit. 964 */ 965 static int __init parse_ioapics_under_ir(void) 966 { 967 struct dmar_drhd_unit *drhd; 968 struct intel_iommu *iommu; 969 bool ir_supported = false; 970 int ioapic_idx; 971 972 for_each_iommu(iommu, drhd) { 973 int ret; 974 975 if (!ecap_ir_support(iommu->ecap)) 976 continue; 977 978 ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu); 979 if (ret) 980 return ret; 981 982 ir_supported = true; 983 } 984 985 if (!ir_supported) 986 return -ENODEV; 987 988 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) { 989 int ioapic_id = mpc_ioapic_id(ioapic_idx); 990 if (!map_ioapic_to_ir(ioapic_id)) { 991 pr_err(FW_BUG "ioapic %d has no mapping iommu, " 992 "interrupt remapping will be disabled\n", 993 ioapic_id); 994 return -1; 995 } 996 } 997 998 return 0; 999 } 1000 1001 static int __init ir_dev_scope_init(void) 1002 { 1003 int ret; 1004 1005 if (!irq_remapping_enabled) 1006 return 0; 1007 1008 down_write(&dmar_global_lock); 1009 ret = dmar_dev_scope_init(); 1010 up_write(&dmar_global_lock); 1011 1012 return ret; 1013 } 1014 rootfs_initcall(ir_dev_scope_init); 1015 1016 static void disable_irq_remapping(void) 1017 { 1018 struct dmar_drhd_unit *drhd; 1019 struct intel_iommu *iommu = NULL; 1020 1021 /* 1022 * Disable Interrupt-remapping for all the DRHD's now. 1023 */ 1024 for_each_iommu(iommu, drhd) { 1025 if (!ecap_ir_support(iommu->ecap)) 1026 continue; 1027 1028 iommu_disable_irq_remapping(iommu); 1029 } 1030 1031 /* 1032 * Clear Posted-Interrupts capability. 1033 */ 1034 if (!disable_irq_post) 1035 intel_irq_remap_ops.capability &= ~(1 << IRQ_POSTING_CAP); 1036 } 1037 1038 static int reenable_irq_remapping(int eim) 1039 { 1040 struct dmar_drhd_unit *drhd; 1041 bool setup = false; 1042 struct intel_iommu *iommu = NULL; 1043 1044 for_each_iommu(iommu, drhd) 1045 if (iommu->qi) 1046 dmar_reenable_qi(iommu); 1047 1048 /* 1049 * Setup Interrupt-remapping for all the DRHD's now. 1050 */ 1051 for_each_iommu(iommu, drhd) { 1052 if (!ecap_ir_support(iommu->ecap)) 1053 continue; 1054 1055 /* Set up interrupt remapping for iommu.*/ 1056 iommu_set_irq_remapping(iommu, eim); 1057 iommu_enable_irq_remapping(iommu); 1058 setup = true; 1059 } 1060 1061 if (!setup) 1062 goto error; 1063 1064 set_irq_posting_cap(); 1065 1066 return 0; 1067 1068 error: 1069 /* 1070 * handle error condition gracefully here! 1071 */ 1072 return -1; 1073 } 1074 1075 static void prepare_irte(struct irte *irte, int vector, unsigned int dest) 1076 { 1077 memset(irte, 0, sizeof(*irte)); 1078 1079 irte->present = 1; 1080 irte->dst_mode = apic->irq_dest_mode; 1081 /* 1082 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the 1083 * actual level or edge trigger will be setup in the IO-APIC 1084 * RTE. This will help simplify level triggered irq migration. 1085 * For more details, see the comments (in io_apic.c) explainig IO-APIC 1086 * irq migration in the presence of interrupt-remapping. 1087 */ 1088 irte->trigger_mode = 0; 1089 irte->dlvry_mode = apic->irq_delivery_mode; 1090 irte->vector = vector; 1091 irte->dest_id = IRTE_DEST(dest); 1092 irte->redir_hint = 1; 1093 } 1094 1095 static struct irq_domain *intel_get_ir_irq_domain(struct irq_alloc_info *info) 1096 { 1097 struct intel_iommu *iommu = NULL; 1098 1099 if (!info) 1100 return NULL; 1101 1102 switch (info->type) { 1103 case X86_IRQ_ALLOC_TYPE_IOAPIC: 1104 iommu = map_ioapic_to_ir(info->ioapic_id); 1105 break; 1106 case X86_IRQ_ALLOC_TYPE_HPET: 1107 iommu = map_hpet_to_ir(info->hpet_id); 1108 break; 1109 case X86_IRQ_ALLOC_TYPE_MSI: 1110 case X86_IRQ_ALLOC_TYPE_MSIX: 1111 iommu = map_dev_to_ir(info->msi_dev); 1112 break; 1113 default: 1114 BUG_ON(1); 1115 break; 1116 } 1117 1118 return iommu ? iommu->ir_domain : NULL; 1119 } 1120 1121 static struct irq_domain *intel_get_irq_domain(struct irq_alloc_info *info) 1122 { 1123 struct intel_iommu *iommu; 1124 1125 if (!info) 1126 return NULL; 1127 1128 switch (info->type) { 1129 case X86_IRQ_ALLOC_TYPE_MSI: 1130 case X86_IRQ_ALLOC_TYPE_MSIX: 1131 iommu = map_dev_to_ir(info->msi_dev); 1132 if (iommu) 1133 return iommu->ir_msi_domain; 1134 break; 1135 default: 1136 break; 1137 } 1138 1139 return NULL; 1140 } 1141 1142 struct irq_remap_ops intel_irq_remap_ops = { 1143 .prepare = intel_prepare_irq_remapping, 1144 .enable = intel_enable_irq_remapping, 1145 .disable = disable_irq_remapping, 1146 .reenable = reenable_irq_remapping, 1147 .enable_faulting = enable_drhd_fault_handling, 1148 .get_ir_irq_domain = intel_get_ir_irq_domain, 1149 .get_irq_domain = intel_get_irq_domain, 1150 }; 1151 1152 static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force) 1153 { 1154 struct intel_ir_data *ir_data = irqd->chip_data; 1155 struct irte *irte = &ir_data->irte_entry; 1156 struct irq_cfg *cfg = irqd_cfg(irqd); 1157 1158 /* 1159 * Atomically updates the IRTE with the new destination, vector 1160 * and flushes the interrupt entry cache. 1161 */ 1162 irte->vector = cfg->vector; 1163 irte->dest_id = IRTE_DEST(cfg->dest_apicid); 1164 1165 /* Update the hardware only if the interrupt is in remapped mode. */ 1166 if (force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING) 1167 modify_irte(&ir_data->irq_2_iommu, irte); 1168 } 1169 1170 /* 1171 * Migrate the IO-APIC irq in the presence of intr-remapping. 1172 * 1173 * For both level and edge triggered, irq migration is a simple atomic 1174 * update(of vector and cpu destination) of IRTE and flush the hardware cache. 1175 * 1176 * For level triggered, we eliminate the io-apic RTE modification (with the 1177 * updated vector information), by using a virtual vector (io-apic pin number). 1178 * Real vector that is used for interrupting cpu will be coming from 1179 * the interrupt-remapping table entry. 1180 * 1181 * As the migration is a simple atomic update of IRTE, the same mechanism 1182 * is used to migrate MSI irq's in the presence of interrupt-remapping. 1183 */ 1184 static int 1185 intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask, 1186 bool force) 1187 { 1188 struct irq_data *parent = data->parent_data; 1189 struct irq_cfg *cfg = irqd_cfg(data); 1190 int ret; 1191 1192 ret = parent->chip->irq_set_affinity(parent, mask, force); 1193 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 1194 return ret; 1195 1196 intel_ir_reconfigure_irte(data, false); 1197 /* 1198 * After this point, all the interrupts will start arriving 1199 * at the new destination. So, time to cleanup the previous 1200 * vector allocation. 1201 */ 1202 send_cleanup_vector(cfg); 1203 1204 return IRQ_SET_MASK_OK_DONE; 1205 } 1206 1207 static void intel_ir_compose_msi_msg(struct irq_data *irq_data, 1208 struct msi_msg *msg) 1209 { 1210 struct intel_ir_data *ir_data = irq_data->chip_data; 1211 1212 *msg = ir_data->msi_entry; 1213 } 1214 1215 static int intel_ir_set_vcpu_affinity(struct irq_data *data, void *info) 1216 { 1217 struct intel_ir_data *ir_data = data->chip_data; 1218 struct vcpu_data *vcpu_pi_info = info; 1219 1220 /* stop posting interrupts, back to remapping mode */ 1221 if (!vcpu_pi_info) { 1222 modify_irte(&ir_data->irq_2_iommu, &ir_data->irte_entry); 1223 } else { 1224 struct irte irte_pi; 1225 1226 /* 1227 * We are not caching the posted interrupt entry. We 1228 * copy the data from the remapped entry and modify 1229 * the fields which are relevant for posted mode. The 1230 * cached remapped entry is used for switching back to 1231 * remapped mode. 1232 */ 1233 memset(&irte_pi, 0, sizeof(irte_pi)); 1234 dmar_copy_shared_irte(&irte_pi, &ir_data->irte_entry); 1235 1236 /* Update the posted mode fields */ 1237 irte_pi.p_pst = 1; 1238 irte_pi.p_urgent = 0; 1239 irte_pi.p_vector = vcpu_pi_info->vector; 1240 irte_pi.pda_l = (vcpu_pi_info->pi_desc_addr >> 1241 (32 - PDA_LOW_BIT)) & ~(-1UL << PDA_LOW_BIT); 1242 irte_pi.pda_h = (vcpu_pi_info->pi_desc_addr >> 32) & 1243 ~(-1UL << PDA_HIGH_BIT); 1244 1245 modify_irte(&ir_data->irq_2_iommu, &irte_pi); 1246 } 1247 1248 return 0; 1249 } 1250 1251 static struct irq_chip intel_ir_chip = { 1252 .name = "INTEL-IR", 1253 .irq_ack = apic_ack_irq, 1254 .irq_set_affinity = intel_ir_set_affinity, 1255 .irq_compose_msi_msg = intel_ir_compose_msi_msg, 1256 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, 1257 }; 1258 1259 static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, 1260 struct irq_cfg *irq_cfg, 1261 struct irq_alloc_info *info, 1262 int index, int sub_handle) 1263 { 1264 struct IR_IO_APIC_route_entry *entry; 1265 struct irte *irte = &data->irte_entry; 1266 struct msi_msg *msg = &data->msi_entry; 1267 1268 prepare_irte(irte, irq_cfg->vector, irq_cfg->dest_apicid); 1269 switch (info->type) { 1270 case X86_IRQ_ALLOC_TYPE_IOAPIC: 1271 /* Set source-id of interrupt request */ 1272 set_ioapic_sid(irte, info->ioapic_id); 1273 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n", 1274 info->ioapic_id, irte->present, irte->fpd, 1275 irte->dst_mode, irte->redir_hint, 1276 irte->trigger_mode, irte->dlvry_mode, 1277 irte->avail, irte->vector, irte->dest_id, 1278 irte->sid, irte->sq, irte->svt); 1279 1280 entry = (struct IR_IO_APIC_route_entry *)info->ioapic_entry; 1281 info->ioapic_entry = NULL; 1282 memset(entry, 0, sizeof(*entry)); 1283 entry->index2 = (index >> 15) & 0x1; 1284 entry->zero = 0; 1285 entry->format = 1; 1286 entry->index = (index & 0x7fff); 1287 /* 1288 * IO-APIC RTE will be configured with virtual vector. 1289 * irq handler will do the explicit EOI to the io-apic. 1290 */ 1291 entry->vector = info->ioapic_pin; 1292 entry->mask = 0; /* enable IRQ */ 1293 entry->trigger = info->ioapic_trigger; 1294 entry->polarity = info->ioapic_polarity; 1295 if (info->ioapic_trigger) 1296 entry->mask = 1; /* Mask level triggered irqs. */ 1297 break; 1298 1299 case X86_IRQ_ALLOC_TYPE_HPET: 1300 case X86_IRQ_ALLOC_TYPE_MSI: 1301 case X86_IRQ_ALLOC_TYPE_MSIX: 1302 if (info->type == X86_IRQ_ALLOC_TYPE_HPET) 1303 set_hpet_sid(irte, info->hpet_id); 1304 else 1305 set_msi_sid(irte, info->msi_dev); 1306 1307 msg->address_hi = MSI_ADDR_BASE_HI; 1308 msg->data = sub_handle; 1309 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | 1310 MSI_ADDR_IR_SHV | 1311 MSI_ADDR_IR_INDEX1(index) | 1312 MSI_ADDR_IR_INDEX2(index); 1313 break; 1314 1315 default: 1316 BUG_ON(1); 1317 break; 1318 } 1319 } 1320 1321 static void intel_free_irq_resources(struct irq_domain *domain, 1322 unsigned int virq, unsigned int nr_irqs) 1323 { 1324 struct irq_data *irq_data; 1325 struct intel_ir_data *data; 1326 struct irq_2_iommu *irq_iommu; 1327 unsigned long flags; 1328 int i; 1329 for (i = 0; i < nr_irqs; i++) { 1330 irq_data = irq_domain_get_irq_data(domain, virq + i); 1331 if (irq_data && irq_data->chip_data) { 1332 data = irq_data->chip_data; 1333 irq_iommu = &data->irq_2_iommu; 1334 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 1335 clear_entries(irq_iommu); 1336 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 1337 irq_domain_reset_irq_data(irq_data); 1338 kfree(data); 1339 } 1340 } 1341 } 1342 1343 static int intel_irq_remapping_alloc(struct irq_domain *domain, 1344 unsigned int virq, unsigned int nr_irqs, 1345 void *arg) 1346 { 1347 struct intel_iommu *iommu = domain->host_data; 1348 struct irq_alloc_info *info = arg; 1349 struct intel_ir_data *data, *ird; 1350 struct irq_data *irq_data; 1351 struct irq_cfg *irq_cfg; 1352 int i, ret, index; 1353 1354 if (!info || !iommu) 1355 return -EINVAL; 1356 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI && 1357 info->type != X86_IRQ_ALLOC_TYPE_MSIX) 1358 return -EINVAL; 1359 1360 /* 1361 * With IRQ remapping enabled, don't need contiguous CPU vectors 1362 * to support multiple MSI interrupts. 1363 */ 1364 if (info->type == X86_IRQ_ALLOC_TYPE_MSI) 1365 info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; 1366 1367 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 1368 if (ret < 0) 1369 return ret; 1370 1371 ret = -ENOMEM; 1372 data = kzalloc(sizeof(*data), GFP_KERNEL); 1373 if (!data) 1374 goto out_free_parent; 1375 1376 down_read(&dmar_global_lock); 1377 index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); 1378 up_read(&dmar_global_lock); 1379 if (index < 0) { 1380 pr_warn("Failed to allocate IRTE\n"); 1381 kfree(data); 1382 goto out_free_parent; 1383 } 1384 1385 for (i = 0; i < nr_irqs; i++) { 1386 irq_data = irq_domain_get_irq_data(domain, virq + i); 1387 irq_cfg = irqd_cfg(irq_data); 1388 if (!irq_data || !irq_cfg) { 1389 ret = -EINVAL; 1390 goto out_free_data; 1391 } 1392 1393 if (i > 0) { 1394 ird = kzalloc(sizeof(*ird), GFP_KERNEL); 1395 if (!ird) 1396 goto out_free_data; 1397 /* Initialize the common data */ 1398 ird->irq_2_iommu = data->irq_2_iommu; 1399 ird->irq_2_iommu.sub_handle = i; 1400 } else { 1401 ird = data; 1402 } 1403 1404 irq_data->hwirq = (index << 16) + i; 1405 irq_data->chip_data = ird; 1406 irq_data->chip = &intel_ir_chip; 1407 intel_irq_remapping_prepare_irte(ird, irq_cfg, info, index, i); 1408 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); 1409 } 1410 return 0; 1411 1412 out_free_data: 1413 intel_free_irq_resources(domain, virq, i); 1414 out_free_parent: 1415 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1416 return ret; 1417 } 1418 1419 static void intel_irq_remapping_free(struct irq_domain *domain, 1420 unsigned int virq, unsigned int nr_irqs) 1421 { 1422 intel_free_irq_resources(domain, virq, nr_irqs); 1423 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1424 } 1425 1426 static int intel_irq_remapping_activate(struct irq_domain *domain, 1427 struct irq_data *irq_data, bool reserve) 1428 { 1429 intel_ir_reconfigure_irte(irq_data, true); 1430 return 0; 1431 } 1432 1433 static void intel_irq_remapping_deactivate(struct irq_domain *domain, 1434 struct irq_data *irq_data) 1435 { 1436 struct intel_ir_data *data = irq_data->chip_data; 1437 struct irte entry; 1438 1439 memset(&entry, 0, sizeof(entry)); 1440 modify_irte(&data->irq_2_iommu, &entry); 1441 } 1442 1443 static const struct irq_domain_ops intel_ir_domain_ops = { 1444 .alloc = intel_irq_remapping_alloc, 1445 .free = intel_irq_remapping_free, 1446 .activate = intel_irq_remapping_activate, 1447 .deactivate = intel_irq_remapping_deactivate, 1448 }; 1449 1450 /* 1451 * Support of Interrupt Remapping Unit Hotplug 1452 */ 1453 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) 1454 { 1455 int ret; 1456 int eim = x2apic_enabled(); 1457 1458 if (eim && !ecap_eim_support(iommu->ecap)) { 1459 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", 1460 iommu->reg_phys, iommu->ecap); 1461 return -ENODEV; 1462 } 1463 1464 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { 1465 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n", 1466 iommu->reg_phys); 1467 return -ENODEV; 1468 } 1469 1470 /* TODO: check all IOAPICs are covered by IOMMU */ 1471 1472 /* Setup Interrupt-remapping now. */ 1473 ret = intel_setup_irq_remapping(iommu); 1474 if (ret) { 1475 pr_err("Failed to setup irq remapping for %s\n", 1476 iommu->name); 1477 intel_teardown_irq_remapping(iommu); 1478 ir_remove_ioapic_hpet_scope(iommu); 1479 } else { 1480 iommu_enable_irq_remapping(iommu); 1481 } 1482 1483 return ret; 1484 } 1485 1486 int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert) 1487 { 1488 int ret = 0; 1489 struct intel_iommu *iommu = dmaru->iommu; 1490 1491 if (!irq_remapping_enabled) 1492 return 0; 1493 if (iommu == NULL) 1494 return -EINVAL; 1495 if (!ecap_ir_support(iommu->ecap)) 1496 return 0; 1497 if (irq_remapping_cap(IRQ_POSTING_CAP) && 1498 !cap_pi_support(iommu->cap)) 1499 return -EBUSY; 1500 1501 if (insert) { 1502 if (!iommu->ir_table) 1503 ret = dmar_ir_add(dmaru, iommu); 1504 } else { 1505 if (iommu->ir_table) { 1506 if (!bitmap_empty(iommu->ir_table->bitmap, 1507 INTR_REMAP_TABLE_ENTRIES)) { 1508 ret = -EBUSY; 1509 } else { 1510 iommu_disable_irq_remapping(iommu); 1511 intel_teardown_irq_remapping(iommu); 1512 ir_remove_ioapic_hpet_scope(iommu); 1513 } 1514 } 1515 } 1516 1517 return ret; 1518 } 1519