1 // SPDX-License-Identifier: GPL-2.0 2 3 #define pr_fmt(fmt) "DMAR-IR: " fmt 4 5 #include <linux/interrupt.h> 6 #include <linux/dmar.h> 7 #include <linux/spinlock.h> 8 #include <linux/slab.h> 9 #include <linux/jiffies.h> 10 #include <linux/hpet.h> 11 #include <linux/pci.h> 12 #include <linux/irq.h> 13 #include <linux/intel-iommu.h> 14 #include <linux/acpi.h> 15 #include <linux/irqdomain.h> 16 #include <linux/crash_dump.h> 17 #include <asm/io_apic.h> 18 #include <asm/apic.h> 19 #include <asm/smp.h> 20 #include <asm/cpu.h> 21 #include <asm/irq_remapping.h> 22 #include <asm/pci-direct.h> 23 #include <asm/msidef.h> 24 25 #include "../irq_remapping.h" 26 27 enum irq_mode { 28 IRQ_REMAPPING, 29 IRQ_POSTING, 30 }; 31 32 struct ioapic_scope { 33 struct intel_iommu *iommu; 34 unsigned int id; 35 unsigned int bus; /* PCI bus number */ 36 unsigned int devfn; /* PCI devfn number */ 37 }; 38 39 struct hpet_scope { 40 struct intel_iommu *iommu; 41 u8 id; 42 unsigned int bus; 43 unsigned int devfn; 44 }; 45 46 struct irq_2_iommu { 47 struct intel_iommu *iommu; 48 u16 irte_index; 49 u16 sub_handle; 50 u8 irte_mask; 51 enum irq_mode mode; 52 }; 53 54 struct intel_ir_data { 55 struct irq_2_iommu irq_2_iommu; 56 struct irte irte_entry; 57 union { 58 struct msi_msg msi_entry; 59 }; 60 }; 61 62 #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) 63 #define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8) 64 65 static int __read_mostly eim_mode; 66 static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; 67 static struct hpet_scope ir_hpet[MAX_HPET_TBS]; 68 69 /* 70 * Lock ordering: 71 * ->dmar_global_lock 72 * ->irq_2_ir_lock 73 * ->qi->q_lock 74 * ->iommu->register_lock 75 * Note: 76 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called 77 * in single-threaded environment with interrupt disabled, so no need to tabke 78 * the dmar_global_lock. 79 */ 80 DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 81 static const struct irq_domain_ops intel_ir_domain_ops; 82 83 static void iommu_disable_irq_remapping(struct intel_iommu *iommu); 84 static int __init parse_ioapics_under_ir(void); 85 86 static bool ir_pre_enabled(struct intel_iommu *iommu) 87 { 88 return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); 89 } 90 91 static void clear_ir_pre_enabled(struct intel_iommu *iommu) 92 { 93 iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 94 } 95 96 static void init_ir_status(struct intel_iommu *iommu) 97 { 98 u32 gsts; 99 100 gsts = readl(iommu->reg + DMAR_GSTS_REG); 101 if (gsts & DMA_GSTS_IRES) 102 iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 103 } 104 105 static int alloc_irte(struct intel_iommu *iommu, 106 struct irq_2_iommu *irq_iommu, u16 count) 107 { 108 struct ir_table *table = iommu->ir_table; 109 unsigned int mask = 0; 110 unsigned long flags; 111 int index; 112 113 if (!count || !irq_iommu) 114 return -1; 115 116 if (count > 1) { 117 count = __roundup_pow_of_two(count); 118 mask = ilog2(count); 119 } 120 121 if (mask > ecap_max_handle_mask(iommu->ecap)) { 122 pr_err("Requested mask %x exceeds the max invalidation handle" 123 " mask value %Lx\n", mask, 124 ecap_max_handle_mask(iommu->ecap)); 125 return -1; 126 } 127 128 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 129 index = bitmap_find_free_region(table->bitmap, 130 INTR_REMAP_TABLE_ENTRIES, mask); 131 if (index < 0) { 132 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); 133 } else { 134 irq_iommu->iommu = iommu; 135 irq_iommu->irte_index = index; 136 irq_iommu->sub_handle = 0; 137 irq_iommu->irte_mask = mask; 138 irq_iommu->mode = IRQ_REMAPPING; 139 } 140 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 141 142 return index; 143 } 144 145 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) 146 { 147 struct qi_desc desc; 148 149 desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) 150 | QI_IEC_SELECTIVE; 151 desc.qw1 = 0; 152 desc.qw2 = 0; 153 desc.qw3 = 0; 154 155 return qi_submit_sync(iommu, &desc, 1, 0); 156 } 157 158 static int modify_irte(struct irq_2_iommu *irq_iommu, 159 struct irte *irte_modified) 160 { 161 struct intel_iommu *iommu; 162 unsigned long flags; 163 struct irte *irte; 164 int rc, index; 165 166 if (!irq_iommu) 167 return -1; 168 169 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 170 171 iommu = irq_iommu->iommu; 172 173 index = irq_iommu->irte_index + irq_iommu->sub_handle; 174 irte = &iommu->ir_table->base[index]; 175 176 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) 177 if ((irte->pst == 1) || (irte_modified->pst == 1)) { 178 bool ret; 179 180 ret = cmpxchg_double(&irte->low, &irte->high, 181 irte->low, irte->high, 182 irte_modified->low, irte_modified->high); 183 /* 184 * We use cmpxchg16 to atomically update the 128-bit IRTE, 185 * and it cannot be updated by the hardware or other processors 186 * behind us, so the return value of cmpxchg16 should be the 187 * same as the old value. 188 */ 189 WARN_ON(!ret); 190 } else 191 #endif 192 { 193 set_64bit(&irte->low, irte_modified->low); 194 set_64bit(&irte->high, irte_modified->high); 195 } 196 __iommu_flush_cache(iommu, irte, sizeof(*irte)); 197 198 rc = qi_flush_iec(iommu, index, 0); 199 200 /* Update iommu mode according to the IRTE mode */ 201 irq_iommu->mode = irte->pst ? IRQ_POSTING : IRQ_REMAPPING; 202 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 203 204 return rc; 205 } 206 207 static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) 208 { 209 int i; 210 211 for (i = 0; i < MAX_HPET_TBS; i++) 212 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) 213 return ir_hpet[i].iommu; 214 return NULL; 215 } 216 217 static struct intel_iommu *map_ioapic_to_ir(int apic) 218 { 219 int i; 220 221 for (i = 0; i < MAX_IO_APICS; i++) 222 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) 223 return ir_ioapic[i].iommu; 224 return NULL; 225 } 226 227 static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) 228 { 229 struct dmar_drhd_unit *drhd; 230 231 drhd = dmar_find_matched_drhd_unit(dev); 232 if (!drhd) 233 return NULL; 234 235 return drhd->iommu; 236 } 237 238 static int clear_entries(struct irq_2_iommu *irq_iommu) 239 { 240 struct irte *start, *entry, *end; 241 struct intel_iommu *iommu; 242 int index; 243 244 if (irq_iommu->sub_handle) 245 return 0; 246 247 iommu = irq_iommu->iommu; 248 index = irq_iommu->irte_index; 249 250 start = iommu->ir_table->base + index; 251 end = start + (1 << irq_iommu->irte_mask); 252 253 for (entry = start; entry < end; entry++) { 254 set_64bit(&entry->low, 0); 255 set_64bit(&entry->high, 0); 256 } 257 bitmap_release_region(iommu->ir_table->bitmap, index, 258 irq_iommu->irte_mask); 259 260 return qi_flush_iec(iommu, index, irq_iommu->irte_mask); 261 } 262 263 /* 264 * source validation type 265 */ 266 #define SVT_NO_VERIFY 0x0 /* no verification is required */ 267 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ 268 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ 269 270 /* 271 * source-id qualifier 272 */ 273 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ 274 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore 275 * the third least significant bit 276 */ 277 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore 278 * the second and third least significant bits 279 */ 280 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore 281 * the least three significant bits 282 */ 283 284 /* 285 * set SVT, SQ and SID fields of irte to verify 286 * source ids of interrupt requests 287 */ 288 static void set_irte_sid(struct irte *irte, unsigned int svt, 289 unsigned int sq, unsigned int sid) 290 { 291 if (disable_sourceid_checking) 292 svt = SVT_NO_VERIFY; 293 irte->svt = svt; 294 irte->sq = sq; 295 irte->sid = sid; 296 } 297 298 /* 299 * Set an IRTE to match only the bus number. Interrupt requests that reference 300 * this IRTE must have a requester-id whose bus number is between or equal 301 * to the start_bus and end_bus arguments. 302 */ 303 static void set_irte_verify_bus(struct irte *irte, unsigned int start_bus, 304 unsigned int end_bus) 305 { 306 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, 307 (start_bus << 8) | end_bus); 308 } 309 310 static int set_ioapic_sid(struct irte *irte, int apic) 311 { 312 int i; 313 u16 sid = 0; 314 315 if (!irte) 316 return -1; 317 318 down_read(&dmar_global_lock); 319 for (i = 0; i < MAX_IO_APICS; i++) { 320 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { 321 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; 322 break; 323 } 324 } 325 up_read(&dmar_global_lock); 326 327 if (sid == 0) { 328 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); 329 return -1; 330 } 331 332 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid); 333 334 return 0; 335 } 336 337 static int set_hpet_sid(struct irte *irte, u8 id) 338 { 339 int i; 340 u16 sid = 0; 341 342 if (!irte) 343 return -1; 344 345 down_read(&dmar_global_lock); 346 for (i = 0; i < MAX_HPET_TBS; i++) { 347 if (ir_hpet[i].iommu && ir_hpet[i].id == id) { 348 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; 349 break; 350 } 351 } 352 up_read(&dmar_global_lock); 353 354 if (sid == 0) { 355 pr_warn("Failed to set source-id of HPET block (%d)\n", id); 356 return -1; 357 } 358 359 /* 360 * Should really use SQ_ALL_16. Some platforms are broken. 361 * While we figure out the right quirks for these broken platforms, use 362 * SQ_13_IGNORE_3 for now. 363 */ 364 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); 365 366 return 0; 367 } 368 369 struct set_msi_sid_data { 370 struct pci_dev *pdev; 371 u16 alias; 372 int count; 373 int busmatch_count; 374 }; 375 376 static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque) 377 { 378 struct set_msi_sid_data *data = opaque; 379 380 if (data->count == 0 || PCI_BUS_NUM(alias) == PCI_BUS_NUM(data->alias)) 381 data->busmatch_count++; 382 383 data->pdev = pdev; 384 data->alias = alias; 385 data->count++; 386 387 return 0; 388 } 389 390 static int set_msi_sid(struct irte *irte, struct pci_dev *dev) 391 { 392 struct set_msi_sid_data data; 393 394 if (!irte || !dev) 395 return -1; 396 397 data.count = 0; 398 data.busmatch_count = 0; 399 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data); 400 401 /* 402 * DMA alias provides us with a PCI device and alias. The only case 403 * where the it will return an alias on a different bus than the 404 * device is the case of a PCIe-to-PCI bridge, where the alias is for 405 * the subordinate bus. In this case we can only verify the bus. 406 * 407 * If there are multiple aliases, all with the same bus number, 408 * then all we can do is verify the bus. This is typical in NTB 409 * hardware which use proxy IDs where the device will generate traffic 410 * from multiple devfn numbers on the same bus. 411 * 412 * If the alias device is on a different bus than our source device 413 * then we have a topology based alias, use it. 414 * 415 * Otherwise, the alias is for a device DMA quirk and we cannot 416 * assume that MSI uses the same requester ID. Therefore use the 417 * original device. 418 */ 419 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number) 420 set_irte_verify_bus(irte, PCI_BUS_NUM(data.alias), 421 dev->bus->number); 422 else if (data.count >= 2 && data.busmatch_count == data.count) 423 set_irte_verify_bus(irte, dev->bus->number, dev->bus->number); 424 else if (data.pdev->bus->number != dev->bus->number) 425 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias); 426 else 427 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, 428 pci_dev_id(dev)); 429 430 return 0; 431 } 432 433 static int iommu_load_old_irte(struct intel_iommu *iommu) 434 { 435 struct irte *old_ir_table; 436 phys_addr_t irt_phys; 437 unsigned int i; 438 size_t size; 439 u64 irta; 440 441 /* Check whether the old ir-table has the same size as ours */ 442 irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); 443 if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) 444 != INTR_REMAP_TABLE_REG_SIZE) 445 return -EINVAL; 446 447 irt_phys = irta & VTD_PAGE_MASK; 448 size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte); 449 450 /* Map the old IR table */ 451 old_ir_table = memremap(irt_phys, size, MEMREMAP_WB); 452 if (!old_ir_table) 453 return -ENOMEM; 454 455 /* Copy data over */ 456 memcpy(iommu->ir_table->base, old_ir_table, size); 457 458 __iommu_flush_cache(iommu, iommu->ir_table->base, size); 459 460 /* 461 * Now check the table for used entries and mark those as 462 * allocated in the bitmap 463 */ 464 for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) { 465 if (iommu->ir_table->base[i].present) 466 bitmap_set(iommu->ir_table->bitmap, i, 1); 467 } 468 469 memunmap(old_ir_table); 470 471 return 0; 472 } 473 474 475 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) 476 { 477 unsigned long flags; 478 u64 addr; 479 u32 sts; 480 481 addr = virt_to_phys((void *)iommu->ir_table->base); 482 483 raw_spin_lock_irqsave(&iommu->register_lock, flags); 484 485 dmar_writeq(iommu->reg + DMAR_IRTA_REG, 486 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); 487 488 /* Set interrupt-remapping table pointer */ 489 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); 490 491 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 492 readl, (sts & DMA_GSTS_IRTPS), sts); 493 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 494 495 /* 496 * Global invalidation of interrupt entry cache to make sure the 497 * hardware uses the new irq remapping table. 498 */ 499 qi_global_iec(iommu); 500 } 501 502 static void iommu_enable_irq_remapping(struct intel_iommu *iommu) 503 { 504 unsigned long flags; 505 u32 sts; 506 507 raw_spin_lock_irqsave(&iommu->register_lock, flags); 508 509 /* Enable interrupt-remapping */ 510 iommu->gcmd |= DMA_GCMD_IRE; 511 iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ 512 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 513 514 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 515 readl, (sts & DMA_GSTS_IRES), sts); 516 517 /* 518 * With CFI clear in the Global Command register, we should be 519 * protected from dangerous (i.e. compatibility) interrupts 520 * regardless of x2apic status. Check just to be sure. 521 */ 522 if (sts & DMA_GSTS_CFIS) 523 WARN(1, KERN_WARNING 524 "Compatibility-format IRQs enabled despite intr remapping;\n" 525 "you are vulnerable to IRQ injection.\n"); 526 527 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 528 } 529 530 static int intel_setup_irq_remapping(struct intel_iommu *iommu) 531 { 532 struct ir_table *ir_table; 533 struct fwnode_handle *fn; 534 unsigned long *bitmap; 535 struct page *pages; 536 537 if (iommu->ir_table) 538 return 0; 539 540 ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL); 541 if (!ir_table) 542 return -ENOMEM; 543 544 pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 545 INTR_REMAP_PAGE_ORDER); 546 if (!pages) { 547 pr_err("IR%d: failed to allocate pages of order %d\n", 548 iommu->seq_id, INTR_REMAP_PAGE_ORDER); 549 goto out_free_table; 550 } 551 552 bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_ATOMIC); 553 if (bitmap == NULL) { 554 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); 555 goto out_free_pages; 556 } 557 558 fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id); 559 if (!fn) 560 goto out_free_bitmap; 561 562 iommu->ir_domain = 563 irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 564 0, INTR_REMAP_TABLE_ENTRIES, 565 fn, &intel_ir_domain_ops, 566 iommu); 567 if (!iommu->ir_domain) { 568 irq_domain_free_fwnode(fn); 569 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id); 570 goto out_free_bitmap; 571 } 572 iommu->ir_msi_domain = 573 arch_create_remap_msi_irq_domain(iommu->ir_domain, 574 "INTEL-IR-MSI", 575 iommu->seq_id); 576 577 ir_table->base = page_address(pages); 578 ir_table->bitmap = bitmap; 579 iommu->ir_table = ir_table; 580 581 /* 582 * If the queued invalidation is already initialized, 583 * shouldn't disable it. 584 */ 585 if (!iommu->qi) { 586 /* 587 * Clear previous faults. 588 */ 589 dmar_fault(-1, iommu); 590 dmar_disable_qi(iommu); 591 592 if (dmar_enable_qi(iommu)) { 593 pr_err("Failed to enable queued invalidation\n"); 594 goto out_free_bitmap; 595 } 596 } 597 598 init_ir_status(iommu); 599 600 if (ir_pre_enabled(iommu)) { 601 if (!is_kdump_kernel()) { 602 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", 603 iommu->name); 604 clear_ir_pre_enabled(iommu); 605 iommu_disable_irq_remapping(iommu); 606 } else if (iommu_load_old_irte(iommu)) 607 pr_err("Failed to copy IR table for %s from previous kernel\n", 608 iommu->name); 609 else 610 pr_info("Copied IR table for %s from previous kernel\n", 611 iommu->name); 612 } 613 614 iommu_set_irq_remapping(iommu, eim_mode); 615 616 return 0; 617 618 out_free_bitmap: 619 bitmap_free(bitmap); 620 out_free_pages: 621 __free_pages(pages, INTR_REMAP_PAGE_ORDER); 622 out_free_table: 623 kfree(ir_table); 624 625 iommu->ir_table = NULL; 626 627 return -ENOMEM; 628 } 629 630 static void intel_teardown_irq_remapping(struct intel_iommu *iommu) 631 { 632 struct fwnode_handle *fn; 633 634 if (iommu && iommu->ir_table) { 635 if (iommu->ir_msi_domain) { 636 fn = iommu->ir_msi_domain->fwnode; 637 638 irq_domain_remove(iommu->ir_msi_domain); 639 irq_domain_free_fwnode(fn); 640 iommu->ir_msi_domain = NULL; 641 } 642 if (iommu->ir_domain) { 643 fn = iommu->ir_domain->fwnode; 644 645 irq_domain_remove(iommu->ir_domain); 646 irq_domain_free_fwnode(fn); 647 iommu->ir_domain = NULL; 648 } 649 free_pages((unsigned long)iommu->ir_table->base, 650 INTR_REMAP_PAGE_ORDER); 651 bitmap_free(iommu->ir_table->bitmap); 652 kfree(iommu->ir_table); 653 iommu->ir_table = NULL; 654 } 655 } 656 657 /* 658 * Disable Interrupt Remapping. 659 */ 660 static void iommu_disable_irq_remapping(struct intel_iommu *iommu) 661 { 662 unsigned long flags; 663 u32 sts; 664 665 if (!ecap_ir_support(iommu->ecap)) 666 return; 667 668 /* 669 * global invalidation of interrupt entry cache before disabling 670 * interrupt-remapping. 671 */ 672 qi_global_iec(iommu); 673 674 raw_spin_lock_irqsave(&iommu->register_lock, flags); 675 676 sts = readl(iommu->reg + DMAR_GSTS_REG); 677 if (!(sts & DMA_GSTS_IRES)) 678 goto end; 679 680 iommu->gcmd &= ~DMA_GCMD_IRE; 681 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 682 683 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 684 readl, !(sts & DMA_GSTS_IRES), sts); 685 686 end: 687 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 688 } 689 690 static int __init dmar_x2apic_optout(void) 691 { 692 struct acpi_table_dmar *dmar; 693 dmar = (struct acpi_table_dmar *)dmar_tbl; 694 if (!dmar || no_x2apic_optout) 695 return 0; 696 return dmar->flags & DMAR_X2APIC_OPT_OUT; 697 } 698 699 static void __init intel_cleanup_irq_remapping(void) 700 { 701 struct dmar_drhd_unit *drhd; 702 struct intel_iommu *iommu; 703 704 for_each_iommu(iommu, drhd) { 705 if (ecap_ir_support(iommu->ecap)) { 706 iommu_disable_irq_remapping(iommu); 707 intel_teardown_irq_remapping(iommu); 708 } 709 } 710 711 if (x2apic_supported()) 712 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); 713 } 714 715 static int __init intel_prepare_irq_remapping(void) 716 { 717 struct dmar_drhd_unit *drhd; 718 struct intel_iommu *iommu; 719 int eim = 0; 720 721 if (irq_remap_broken) { 722 pr_warn("This system BIOS has enabled interrupt remapping\n" 723 "on a chipset that contains an erratum making that\n" 724 "feature unstable. To maintain system stability\n" 725 "interrupt remapping is being disabled. Please\n" 726 "contact your BIOS vendor for an update\n"); 727 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 728 return -ENODEV; 729 } 730 731 if (dmar_table_init() < 0) 732 return -ENODEV; 733 734 if (!dmar_ir_support()) 735 return -ENODEV; 736 737 if (parse_ioapics_under_ir()) { 738 pr_info("Not enabling interrupt remapping\n"); 739 goto error; 740 } 741 742 /* First make sure all IOMMUs support IRQ remapping */ 743 for_each_iommu(iommu, drhd) 744 if (!ecap_ir_support(iommu->ecap)) 745 goto error; 746 747 /* Detect remapping mode: lapic or x2apic */ 748 if (x2apic_supported()) { 749 eim = !dmar_x2apic_optout(); 750 if (!eim) { 751 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); 752 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); 753 } 754 } 755 756 for_each_iommu(iommu, drhd) { 757 if (eim && !ecap_eim_support(iommu->ecap)) { 758 pr_info("%s does not support EIM\n", iommu->name); 759 eim = 0; 760 } 761 } 762 763 eim_mode = eim; 764 if (eim) 765 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); 766 767 /* Do the initializations early */ 768 for_each_iommu(iommu, drhd) { 769 if (intel_setup_irq_remapping(iommu)) { 770 pr_err("Failed to setup irq remapping for %s\n", 771 iommu->name); 772 goto error; 773 } 774 } 775 776 return 0; 777 778 error: 779 intel_cleanup_irq_remapping(); 780 return -ENODEV; 781 } 782 783 /* 784 * Set Posted-Interrupts capability. 785 */ 786 static inline void set_irq_posting_cap(void) 787 { 788 struct dmar_drhd_unit *drhd; 789 struct intel_iommu *iommu; 790 791 if (!disable_irq_post) { 792 /* 793 * If IRTE is in posted format, the 'pda' field goes across the 794 * 64-bit boundary, we need use cmpxchg16b to atomically update 795 * it. We only expose posted-interrupt when X86_FEATURE_CX16 796 * is supported. Actually, hardware platforms supporting PI 797 * should have X86_FEATURE_CX16 support, this has been confirmed 798 * with Intel hardware guys. 799 */ 800 if (boot_cpu_has(X86_FEATURE_CX16)) 801 intel_irq_remap_ops.capability |= 1 << IRQ_POSTING_CAP; 802 803 for_each_iommu(iommu, drhd) 804 if (!cap_pi_support(iommu->cap)) { 805 intel_irq_remap_ops.capability &= 806 ~(1 << IRQ_POSTING_CAP); 807 break; 808 } 809 } 810 } 811 812 static int __init intel_enable_irq_remapping(void) 813 { 814 struct dmar_drhd_unit *drhd; 815 struct intel_iommu *iommu; 816 bool setup = false; 817 818 /* 819 * Setup Interrupt-remapping for all the DRHD's now. 820 */ 821 for_each_iommu(iommu, drhd) { 822 if (!ir_pre_enabled(iommu)) 823 iommu_enable_irq_remapping(iommu); 824 setup = true; 825 } 826 827 if (!setup) 828 goto error; 829 830 irq_remapping_enabled = 1; 831 832 set_irq_posting_cap(); 833 834 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); 835 836 return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; 837 838 error: 839 intel_cleanup_irq_remapping(); 840 return -1; 841 } 842 843 static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, 844 struct intel_iommu *iommu, 845 struct acpi_dmar_hardware_unit *drhd) 846 { 847 struct acpi_dmar_pci_path *path; 848 u8 bus; 849 int count, free = -1; 850 851 bus = scope->bus; 852 path = (struct acpi_dmar_pci_path *)(scope + 1); 853 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 854 / sizeof(struct acpi_dmar_pci_path); 855 856 while (--count > 0) { 857 /* 858 * Access PCI directly due to the PCI 859 * subsystem isn't initialized yet. 860 */ 861 bus = read_pci_config_byte(bus, path->device, path->function, 862 PCI_SECONDARY_BUS); 863 path++; 864 } 865 866 for (count = 0; count < MAX_HPET_TBS; count++) { 867 if (ir_hpet[count].iommu == iommu && 868 ir_hpet[count].id == scope->enumeration_id) 869 return 0; 870 else if (ir_hpet[count].iommu == NULL && free == -1) 871 free = count; 872 } 873 if (free == -1) { 874 pr_warn("Exceeded Max HPET blocks\n"); 875 return -ENOSPC; 876 } 877 878 ir_hpet[free].iommu = iommu; 879 ir_hpet[free].id = scope->enumeration_id; 880 ir_hpet[free].bus = bus; 881 ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function); 882 pr_info("HPET id %d under DRHD base 0x%Lx\n", 883 scope->enumeration_id, drhd->address); 884 885 return 0; 886 } 887 888 static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, 889 struct intel_iommu *iommu, 890 struct acpi_dmar_hardware_unit *drhd) 891 { 892 struct acpi_dmar_pci_path *path; 893 u8 bus; 894 int count, free = -1; 895 896 bus = scope->bus; 897 path = (struct acpi_dmar_pci_path *)(scope + 1); 898 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 899 / sizeof(struct acpi_dmar_pci_path); 900 901 while (--count > 0) { 902 /* 903 * Access PCI directly due to the PCI 904 * subsystem isn't initialized yet. 905 */ 906 bus = read_pci_config_byte(bus, path->device, path->function, 907 PCI_SECONDARY_BUS); 908 path++; 909 } 910 911 for (count = 0; count < MAX_IO_APICS; count++) { 912 if (ir_ioapic[count].iommu == iommu && 913 ir_ioapic[count].id == scope->enumeration_id) 914 return 0; 915 else if (ir_ioapic[count].iommu == NULL && free == -1) 916 free = count; 917 } 918 if (free == -1) { 919 pr_warn("Exceeded Max IO APICS\n"); 920 return -ENOSPC; 921 } 922 923 ir_ioapic[free].bus = bus; 924 ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function); 925 ir_ioapic[free].iommu = iommu; 926 ir_ioapic[free].id = scope->enumeration_id; 927 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", 928 scope->enumeration_id, drhd->address, iommu->seq_id); 929 930 return 0; 931 } 932 933 static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, 934 struct intel_iommu *iommu) 935 { 936 int ret = 0; 937 struct acpi_dmar_hardware_unit *drhd; 938 struct acpi_dmar_device_scope *scope; 939 void *start, *end; 940 941 drhd = (struct acpi_dmar_hardware_unit *)header; 942 start = (void *)(drhd + 1); 943 end = ((void *)drhd) + header->length; 944 945 while (start < end && ret == 0) { 946 scope = start; 947 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) 948 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); 949 else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) 950 ret = ir_parse_one_hpet_scope(scope, iommu, drhd); 951 start += scope->length; 952 } 953 954 return ret; 955 } 956 957 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) 958 { 959 int i; 960 961 for (i = 0; i < MAX_HPET_TBS; i++) 962 if (ir_hpet[i].iommu == iommu) 963 ir_hpet[i].iommu = NULL; 964 965 for (i = 0; i < MAX_IO_APICS; i++) 966 if (ir_ioapic[i].iommu == iommu) 967 ir_ioapic[i].iommu = NULL; 968 } 969 970 /* 971 * Finds the assocaition between IOAPIC's and its Interrupt-remapping 972 * hardware unit. 973 */ 974 static int __init parse_ioapics_under_ir(void) 975 { 976 struct dmar_drhd_unit *drhd; 977 struct intel_iommu *iommu; 978 bool ir_supported = false; 979 int ioapic_idx; 980 981 for_each_iommu(iommu, drhd) { 982 int ret; 983 984 if (!ecap_ir_support(iommu->ecap)) 985 continue; 986 987 ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu); 988 if (ret) 989 return ret; 990 991 ir_supported = true; 992 } 993 994 if (!ir_supported) 995 return -ENODEV; 996 997 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) { 998 int ioapic_id = mpc_ioapic_id(ioapic_idx); 999 if (!map_ioapic_to_ir(ioapic_id)) { 1000 pr_err(FW_BUG "ioapic %d has no mapping iommu, " 1001 "interrupt remapping will be disabled\n", 1002 ioapic_id); 1003 return -1; 1004 } 1005 } 1006 1007 return 0; 1008 } 1009 1010 static int __init ir_dev_scope_init(void) 1011 { 1012 int ret; 1013 1014 if (!irq_remapping_enabled) 1015 return 0; 1016 1017 down_write(&dmar_global_lock); 1018 ret = dmar_dev_scope_init(); 1019 up_write(&dmar_global_lock); 1020 1021 return ret; 1022 } 1023 rootfs_initcall(ir_dev_scope_init); 1024 1025 static void disable_irq_remapping(void) 1026 { 1027 struct dmar_drhd_unit *drhd; 1028 struct intel_iommu *iommu = NULL; 1029 1030 /* 1031 * Disable Interrupt-remapping for all the DRHD's now. 1032 */ 1033 for_each_iommu(iommu, drhd) { 1034 if (!ecap_ir_support(iommu->ecap)) 1035 continue; 1036 1037 iommu_disable_irq_remapping(iommu); 1038 } 1039 1040 /* 1041 * Clear Posted-Interrupts capability. 1042 */ 1043 if (!disable_irq_post) 1044 intel_irq_remap_ops.capability &= ~(1 << IRQ_POSTING_CAP); 1045 } 1046 1047 static int reenable_irq_remapping(int eim) 1048 { 1049 struct dmar_drhd_unit *drhd; 1050 bool setup = false; 1051 struct intel_iommu *iommu = NULL; 1052 1053 for_each_iommu(iommu, drhd) 1054 if (iommu->qi) 1055 dmar_reenable_qi(iommu); 1056 1057 /* 1058 * Setup Interrupt-remapping for all the DRHD's now. 1059 */ 1060 for_each_iommu(iommu, drhd) { 1061 if (!ecap_ir_support(iommu->ecap)) 1062 continue; 1063 1064 /* Set up interrupt remapping for iommu.*/ 1065 iommu_set_irq_remapping(iommu, eim); 1066 iommu_enable_irq_remapping(iommu); 1067 setup = true; 1068 } 1069 1070 if (!setup) 1071 goto error; 1072 1073 set_irq_posting_cap(); 1074 1075 return 0; 1076 1077 error: 1078 /* 1079 * handle error condition gracefully here! 1080 */ 1081 return -1; 1082 } 1083 1084 static void prepare_irte(struct irte *irte, int vector, unsigned int dest) 1085 { 1086 memset(irte, 0, sizeof(*irte)); 1087 1088 irte->present = 1; 1089 irte->dst_mode = apic->irq_dest_mode; 1090 /* 1091 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the 1092 * actual level or edge trigger will be setup in the IO-APIC 1093 * RTE. This will help simplify level triggered irq migration. 1094 * For more details, see the comments (in io_apic.c) explainig IO-APIC 1095 * irq migration in the presence of interrupt-remapping. 1096 */ 1097 irte->trigger_mode = 0; 1098 irte->dlvry_mode = apic->irq_delivery_mode; 1099 irte->vector = vector; 1100 irte->dest_id = IRTE_DEST(dest); 1101 irte->redir_hint = 1; 1102 } 1103 1104 static struct irq_domain *intel_get_ir_irq_domain(struct irq_alloc_info *info) 1105 { 1106 struct intel_iommu *iommu = NULL; 1107 1108 if (!info) 1109 return NULL; 1110 1111 switch (info->type) { 1112 case X86_IRQ_ALLOC_TYPE_IOAPIC: 1113 iommu = map_ioapic_to_ir(info->ioapic_id); 1114 break; 1115 case X86_IRQ_ALLOC_TYPE_HPET: 1116 iommu = map_hpet_to_ir(info->hpet_id); 1117 break; 1118 case X86_IRQ_ALLOC_TYPE_MSI: 1119 case X86_IRQ_ALLOC_TYPE_MSIX: 1120 iommu = map_dev_to_ir(info->msi_dev); 1121 break; 1122 default: 1123 BUG_ON(1); 1124 break; 1125 } 1126 1127 return iommu ? iommu->ir_domain : NULL; 1128 } 1129 1130 static struct irq_domain *intel_get_irq_domain(struct irq_alloc_info *info) 1131 { 1132 struct intel_iommu *iommu; 1133 1134 if (!info) 1135 return NULL; 1136 1137 switch (info->type) { 1138 case X86_IRQ_ALLOC_TYPE_MSI: 1139 case X86_IRQ_ALLOC_TYPE_MSIX: 1140 iommu = map_dev_to_ir(info->msi_dev); 1141 if (iommu) 1142 return iommu->ir_msi_domain; 1143 break; 1144 default: 1145 break; 1146 } 1147 1148 return NULL; 1149 } 1150 1151 struct irq_remap_ops intel_irq_remap_ops = { 1152 .prepare = intel_prepare_irq_remapping, 1153 .enable = intel_enable_irq_remapping, 1154 .disable = disable_irq_remapping, 1155 .reenable = reenable_irq_remapping, 1156 .enable_faulting = enable_drhd_fault_handling, 1157 .get_ir_irq_domain = intel_get_ir_irq_domain, 1158 .get_irq_domain = intel_get_irq_domain, 1159 }; 1160 1161 static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force) 1162 { 1163 struct intel_ir_data *ir_data = irqd->chip_data; 1164 struct irte *irte = &ir_data->irte_entry; 1165 struct irq_cfg *cfg = irqd_cfg(irqd); 1166 1167 /* 1168 * Atomically updates the IRTE with the new destination, vector 1169 * and flushes the interrupt entry cache. 1170 */ 1171 irte->vector = cfg->vector; 1172 irte->dest_id = IRTE_DEST(cfg->dest_apicid); 1173 1174 /* Update the hardware only if the interrupt is in remapped mode. */ 1175 if (force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING) 1176 modify_irte(&ir_data->irq_2_iommu, irte); 1177 } 1178 1179 /* 1180 * Migrate the IO-APIC irq in the presence of intr-remapping. 1181 * 1182 * For both level and edge triggered, irq migration is a simple atomic 1183 * update(of vector and cpu destination) of IRTE and flush the hardware cache. 1184 * 1185 * For level triggered, we eliminate the io-apic RTE modification (with the 1186 * updated vector information), by using a virtual vector (io-apic pin number). 1187 * Real vector that is used for interrupting cpu will be coming from 1188 * the interrupt-remapping table entry. 1189 * 1190 * As the migration is a simple atomic update of IRTE, the same mechanism 1191 * is used to migrate MSI irq's in the presence of interrupt-remapping. 1192 */ 1193 static int 1194 intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask, 1195 bool force) 1196 { 1197 struct irq_data *parent = data->parent_data; 1198 struct irq_cfg *cfg = irqd_cfg(data); 1199 int ret; 1200 1201 ret = parent->chip->irq_set_affinity(parent, mask, force); 1202 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 1203 return ret; 1204 1205 intel_ir_reconfigure_irte(data, false); 1206 /* 1207 * After this point, all the interrupts will start arriving 1208 * at the new destination. So, time to cleanup the previous 1209 * vector allocation. 1210 */ 1211 send_cleanup_vector(cfg); 1212 1213 return IRQ_SET_MASK_OK_DONE; 1214 } 1215 1216 static void intel_ir_compose_msi_msg(struct irq_data *irq_data, 1217 struct msi_msg *msg) 1218 { 1219 struct intel_ir_data *ir_data = irq_data->chip_data; 1220 1221 *msg = ir_data->msi_entry; 1222 } 1223 1224 static int intel_ir_set_vcpu_affinity(struct irq_data *data, void *info) 1225 { 1226 struct intel_ir_data *ir_data = data->chip_data; 1227 struct vcpu_data *vcpu_pi_info = info; 1228 1229 /* stop posting interrupts, back to remapping mode */ 1230 if (!vcpu_pi_info) { 1231 modify_irte(&ir_data->irq_2_iommu, &ir_data->irte_entry); 1232 } else { 1233 struct irte irte_pi; 1234 1235 /* 1236 * We are not caching the posted interrupt entry. We 1237 * copy the data from the remapped entry and modify 1238 * the fields which are relevant for posted mode. The 1239 * cached remapped entry is used for switching back to 1240 * remapped mode. 1241 */ 1242 memset(&irte_pi, 0, sizeof(irte_pi)); 1243 dmar_copy_shared_irte(&irte_pi, &ir_data->irte_entry); 1244 1245 /* Update the posted mode fields */ 1246 irte_pi.p_pst = 1; 1247 irte_pi.p_urgent = 0; 1248 irte_pi.p_vector = vcpu_pi_info->vector; 1249 irte_pi.pda_l = (vcpu_pi_info->pi_desc_addr >> 1250 (32 - PDA_LOW_BIT)) & ~(-1UL << PDA_LOW_BIT); 1251 irte_pi.pda_h = (vcpu_pi_info->pi_desc_addr >> 32) & 1252 ~(-1UL << PDA_HIGH_BIT); 1253 1254 modify_irte(&ir_data->irq_2_iommu, &irte_pi); 1255 } 1256 1257 return 0; 1258 } 1259 1260 static struct irq_chip intel_ir_chip = { 1261 .name = "INTEL-IR", 1262 .irq_ack = apic_ack_irq, 1263 .irq_set_affinity = intel_ir_set_affinity, 1264 .irq_compose_msi_msg = intel_ir_compose_msi_msg, 1265 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, 1266 }; 1267 1268 static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, 1269 struct irq_cfg *irq_cfg, 1270 struct irq_alloc_info *info, 1271 int index, int sub_handle) 1272 { 1273 struct IR_IO_APIC_route_entry *entry; 1274 struct irte *irte = &data->irte_entry; 1275 struct msi_msg *msg = &data->msi_entry; 1276 1277 prepare_irte(irte, irq_cfg->vector, irq_cfg->dest_apicid); 1278 switch (info->type) { 1279 case X86_IRQ_ALLOC_TYPE_IOAPIC: 1280 /* Set source-id of interrupt request */ 1281 set_ioapic_sid(irte, info->ioapic_id); 1282 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n", 1283 info->ioapic_id, irte->present, irte->fpd, 1284 irte->dst_mode, irte->redir_hint, 1285 irte->trigger_mode, irte->dlvry_mode, 1286 irte->avail, irte->vector, irte->dest_id, 1287 irte->sid, irte->sq, irte->svt); 1288 1289 entry = (struct IR_IO_APIC_route_entry *)info->ioapic_entry; 1290 info->ioapic_entry = NULL; 1291 memset(entry, 0, sizeof(*entry)); 1292 entry->index2 = (index >> 15) & 0x1; 1293 entry->zero = 0; 1294 entry->format = 1; 1295 entry->index = (index & 0x7fff); 1296 /* 1297 * IO-APIC RTE will be configured with virtual vector. 1298 * irq handler will do the explicit EOI to the io-apic. 1299 */ 1300 entry->vector = info->ioapic_pin; 1301 entry->mask = 0; /* enable IRQ */ 1302 entry->trigger = info->ioapic_trigger; 1303 entry->polarity = info->ioapic_polarity; 1304 if (info->ioapic_trigger) 1305 entry->mask = 1; /* Mask level triggered irqs. */ 1306 break; 1307 1308 case X86_IRQ_ALLOC_TYPE_HPET: 1309 case X86_IRQ_ALLOC_TYPE_MSI: 1310 case X86_IRQ_ALLOC_TYPE_MSIX: 1311 if (info->type == X86_IRQ_ALLOC_TYPE_HPET) 1312 set_hpet_sid(irte, info->hpet_id); 1313 else 1314 set_msi_sid(irte, info->msi_dev); 1315 1316 msg->address_hi = MSI_ADDR_BASE_HI; 1317 msg->data = sub_handle; 1318 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | 1319 MSI_ADDR_IR_SHV | 1320 MSI_ADDR_IR_INDEX1(index) | 1321 MSI_ADDR_IR_INDEX2(index); 1322 break; 1323 1324 default: 1325 BUG_ON(1); 1326 break; 1327 } 1328 } 1329 1330 static void intel_free_irq_resources(struct irq_domain *domain, 1331 unsigned int virq, unsigned int nr_irqs) 1332 { 1333 struct irq_data *irq_data; 1334 struct intel_ir_data *data; 1335 struct irq_2_iommu *irq_iommu; 1336 unsigned long flags; 1337 int i; 1338 for (i = 0; i < nr_irqs; i++) { 1339 irq_data = irq_domain_get_irq_data(domain, virq + i); 1340 if (irq_data && irq_data->chip_data) { 1341 data = irq_data->chip_data; 1342 irq_iommu = &data->irq_2_iommu; 1343 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 1344 clear_entries(irq_iommu); 1345 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 1346 irq_domain_reset_irq_data(irq_data); 1347 kfree(data); 1348 } 1349 } 1350 } 1351 1352 static int intel_irq_remapping_alloc(struct irq_domain *domain, 1353 unsigned int virq, unsigned int nr_irqs, 1354 void *arg) 1355 { 1356 struct intel_iommu *iommu = domain->host_data; 1357 struct irq_alloc_info *info = arg; 1358 struct intel_ir_data *data, *ird; 1359 struct irq_data *irq_data; 1360 struct irq_cfg *irq_cfg; 1361 int i, ret, index; 1362 1363 if (!info || !iommu) 1364 return -EINVAL; 1365 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI && 1366 info->type != X86_IRQ_ALLOC_TYPE_MSIX) 1367 return -EINVAL; 1368 1369 /* 1370 * With IRQ remapping enabled, don't need contiguous CPU vectors 1371 * to support multiple MSI interrupts. 1372 */ 1373 if (info->type == X86_IRQ_ALLOC_TYPE_MSI) 1374 info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; 1375 1376 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 1377 if (ret < 0) 1378 return ret; 1379 1380 ret = -ENOMEM; 1381 data = kzalloc(sizeof(*data), GFP_KERNEL); 1382 if (!data) 1383 goto out_free_parent; 1384 1385 down_read(&dmar_global_lock); 1386 index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); 1387 up_read(&dmar_global_lock); 1388 if (index < 0) { 1389 pr_warn("Failed to allocate IRTE\n"); 1390 kfree(data); 1391 goto out_free_parent; 1392 } 1393 1394 for (i = 0; i < nr_irqs; i++) { 1395 irq_data = irq_domain_get_irq_data(domain, virq + i); 1396 irq_cfg = irqd_cfg(irq_data); 1397 if (!irq_data || !irq_cfg) { 1398 ret = -EINVAL; 1399 goto out_free_data; 1400 } 1401 1402 if (i > 0) { 1403 ird = kzalloc(sizeof(*ird), GFP_KERNEL); 1404 if (!ird) 1405 goto out_free_data; 1406 /* Initialize the common data */ 1407 ird->irq_2_iommu = data->irq_2_iommu; 1408 ird->irq_2_iommu.sub_handle = i; 1409 } else { 1410 ird = data; 1411 } 1412 1413 irq_data->hwirq = (index << 16) + i; 1414 irq_data->chip_data = ird; 1415 irq_data->chip = &intel_ir_chip; 1416 intel_irq_remapping_prepare_irte(ird, irq_cfg, info, index, i); 1417 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); 1418 } 1419 return 0; 1420 1421 out_free_data: 1422 intel_free_irq_resources(domain, virq, i); 1423 out_free_parent: 1424 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1425 return ret; 1426 } 1427 1428 static void intel_irq_remapping_free(struct irq_domain *domain, 1429 unsigned int virq, unsigned int nr_irqs) 1430 { 1431 intel_free_irq_resources(domain, virq, nr_irqs); 1432 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1433 } 1434 1435 static int intel_irq_remapping_activate(struct irq_domain *domain, 1436 struct irq_data *irq_data, bool reserve) 1437 { 1438 intel_ir_reconfigure_irte(irq_data, true); 1439 return 0; 1440 } 1441 1442 static void intel_irq_remapping_deactivate(struct irq_domain *domain, 1443 struct irq_data *irq_data) 1444 { 1445 struct intel_ir_data *data = irq_data->chip_data; 1446 struct irte entry; 1447 1448 memset(&entry, 0, sizeof(entry)); 1449 modify_irte(&data->irq_2_iommu, &entry); 1450 } 1451 1452 static const struct irq_domain_ops intel_ir_domain_ops = { 1453 .alloc = intel_irq_remapping_alloc, 1454 .free = intel_irq_remapping_free, 1455 .activate = intel_irq_remapping_activate, 1456 .deactivate = intel_irq_remapping_deactivate, 1457 }; 1458 1459 /* 1460 * Support of Interrupt Remapping Unit Hotplug 1461 */ 1462 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) 1463 { 1464 int ret; 1465 int eim = x2apic_enabled(); 1466 1467 if (eim && !ecap_eim_support(iommu->ecap)) { 1468 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", 1469 iommu->reg_phys, iommu->ecap); 1470 return -ENODEV; 1471 } 1472 1473 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { 1474 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n", 1475 iommu->reg_phys); 1476 return -ENODEV; 1477 } 1478 1479 /* TODO: check all IOAPICs are covered by IOMMU */ 1480 1481 /* Setup Interrupt-remapping now. */ 1482 ret = intel_setup_irq_remapping(iommu); 1483 if (ret) { 1484 pr_err("Failed to setup irq remapping for %s\n", 1485 iommu->name); 1486 intel_teardown_irq_remapping(iommu); 1487 ir_remove_ioapic_hpet_scope(iommu); 1488 } else { 1489 iommu_enable_irq_remapping(iommu); 1490 } 1491 1492 return ret; 1493 } 1494 1495 int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert) 1496 { 1497 int ret = 0; 1498 struct intel_iommu *iommu = dmaru->iommu; 1499 1500 if (!irq_remapping_enabled) 1501 return 0; 1502 if (iommu == NULL) 1503 return -EINVAL; 1504 if (!ecap_ir_support(iommu->ecap)) 1505 return 0; 1506 if (irq_remapping_cap(IRQ_POSTING_CAP) && 1507 !cap_pi_support(iommu->cap)) 1508 return -EBUSY; 1509 1510 if (insert) { 1511 if (!iommu->ir_table) 1512 ret = dmar_ir_add(dmaru, iommu); 1513 } else { 1514 if (iommu->ir_table) { 1515 if (!bitmap_empty(iommu->ir_table->bitmap, 1516 INTR_REMAP_TABLE_ENTRIES)) { 1517 ret = -EBUSY; 1518 } else { 1519 iommu_disable_irq_remapping(iommu); 1520 intel_teardown_irq_remapping(iommu); 1521 ir_remove_ioapic_hpet_scope(iommu); 1522 } 1523 } 1524 } 1525 1526 return ret; 1527 } 1528