1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #define pr_fmt(fmt) "AMD-Vi: " fmt 9 #define dev_fmt(fmt) pr_fmt(fmt) 10 11 #include <linux/pci.h> 12 #include <linux/acpi.h> 13 #include <linux/list.h> 14 #include <linux/bitmap.h> 15 #include <linux/slab.h> 16 #include <linux/syscore_ops.h> 17 #include <linux/interrupt.h> 18 #include <linux/msi.h> 19 #include <linux/irq.h> 20 #include <linux/amd-iommu.h> 21 #include <linux/export.h> 22 #include <linux/kmemleak.h> 23 #include <linux/mem_encrypt.h> 24 #include <asm/pci-direct.h> 25 #include <asm/iommu.h> 26 #include <asm/apic.h> 27 #include <asm/gart.h> 28 #include <asm/x86_init.h> 29 #include <asm/iommu_table.h> 30 #include <asm/io_apic.h> 31 #include <asm/irq_remapping.h> 32 #include <asm/set_memory.h> 33 34 #include <linux/crash_dump.h> 35 36 #include "amd_iommu.h" 37 #include "../irq_remapping.h" 38 39 /* 40 * definitions for the ACPI scanning code 41 */ 42 #define IVRS_HEADER_LENGTH 48 43 44 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 45 #define ACPI_IVMD_TYPE_ALL 0x20 46 #define ACPI_IVMD_TYPE 0x21 47 #define ACPI_IVMD_TYPE_RANGE 0x22 48 49 #define IVHD_DEV_ALL 0x01 50 #define IVHD_DEV_SELECT 0x02 51 #define IVHD_DEV_SELECT_RANGE_START 0x03 52 #define IVHD_DEV_RANGE_END 0x04 53 #define IVHD_DEV_ALIAS 0x42 54 #define IVHD_DEV_ALIAS_RANGE 0x43 55 #define IVHD_DEV_EXT_SELECT 0x46 56 #define IVHD_DEV_EXT_SELECT_RANGE 0x47 57 #define IVHD_DEV_SPECIAL 0x48 58 #define IVHD_DEV_ACPI_HID 0xf0 59 60 #define UID_NOT_PRESENT 0 61 #define UID_IS_INTEGER 1 62 #define UID_IS_CHARACTER 2 63 64 #define IVHD_SPECIAL_IOAPIC 1 65 #define IVHD_SPECIAL_HPET 2 66 67 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01 68 #define IVHD_FLAG_PASSPW_EN_MASK 0x02 69 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 70 #define IVHD_FLAG_ISOC_EN_MASK 0x08 71 72 #define IVMD_FLAG_EXCL_RANGE 0x08 73 #define IVMD_FLAG_IW 0x04 74 #define IVMD_FLAG_IR 0x02 75 #define IVMD_FLAG_UNITY_MAP 0x01 76 77 #define ACPI_DEVFLAG_INITPASS 0x01 78 #define ACPI_DEVFLAG_EXTINT 0x02 79 #define ACPI_DEVFLAG_NMI 0x04 80 #define ACPI_DEVFLAG_SYSMGT1 0x10 81 #define ACPI_DEVFLAG_SYSMGT2 0x20 82 #define ACPI_DEVFLAG_LINT0 0x40 83 #define ACPI_DEVFLAG_LINT1 0x80 84 #define ACPI_DEVFLAG_ATSDIS 0x10000000 85 86 #define LOOP_TIMEOUT 100000 87 /* 88 * ACPI table definitions 89 * 90 * These data structures are laid over the table to parse the important values 91 * out of it. 92 */ 93 94 extern const struct iommu_ops amd_iommu_ops; 95 96 /* 97 * structure describing one IOMMU in the ACPI table. Typically followed by one 98 * or more ivhd_entrys. 99 */ 100 struct ivhd_header { 101 u8 type; 102 u8 flags; 103 u16 length; 104 u16 devid; 105 u16 cap_ptr; 106 u64 mmio_phys; 107 u16 pci_seg; 108 u16 info; 109 u32 efr_attr; 110 111 /* Following only valid on IVHD type 11h and 40h */ 112 u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ 113 u64 res; 114 } __attribute__((packed)); 115 116 /* 117 * A device entry describing which devices a specific IOMMU translates and 118 * which requestor ids they use. 119 */ 120 struct ivhd_entry { 121 u8 type; 122 u16 devid; 123 u8 flags; 124 u32 ext; 125 u32 hidh; 126 u64 cid; 127 u8 uidf; 128 u8 uidl; 129 u8 uid; 130 } __attribute__((packed)); 131 132 /* 133 * An AMD IOMMU memory definition structure. It defines things like exclusion 134 * ranges for devices and regions that should be unity mapped. 135 */ 136 struct ivmd_header { 137 u8 type; 138 u8 flags; 139 u16 length; 140 u16 devid; 141 u16 aux; 142 u64 resv; 143 u64 range_start; 144 u64 range_length; 145 } __attribute__((packed)); 146 147 bool amd_iommu_dump; 148 bool amd_iommu_irq_remap __read_mostly; 149 150 enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1; 151 152 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 153 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; 154 155 static bool amd_iommu_detected; 156 static bool amd_iommu_disabled __initdata; 157 static bool amd_iommu_force_enable __initdata; 158 static int amd_iommu_target_ivhd_type; 159 160 u16 amd_iommu_last_bdf; /* largest PCI device id we have 161 to handle */ 162 LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings 163 we find in ACPI */ 164 bool amd_iommu_unmap_flush; /* if true, flush on every unmap */ 165 166 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the 167 system */ 168 169 /* Array to assign indices to IOMMUs*/ 170 struct amd_iommu *amd_iommus[MAX_IOMMUS]; 171 172 /* Number of IOMMUs present in the system */ 173 static int amd_iommus_present; 174 175 /* IOMMUs have a non-present cache? */ 176 bool amd_iommu_np_cache __read_mostly; 177 bool amd_iommu_iotlb_sup __read_mostly = true; 178 179 u32 amd_iommu_max_pasid __read_mostly = ~0; 180 181 bool amd_iommu_v2_present __read_mostly; 182 static bool amd_iommu_pc_present __read_mostly; 183 184 bool amd_iommu_force_isolation __read_mostly; 185 186 /* 187 * Pointer to the device table which is shared by all AMD IOMMUs 188 * it is indexed by the PCI device id or the HT unit id and contains 189 * information about the domain the device belongs to as well as the 190 * page table root pointer. 191 */ 192 struct dev_table_entry *amd_iommu_dev_table; 193 /* 194 * Pointer to a device table which the content of old device table 195 * will be copied to. It's only be used in kdump kernel. 196 */ 197 static struct dev_table_entry *old_dev_tbl_cpy; 198 199 /* 200 * The alias table is a driver specific data structure which contains the 201 * mappings of the PCI device ids to the actual requestor ids on the IOMMU. 202 * More than one device can share the same requestor id. 203 */ 204 u16 *amd_iommu_alias_table; 205 206 /* 207 * The rlookup table is used to find the IOMMU which is responsible 208 * for a specific device. It is also indexed by the PCI device id. 209 */ 210 struct amd_iommu **amd_iommu_rlookup_table; 211 212 /* 213 * This table is used to find the irq remapping table for a given device id 214 * quickly. 215 */ 216 struct irq_remap_table **irq_lookup_table; 217 218 /* 219 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap 220 * to know which ones are already in use. 221 */ 222 unsigned long *amd_iommu_pd_alloc_bitmap; 223 224 static u32 dev_table_size; /* size of the device table */ 225 static u32 alias_table_size; /* size of the alias table */ 226 static u32 rlookup_table_size; /* size if the rlookup table */ 227 228 enum iommu_init_state { 229 IOMMU_START_STATE, 230 IOMMU_IVRS_DETECTED, 231 IOMMU_ACPI_FINISHED, 232 IOMMU_ENABLED, 233 IOMMU_PCI_INIT, 234 IOMMU_INTERRUPTS_EN, 235 IOMMU_DMA_OPS, 236 IOMMU_INITIALIZED, 237 IOMMU_NOT_FOUND, 238 IOMMU_INIT_ERROR, 239 IOMMU_CMDLINE_DISABLED, 240 }; 241 242 /* Early ioapic and hpet maps from kernel command line */ 243 #define EARLY_MAP_SIZE 4 244 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; 245 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; 246 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; 247 248 static int __initdata early_ioapic_map_size; 249 static int __initdata early_hpet_map_size; 250 static int __initdata early_acpihid_map_size; 251 252 static bool __initdata cmdline_maps; 253 254 static enum iommu_init_state init_state = IOMMU_START_STATE; 255 256 static int amd_iommu_enable_interrupts(void); 257 static int __init iommu_go_to_state(enum iommu_init_state state); 258 static void init_device_table_dma(void); 259 260 static bool amd_iommu_pre_enabled = true; 261 262 static u32 amd_iommu_ivinfo __initdata; 263 264 bool translation_pre_enabled(struct amd_iommu *iommu) 265 { 266 return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); 267 } 268 269 static void clear_translation_pre_enabled(struct amd_iommu *iommu) 270 { 271 iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 272 } 273 274 static void init_translation_status(struct amd_iommu *iommu) 275 { 276 u64 ctrl; 277 278 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 279 if (ctrl & (1<<CONTROL_IOMMU_EN)) 280 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 281 } 282 283 static inline void update_last_devid(u16 devid) 284 { 285 if (devid > amd_iommu_last_bdf) 286 amd_iommu_last_bdf = devid; 287 } 288 289 static inline unsigned long tbl_size(int entry_size) 290 { 291 unsigned shift = PAGE_SHIFT + 292 get_order(((int)amd_iommu_last_bdf + 1) * entry_size); 293 294 return 1UL << shift; 295 } 296 297 int amd_iommu_get_num_iommus(void) 298 { 299 return amd_iommus_present; 300 } 301 302 /* 303 * For IVHD type 0x11/0x40, EFR is also available via IVHD. 304 * Default to IVHD EFR since it is available sooner 305 * (i.e. before PCI init). 306 */ 307 static void __init early_iommu_features_init(struct amd_iommu *iommu, 308 struct ivhd_header *h) 309 { 310 if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) 311 iommu->features = h->efr_reg; 312 } 313 314 /* Access to l1 and l2 indexed register spaces */ 315 316 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) 317 { 318 u32 val; 319 320 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 321 pci_read_config_dword(iommu->dev, 0xfc, &val); 322 return val; 323 } 324 325 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) 326 { 327 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); 328 pci_write_config_dword(iommu->dev, 0xfc, val); 329 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 330 } 331 332 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) 333 { 334 u32 val; 335 336 pci_write_config_dword(iommu->dev, 0xf0, address); 337 pci_read_config_dword(iommu->dev, 0xf4, &val); 338 return val; 339 } 340 341 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) 342 { 343 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); 344 pci_write_config_dword(iommu->dev, 0xf4, val); 345 } 346 347 /**************************************************************************** 348 * 349 * AMD IOMMU MMIO register space handling functions 350 * 351 * These functions are used to program the IOMMU device registers in 352 * MMIO space required for that driver. 353 * 354 ****************************************************************************/ 355 356 /* 357 * This function set the exclusion range in the IOMMU. DMA accesses to the 358 * exclusion range are passed through untranslated 359 */ 360 static void iommu_set_exclusion_range(struct amd_iommu *iommu) 361 { 362 u64 start = iommu->exclusion_start & PAGE_MASK; 363 u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK; 364 u64 entry; 365 366 if (!iommu->exclusion_start) 367 return; 368 369 entry = start | MMIO_EXCL_ENABLE_MASK; 370 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 371 &entry, sizeof(entry)); 372 373 entry = limit; 374 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 375 &entry, sizeof(entry)); 376 } 377 378 static void iommu_set_cwwb_range(struct amd_iommu *iommu) 379 { 380 u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); 381 u64 entry = start & PM_ADDR_MASK; 382 383 if (!iommu_feature(iommu, FEATURE_SNP)) 384 return; 385 386 /* Note: 387 * Re-purpose Exclusion base/limit registers for Completion wait 388 * write-back base/limit. 389 */ 390 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 391 &entry, sizeof(entry)); 392 393 /* Note: 394 * Default to 4 Kbytes, which can be specified by setting base 395 * address equal to the limit address. 396 */ 397 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 398 &entry, sizeof(entry)); 399 } 400 401 /* Programs the physical address of the device table into the IOMMU hardware */ 402 static void iommu_set_device_table(struct amd_iommu *iommu) 403 { 404 u64 entry; 405 406 BUG_ON(iommu->mmio_base == NULL); 407 408 entry = iommu_virt_to_phys(amd_iommu_dev_table); 409 entry |= (dev_table_size >> 12) - 1; 410 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, 411 &entry, sizeof(entry)); 412 } 413 414 /* Generic functions to enable/disable certain features of the IOMMU. */ 415 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) 416 { 417 u64 ctrl; 418 419 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 420 ctrl |= (1ULL << bit); 421 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 422 } 423 424 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) 425 { 426 u64 ctrl; 427 428 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 429 ctrl &= ~(1ULL << bit); 430 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 431 } 432 433 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout) 434 { 435 u64 ctrl; 436 437 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 438 ctrl &= ~CTRL_INV_TO_MASK; 439 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK; 440 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 441 } 442 443 /* Function to enable the hardware */ 444 static void iommu_enable(struct amd_iommu *iommu) 445 { 446 iommu_feature_enable(iommu, CONTROL_IOMMU_EN); 447 } 448 449 static void iommu_disable(struct amd_iommu *iommu) 450 { 451 if (!iommu->mmio_base) 452 return; 453 454 /* Disable command buffer */ 455 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 456 457 /* Disable event logging and event interrupts */ 458 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); 459 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 460 461 /* Disable IOMMU GA_LOG */ 462 iommu_feature_disable(iommu, CONTROL_GALOG_EN); 463 iommu_feature_disable(iommu, CONTROL_GAINT_EN); 464 465 /* Disable IOMMU hardware itself */ 466 iommu_feature_disable(iommu, CONTROL_IOMMU_EN); 467 } 468 469 /* 470 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in 471 * the system has one. 472 */ 473 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) 474 { 475 if (!request_mem_region(address, end, "amd_iommu")) { 476 pr_err("Can not reserve memory region %llx-%llx for mmio\n", 477 address, end); 478 pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); 479 return NULL; 480 } 481 482 return (u8 __iomem *)ioremap(address, end); 483 } 484 485 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) 486 { 487 if (iommu->mmio_base) 488 iounmap(iommu->mmio_base); 489 release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); 490 } 491 492 static inline u32 get_ivhd_header_size(struct ivhd_header *h) 493 { 494 u32 size = 0; 495 496 switch (h->type) { 497 case 0x10: 498 size = 24; 499 break; 500 case 0x11: 501 case 0x40: 502 size = 40; 503 break; 504 } 505 return size; 506 } 507 508 /**************************************************************************** 509 * 510 * The functions below belong to the first pass of AMD IOMMU ACPI table 511 * parsing. In this pass we try to find out the highest device id this 512 * code has to handle. Upon this information the size of the shared data 513 * structures is determined later. 514 * 515 ****************************************************************************/ 516 517 /* 518 * This function calculates the length of a given IVHD entry 519 */ 520 static inline int ivhd_entry_length(u8 *ivhd) 521 { 522 u32 type = ((struct ivhd_entry *)ivhd)->type; 523 524 if (type < 0x80) { 525 return 0x04 << (*ivhd >> 6); 526 } else if (type == IVHD_DEV_ACPI_HID) { 527 /* For ACPI_HID, offset 21 is uid len */ 528 return *((u8 *)ivhd + 21) + 22; 529 } 530 return 0; 531 } 532 533 /* 534 * After reading the highest device id from the IOMMU PCI capability header 535 * this function looks if there is a higher device id defined in the ACPI table 536 */ 537 static int __init find_last_devid_from_ivhd(struct ivhd_header *h) 538 { 539 u8 *p = (void *)h, *end = (void *)h; 540 struct ivhd_entry *dev; 541 542 u32 ivhd_size = get_ivhd_header_size(h); 543 544 if (!ivhd_size) { 545 pr_err("Unsupported IVHD type %#x\n", h->type); 546 return -EINVAL; 547 } 548 549 p += ivhd_size; 550 end += h->length; 551 552 while (p < end) { 553 dev = (struct ivhd_entry *)p; 554 switch (dev->type) { 555 case IVHD_DEV_ALL: 556 /* Use maximum BDF value for DEV_ALL */ 557 update_last_devid(0xffff); 558 break; 559 case IVHD_DEV_SELECT: 560 case IVHD_DEV_RANGE_END: 561 case IVHD_DEV_ALIAS: 562 case IVHD_DEV_EXT_SELECT: 563 /* all the above subfield types refer to device ids */ 564 update_last_devid(dev->devid); 565 break; 566 default: 567 break; 568 } 569 p += ivhd_entry_length(p); 570 } 571 572 WARN_ON(p != end); 573 574 return 0; 575 } 576 577 static int __init check_ivrs_checksum(struct acpi_table_header *table) 578 { 579 int i; 580 u8 checksum = 0, *p = (u8 *)table; 581 582 for (i = 0; i < table->length; ++i) 583 checksum += p[i]; 584 if (checksum != 0) { 585 /* ACPI table corrupt */ 586 pr_err(FW_BUG "IVRS invalid checksum\n"); 587 return -ENODEV; 588 } 589 590 return 0; 591 } 592 593 /* 594 * Iterate over all IVHD entries in the ACPI table and find the highest device 595 * id which we need to handle. This is the first of three functions which parse 596 * the ACPI table. So we check the checksum here. 597 */ 598 static int __init find_last_devid_acpi(struct acpi_table_header *table) 599 { 600 u8 *p = (u8 *)table, *end = (u8 *)table; 601 struct ivhd_header *h; 602 603 p += IVRS_HEADER_LENGTH; 604 605 end += table->length; 606 while (p < end) { 607 h = (struct ivhd_header *)p; 608 if (h->type == amd_iommu_target_ivhd_type) { 609 int ret = find_last_devid_from_ivhd(h); 610 611 if (ret) 612 return ret; 613 } 614 p += h->length; 615 } 616 WARN_ON(p != end); 617 618 return 0; 619 } 620 621 /**************************************************************************** 622 * 623 * The following functions belong to the code path which parses the ACPI table 624 * the second time. In this ACPI parsing iteration we allocate IOMMU specific 625 * data structures, initialize the device/alias/rlookup table and also 626 * basically initialize the hardware. 627 * 628 ****************************************************************************/ 629 630 /* 631 * Allocates the command buffer. This buffer is per AMD IOMMU. We can 632 * write commands to that buffer later and the IOMMU will execute them 633 * asynchronously 634 */ 635 static int __init alloc_command_buffer(struct amd_iommu *iommu) 636 { 637 iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 638 get_order(CMD_BUFFER_SIZE)); 639 640 return iommu->cmd_buf ? 0 : -ENOMEM; 641 } 642 643 /* 644 * This function resets the command buffer if the IOMMU stopped fetching 645 * commands from it. 646 */ 647 void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) 648 { 649 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 650 651 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); 652 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 653 iommu->cmd_buf_head = 0; 654 iommu->cmd_buf_tail = 0; 655 656 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); 657 } 658 659 /* 660 * This function writes the command buffer address to the hardware and 661 * enables it. 662 */ 663 static void iommu_enable_command_buffer(struct amd_iommu *iommu) 664 { 665 u64 entry; 666 667 BUG_ON(iommu->cmd_buf == NULL); 668 669 entry = iommu_virt_to_phys(iommu->cmd_buf); 670 entry |= MMIO_CMD_SIZE_512; 671 672 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, 673 &entry, sizeof(entry)); 674 675 amd_iommu_reset_cmd_buffer(iommu); 676 } 677 678 /* 679 * This function disables the command buffer 680 */ 681 static void iommu_disable_command_buffer(struct amd_iommu *iommu) 682 { 683 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 684 } 685 686 static void __init free_command_buffer(struct amd_iommu *iommu) 687 { 688 free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); 689 } 690 691 static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, 692 gfp_t gfp, size_t size) 693 { 694 int order = get_order(size); 695 void *buf = (void *)__get_free_pages(gfp, order); 696 697 if (buf && 698 iommu_feature(iommu, FEATURE_SNP) && 699 set_memory_4k((unsigned long)buf, (1 << order))) { 700 free_pages((unsigned long)buf, order); 701 buf = NULL; 702 } 703 704 return buf; 705 } 706 707 /* allocates the memory where the IOMMU will log its events to */ 708 static int __init alloc_event_buffer(struct amd_iommu *iommu) 709 { 710 iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 711 EVT_BUFFER_SIZE); 712 713 return iommu->evt_buf ? 0 : -ENOMEM; 714 } 715 716 static void iommu_enable_event_buffer(struct amd_iommu *iommu) 717 { 718 u64 entry; 719 720 BUG_ON(iommu->evt_buf == NULL); 721 722 entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; 723 724 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, 725 &entry, sizeof(entry)); 726 727 /* set head and tail to zero manually */ 728 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 729 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 730 731 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); 732 } 733 734 /* 735 * This function disables the event log buffer 736 */ 737 static void iommu_disable_event_buffer(struct amd_iommu *iommu) 738 { 739 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 740 } 741 742 static void __init free_event_buffer(struct amd_iommu *iommu) 743 { 744 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); 745 } 746 747 /* allocates the memory where the IOMMU will log its events to */ 748 static int __init alloc_ppr_log(struct amd_iommu *iommu) 749 { 750 iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 751 PPR_LOG_SIZE); 752 753 return iommu->ppr_log ? 0 : -ENOMEM; 754 } 755 756 static void iommu_enable_ppr_log(struct amd_iommu *iommu) 757 { 758 u64 entry; 759 760 if (iommu->ppr_log == NULL) 761 return; 762 763 entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; 764 765 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, 766 &entry, sizeof(entry)); 767 768 /* set head and tail to zero manually */ 769 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); 770 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); 771 772 iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); 773 iommu_feature_enable(iommu, CONTROL_PPR_EN); 774 } 775 776 static void __init free_ppr_log(struct amd_iommu *iommu) 777 { 778 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); 779 } 780 781 static void free_ga_log(struct amd_iommu *iommu) 782 { 783 #ifdef CONFIG_IRQ_REMAP 784 free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE)); 785 free_pages((unsigned long)iommu->ga_log_tail, get_order(8)); 786 #endif 787 } 788 789 static int iommu_ga_log_enable(struct amd_iommu *iommu) 790 { 791 #ifdef CONFIG_IRQ_REMAP 792 u32 status, i; 793 794 if (!iommu->ga_log) 795 return -EINVAL; 796 797 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 798 799 /* Check if already running */ 800 if (status & (MMIO_STATUS_GALOG_RUN_MASK)) 801 return 0; 802 803 iommu_feature_enable(iommu, CONTROL_GAINT_EN); 804 iommu_feature_enable(iommu, CONTROL_GALOG_EN); 805 806 for (i = 0; i < LOOP_TIMEOUT; ++i) { 807 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 808 if (status & (MMIO_STATUS_GALOG_RUN_MASK)) 809 break; 810 } 811 812 if (i >= LOOP_TIMEOUT) 813 return -EINVAL; 814 #endif /* CONFIG_IRQ_REMAP */ 815 return 0; 816 } 817 818 #ifdef CONFIG_IRQ_REMAP 819 static int iommu_init_ga_log(struct amd_iommu *iommu) 820 { 821 u64 entry; 822 823 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 824 return 0; 825 826 iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 827 get_order(GA_LOG_SIZE)); 828 if (!iommu->ga_log) 829 goto err_out; 830 831 iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 832 get_order(8)); 833 if (!iommu->ga_log_tail) 834 goto err_out; 835 836 entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; 837 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, 838 &entry, sizeof(entry)); 839 entry = (iommu_virt_to_phys(iommu->ga_log_tail) & 840 (BIT_ULL(52)-1)) & ~7ULL; 841 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, 842 &entry, sizeof(entry)); 843 writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 844 writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 845 846 return 0; 847 err_out: 848 free_ga_log(iommu); 849 return -EINVAL; 850 } 851 #endif /* CONFIG_IRQ_REMAP */ 852 853 static int iommu_init_ga(struct amd_iommu *iommu) 854 { 855 int ret = 0; 856 857 #ifdef CONFIG_IRQ_REMAP 858 /* Note: We have already checked GASup from IVRS table. 859 * Now, we need to make sure that GAMSup is set. 860 */ 861 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && 862 !iommu_feature(iommu, FEATURE_GAM_VAPIC)) 863 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 864 865 ret = iommu_init_ga_log(iommu); 866 #endif /* CONFIG_IRQ_REMAP */ 867 868 return ret; 869 } 870 871 static int __init alloc_cwwb_sem(struct amd_iommu *iommu) 872 { 873 iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1); 874 875 return iommu->cmd_sem ? 0 : -ENOMEM; 876 } 877 878 static void __init free_cwwb_sem(struct amd_iommu *iommu) 879 { 880 if (iommu->cmd_sem) 881 free_page((unsigned long)iommu->cmd_sem); 882 } 883 884 static void iommu_enable_xt(struct amd_iommu *iommu) 885 { 886 #ifdef CONFIG_IRQ_REMAP 887 /* 888 * XT mode (32-bit APIC destination ID) requires 889 * GA mode (128-bit IRTE support) as a prerequisite. 890 */ 891 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) && 892 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 893 iommu_feature_enable(iommu, CONTROL_XT_EN); 894 #endif /* CONFIG_IRQ_REMAP */ 895 } 896 897 static void iommu_enable_gt(struct amd_iommu *iommu) 898 { 899 if (!iommu_feature(iommu, FEATURE_GT)) 900 return; 901 902 iommu_feature_enable(iommu, CONTROL_GT_EN); 903 } 904 905 /* sets a specific bit in the device table entry. */ 906 static void set_dev_entry_bit(u16 devid, u8 bit) 907 { 908 int i = (bit >> 6) & 0x03; 909 int _bit = bit & 0x3f; 910 911 amd_iommu_dev_table[devid].data[i] |= (1UL << _bit); 912 } 913 914 static int get_dev_entry_bit(u16 devid, u8 bit) 915 { 916 int i = (bit >> 6) & 0x03; 917 int _bit = bit & 0x3f; 918 919 return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit; 920 } 921 922 923 static bool copy_device_table(void) 924 { 925 u64 int_ctl, int_tab_len, entry = 0, last_entry = 0; 926 struct dev_table_entry *old_devtb = NULL; 927 u32 lo, hi, devid, old_devtb_size; 928 phys_addr_t old_devtb_phys; 929 struct amd_iommu *iommu; 930 u16 dom_id, dte_v, irq_v; 931 gfp_t gfp_flag; 932 u64 tmp; 933 934 if (!amd_iommu_pre_enabled) 935 return false; 936 937 pr_warn("Translation is already enabled - trying to copy translation structures\n"); 938 for_each_iommu(iommu) { 939 /* All IOMMUs should use the same device table with the same size */ 940 lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); 941 hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); 942 entry = (((u64) hi) << 32) + lo; 943 if (last_entry && last_entry != entry) { 944 pr_err("IOMMU:%d should use the same dev table as others!\n", 945 iommu->index); 946 return false; 947 } 948 last_entry = entry; 949 950 old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; 951 if (old_devtb_size != dev_table_size) { 952 pr_err("The device table size of IOMMU:%d is not expected!\n", 953 iommu->index); 954 return false; 955 } 956 } 957 958 /* 959 * When SME is enabled in the first kernel, the entry includes the 960 * memory encryption mask(sme_me_mask), we must remove the memory 961 * encryption mask to obtain the true physical address in kdump kernel. 962 */ 963 old_devtb_phys = __sme_clr(entry) & PAGE_MASK; 964 965 if (old_devtb_phys >= 0x100000000ULL) { 966 pr_err("The address of old device table is above 4G, not trustworthy!\n"); 967 return false; 968 } 969 old_devtb = (sme_active() && is_kdump_kernel()) 970 ? (__force void *)ioremap_encrypted(old_devtb_phys, 971 dev_table_size) 972 : memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB); 973 974 if (!old_devtb) 975 return false; 976 977 gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; 978 old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, 979 get_order(dev_table_size)); 980 if (old_dev_tbl_cpy == NULL) { 981 pr_err("Failed to allocate memory for copying old device table!\n"); 982 return false; 983 } 984 985 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { 986 old_dev_tbl_cpy[devid] = old_devtb[devid]; 987 dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK; 988 dte_v = old_devtb[devid].data[0] & DTE_FLAG_V; 989 990 if (dte_v && dom_id) { 991 old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0]; 992 old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1]; 993 __set_bit(dom_id, amd_iommu_pd_alloc_bitmap); 994 /* If gcr3 table existed, mask it out */ 995 if (old_devtb[devid].data[0] & DTE_FLAG_GV) { 996 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B; 997 tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C; 998 old_dev_tbl_cpy[devid].data[1] &= ~tmp; 999 tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A; 1000 tmp |= DTE_FLAG_GV; 1001 old_dev_tbl_cpy[devid].data[0] &= ~tmp; 1002 } 1003 } 1004 1005 irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE; 1006 int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK; 1007 int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK; 1008 if (irq_v && (int_ctl || int_tab_len)) { 1009 if ((int_ctl != DTE_IRQ_REMAP_INTCTL) || 1010 (int_tab_len != DTE_INTTABLEN)) { 1011 pr_err("Wrong old irq remapping flag: %#x\n", devid); 1012 return false; 1013 } 1014 1015 old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2]; 1016 } 1017 } 1018 memunmap(old_devtb); 1019 1020 return true; 1021 } 1022 1023 void amd_iommu_apply_erratum_63(u16 devid) 1024 { 1025 int sysmgt; 1026 1027 sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) | 1028 (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1); 1029 1030 if (sysmgt == 0x01) 1031 set_dev_entry_bit(devid, DEV_ENTRY_IW); 1032 } 1033 1034 /* Writes the specific IOMMU for a device into the rlookup table */ 1035 static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) 1036 { 1037 amd_iommu_rlookup_table[devid] = iommu; 1038 } 1039 1040 /* 1041 * This function takes the device specific flags read from the ACPI 1042 * table and sets up the device table entry with that information 1043 */ 1044 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, 1045 u16 devid, u32 flags, u32 ext_flags) 1046 { 1047 if (flags & ACPI_DEVFLAG_INITPASS) 1048 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS); 1049 if (flags & ACPI_DEVFLAG_EXTINT) 1050 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS); 1051 if (flags & ACPI_DEVFLAG_NMI) 1052 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS); 1053 if (flags & ACPI_DEVFLAG_SYSMGT1) 1054 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1); 1055 if (flags & ACPI_DEVFLAG_SYSMGT2) 1056 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2); 1057 if (flags & ACPI_DEVFLAG_LINT0) 1058 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS); 1059 if (flags & ACPI_DEVFLAG_LINT1) 1060 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS); 1061 1062 amd_iommu_apply_erratum_63(devid); 1063 1064 set_iommu_for_device(iommu, devid); 1065 } 1066 1067 int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) 1068 { 1069 struct devid_map *entry; 1070 struct list_head *list; 1071 1072 if (type == IVHD_SPECIAL_IOAPIC) 1073 list = &ioapic_map; 1074 else if (type == IVHD_SPECIAL_HPET) 1075 list = &hpet_map; 1076 else 1077 return -EINVAL; 1078 1079 list_for_each_entry(entry, list, list) { 1080 if (!(entry->id == id && entry->cmd_line)) 1081 continue; 1082 1083 pr_info("Command-line override present for %s id %d - ignoring\n", 1084 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); 1085 1086 *devid = entry->devid; 1087 1088 return 0; 1089 } 1090 1091 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1092 if (!entry) 1093 return -ENOMEM; 1094 1095 entry->id = id; 1096 entry->devid = *devid; 1097 entry->cmd_line = cmd_line; 1098 1099 list_add_tail(&entry->list, list); 1100 1101 return 0; 1102 } 1103 1104 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, 1105 bool cmd_line) 1106 { 1107 struct acpihid_map_entry *entry; 1108 struct list_head *list = &acpihid_map; 1109 1110 list_for_each_entry(entry, list, list) { 1111 if (strcmp(entry->hid, hid) || 1112 (*uid && *entry->uid && strcmp(entry->uid, uid)) || 1113 !entry->cmd_line) 1114 continue; 1115 1116 pr_info("Command-line override for hid:%s uid:%s\n", 1117 hid, uid); 1118 *devid = entry->devid; 1119 return 0; 1120 } 1121 1122 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1123 if (!entry) 1124 return -ENOMEM; 1125 1126 memcpy(entry->uid, uid, strlen(uid)); 1127 memcpy(entry->hid, hid, strlen(hid)); 1128 entry->devid = *devid; 1129 entry->cmd_line = cmd_line; 1130 entry->root_devid = (entry->devid & (~0x7)); 1131 1132 pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n", 1133 entry->cmd_line ? "cmd" : "ivrs", 1134 entry->hid, entry->uid, entry->root_devid); 1135 1136 list_add_tail(&entry->list, list); 1137 return 0; 1138 } 1139 1140 static int __init add_early_maps(void) 1141 { 1142 int i, ret; 1143 1144 for (i = 0; i < early_ioapic_map_size; ++i) { 1145 ret = add_special_device(IVHD_SPECIAL_IOAPIC, 1146 early_ioapic_map[i].id, 1147 &early_ioapic_map[i].devid, 1148 early_ioapic_map[i].cmd_line); 1149 if (ret) 1150 return ret; 1151 } 1152 1153 for (i = 0; i < early_hpet_map_size; ++i) { 1154 ret = add_special_device(IVHD_SPECIAL_HPET, 1155 early_hpet_map[i].id, 1156 &early_hpet_map[i].devid, 1157 early_hpet_map[i].cmd_line); 1158 if (ret) 1159 return ret; 1160 } 1161 1162 for (i = 0; i < early_acpihid_map_size; ++i) { 1163 ret = add_acpi_hid_device(early_acpihid_map[i].hid, 1164 early_acpihid_map[i].uid, 1165 &early_acpihid_map[i].devid, 1166 early_acpihid_map[i].cmd_line); 1167 if (ret) 1168 return ret; 1169 } 1170 1171 return 0; 1172 } 1173 1174 /* 1175 * Takes a pointer to an AMD IOMMU entry in the ACPI table and 1176 * initializes the hardware and our data structures with it. 1177 */ 1178 static int __init init_iommu_from_acpi(struct amd_iommu *iommu, 1179 struct ivhd_header *h) 1180 { 1181 u8 *p = (u8 *)h; 1182 u8 *end = p, flags = 0; 1183 u16 devid = 0, devid_start = 0, devid_to = 0; 1184 u32 dev_i, ext_flags = 0; 1185 bool alias = false; 1186 struct ivhd_entry *e; 1187 u32 ivhd_size; 1188 int ret; 1189 1190 1191 ret = add_early_maps(); 1192 if (ret) 1193 return ret; 1194 1195 amd_iommu_apply_ivrs_quirks(); 1196 1197 /* 1198 * First save the recommended feature enable bits from ACPI 1199 */ 1200 iommu->acpi_flags = h->flags; 1201 1202 /* 1203 * Done. Now parse the device entries 1204 */ 1205 ivhd_size = get_ivhd_header_size(h); 1206 if (!ivhd_size) { 1207 pr_err("Unsupported IVHD type %#x\n", h->type); 1208 return -EINVAL; 1209 } 1210 1211 p += ivhd_size; 1212 1213 end += h->length; 1214 1215 1216 while (p < end) { 1217 e = (struct ivhd_entry *)p; 1218 switch (e->type) { 1219 case IVHD_DEV_ALL: 1220 1221 DUMP_printk(" DEV_ALL\t\t\tflags: %02x\n", e->flags); 1222 1223 for (dev_i = 0; dev_i <= amd_iommu_last_bdf; ++dev_i) 1224 set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0); 1225 break; 1226 case IVHD_DEV_SELECT: 1227 1228 DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " 1229 "flags: %02x\n", 1230 PCI_BUS_NUM(e->devid), 1231 PCI_SLOT(e->devid), 1232 PCI_FUNC(e->devid), 1233 e->flags); 1234 1235 devid = e->devid; 1236 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1237 break; 1238 case IVHD_DEV_SELECT_RANGE_START: 1239 1240 DUMP_printk(" DEV_SELECT_RANGE_START\t " 1241 "devid: %02x:%02x.%x flags: %02x\n", 1242 PCI_BUS_NUM(e->devid), 1243 PCI_SLOT(e->devid), 1244 PCI_FUNC(e->devid), 1245 e->flags); 1246 1247 devid_start = e->devid; 1248 flags = e->flags; 1249 ext_flags = 0; 1250 alias = false; 1251 break; 1252 case IVHD_DEV_ALIAS: 1253 1254 DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " 1255 "flags: %02x devid_to: %02x:%02x.%x\n", 1256 PCI_BUS_NUM(e->devid), 1257 PCI_SLOT(e->devid), 1258 PCI_FUNC(e->devid), 1259 e->flags, 1260 PCI_BUS_NUM(e->ext >> 8), 1261 PCI_SLOT(e->ext >> 8), 1262 PCI_FUNC(e->ext >> 8)); 1263 1264 devid = e->devid; 1265 devid_to = e->ext >> 8; 1266 set_dev_entry_from_acpi(iommu, devid , e->flags, 0); 1267 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); 1268 amd_iommu_alias_table[devid] = devid_to; 1269 break; 1270 case IVHD_DEV_ALIAS_RANGE: 1271 1272 DUMP_printk(" DEV_ALIAS_RANGE\t\t " 1273 "devid: %02x:%02x.%x flags: %02x " 1274 "devid_to: %02x:%02x.%x\n", 1275 PCI_BUS_NUM(e->devid), 1276 PCI_SLOT(e->devid), 1277 PCI_FUNC(e->devid), 1278 e->flags, 1279 PCI_BUS_NUM(e->ext >> 8), 1280 PCI_SLOT(e->ext >> 8), 1281 PCI_FUNC(e->ext >> 8)); 1282 1283 devid_start = e->devid; 1284 flags = e->flags; 1285 devid_to = e->ext >> 8; 1286 ext_flags = 0; 1287 alias = true; 1288 break; 1289 case IVHD_DEV_EXT_SELECT: 1290 1291 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " 1292 "flags: %02x ext: %08x\n", 1293 PCI_BUS_NUM(e->devid), 1294 PCI_SLOT(e->devid), 1295 PCI_FUNC(e->devid), 1296 e->flags, e->ext); 1297 1298 devid = e->devid; 1299 set_dev_entry_from_acpi(iommu, devid, e->flags, 1300 e->ext); 1301 break; 1302 case IVHD_DEV_EXT_SELECT_RANGE: 1303 1304 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " 1305 "%02x:%02x.%x flags: %02x ext: %08x\n", 1306 PCI_BUS_NUM(e->devid), 1307 PCI_SLOT(e->devid), 1308 PCI_FUNC(e->devid), 1309 e->flags, e->ext); 1310 1311 devid_start = e->devid; 1312 flags = e->flags; 1313 ext_flags = e->ext; 1314 alias = false; 1315 break; 1316 case IVHD_DEV_RANGE_END: 1317 1318 DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", 1319 PCI_BUS_NUM(e->devid), 1320 PCI_SLOT(e->devid), 1321 PCI_FUNC(e->devid)); 1322 1323 devid = e->devid; 1324 for (dev_i = devid_start; dev_i <= devid; ++dev_i) { 1325 if (alias) { 1326 amd_iommu_alias_table[dev_i] = devid_to; 1327 set_dev_entry_from_acpi(iommu, 1328 devid_to, flags, ext_flags); 1329 } 1330 set_dev_entry_from_acpi(iommu, dev_i, 1331 flags, ext_flags); 1332 } 1333 break; 1334 case IVHD_DEV_SPECIAL: { 1335 u8 handle, type; 1336 const char *var; 1337 u16 devid; 1338 int ret; 1339 1340 handle = e->ext & 0xff; 1341 devid = (e->ext >> 8) & 0xffff; 1342 type = (e->ext >> 24) & 0xff; 1343 1344 if (type == IVHD_SPECIAL_IOAPIC) 1345 var = "IOAPIC"; 1346 else if (type == IVHD_SPECIAL_HPET) 1347 var = "HPET"; 1348 else 1349 var = "UNKNOWN"; 1350 1351 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n", 1352 var, (int)handle, 1353 PCI_BUS_NUM(devid), 1354 PCI_SLOT(devid), 1355 PCI_FUNC(devid)); 1356 1357 ret = add_special_device(type, handle, &devid, false); 1358 if (ret) 1359 return ret; 1360 1361 /* 1362 * add_special_device might update the devid in case a 1363 * command-line override is present. So call 1364 * set_dev_entry_from_acpi after add_special_device. 1365 */ 1366 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1367 1368 break; 1369 } 1370 case IVHD_DEV_ACPI_HID: { 1371 u16 devid; 1372 u8 hid[ACPIHID_HID_LEN]; 1373 u8 uid[ACPIHID_UID_LEN]; 1374 int ret; 1375 1376 if (h->type != 0x40) { 1377 pr_err(FW_BUG "Invalid IVHD device type %#x\n", 1378 e->type); 1379 break; 1380 } 1381 1382 memcpy(hid, (u8 *)(&e->ext), ACPIHID_HID_LEN - 1); 1383 hid[ACPIHID_HID_LEN - 1] = '\0'; 1384 1385 if (!(*hid)) { 1386 pr_err(FW_BUG "Invalid HID.\n"); 1387 break; 1388 } 1389 1390 uid[0] = '\0'; 1391 switch (e->uidf) { 1392 case UID_NOT_PRESENT: 1393 1394 if (e->uidl != 0) 1395 pr_warn(FW_BUG "Invalid UID length.\n"); 1396 1397 break; 1398 case UID_IS_INTEGER: 1399 1400 sprintf(uid, "%d", e->uid); 1401 1402 break; 1403 case UID_IS_CHARACTER: 1404 1405 memcpy(uid, &e->uid, e->uidl); 1406 uid[e->uidl] = '\0'; 1407 1408 break; 1409 default: 1410 break; 1411 } 1412 1413 devid = e->devid; 1414 DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", 1415 hid, uid, 1416 PCI_BUS_NUM(devid), 1417 PCI_SLOT(devid), 1418 PCI_FUNC(devid)); 1419 1420 flags = e->flags; 1421 1422 ret = add_acpi_hid_device(hid, uid, &devid, false); 1423 if (ret) 1424 return ret; 1425 1426 /* 1427 * add_special_device might update the devid in case a 1428 * command-line override is present. So call 1429 * set_dev_entry_from_acpi after add_special_device. 1430 */ 1431 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1432 1433 break; 1434 } 1435 default: 1436 break; 1437 } 1438 1439 p += ivhd_entry_length(p); 1440 } 1441 1442 return 0; 1443 } 1444 1445 static void __init free_iommu_one(struct amd_iommu *iommu) 1446 { 1447 free_cwwb_sem(iommu); 1448 free_command_buffer(iommu); 1449 free_event_buffer(iommu); 1450 free_ppr_log(iommu); 1451 free_ga_log(iommu); 1452 iommu_unmap_mmio_space(iommu); 1453 } 1454 1455 static void __init free_iommu_all(void) 1456 { 1457 struct amd_iommu *iommu, *next; 1458 1459 for_each_iommu_safe(iommu, next) { 1460 list_del(&iommu->list); 1461 free_iommu_one(iommu); 1462 kfree(iommu); 1463 } 1464 } 1465 1466 /* 1467 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations) 1468 * Workaround: 1469 * BIOS should disable L2B micellaneous clock gating by setting 1470 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b 1471 */ 1472 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) 1473 { 1474 u32 value; 1475 1476 if ((boot_cpu_data.x86 != 0x15) || 1477 (boot_cpu_data.x86_model < 0x10) || 1478 (boot_cpu_data.x86_model > 0x1f)) 1479 return; 1480 1481 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1482 pci_read_config_dword(iommu->dev, 0xf4, &value); 1483 1484 if (value & BIT(2)) 1485 return; 1486 1487 /* Select NB indirect register 0x90 and enable writing */ 1488 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); 1489 1490 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); 1491 pci_info(iommu->dev, "Applying erratum 746 workaround\n"); 1492 1493 /* Clear the enable writing bit */ 1494 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1495 } 1496 1497 /* 1498 * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission) 1499 * Workaround: 1500 * BIOS should enable ATS write permission check by setting 1501 * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b 1502 */ 1503 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) 1504 { 1505 u32 value; 1506 1507 if ((boot_cpu_data.x86 != 0x15) || 1508 (boot_cpu_data.x86_model < 0x30) || 1509 (boot_cpu_data.x86_model > 0x3f)) 1510 return; 1511 1512 /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */ 1513 value = iommu_read_l2(iommu, 0x47); 1514 1515 if (value & BIT(0)) 1516 return; 1517 1518 /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ 1519 iommu_write_l2(iommu, 0x47, value | BIT(0)); 1520 1521 pci_info(iommu->dev, "Applying ATS write check workaround\n"); 1522 } 1523 1524 /* 1525 * This function clues the initialization function for one IOMMU 1526 * together and also allocates the command buffer and programs the 1527 * hardware. It does NOT enable the IOMMU. This is done afterwards. 1528 */ 1529 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) 1530 { 1531 int ret; 1532 1533 raw_spin_lock_init(&iommu->lock); 1534 iommu->cmd_sem_val = 0; 1535 1536 /* Add IOMMU to internal data structures */ 1537 list_add_tail(&iommu->list, &amd_iommu_list); 1538 iommu->index = amd_iommus_present++; 1539 1540 if (unlikely(iommu->index >= MAX_IOMMUS)) { 1541 WARN(1, "System has more IOMMUs than supported by this driver\n"); 1542 return -ENOSYS; 1543 } 1544 1545 /* Index is fine - add IOMMU to the array */ 1546 amd_iommus[iommu->index] = iommu; 1547 1548 /* 1549 * Copy data from ACPI table entry to the iommu struct 1550 */ 1551 iommu->devid = h->devid; 1552 iommu->cap_ptr = h->cap_ptr; 1553 iommu->pci_seg = h->pci_seg; 1554 iommu->mmio_phys = h->mmio_phys; 1555 1556 switch (h->type) { 1557 case 0x10: 1558 /* Check if IVHD EFR contains proper max banks/counters */ 1559 if ((h->efr_attr != 0) && 1560 ((h->efr_attr & (0xF << 13)) != 0) && 1561 ((h->efr_attr & (0x3F << 17)) != 0)) 1562 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1563 else 1564 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1565 1566 /* 1567 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports. 1568 * GAM also requires GA mode. Therefore, we need to 1569 * check cmpxchg16b support before enabling it. 1570 */ 1571 if (!boot_cpu_has(X86_FEATURE_CX16) || 1572 ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) 1573 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1574 break; 1575 case 0x11: 1576 case 0x40: 1577 if (h->efr_reg & (1 << 9)) 1578 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1579 else 1580 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1581 1582 /* 1583 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports. 1584 * XT, GAM also requires GA mode. Therefore, we need to 1585 * check cmpxchg16b support before enabling them. 1586 */ 1587 if (!boot_cpu_has(X86_FEATURE_CX16) || 1588 ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) { 1589 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1590 break; 1591 } 1592 1593 if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) 1594 amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; 1595 1596 early_iommu_features_init(iommu, h); 1597 1598 break; 1599 default: 1600 return -EINVAL; 1601 } 1602 1603 iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys, 1604 iommu->mmio_phys_end); 1605 if (!iommu->mmio_base) 1606 return -ENOMEM; 1607 1608 if (alloc_cwwb_sem(iommu)) 1609 return -ENOMEM; 1610 1611 if (alloc_command_buffer(iommu)) 1612 return -ENOMEM; 1613 1614 if (alloc_event_buffer(iommu)) 1615 return -ENOMEM; 1616 1617 iommu->int_enabled = false; 1618 1619 init_translation_status(iommu); 1620 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { 1621 iommu_disable(iommu); 1622 clear_translation_pre_enabled(iommu); 1623 pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n", 1624 iommu->index); 1625 } 1626 if (amd_iommu_pre_enabled) 1627 amd_iommu_pre_enabled = translation_pre_enabled(iommu); 1628 1629 ret = init_iommu_from_acpi(iommu, h); 1630 if (ret) 1631 return ret; 1632 1633 if (amd_iommu_irq_remap) { 1634 ret = amd_iommu_create_irq_domain(iommu); 1635 if (ret) 1636 return ret; 1637 } 1638 1639 /* 1640 * Make sure IOMMU is not considered to translate itself. The IVRS 1641 * table tells us so, but this is a lie! 1642 */ 1643 amd_iommu_rlookup_table[iommu->devid] = NULL; 1644 1645 return 0; 1646 } 1647 1648 /** 1649 * get_highest_supported_ivhd_type - Look up the appropriate IVHD type 1650 * @ivrs: Pointer to the IVRS header 1651 * 1652 * This function search through all IVDB of the maximum supported IVHD 1653 */ 1654 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) 1655 { 1656 u8 *base = (u8 *)ivrs; 1657 struct ivhd_header *ivhd = (struct ivhd_header *) 1658 (base + IVRS_HEADER_LENGTH); 1659 u8 last_type = ivhd->type; 1660 u16 devid = ivhd->devid; 1661 1662 while (((u8 *)ivhd - base < ivrs->length) && 1663 (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) { 1664 u8 *p = (u8 *) ivhd; 1665 1666 if (ivhd->devid == devid) 1667 last_type = ivhd->type; 1668 ivhd = (struct ivhd_header *)(p + ivhd->length); 1669 } 1670 1671 return last_type; 1672 } 1673 1674 /* 1675 * Iterates over all IOMMU entries in the ACPI table, allocates the 1676 * IOMMU structure and initializes it with init_iommu_one() 1677 */ 1678 static int __init init_iommu_all(struct acpi_table_header *table) 1679 { 1680 u8 *p = (u8 *)table, *end = (u8 *)table; 1681 struct ivhd_header *h; 1682 struct amd_iommu *iommu; 1683 int ret; 1684 1685 end += table->length; 1686 p += IVRS_HEADER_LENGTH; 1687 1688 while (p < end) { 1689 h = (struct ivhd_header *)p; 1690 if (*p == amd_iommu_target_ivhd_type) { 1691 1692 DUMP_printk("device: %02x:%02x.%01x cap: %04x " 1693 "seg: %d flags: %01x info %04x\n", 1694 PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), 1695 PCI_FUNC(h->devid), h->cap_ptr, 1696 h->pci_seg, h->flags, h->info); 1697 DUMP_printk(" mmio-addr: %016llx\n", 1698 h->mmio_phys); 1699 1700 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL); 1701 if (iommu == NULL) 1702 return -ENOMEM; 1703 1704 ret = init_iommu_one(iommu, h); 1705 if (ret) 1706 return ret; 1707 } 1708 p += h->length; 1709 1710 } 1711 WARN_ON(p != end); 1712 1713 return 0; 1714 } 1715 1716 static void init_iommu_perf_ctr(struct amd_iommu *iommu) 1717 { 1718 u64 val; 1719 struct pci_dev *pdev = iommu->dev; 1720 1721 if (!iommu_feature(iommu, FEATURE_PC)) 1722 return; 1723 1724 amd_iommu_pc_present = true; 1725 1726 pci_info(pdev, "IOMMU performance counters supported\n"); 1727 1728 val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); 1729 iommu->max_banks = (u8) ((val >> 12) & 0x3f); 1730 iommu->max_counters = (u8) ((val >> 7) & 0xf); 1731 1732 return; 1733 } 1734 1735 static ssize_t amd_iommu_show_cap(struct device *dev, 1736 struct device_attribute *attr, 1737 char *buf) 1738 { 1739 struct amd_iommu *iommu = dev_to_amd_iommu(dev); 1740 return sprintf(buf, "%x\n", iommu->cap); 1741 } 1742 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL); 1743 1744 static ssize_t amd_iommu_show_features(struct device *dev, 1745 struct device_attribute *attr, 1746 char *buf) 1747 { 1748 struct amd_iommu *iommu = dev_to_amd_iommu(dev); 1749 return sprintf(buf, "%llx\n", iommu->features); 1750 } 1751 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL); 1752 1753 static struct attribute *amd_iommu_attrs[] = { 1754 &dev_attr_cap.attr, 1755 &dev_attr_features.attr, 1756 NULL, 1757 }; 1758 1759 static struct attribute_group amd_iommu_group = { 1760 .name = "amd-iommu", 1761 .attrs = amd_iommu_attrs, 1762 }; 1763 1764 static const struct attribute_group *amd_iommu_groups[] = { 1765 &amd_iommu_group, 1766 NULL, 1767 }; 1768 1769 /* 1770 * Note: IVHD 0x11 and 0x40 also contains exact copy 1771 * of the IOMMU Extended Feature Register [MMIO Offset 0030h]. 1772 * Default to EFR in IVHD since it is available sooner (i.e. before PCI init). 1773 */ 1774 static void __init late_iommu_features_init(struct amd_iommu *iommu) 1775 { 1776 u64 features; 1777 1778 if (!(iommu->cap & (1 << IOMMU_CAP_EFR))) 1779 return; 1780 1781 /* read extended feature bits */ 1782 features = readq(iommu->mmio_base + MMIO_EXT_FEATURES); 1783 1784 if (!iommu->features) { 1785 iommu->features = features; 1786 return; 1787 } 1788 1789 /* 1790 * Sanity check and warn if EFR values from 1791 * IVHD and MMIO conflict. 1792 */ 1793 if (features != iommu->features) 1794 pr_warn(FW_WARN "EFR mismatch. Use IVHD EFR (%#llx : %#llx).\n", 1795 features, iommu->features); 1796 } 1797 1798 static int __init iommu_init_pci(struct amd_iommu *iommu) 1799 { 1800 int cap_ptr = iommu->cap_ptr; 1801 int ret; 1802 1803 iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid), 1804 iommu->devid & 0xff); 1805 if (!iommu->dev) 1806 return -ENODEV; 1807 1808 /* Prevent binding other PCI device drivers to IOMMU devices */ 1809 iommu->dev->match_driver = false; 1810 1811 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, 1812 &iommu->cap); 1813 1814 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) 1815 amd_iommu_iotlb_sup = false; 1816 1817 late_iommu_features_init(iommu); 1818 1819 if (iommu_feature(iommu, FEATURE_GT)) { 1820 int glxval; 1821 u32 max_pasid; 1822 u64 pasmax; 1823 1824 pasmax = iommu->features & FEATURE_PASID_MASK; 1825 pasmax >>= FEATURE_PASID_SHIFT; 1826 max_pasid = (1 << (pasmax + 1)) - 1; 1827 1828 amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid); 1829 1830 BUG_ON(amd_iommu_max_pasid & ~PASID_MASK); 1831 1832 glxval = iommu->features & FEATURE_GLXVAL_MASK; 1833 glxval >>= FEATURE_GLXVAL_SHIFT; 1834 1835 if (amd_iommu_max_glx_val == -1) 1836 amd_iommu_max_glx_val = glxval; 1837 else 1838 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); 1839 } 1840 1841 if (iommu_feature(iommu, FEATURE_GT) && 1842 iommu_feature(iommu, FEATURE_PPR)) { 1843 iommu->is_iommu_v2 = true; 1844 amd_iommu_v2_present = true; 1845 } 1846 1847 if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu)) 1848 return -ENOMEM; 1849 1850 ret = iommu_init_ga(iommu); 1851 if (ret) 1852 return ret; 1853 1854 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) 1855 amd_iommu_np_cache = true; 1856 1857 init_iommu_perf_ctr(iommu); 1858 1859 if (is_rd890_iommu(iommu->dev)) { 1860 int i, j; 1861 1862 iommu->root_pdev = 1863 pci_get_domain_bus_and_slot(0, iommu->dev->bus->number, 1864 PCI_DEVFN(0, 0)); 1865 1866 /* 1867 * Some rd890 systems may not be fully reconfigured by the 1868 * BIOS, so it's necessary for us to store this information so 1869 * it can be reprogrammed on resume 1870 */ 1871 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, 1872 &iommu->stored_addr_lo); 1873 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, 1874 &iommu->stored_addr_hi); 1875 1876 /* Low bit locks writes to configuration space */ 1877 iommu->stored_addr_lo &= ~1; 1878 1879 for (i = 0; i < 6; i++) 1880 for (j = 0; j < 0x12; j++) 1881 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); 1882 1883 for (i = 0; i < 0x83; i++) 1884 iommu->stored_l2[i] = iommu_read_l2(iommu, i); 1885 } 1886 1887 amd_iommu_erratum_746_workaround(iommu); 1888 amd_iommu_ats_write_check_workaround(iommu); 1889 1890 iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev, 1891 amd_iommu_groups, "ivhd%d", iommu->index); 1892 iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL); 1893 1894 return pci_enable_device(iommu->dev); 1895 } 1896 1897 static void print_iommu_info(void) 1898 { 1899 static const char * const feat_str[] = { 1900 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", 1901 "IA", "GA", "HE", "PC" 1902 }; 1903 struct amd_iommu *iommu; 1904 1905 for_each_iommu(iommu) { 1906 struct pci_dev *pdev = iommu->dev; 1907 int i; 1908 1909 pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr); 1910 1911 if (iommu->cap & (1 << IOMMU_CAP_EFR)) { 1912 pr_info("Extended features (%#llx):", iommu->features); 1913 1914 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { 1915 if (iommu_feature(iommu, (1ULL << i))) 1916 pr_cont(" %s", feat_str[i]); 1917 } 1918 1919 if (iommu->features & FEATURE_GAM_VAPIC) 1920 pr_cont(" GA_vAPIC"); 1921 1922 pr_cont("\n"); 1923 } 1924 } 1925 if (irq_remapping_enabled) { 1926 pr_info("Interrupt remapping enabled\n"); 1927 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 1928 pr_info("Virtual APIC enabled\n"); 1929 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 1930 pr_info("X2APIC enabled\n"); 1931 } 1932 } 1933 1934 static int __init amd_iommu_init_pci(void) 1935 { 1936 struct amd_iommu *iommu; 1937 int ret; 1938 1939 for_each_iommu(iommu) { 1940 ret = iommu_init_pci(iommu); 1941 if (ret) 1942 break; 1943 1944 /* Need to setup range after PCI init */ 1945 iommu_set_cwwb_range(iommu); 1946 } 1947 1948 /* 1949 * Order is important here to make sure any unity map requirements are 1950 * fulfilled. The unity mappings are created and written to the device 1951 * table during the amd_iommu_init_api() call. 1952 * 1953 * After that we call init_device_table_dma() to make sure any 1954 * uninitialized DTE will block DMA, and in the end we flush the caches 1955 * of all IOMMUs to make sure the changes to the device table are 1956 * active. 1957 */ 1958 ret = amd_iommu_init_api(); 1959 1960 init_device_table_dma(); 1961 1962 for_each_iommu(iommu) 1963 iommu_flush_all_caches(iommu); 1964 1965 if (!ret) 1966 print_iommu_info(); 1967 1968 return ret; 1969 } 1970 1971 /**************************************************************************** 1972 * 1973 * The following functions initialize the MSI interrupts for all IOMMUs 1974 * in the system. It's a bit challenging because there could be multiple 1975 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per 1976 * pci_dev. 1977 * 1978 ****************************************************************************/ 1979 1980 static int iommu_setup_msi(struct amd_iommu *iommu) 1981 { 1982 int r; 1983 1984 r = pci_enable_msi(iommu->dev); 1985 if (r) 1986 return r; 1987 1988 r = request_threaded_irq(iommu->dev->irq, 1989 amd_iommu_int_handler, 1990 amd_iommu_int_thread, 1991 0, "AMD-Vi", 1992 iommu); 1993 1994 if (r) { 1995 pci_disable_msi(iommu->dev); 1996 return r; 1997 } 1998 1999 return 0; 2000 } 2001 2002 union intcapxt { 2003 u64 capxt; 2004 struct { 2005 u64 reserved_0 : 2, 2006 dest_mode_logical : 1, 2007 reserved_1 : 5, 2008 destid_0_23 : 24, 2009 vector : 8, 2010 reserved_2 : 16, 2011 destid_24_31 : 8; 2012 }; 2013 } __attribute__ ((packed)); 2014 2015 /* 2016 * There isn't really any need to mask/unmask at the irqchip level because 2017 * the 64-bit INTCAPXT registers can be updated atomically without tearing 2018 * when the affinity is being updated. 2019 */ 2020 static void intcapxt_unmask_irq(struct irq_data *data) 2021 { 2022 } 2023 2024 static void intcapxt_mask_irq(struct irq_data *data) 2025 { 2026 } 2027 2028 static struct irq_chip intcapxt_controller; 2029 2030 static int intcapxt_irqdomain_activate(struct irq_domain *domain, 2031 struct irq_data *irqd, bool reserve) 2032 { 2033 struct amd_iommu *iommu = irqd->chip_data; 2034 struct irq_cfg *cfg = irqd_cfg(irqd); 2035 union intcapxt xt; 2036 2037 xt.capxt = 0ULL; 2038 xt.dest_mode_logical = apic->dest_mode_logical; 2039 xt.vector = cfg->vector; 2040 xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); 2041 xt.destid_24_31 = cfg->dest_apicid >> 24; 2042 2043 /** 2044 * Current IOMMU implemtation uses the same IRQ for all 2045 * 3 IOMMU interrupts. 2046 */ 2047 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); 2048 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); 2049 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); 2050 return 0; 2051 } 2052 2053 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain, 2054 struct irq_data *irqd) 2055 { 2056 intcapxt_mask_irq(irqd); 2057 } 2058 2059 2060 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, 2061 unsigned int nr_irqs, void *arg) 2062 { 2063 struct irq_alloc_info *info = arg; 2064 int i, ret; 2065 2066 if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI) 2067 return -EINVAL; 2068 2069 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 2070 if (ret < 0) 2071 return ret; 2072 2073 for (i = virq; i < virq + nr_irqs; i++) { 2074 struct irq_data *irqd = irq_domain_get_irq_data(domain, i); 2075 2076 irqd->chip = &intcapxt_controller; 2077 irqd->chip_data = info->data; 2078 __irq_set_handler(i, handle_edge_irq, 0, "edge"); 2079 } 2080 2081 return ret; 2082 } 2083 2084 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq, 2085 unsigned int nr_irqs) 2086 { 2087 irq_domain_free_irqs_top(domain, virq, nr_irqs); 2088 } 2089 2090 static int intcapxt_set_affinity(struct irq_data *irqd, 2091 const struct cpumask *mask, bool force) 2092 { 2093 struct irq_data *parent = irqd->parent_data; 2094 int ret; 2095 2096 ret = parent->chip->irq_set_affinity(parent, mask, force); 2097 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 2098 return ret; 2099 2100 return intcapxt_irqdomain_activate(irqd->domain, irqd, false); 2101 } 2102 2103 static struct irq_chip intcapxt_controller = { 2104 .name = "IOMMU-MSI", 2105 .irq_unmask = intcapxt_unmask_irq, 2106 .irq_mask = intcapxt_mask_irq, 2107 .irq_ack = irq_chip_ack_parent, 2108 .irq_retrigger = irq_chip_retrigger_hierarchy, 2109 .irq_set_affinity = intcapxt_set_affinity, 2110 .flags = IRQCHIP_SKIP_SET_WAKE, 2111 }; 2112 2113 static const struct irq_domain_ops intcapxt_domain_ops = { 2114 .alloc = intcapxt_irqdomain_alloc, 2115 .free = intcapxt_irqdomain_free, 2116 .activate = intcapxt_irqdomain_activate, 2117 .deactivate = intcapxt_irqdomain_deactivate, 2118 }; 2119 2120 2121 static struct irq_domain *iommu_irqdomain; 2122 2123 static struct irq_domain *iommu_get_irqdomain(void) 2124 { 2125 struct fwnode_handle *fn; 2126 2127 /* No need for locking here (yet) as the init is single-threaded */ 2128 if (iommu_irqdomain) 2129 return iommu_irqdomain; 2130 2131 fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI"); 2132 if (!fn) 2133 return NULL; 2134 2135 iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0, 2136 fn, &intcapxt_domain_ops, 2137 NULL); 2138 if (!iommu_irqdomain) 2139 irq_domain_free_fwnode(fn); 2140 2141 return iommu_irqdomain; 2142 } 2143 2144 static int iommu_setup_intcapxt(struct amd_iommu *iommu) 2145 { 2146 struct irq_domain *domain; 2147 struct irq_alloc_info info; 2148 int irq, ret; 2149 2150 domain = iommu_get_irqdomain(); 2151 if (!domain) 2152 return -ENXIO; 2153 2154 init_irq_alloc_info(&info, NULL); 2155 info.type = X86_IRQ_ALLOC_TYPE_AMDVI; 2156 info.data = iommu; 2157 2158 irq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info); 2159 if (irq < 0) { 2160 irq_domain_remove(domain); 2161 return irq; 2162 } 2163 2164 ret = request_threaded_irq(irq, amd_iommu_int_handler, 2165 amd_iommu_int_thread, 0, "AMD-Vi", iommu); 2166 if (ret) { 2167 irq_domain_free_irqs(irq, 1); 2168 irq_domain_remove(domain); 2169 return ret; 2170 } 2171 2172 iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN); 2173 return 0; 2174 } 2175 2176 static int iommu_init_irq(struct amd_iommu *iommu) 2177 { 2178 int ret; 2179 2180 if (iommu->int_enabled) 2181 goto enable_faults; 2182 2183 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2184 ret = iommu_setup_intcapxt(iommu); 2185 else if (iommu->dev->msi_cap) 2186 ret = iommu_setup_msi(iommu); 2187 else 2188 ret = -ENODEV; 2189 2190 if (ret) 2191 return ret; 2192 2193 iommu->int_enabled = true; 2194 enable_faults: 2195 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); 2196 2197 if (iommu->ppr_log != NULL) 2198 iommu_feature_enable(iommu, CONTROL_PPRINT_EN); 2199 2200 iommu_ga_log_enable(iommu); 2201 2202 return 0; 2203 } 2204 2205 /**************************************************************************** 2206 * 2207 * The next functions belong to the third pass of parsing the ACPI 2208 * table. In this last pass the memory mapping requirements are 2209 * gathered (like exclusion and unity mapping ranges). 2210 * 2211 ****************************************************************************/ 2212 2213 static void __init free_unity_maps(void) 2214 { 2215 struct unity_map_entry *entry, *next; 2216 2217 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) { 2218 list_del(&entry->list); 2219 kfree(entry); 2220 } 2221 } 2222 2223 /* called for unity map ACPI definition */ 2224 static int __init init_unity_map_range(struct ivmd_header *m) 2225 { 2226 struct unity_map_entry *e = NULL; 2227 char *s; 2228 2229 e = kzalloc(sizeof(*e), GFP_KERNEL); 2230 if (e == NULL) 2231 return -ENOMEM; 2232 2233 switch (m->type) { 2234 default: 2235 kfree(e); 2236 return 0; 2237 case ACPI_IVMD_TYPE: 2238 s = "IVMD_TYPEi\t\t\t"; 2239 e->devid_start = e->devid_end = m->devid; 2240 break; 2241 case ACPI_IVMD_TYPE_ALL: 2242 s = "IVMD_TYPE_ALL\t\t"; 2243 e->devid_start = 0; 2244 e->devid_end = amd_iommu_last_bdf; 2245 break; 2246 case ACPI_IVMD_TYPE_RANGE: 2247 s = "IVMD_TYPE_RANGE\t\t"; 2248 e->devid_start = m->devid; 2249 e->devid_end = m->aux; 2250 break; 2251 } 2252 e->address_start = PAGE_ALIGN(m->range_start); 2253 e->address_end = e->address_start + PAGE_ALIGN(m->range_length); 2254 e->prot = m->flags >> 1; 2255 2256 /* 2257 * Treat per-device exclusion ranges as r/w unity-mapped regions 2258 * since some buggy BIOSes might lead to the overwritten exclusion 2259 * range (exclusion_start and exclusion_length members). This 2260 * happens when there are multiple exclusion ranges (IVMD entries) 2261 * defined in ACPI table. 2262 */ 2263 if (m->flags & IVMD_FLAG_EXCL_RANGE) 2264 e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1; 2265 2266 DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" 2267 " range_start: %016llx range_end: %016llx flags: %x\n", s, 2268 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), 2269 PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end), 2270 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), 2271 e->address_start, e->address_end, m->flags); 2272 2273 list_add_tail(&e->list, &amd_iommu_unity_map); 2274 2275 return 0; 2276 } 2277 2278 /* iterates over all memory definitions we find in the ACPI table */ 2279 static int __init init_memory_definitions(struct acpi_table_header *table) 2280 { 2281 u8 *p = (u8 *)table, *end = (u8 *)table; 2282 struct ivmd_header *m; 2283 2284 end += table->length; 2285 p += IVRS_HEADER_LENGTH; 2286 2287 while (p < end) { 2288 m = (struct ivmd_header *)p; 2289 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) 2290 init_unity_map_range(m); 2291 2292 p += m->length; 2293 } 2294 2295 return 0; 2296 } 2297 2298 /* 2299 * Init the device table to not allow DMA access for devices 2300 */ 2301 static void init_device_table_dma(void) 2302 { 2303 u32 devid; 2304 2305 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { 2306 set_dev_entry_bit(devid, DEV_ENTRY_VALID); 2307 set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION); 2308 } 2309 } 2310 2311 static void __init uninit_device_table_dma(void) 2312 { 2313 u32 devid; 2314 2315 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { 2316 amd_iommu_dev_table[devid].data[0] = 0ULL; 2317 amd_iommu_dev_table[devid].data[1] = 0ULL; 2318 } 2319 } 2320 2321 static void init_device_table(void) 2322 { 2323 u32 devid; 2324 2325 if (!amd_iommu_irq_remap) 2326 return; 2327 2328 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) 2329 set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN); 2330 } 2331 2332 static void iommu_init_flags(struct amd_iommu *iommu) 2333 { 2334 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? 2335 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : 2336 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); 2337 2338 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? 2339 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : 2340 iommu_feature_disable(iommu, CONTROL_PASSPW_EN); 2341 2342 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? 2343 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : 2344 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); 2345 2346 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? 2347 iommu_feature_enable(iommu, CONTROL_ISOC_EN) : 2348 iommu_feature_disable(iommu, CONTROL_ISOC_EN); 2349 2350 /* 2351 * make IOMMU memory accesses cache coherent 2352 */ 2353 iommu_feature_enable(iommu, CONTROL_COHERENT_EN); 2354 2355 /* Set IOTLB invalidation timeout to 1s */ 2356 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S); 2357 } 2358 2359 static void iommu_apply_resume_quirks(struct amd_iommu *iommu) 2360 { 2361 int i, j; 2362 u32 ioc_feature_control; 2363 struct pci_dev *pdev = iommu->root_pdev; 2364 2365 /* RD890 BIOSes may not have completely reconfigured the iommu */ 2366 if (!is_rd890_iommu(iommu->dev) || !pdev) 2367 return; 2368 2369 /* 2370 * First, we need to ensure that the iommu is enabled. This is 2371 * controlled by a register in the northbridge 2372 */ 2373 2374 /* Select Northbridge indirect register 0x75 and enable writing */ 2375 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); 2376 pci_read_config_dword(pdev, 0x64, &ioc_feature_control); 2377 2378 /* Enable the iommu */ 2379 if (!(ioc_feature_control & 0x1)) 2380 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); 2381 2382 /* Restore the iommu BAR */ 2383 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2384 iommu->stored_addr_lo); 2385 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, 2386 iommu->stored_addr_hi); 2387 2388 /* Restore the l1 indirect regs for each of the 6 l1s */ 2389 for (i = 0; i < 6; i++) 2390 for (j = 0; j < 0x12; j++) 2391 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); 2392 2393 /* Restore the l2 indirect regs */ 2394 for (i = 0; i < 0x83; i++) 2395 iommu_write_l2(iommu, i, iommu->stored_l2[i]); 2396 2397 /* Lock PCI setup registers */ 2398 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2399 iommu->stored_addr_lo | 1); 2400 } 2401 2402 static void iommu_enable_ga(struct amd_iommu *iommu) 2403 { 2404 #ifdef CONFIG_IRQ_REMAP 2405 switch (amd_iommu_guest_ir) { 2406 case AMD_IOMMU_GUEST_IR_VAPIC: 2407 iommu_feature_enable(iommu, CONTROL_GAM_EN); 2408 fallthrough; 2409 case AMD_IOMMU_GUEST_IR_LEGACY_GA: 2410 iommu_feature_enable(iommu, CONTROL_GA_EN); 2411 iommu->irte_ops = &irte_128_ops; 2412 break; 2413 default: 2414 iommu->irte_ops = &irte_32_ops; 2415 break; 2416 } 2417 #endif 2418 } 2419 2420 static void early_enable_iommu(struct amd_iommu *iommu) 2421 { 2422 iommu_disable(iommu); 2423 iommu_init_flags(iommu); 2424 iommu_set_device_table(iommu); 2425 iommu_enable_command_buffer(iommu); 2426 iommu_enable_event_buffer(iommu); 2427 iommu_set_exclusion_range(iommu); 2428 iommu_enable_ga(iommu); 2429 iommu_enable_xt(iommu); 2430 iommu_enable(iommu); 2431 iommu_flush_all_caches(iommu); 2432 } 2433 2434 /* 2435 * This function finally enables all IOMMUs found in the system after 2436 * they have been initialized. 2437 * 2438 * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy 2439 * the old content of device table entries. Not this case or copy failed, 2440 * just continue as normal kernel does. 2441 */ 2442 static void early_enable_iommus(void) 2443 { 2444 struct amd_iommu *iommu; 2445 2446 2447 if (!copy_device_table()) { 2448 /* 2449 * If come here because of failure in copying device table from old 2450 * kernel with all IOMMUs enabled, print error message and try to 2451 * free allocated old_dev_tbl_cpy. 2452 */ 2453 if (amd_iommu_pre_enabled) 2454 pr_err("Failed to copy DEV table from previous kernel.\n"); 2455 if (old_dev_tbl_cpy != NULL) 2456 free_pages((unsigned long)old_dev_tbl_cpy, 2457 get_order(dev_table_size)); 2458 2459 for_each_iommu(iommu) { 2460 clear_translation_pre_enabled(iommu); 2461 early_enable_iommu(iommu); 2462 } 2463 } else { 2464 pr_info("Copied DEV table from previous kernel.\n"); 2465 free_pages((unsigned long)amd_iommu_dev_table, 2466 get_order(dev_table_size)); 2467 amd_iommu_dev_table = old_dev_tbl_cpy; 2468 for_each_iommu(iommu) { 2469 iommu_disable_command_buffer(iommu); 2470 iommu_disable_event_buffer(iommu); 2471 iommu_enable_command_buffer(iommu); 2472 iommu_enable_event_buffer(iommu); 2473 iommu_enable_ga(iommu); 2474 iommu_enable_xt(iommu); 2475 iommu_set_device_table(iommu); 2476 iommu_flush_all_caches(iommu); 2477 } 2478 } 2479 2480 #ifdef CONFIG_IRQ_REMAP 2481 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 2482 amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP); 2483 #endif 2484 } 2485 2486 static void enable_iommus_v2(void) 2487 { 2488 struct amd_iommu *iommu; 2489 2490 for_each_iommu(iommu) { 2491 iommu_enable_ppr_log(iommu); 2492 iommu_enable_gt(iommu); 2493 } 2494 } 2495 2496 static void enable_iommus(void) 2497 { 2498 early_enable_iommus(); 2499 2500 enable_iommus_v2(); 2501 } 2502 2503 static void disable_iommus(void) 2504 { 2505 struct amd_iommu *iommu; 2506 2507 for_each_iommu(iommu) 2508 iommu_disable(iommu); 2509 2510 #ifdef CONFIG_IRQ_REMAP 2511 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 2512 amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP); 2513 #endif 2514 } 2515 2516 /* 2517 * Suspend/Resume support 2518 * disable suspend until real resume implemented 2519 */ 2520 2521 static void amd_iommu_resume(void) 2522 { 2523 struct amd_iommu *iommu; 2524 2525 for_each_iommu(iommu) 2526 iommu_apply_resume_quirks(iommu); 2527 2528 /* re-load the hardware */ 2529 enable_iommus(); 2530 2531 amd_iommu_enable_interrupts(); 2532 } 2533 2534 static int amd_iommu_suspend(void) 2535 { 2536 /* disable IOMMUs to go out of the way for BIOS */ 2537 disable_iommus(); 2538 2539 return 0; 2540 } 2541 2542 static struct syscore_ops amd_iommu_syscore_ops = { 2543 .suspend = amd_iommu_suspend, 2544 .resume = amd_iommu_resume, 2545 }; 2546 2547 static void __init free_iommu_resources(void) 2548 { 2549 kmemleak_free(irq_lookup_table); 2550 free_pages((unsigned long)irq_lookup_table, 2551 get_order(rlookup_table_size)); 2552 irq_lookup_table = NULL; 2553 2554 kmem_cache_destroy(amd_iommu_irq_cache); 2555 amd_iommu_irq_cache = NULL; 2556 2557 free_pages((unsigned long)amd_iommu_rlookup_table, 2558 get_order(rlookup_table_size)); 2559 amd_iommu_rlookup_table = NULL; 2560 2561 free_pages((unsigned long)amd_iommu_alias_table, 2562 get_order(alias_table_size)); 2563 amd_iommu_alias_table = NULL; 2564 2565 free_pages((unsigned long)amd_iommu_dev_table, 2566 get_order(dev_table_size)); 2567 amd_iommu_dev_table = NULL; 2568 2569 free_iommu_all(); 2570 } 2571 2572 /* SB IOAPIC is always on this device in AMD systems */ 2573 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) 2574 2575 static bool __init check_ioapic_information(void) 2576 { 2577 const char *fw_bug = FW_BUG; 2578 bool ret, has_sb_ioapic; 2579 int idx; 2580 2581 has_sb_ioapic = false; 2582 ret = false; 2583 2584 /* 2585 * If we have map overrides on the kernel command line the 2586 * messages in this function might not describe firmware bugs 2587 * anymore - so be careful 2588 */ 2589 if (cmdline_maps) 2590 fw_bug = ""; 2591 2592 for (idx = 0; idx < nr_ioapics; idx++) { 2593 int devid, id = mpc_ioapic_id(idx); 2594 2595 devid = get_ioapic_devid(id); 2596 if (devid < 0) { 2597 pr_err("%s: IOAPIC[%d] not in IVRS table\n", 2598 fw_bug, id); 2599 ret = false; 2600 } else if (devid == IOAPIC_SB_DEVID) { 2601 has_sb_ioapic = true; 2602 ret = true; 2603 } 2604 } 2605 2606 if (!has_sb_ioapic) { 2607 /* 2608 * We expect the SB IOAPIC to be listed in the IVRS 2609 * table. The system timer is connected to the SB IOAPIC 2610 * and if we don't have it in the list the system will 2611 * panic at boot time. This situation usually happens 2612 * when the BIOS is buggy and provides us the wrong 2613 * device id for the IOAPIC in the system. 2614 */ 2615 pr_err("%s: No southbridge IOAPIC found\n", fw_bug); 2616 } 2617 2618 if (!ret) 2619 pr_err("Disabling interrupt remapping\n"); 2620 2621 return ret; 2622 } 2623 2624 static void __init free_dma_resources(void) 2625 { 2626 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, 2627 get_order(MAX_DOMAIN_ID/8)); 2628 amd_iommu_pd_alloc_bitmap = NULL; 2629 2630 free_unity_maps(); 2631 } 2632 2633 static void __init ivinfo_init(void *ivrs) 2634 { 2635 amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET)); 2636 } 2637 2638 /* 2639 * This is the hardware init function for AMD IOMMU in the system. 2640 * This function is called either from amd_iommu_init or from the interrupt 2641 * remapping setup code. 2642 * 2643 * This function basically parses the ACPI table for AMD IOMMU (IVRS) 2644 * four times: 2645 * 2646 * 1 pass) Discover the most comprehensive IVHD type to use. 2647 * 2648 * 2 pass) Find the highest PCI device id the driver has to handle. 2649 * Upon this information the size of the data structures is 2650 * determined that needs to be allocated. 2651 * 2652 * 3 pass) Initialize the data structures just allocated with the 2653 * information in the ACPI table about available AMD IOMMUs 2654 * in the system. It also maps the PCI devices in the 2655 * system to specific IOMMUs 2656 * 2657 * 4 pass) After the basic data structures are allocated and 2658 * initialized we update them with information about memory 2659 * remapping requirements parsed out of the ACPI table in 2660 * this last pass. 2661 * 2662 * After everything is set up the IOMMUs are enabled and the necessary 2663 * hotplug and suspend notifiers are registered. 2664 */ 2665 static int __init early_amd_iommu_init(void) 2666 { 2667 struct acpi_table_header *ivrs_base; 2668 int i, remap_cache_sz, ret; 2669 acpi_status status; 2670 2671 if (!amd_iommu_detected) 2672 return -ENODEV; 2673 2674 status = acpi_get_table("IVRS", 0, &ivrs_base); 2675 if (status == AE_NOT_FOUND) 2676 return -ENODEV; 2677 else if (ACPI_FAILURE(status)) { 2678 const char *err = acpi_format_exception(status); 2679 pr_err("IVRS table error: %s\n", err); 2680 return -EINVAL; 2681 } 2682 2683 /* 2684 * Validate checksum here so we don't need to do it when 2685 * we actually parse the table 2686 */ 2687 ret = check_ivrs_checksum(ivrs_base); 2688 if (ret) 2689 goto out; 2690 2691 ivinfo_init(ivrs_base); 2692 2693 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); 2694 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); 2695 2696 /* 2697 * First parse ACPI tables to find the largest Bus/Dev/Func 2698 * we need to handle. Upon this information the shared data 2699 * structures for the IOMMUs in the system will be allocated 2700 */ 2701 ret = find_last_devid_acpi(ivrs_base); 2702 if (ret) 2703 goto out; 2704 2705 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); 2706 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); 2707 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); 2708 2709 /* Device table - directly used by all IOMMUs */ 2710 ret = -ENOMEM; 2711 amd_iommu_dev_table = (void *)__get_free_pages( 2712 GFP_KERNEL | __GFP_ZERO | GFP_DMA32, 2713 get_order(dev_table_size)); 2714 if (amd_iommu_dev_table == NULL) 2715 goto out; 2716 2717 /* 2718 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the 2719 * IOMMU see for that device 2720 */ 2721 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL, 2722 get_order(alias_table_size)); 2723 if (amd_iommu_alias_table == NULL) 2724 goto out; 2725 2726 /* IOMMU rlookup table - find the IOMMU for a specific device */ 2727 amd_iommu_rlookup_table = (void *)__get_free_pages( 2728 GFP_KERNEL | __GFP_ZERO, 2729 get_order(rlookup_table_size)); 2730 if (amd_iommu_rlookup_table == NULL) 2731 goto out; 2732 2733 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( 2734 GFP_KERNEL | __GFP_ZERO, 2735 get_order(MAX_DOMAIN_ID/8)); 2736 if (amd_iommu_pd_alloc_bitmap == NULL) 2737 goto out; 2738 2739 /* 2740 * let all alias entries point to itself 2741 */ 2742 for (i = 0; i <= amd_iommu_last_bdf; ++i) 2743 amd_iommu_alias_table[i] = i; 2744 2745 /* 2746 * never allocate domain 0 because its used as the non-allocated and 2747 * error value placeholder 2748 */ 2749 __set_bit(0, amd_iommu_pd_alloc_bitmap); 2750 2751 /* 2752 * now the data structures are allocated and basically initialized 2753 * start the real acpi table scan 2754 */ 2755 ret = init_iommu_all(ivrs_base); 2756 if (ret) 2757 goto out; 2758 2759 /* Disable any previously enabled IOMMUs */ 2760 if (!is_kdump_kernel() || amd_iommu_disabled) 2761 disable_iommus(); 2762 2763 if (amd_iommu_irq_remap) 2764 amd_iommu_irq_remap = check_ioapic_information(); 2765 2766 if (amd_iommu_irq_remap) { 2767 /* 2768 * Interrupt remapping enabled, create kmem_cache for the 2769 * remapping tables. 2770 */ 2771 ret = -ENOMEM; 2772 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 2773 remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32); 2774 else 2775 remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2); 2776 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache", 2777 remap_cache_sz, 2778 DTE_INTTAB_ALIGNMENT, 2779 0, NULL); 2780 if (!amd_iommu_irq_cache) 2781 goto out; 2782 2783 irq_lookup_table = (void *)__get_free_pages( 2784 GFP_KERNEL | __GFP_ZERO, 2785 get_order(rlookup_table_size)); 2786 kmemleak_alloc(irq_lookup_table, rlookup_table_size, 2787 1, GFP_KERNEL); 2788 if (!irq_lookup_table) 2789 goto out; 2790 } 2791 2792 ret = init_memory_definitions(ivrs_base); 2793 if (ret) 2794 goto out; 2795 2796 /* init the device table */ 2797 init_device_table(); 2798 2799 out: 2800 /* Don't leak any ACPI memory */ 2801 acpi_put_table(ivrs_base); 2802 2803 return ret; 2804 } 2805 2806 static int amd_iommu_enable_interrupts(void) 2807 { 2808 struct amd_iommu *iommu; 2809 int ret = 0; 2810 2811 for_each_iommu(iommu) { 2812 ret = iommu_init_irq(iommu); 2813 if (ret) 2814 goto out; 2815 } 2816 2817 out: 2818 return ret; 2819 } 2820 2821 static bool __init detect_ivrs(void) 2822 { 2823 struct acpi_table_header *ivrs_base; 2824 acpi_status status; 2825 int i; 2826 2827 status = acpi_get_table("IVRS", 0, &ivrs_base); 2828 if (status == AE_NOT_FOUND) 2829 return false; 2830 else if (ACPI_FAILURE(status)) { 2831 const char *err = acpi_format_exception(status); 2832 pr_err("IVRS table error: %s\n", err); 2833 return false; 2834 } 2835 2836 acpi_put_table(ivrs_base); 2837 2838 if (amd_iommu_force_enable) 2839 goto out; 2840 2841 /* Don't use IOMMU if there is Stoney Ridge graphics */ 2842 for (i = 0; i < 32; i++) { 2843 u32 pci_id; 2844 2845 pci_id = read_pci_config(0, i, 0, 0); 2846 if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) { 2847 pr_info("Disable IOMMU on Stoney Ridge\n"); 2848 return false; 2849 } 2850 } 2851 2852 out: 2853 /* Make sure ACS will be enabled during PCI probe */ 2854 pci_request_acs(); 2855 2856 return true; 2857 } 2858 2859 /**************************************************************************** 2860 * 2861 * AMD IOMMU Initialization State Machine 2862 * 2863 ****************************************************************************/ 2864 2865 static int __init state_next(void) 2866 { 2867 int ret = 0; 2868 2869 switch (init_state) { 2870 case IOMMU_START_STATE: 2871 if (!detect_ivrs()) { 2872 init_state = IOMMU_NOT_FOUND; 2873 ret = -ENODEV; 2874 } else { 2875 init_state = IOMMU_IVRS_DETECTED; 2876 } 2877 break; 2878 case IOMMU_IVRS_DETECTED: 2879 if (amd_iommu_disabled) { 2880 init_state = IOMMU_CMDLINE_DISABLED; 2881 ret = -EINVAL; 2882 } else { 2883 ret = early_amd_iommu_init(); 2884 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; 2885 } 2886 break; 2887 case IOMMU_ACPI_FINISHED: 2888 early_enable_iommus(); 2889 x86_platform.iommu_shutdown = disable_iommus; 2890 init_state = IOMMU_ENABLED; 2891 break; 2892 case IOMMU_ENABLED: 2893 register_syscore_ops(&amd_iommu_syscore_ops); 2894 ret = amd_iommu_init_pci(); 2895 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; 2896 enable_iommus_v2(); 2897 break; 2898 case IOMMU_PCI_INIT: 2899 ret = amd_iommu_enable_interrupts(); 2900 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN; 2901 break; 2902 case IOMMU_INTERRUPTS_EN: 2903 ret = amd_iommu_init_dma_ops(); 2904 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS; 2905 break; 2906 case IOMMU_DMA_OPS: 2907 init_state = IOMMU_INITIALIZED; 2908 break; 2909 case IOMMU_INITIALIZED: 2910 /* Nothing to do */ 2911 break; 2912 case IOMMU_NOT_FOUND: 2913 case IOMMU_INIT_ERROR: 2914 case IOMMU_CMDLINE_DISABLED: 2915 /* Error states => do nothing */ 2916 ret = -EINVAL; 2917 break; 2918 default: 2919 /* Unknown state */ 2920 BUG(); 2921 } 2922 2923 if (ret) { 2924 free_dma_resources(); 2925 if (!irq_remapping_enabled) { 2926 disable_iommus(); 2927 free_iommu_resources(); 2928 } else { 2929 struct amd_iommu *iommu; 2930 2931 uninit_device_table_dma(); 2932 for_each_iommu(iommu) 2933 iommu_flush_all_caches(iommu); 2934 } 2935 } 2936 return ret; 2937 } 2938 2939 static int __init iommu_go_to_state(enum iommu_init_state state) 2940 { 2941 int ret = -EINVAL; 2942 2943 while (init_state != state) { 2944 if (init_state == IOMMU_NOT_FOUND || 2945 init_state == IOMMU_INIT_ERROR || 2946 init_state == IOMMU_CMDLINE_DISABLED) 2947 break; 2948 ret = state_next(); 2949 } 2950 2951 return ret; 2952 } 2953 2954 #ifdef CONFIG_IRQ_REMAP 2955 int __init amd_iommu_prepare(void) 2956 { 2957 int ret; 2958 2959 amd_iommu_irq_remap = true; 2960 2961 ret = iommu_go_to_state(IOMMU_ACPI_FINISHED); 2962 if (ret) { 2963 amd_iommu_irq_remap = false; 2964 return ret; 2965 } 2966 2967 return amd_iommu_irq_remap ? 0 : -ENODEV; 2968 } 2969 2970 int __init amd_iommu_enable(void) 2971 { 2972 int ret; 2973 2974 ret = iommu_go_to_state(IOMMU_ENABLED); 2975 if (ret) 2976 return ret; 2977 2978 irq_remapping_enabled = 1; 2979 return amd_iommu_xt_mode; 2980 } 2981 2982 void amd_iommu_disable(void) 2983 { 2984 amd_iommu_suspend(); 2985 } 2986 2987 int amd_iommu_reenable(int mode) 2988 { 2989 amd_iommu_resume(); 2990 2991 return 0; 2992 } 2993 2994 int __init amd_iommu_enable_faulting(void) 2995 { 2996 /* We enable MSI later when PCI is initialized */ 2997 return 0; 2998 } 2999 #endif 3000 3001 /* 3002 * This is the core init function for AMD IOMMU hardware in the system. 3003 * This function is called from the generic x86 DMA layer initialization 3004 * code. 3005 */ 3006 static int __init amd_iommu_init(void) 3007 { 3008 struct amd_iommu *iommu; 3009 int ret; 3010 3011 ret = iommu_go_to_state(IOMMU_INITIALIZED); 3012 #ifdef CONFIG_GART_IOMMU 3013 if (ret && list_empty(&amd_iommu_list)) { 3014 /* 3015 * We failed to initialize the AMD IOMMU - try fallback 3016 * to GART if possible. 3017 */ 3018 gart_iommu_init(); 3019 } 3020 #endif 3021 3022 for_each_iommu(iommu) 3023 amd_iommu_debugfs_setup(iommu); 3024 3025 return ret; 3026 } 3027 3028 static bool amd_iommu_sme_check(void) 3029 { 3030 if (!sme_active() || (boot_cpu_data.x86 != 0x17)) 3031 return true; 3032 3033 /* For Fam17h, a specific level of support is required */ 3034 if (boot_cpu_data.microcode >= 0x08001205) 3035 return true; 3036 3037 if ((boot_cpu_data.microcode >= 0x08001126) && 3038 (boot_cpu_data.microcode <= 0x080011ff)) 3039 return true; 3040 3041 pr_notice("IOMMU not currently supported when SME is active\n"); 3042 3043 return false; 3044 } 3045 3046 /**************************************************************************** 3047 * 3048 * Early detect code. This code runs at IOMMU detection time in the DMA 3049 * layer. It just looks if there is an IVRS ACPI table to detect AMD 3050 * IOMMUs 3051 * 3052 ****************************************************************************/ 3053 int __init amd_iommu_detect(void) 3054 { 3055 int ret; 3056 3057 if (no_iommu || (iommu_detected && !gart_iommu_aperture)) 3058 return -ENODEV; 3059 3060 if (!amd_iommu_sme_check()) 3061 return -ENODEV; 3062 3063 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); 3064 if (ret) 3065 return ret; 3066 3067 amd_iommu_detected = true; 3068 iommu_detected = 1; 3069 x86_init.iommu.iommu_init = amd_iommu_init; 3070 3071 return 1; 3072 } 3073 3074 /**************************************************************************** 3075 * 3076 * Parsing functions for the AMD IOMMU specific kernel command line 3077 * options. 3078 * 3079 ****************************************************************************/ 3080 3081 static int __init parse_amd_iommu_dump(char *str) 3082 { 3083 amd_iommu_dump = true; 3084 3085 return 1; 3086 } 3087 3088 static int __init parse_amd_iommu_intr(char *str) 3089 { 3090 for (; *str; ++str) { 3091 if (strncmp(str, "legacy", 6) == 0) { 3092 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 3093 break; 3094 } 3095 if (strncmp(str, "vapic", 5) == 0) { 3096 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 3097 break; 3098 } 3099 } 3100 return 1; 3101 } 3102 3103 static int __init parse_amd_iommu_options(char *str) 3104 { 3105 for (; *str; ++str) { 3106 if (strncmp(str, "fullflush", 9) == 0) 3107 amd_iommu_unmap_flush = true; 3108 if (strncmp(str, "force_enable", 12) == 0) 3109 amd_iommu_force_enable = true; 3110 if (strncmp(str, "off", 3) == 0) 3111 amd_iommu_disabled = true; 3112 if (strncmp(str, "force_isolation", 15) == 0) 3113 amd_iommu_force_isolation = true; 3114 } 3115 3116 return 1; 3117 } 3118 3119 static int __init parse_ivrs_ioapic(char *str) 3120 { 3121 unsigned int bus, dev, fn; 3122 int ret, id, i; 3123 u16 devid; 3124 3125 ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); 3126 3127 if (ret != 4) { 3128 pr_err("Invalid command line: ivrs_ioapic%s\n", str); 3129 return 1; 3130 } 3131 3132 if (early_ioapic_map_size == EARLY_MAP_SIZE) { 3133 pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", 3134 str); 3135 return 1; 3136 } 3137 3138 devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); 3139 3140 cmdline_maps = true; 3141 i = early_ioapic_map_size++; 3142 early_ioapic_map[i].id = id; 3143 early_ioapic_map[i].devid = devid; 3144 early_ioapic_map[i].cmd_line = true; 3145 3146 return 1; 3147 } 3148 3149 static int __init parse_ivrs_hpet(char *str) 3150 { 3151 unsigned int bus, dev, fn; 3152 int ret, id, i; 3153 u16 devid; 3154 3155 ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); 3156 3157 if (ret != 4) { 3158 pr_err("Invalid command line: ivrs_hpet%s\n", str); 3159 return 1; 3160 } 3161 3162 if (early_hpet_map_size == EARLY_MAP_SIZE) { 3163 pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n", 3164 str); 3165 return 1; 3166 } 3167 3168 devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); 3169 3170 cmdline_maps = true; 3171 i = early_hpet_map_size++; 3172 early_hpet_map[i].id = id; 3173 early_hpet_map[i].devid = devid; 3174 early_hpet_map[i].cmd_line = true; 3175 3176 return 1; 3177 } 3178 3179 static int __init parse_ivrs_acpihid(char *str) 3180 { 3181 u32 bus, dev, fn; 3182 char *hid, *uid, *p; 3183 char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0}; 3184 int ret, i; 3185 3186 ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid); 3187 if (ret != 4) { 3188 pr_err("Invalid command line: ivrs_acpihid(%s)\n", str); 3189 return 1; 3190 } 3191 3192 p = acpiid; 3193 hid = strsep(&p, ":"); 3194 uid = p; 3195 3196 if (!hid || !(*hid) || !uid) { 3197 pr_err("Invalid command line: hid or uid\n"); 3198 return 1; 3199 } 3200 3201 i = early_acpihid_map_size++; 3202 memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); 3203 memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); 3204 early_acpihid_map[i].devid = 3205 ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); 3206 early_acpihid_map[i].cmd_line = true; 3207 3208 return 1; 3209 } 3210 3211 __setup("amd_iommu_dump", parse_amd_iommu_dump); 3212 __setup("amd_iommu=", parse_amd_iommu_options); 3213 __setup("amd_iommu_intr=", parse_amd_iommu_intr); 3214 __setup("ivrs_ioapic", parse_ivrs_ioapic); 3215 __setup("ivrs_hpet", parse_ivrs_hpet); 3216 __setup("ivrs_acpihid", parse_ivrs_acpihid); 3217 3218 IOMMU_INIT_FINISH(amd_iommu_detect, 3219 gart_iommu_hole_init, 3220 NULL, 3221 NULL); 3222 3223 bool amd_iommu_v2_supported(void) 3224 { 3225 return amd_iommu_v2_present; 3226 } 3227 EXPORT_SYMBOL(amd_iommu_v2_supported); 3228 3229 struct amd_iommu *get_amd_iommu(unsigned int idx) 3230 { 3231 unsigned int i = 0; 3232 struct amd_iommu *iommu; 3233 3234 for_each_iommu(iommu) 3235 if (i++ == idx) 3236 return iommu; 3237 return NULL; 3238 } 3239 3240 /**************************************************************************** 3241 * 3242 * IOMMU EFR Performance Counter support functionality. This code allows 3243 * access to the IOMMU PC functionality. 3244 * 3245 ****************************************************************************/ 3246 3247 u8 amd_iommu_pc_get_max_banks(unsigned int idx) 3248 { 3249 struct amd_iommu *iommu = get_amd_iommu(idx); 3250 3251 if (iommu) 3252 return iommu->max_banks; 3253 3254 return 0; 3255 } 3256 EXPORT_SYMBOL(amd_iommu_pc_get_max_banks); 3257 3258 bool amd_iommu_pc_supported(void) 3259 { 3260 return amd_iommu_pc_present; 3261 } 3262 EXPORT_SYMBOL(amd_iommu_pc_supported); 3263 3264 u8 amd_iommu_pc_get_max_counters(unsigned int idx) 3265 { 3266 struct amd_iommu *iommu = get_amd_iommu(idx); 3267 3268 if (iommu) 3269 return iommu->max_counters; 3270 3271 return 0; 3272 } 3273 EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); 3274 3275 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, 3276 u8 fxn, u64 *value, bool is_write) 3277 { 3278 u32 offset; 3279 u32 max_offset_lim; 3280 3281 /* Make sure the IOMMU PC resource is available */ 3282 if (!amd_iommu_pc_present) 3283 return -ENODEV; 3284 3285 /* Check for valid iommu and pc register indexing */ 3286 if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7))) 3287 return -ENODEV; 3288 3289 offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn); 3290 3291 /* Limit the offset to the hw defined mmio region aperture */ 3292 max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) | 3293 (iommu->max_counters << 8) | 0x28); 3294 if ((offset < MMIO_CNTR_REG_OFFSET) || 3295 (offset > max_offset_lim)) 3296 return -EINVAL; 3297 3298 if (is_write) { 3299 u64 val = *value & GENMASK_ULL(47, 0); 3300 3301 writel((u32)val, iommu->mmio_base + offset); 3302 writel((val >> 32), iommu->mmio_base + offset + 4); 3303 } else { 3304 *value = readl(iommu->mmio_base + offset + 4); 3305 *value <<= 32; 3306 *value |= readl(iommu->mmio_base + offset); 3307 *value &= GENMASK_ULL(47, 0); 3308 } 3309 3310 return 0; 3311 } 3312 3313 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3314 { 3315 if (!iommu) 3316 return -EINVAL; 3317 3318 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); 3319 } 3320 3321 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3322 { 3323 if (!iommu) 3324 return -EINVAL; 3325 3326 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); 3327 } 3328