1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * intel-pasid.c - PASID idr, table and entry manipulation 4 * 5 * Copyright (C) 2018 Intel Corporation 6 * 7 * Author: Lu Baolu <baolu.lu@linux.intel.com> 8 */ 9 10 #define pr_fmt(fmt) "DMAR: " fmt 11 12 #include <linux/bitops.h> 13 #include <linux/cpufeature.h> 14 #include <linux/dmar.h> 15 #include <linux/intel-iommu.h> 16 #include <linux/iommu.h> 17 #include <linux/memory.h> 18 #include <linux/pci.h> 19 #include <linux/pci-ats.h> 20 #include <linux/spinlock.h> 21 22 #include "pasid.h" 23 24 /* 25 * Intel IOMMU system wide PASID name space: 26 */ 27 u32 intel_pasid_max_id = PASID_MAX; 28 29 int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid) 30 { 31 unsigned long flags; 32 u8 status_code; 33 int ret = 0; 34 u64 res; 35 36 raw_spin_lock_irqsave(&iommu->register_lock, flags); 37 dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC); 38 IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq, 39 !(res & VCMD_VRSP_IP), res); 40 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 41 42 status_code = VCMD_VRSP_SC(res); 43 switch (status_code) { 44 case VCMD_VRSP_SC_SUCCESS: 45 *pasid = VCMD_VRSP_RESULT_PASID(res); 46 break; 47 case VCMD_VRSP_SC_NO_PASID_AVAIL: 48 pr_info("IOMMU: %s: No PASID available\n", iommu->name); 49 ret = -ENOSPC; 50 break; 51 default: 52 ret = -ENODEV; 53 pr_warn("IOMMU: %s: Unexpected error code %d\n", 54 iommu->name, status_code); 55 } 56 57 return ret; 58 } 59 60 void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid) 61 { 62 unsigned long flags; 63 u8 status_code; 64 u64 res; 65 66 raw_spin_lock_irqsave(&iommu->register_lock, flags); 67 dmar_writeq(iommu->reg + DMAR_VCMD_REG, 68 VCMD_CMD_OPERAND(pasid) | VCMD_CMD_FREE); 69 IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq, 70 !(res & VCMD_VRSP_IP), res); 71 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 72 73 status_code = VCMD_VRSP_SC(res); 74 switch (status_code) { 75 case VCMD_VRSP_SC_SUCCESS: 76 break; 77 case VCMD_VRSP_SC_INVALID_PASID: 78 pr_info("IOMMU: %s: Invalid PASID\n", iommu->name); 79 break; 80 default: 81 pr_warn("IOMMU: %s: Unexpected error code %d\n", 82 iommu->name, status_code); 83 } 84 } 85 86 /* 87 * Per device pasid table management: 88 */ 89 static inline void 90 device_attach_pasid_table(struct device_domain_info *info, 91 struct pasid_table *pasid_table) 92 { 93 info->pasid_table = pasid_table; 94 list_add(&info->table, &pasid_table->dev); 95 } 96 97 static inline void 98 device_detach_pasid_table(struct device_domain_info *info, 99 struct pasid_table *pasid_table) 100 { 101 info->pasid_table = NULL; 102 list_del(&info->table); 103 } 104 105 struct pasid_table_opaque { 106 struct pasid_table **pasid_table; 107 int segment; 108 int bus; 109 int devfn; 110 }; 111 112 static int search_pasid_table(struct device_domain_info *info, void *opaque) 113 { 114 struct pasid_table_opaque *data = opaque; 115 116 if (info->iommu->segment == data->segment && 117 info->bus == data->bus && 118 info->devfn == data->devfn && 119 info->pasid_table) { 120 *data->pasid_table = info->pasid_table; 121 return 1; 122 } 123 124 return 0; 125 } 126 127 static int get_alias_pasid_table(struct pci_dev *pdev, u16 alias, void *opaque) 128 { 129 struct pasid_table_opaque *data = opaque; 130 131 data->segment = pci_domain_nr(pdev->bus); 132 data->bus = PCI_BUS_NUM(alias); 133 data->devfn = alias & 0xff; 134 135 return for_each_device_domain(&search_pasid_table, data); 136 } 137 138 /* 139 * Allocate a pasid table for @dev. It should be called in a 140 * single-thread context. 141 */ 142 int intel_pasid_alloc_table(struct device *dev) 143 { 144 struct device_domain_info *info; 145 struct pasid_table *pasid_table; 146 struct pasid_table_opaque data; 147 struct page *pages; 148 u32 max_pasid = 0; 149 int ret, order; 150 int size; 151 152 might_sleep(); 153 info = get_domain_info(dev); 154 if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table)) 155 return -EINVAL; 156 157 /* DMA alias device already has a pasid table, use it: */ 158 data.pasid_table = &pasid_table; 159 ret = pci_for_each_dma_alias(to_pci_dev(dev), 160 &get_alias_pasid_table, &data); 161 if (ret) 162 goto attach_out; 163 164 pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL); 165 if (!pasid_table) 166 return -ENOMEM; 167 INIT_LIST_HEAD(&pasid_table->dev); 168 169 if (info->pasid_supported) 170 max_pasid = min_t(u32, pci_max_pasids(to_pci_dev(dev)), 171 intel_pasid_max_id); 172 173 size = max_pasid >> (PASID_PDE_SHIFT - 3); 174 order = size ? get_order(size) : 0; 175 pages = alloc_pages_node(info->iommu->node, 176 GFP_KERNEL | __GFP_ZERO, order); 177 if (!pages) { 178 kfree(pasid_table); 179 return -ENOMEM; 180 } 181 182 pasid_table->table = page_address(pages); 183 pasid_table->order = order; 184 pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); 185 186 attach_out: 187 device_attach_pasid_table(info, pasid_table); 188 189 return 0; 190 } 191 192 void intel_pasid_free_table(struct device *dev) 193 { 194 struct device_domain_info *info; 195 struct pasid_table *pasid_table; 196 struct pasid_dir_entry *dir; 197 struct pasid_entry *table; 198 int i, max_pde; 199 200 info = get_domain_info(dev); 201 if (!info || !dev_is_pci(dev) || !info->pasid_table) 202 return; 203 204 pasid_table = info->pasid_table; 205 device_detach_pasid_table(info, pasid_table); 206 207 if (!list_empty(&pasid_table->dev)) 208 return; 209 210 /* Free scalable mode PASID directory tables: */ 211 dir = pasid_table->table; 212 max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; 213 for (i = 0; i < max_pde; i++) { 214 table = get_pasid_table_from_pde(&dir[i]); 215 free_pgtable_page(table); 216 } 217 218 free_pages((unsigned long)pasid_table->table, pasid_table->order); 219 kfree(pasid_table); 220 } 221 222 struct pasid_table *intel_pasid_get_table(struct device *dev) 223 { 224 struct device_domain_info *info; 225 226 info = get_domain_info(dev); 227 if (!info) 228 return NULL; 229 230 return info->pasid_table; 231 } 232 233 static int intel_pasid_get_dev_max_id(struct device *dev) 234 { 235 struct device_domain_info *info; 236 237 info = get_domain_info(dev); 238 if (!info || !info->pasid_table) 239 return 0; 240 241 return info->pasid_table->max_pasid; 242 } 243 244 static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) 245 { 246 struct device_domain_info *info; 247 struct pasid_table *pasid_table; 248 struct pasid_dir_entry *dir; 249 struct pasid_entry *entries; 250 int dir_index, index; 251 252 pasid_table = intel_pasid_get_table(dev); 253 if (WARN_ON(!pasid_table || pasid >= intel_pasid_get_dev_max_id(dev))) 254 return NULL; 255 256 dir = pasid_table->table; 257 info = get_domain_info(dev); 258 dir_index = pasid >> PASID_PDE_SHIFT; 259 index = pasid & PASID_PTE_MASK; 260 261 retry: 262 entries = get_pasid_table_from_pde(&dir[dir_index]); 263 if (!entries) { 264 entries = alloc_pgtable_page(info->iommu->node); 265 if (!entries) 266 return NULL; 267 268 /* 269 * The pasid directory table entry won't be freed after 270 * allocation. No worry about the race with free and 271 * clear. However, this entry might be populated by others 272 * while we are preparing it. Use theirs with a retry. 273 */ 274 if (cmpxchg64(&dir[dir_index].val, 0ULL, 275 (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { 276 free_pgtable_page(entries); 277 goto retry; 278 } 279 } 280 281 return &entries[index]; 282 } 283 284 /* 285 * Interfaces for PASID table entry manipulation: 286 */ 287 static inline void pasid_clear_entry(struct pasid_entry *pe) 288 { 289 WRITE_ONCE(pe->val[0], 0); 290 WRITE_ONCE(pe->val[1], 0); 291 WRITE_ONCE(pe->val[2], 0); 292 WRITE_ONCE(pe->val[3], 0); 293 WRITE_ONCE(pe->val[4], 0); 294 WRITE_ONCE(pe->val[5], 0); 295 WRITE_ONCE(pe->val[6], 0); 296 WRITE_ONCE(pe->val[7], 0); 297 } 298 299 static inline void pasid_clear_entry_with_fpd(struct pasid_entry *pe) 300 { 301 WRITE_ONCE(pe->val[0], PASID_PTE_FPD); 302 WRITE_ONCE(pe->val[1], 0); 303 WRITE_ONCE(pe->val[2], 0); 304 WRITE_ONCE(pe->val[3], 0); 305 WRITE_ONCE(pe->val[4], 0); 306 WRITE_ONCE(pe->val[5], 0); 307 WRITE_ONCE(pe->val[6], 0); 308 WRITE_ONCE(pe->val[7], 0); 309 } 310 311 static void 312 intel_pasid_clear_entry(struct device *dev, u32 pasid, bool fault_ignore) 313 { 314 struct pasid_entry *pe; 315 316 pe = intel_pasid_get_entry(dev, pasid); 317 if (WARN_ON(!pe)) 318 return; 319 320 if (fault_ignore && pasid_pte_is_present(pe)) 321 pasid_clear_entry_with_fpd(pe); 322 else 323 pasid_clear_entry(pe); 324 } 325 326 static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits) 327 { 328 u64 old; 329 330 old = READ_ONCE(*ptr); 331 WRITE_ONCE(*ptr, (old & ~mask) | bits); 332 } 333 334 /* 335 * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode 336 * PASID entry. 337 */ 338 static inline void 339 pasid_set_domain_id(struct pasid_entry *pe, u64 value) 340 { 341 pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value); 342 } 343 344 /* 345 * Get domain ID value of a scalable mode PASID entry. 346 */ 347 static inline u16 348 pasid_get_domain_id(struct pasid_entry *pe) 349 { 350 return (u16)(READ_ONCE(pe->val[1]) & GENMASK_ULL(15, 0)); 351 } 352 353 /* 354 * Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63) 355 * of a scalable mode PASID entry. 356 */ 357 static inline void 358 pasid_set_slptr(struct pasid_entry *pe, u64 value) 359 { 360 pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value); 361 } 362 363 /* 364 * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID 365 * entry. 366 */ 367 static inline void 368 pasid_set_address_width(struct pasid_entry *pe, u64 value) 369 { 370 pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2); 371 } 372 373 /* 374 * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8) 375 * of a scalable mode PASID entry. 376 */ 377 static inline void 378 pasid_set_translation_type(struct pasid_entry *pe, u64 value) 379 { 380 pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6); 381 } 382 383 /* 384 * Enable fault processing by clearing the FPD(Fault Processing 385 * Disable) field (Bit 1) of a scalable mode PASID entry. 386 */ 387 static inline void pasid_set_fault_enable(struct pasid_entry *pe) 388 { 389 pasid_set_bits(&pe->val[0], 1 << 1, 0); 390 } 391 392 /* 393 * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a 394 * scalable mode PASID entry. 395 */ 396 static inline void pasid_set_sre(struct pasid_entry *pe) 397 { 398 pasid_set_bits(&pe->val[2], 1 << 0, 1); 399 } 400 401 /* 402 * Setup the WPE(Write Protect Enable) field (Bit 132) of a 403 * scalable mode PASID entry. 404 */ 405 static inline void pasid_set_wpe(struct pasid_entry *pe) 406 { 407 pasid_set_bits(&pe->val[2], 1 << 4, 1 << 4); 408 } 409 410 /* 411 * Setup the P(Present) field (Bit 0) of a scalable mode PASID 412 * entry. 413 */ 414 static inline void pasid_set_present(struct pasid_entry *pe) 415 { 416 pasid_set_bits(&pe->val[0], 1 << 0, 1); 417 } 418 419 /* 420 * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID 421 * entry. 422 */ 423 static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) 424 { 425 pasid_set_bits(&pe->val[1], 1 << 23, value << 23); 426 } 427 428 /* 429 * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode 430 * PASID entry. 431 */ 432 static inline void 433 pasid_set_pgsnp(struct pasid_entry *pe) 434 { 435 pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24); 436 } 437 438 /* 439 * Setup the First Level Page table Pointer field (Bit 140~191) 440 * of a scalable mode PASID entry. 441 */ 442 static inline void 443 pasid_set_flptr(struct pasid_entry *pe, u64 value) 444 { 445 pasid_set_bits(&pe->val[2], VTD_PAGE_MASK, value); 446 } 447 448 /* 449 * Setup the First Level Paging Mode field (Bit 130~131) of a 450 * scalable mode PASID entry. 451 */ 452 static inline void 453 pasid_set_flpm(struct pasid_entry *pe, u64 value) 454 { 455 pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2); 456 } 457 458 /* 459 * Setup the Extended Access Flag Enable (EAFE) field (Bit 135) 460 * of a scalable mode PASID entry. 461 */ 462 static inline void 463 pasid_set_eafe(struct pasid_entry *pe) 464 { 465 pasid_set_bits(&pe->val[2], 1 << 7, 1 << 7); 466 } 467 468 static void 469 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu, 470 u16 did, u32 pasid) 471 { 472 struct qi_desc desc; 473 474 desc.qw0 = QI_PC_DID(did) | QI_PC_GRAN(QI_PC_PASID_SEL) | 475 QI_PC_PASID(pasid) | QI_PC_TYPE; 476 desc.qw1 = 0; 477 desc.qw2 = 0; 478 desc.qw3 = 0; 479 480 qi_submit_sync(iommu, &desc, 1, 0); 481 } 482 483 static void 484 devtlb_invalidation_with_pasid(struct intel_iommu *iommu, 485 struct device *dev, u32 pasid) 486 { 487 struct device_domain_info *info; 488 u16 sid, qdep, pfsid; 489 490 info = get_domain_info(dev); 491 if (!info || !info->ats_enabled) 492 return; 493 494 sid = info->bus << 8 | info->devfn; 495 qdep = info->ats_qdep; 496 pfsid = info->pfsid; 497 498 /* 499 * When PASID 0 is used, it indicates RID2PASID(DMA request w/o PASID), 500 * devTLB flush w/o PASID should be used. For non-zero PASID under 501 * SVA usage, device could do DMA with multiple PASIDs. It is more 502 * efficient to flush devTLB specific to the PASID. 503 */ 504 if (pasid == PASID_RID2PASID) 505 qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT); 506 else 507 qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT); 508 } 509 510 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, 511 u32 pasid, bool fault_ignore) 512 { 513 struct pasid_entry *pte; 514 u16 did; 515 516 pte = intel_pasid_get_entry(dev, pasid); 517 if (WARN_ON(!pte)) 518 return; 519 520 if (!(pte->val[0] & PASID_PTE_PRESENT)) 521 return; 522 523 did = pasid_get_domain_id(pte); 524 intel_pasid_clear_entry(dev, pasid, fault_ignore); 525 526 if (!ecap_coherent(iommu->ecap)) 527 clflush_cache_range(pte, sizeof(*pte)); 528 529 pasid_cache_invalidation_with_pasid(iommu, did, pasid); 530 qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); 531 532 /* Device IOTLB doesn't need to be flushed in caching mode. */ 533 if (!cap_caching_mode(iommu->cap)) 534 devtlb_invalidation_with_pasid(iommu, dev, pasid); 535 } 536 537 static void pasid_flush_caches(struct intel_iommu *iommu, 538 struct pasid_entry *pte, 539 u32 pasid, u16 did) 540 { 541 if (!ecap_coherent(iommu->ecap)) 542 clflush_cache_range(pte, sizeof(*pte)); 543 544 if (cap_caching_mode(iommu->cap)) { 545 pasid_cache_invalidation_with_pasid(iommu, did, pasid); 546 qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); 547 } else { 548 iommu_flush_write_buffer(iommu); 549 } 550 } 551 552 static inline int pasid_enable_wpe(struct pasid_entry *pte) 553 { 554 #ifdef CONFIG_X86 555 unsigned long cr0 = read_cr0(); 556 557 /* CR0.WP is normally set but just to be sure */ 558 if (unlikely(!(cr0 & X86_CR0_WP))) { 559 pr_err_ratelimited("No CPU write protect!\n"); 560 return -EINVAL; 561 } 562 #endif 563 pasid_set_wpe(pte); 564 565 return 0; 566 }; 567 568 /* 569 * Set up the scalable mode pasid table entry for first only 570 * translation type. 571 */ 572 int intel_pasid_setup_first_level(struct intel_iommu *iommu, 573 struct device *dev, pgd_t *pgd, 574 u32 pasid, u16 did, int flags) 575 { 576 struct pasid_entry *pte; 577 578 if (!ecap_flts(iommu->ecap)) { 579 pr_err("No first level translation support on %s\n", 580 iommu->name); 581 return -EINVAL; 582 } 583 584 pte = intel_pasid_get_entry(dev, pasid); 585 if (WARN_ON(!pte)) 586 return -EINVAL; 587 588 pasid_clear_entry(pte); 589 590 /* Setup the first level page table pointer: */ 591 pasid_set_flptr(pte, (u64)__pa(pgd)); 592 if (flags & PASID_FLAG_SUPERVISOR_MODE) { 593 if (!ecap_srs(iommu->ecap)) { 594 pr_err("No supervisor request support on %s\n", 595 iommu->name); 596 return -EINVAL; 597 } 598 pasid_set_sre(pte); 599 if (pasid_enable_wpe(pte)) 600 return -EINVAL; 601 602 } 603 604 if (flags & PASID_FLAG_FL5LP) { 605 if (cap_5lp_support(iommu->cap)) { 606 pasid_set_flpm(pte, 1); 607 } else { 608 pr_err("No 5-level paging support for first-level\n"); 609 pasid_clear_entry(pte); 610 return -EINVAL; 611 } 612 } 613 614 if (flags & PASID_FLAG_PAGE_SNOOP) 615 pasid_set_pgsnp(pte); 616 617 pasid_set_domain_id(pte, did); 618 pasid_set_address_width(pte, iommu->agaw); 619 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 620 621 /* Setup Present and PASID Granular Transfer Type: */ 622 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY); 623 pasid_set_present(pte); 624 pasid_flush_caches(iommu, pte, pasid, did); 625 626 return 0; 627 } 628 629 /* 630 * Skip top levels of page tables for iommu which has less agaw 631 * than default. Unnecessary for PT mode. 632 */ 633 static inline int iommu_skip_agaw(struct dmar_domain *domain, 634 struct intel_iommu *iommu, 635 struct dma_pte **pgd) 636 { 637 int agaw; 638 639 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { 640 *pgd = phys_to_virt(dma_pte_addr(*pgd)); 641 if (!dma_pte_present(*pgd)) 642 return -EINVAL; 643 } 644 645 return agaw; 646 } 647 648 /* 649 * Set up the scalable mode pasid entry for second only translation type. 650 */ 651 int intel_pasid_setup_second_level(struct intel_iommu *iommu, 652 struct dmar_domain *domain, 653 struct device *dev, u32 pasid) 654 { 655 struct pasid_entry *pte; 656 struct dma_pte *pgd; 657 u64 pgd_val; 658 int agaw; 659 u16 did; 660 661 /* 662 * If hardware advertises no support for second level 663 * translation, return directly. 664 */ 665 if (!ecap_slts(iommu->ecap)) { 666 pr_err("No second level translation support on %s\n", 667 iommu->name); 668 return -EINVAL; 669 } 670 671 pgd = domain->pgd; 672 agaw = iommu_skip_agaw(domain, iommu, &pgd); 673 if (agaw < 0) { 674 dev_err(dev, "Invalid domain page table\n"); 675 return -EINVAL; 676 } 677 678 pgd_val = virt_to_phys(pgd); 679 did = domain->iommu_did[iommu->seq_id]; 680 681 pte = intel_pasid_get_entry(dev, pasid); 682 if (!pte) { 683 dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid); 684 return -ENODEV; 685 } 686 687 pasid_clear_entry(pte); 688 pasid_set_domain_id(pte, did); 689 pasid_set_slptr(pte, pgd_val); 690 pasid_set_address_width(pte, agaw); 691 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_SL_ONLY); 692 pasid_set_fault_enable(pte); 693 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 694 695 if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) 696 pasid_set_pgsnp(pte); 697 698 /* 699 * Since it is a second level only translation setup, we should 700 * set SRE bit as well (addresses are expected to be GPAs). 701 */ 702 if (pasid != PASID_RID2PASID) 703 pasid_set_sre(pte); 704 pasid_set_present(pte); 705 pasid_flush_caches(iommu, pte, pasid, did); 706 707 return 0; 708 } 709 710 /* 711 * Set up the scalable mode pasid entry for passthrough translation type. 712 */ 713 int intel_pasid_setup_pass_through(struct intel_iommu *iommu, 714 struct dmar_domain *domain, 715 struct device *dev, u32 pasid) 716 { 717 u16 did = FLPT_DEFAULT_DID; 718 struct pasid_entry *pte; 719 720 pte = intel_pasid_get_entry(dev, pasid); 721 if (!pte) { 722 dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid); 723 return -ENODEV; 724 } 725 726 pasid_clear_entry(pte); 727 pasid_set_domain_id(pte, did); 728 pasid_set_address_width(pte, iommu->agaw); 729 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_PT); 730 pasid_set_fault_enable(pte); 731 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 732 733 /* 734 * We should set SRE bit as well since the addresses are expected 735 * to be GPAs. 736 */ 737 pasid_set_sre(pte); 738 pasid_set_present(pte); 739 pasid_flush_caches(iommu, pte, pasid, did); 740 741 return 0; 742 } 743 744 static int 745 intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte, 746 struct iommu_gpasid_bind_data_vtd *pasid_data) 747 { 748 /* 749 * Not all guest PASID table entry fields are passed down during bind, 750 * here we only set up the ones that are dependent on guest settings. 751 * Execution related bits such as NXE, SMEP are not supported. 752 * Other fields, such as snoop related, are set based on host needs 753 * regardless of guest settings. 754 */ 755 if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_SRE) { 756 if (!ecap_srs(iommu->ecap)) { 757 pr_err_ratelimited("No supervisor request support on %s\n", 758 iommu->name); 759 return -EINVAL; 760 } 761 pasid_set_sre(pte); 762 /* Enable write protect WP if guest requested */ 763 if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_WPE) 764 pasid_set_wpe(pte); 765 } 766 767 if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) { 768 if (!ecap_eafs(iommu->ecap)) { 769 pr_err_ratelimited("No extended access flag support on %s\n", 770 iommu->name); 771 return -EINVAL; 772 } 773 pasid_set_eafe(pte); 774 } 775 776 /* 777 * Memory type is only applicable to devices inside processor coherent 778 * domain. Will add MTS support once coherent devices are available. 779 */ 780 if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_MTS_MASK) { 781 pr_warn_ratelimited("No memory type support %s\n", 782 iommu->name); 783 return -EINVAL; 784 } 785 786 return 0; 787 } 788 789 /** 790 * intel_pasid_setup_nested() - Set up PASID entry for nested translation. 791 * This could be used for guest shared virtual address. In this case, the 792 * first level page tables are used for GVA-GPA translation in the guest, 793 * second level page tables are used for GPA-HPA translation. 794 * 795 * @iommu: IOMMU which the device belong to 796 * @dev: Device to be set up for translation 797 * @gpgd: FLPTPTR: First Level Page translation pointer in GPA 798 * @pasid: PASID to be programmed in the device PASID table 799 * @pasid_data: Additional PASID info from the guest bind request 800 * @domain: Domain info for setting up second level page tables 801 * @addr_width: Address width of the first level (guest) 802 */ 803 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, 804 pgd_t *gpgd, u32 pasid, 805 struct iommu_gpasid_bind_data_vtd *pasid_data, 806 struct dmar_domain *domain, int addr_width) 807 { 808 struct pasid_entry *pte; 809 struct dma_pte *pgd; 810 int ret = 0; 811 u64 pgd_val; 812 int agaw; 813 u16 did; 814 815 if (!ecap_nest(iommu->ecap)) { 816 pr_err_ratelimited("IOMMU: %s: No nested translation support\n", 817 iommu->name); 818 return -EINVAL; 819 } 820 821 if (!(domain->flags & DOMAIN_FLAG_NESTING_MODE)) { 822 pr_err_ratelimited("Domain is not in nesting mode, %x\n", 823 domain->flags); 824 return -EINVAL; 825 } 826 827 pte = intel_pasid_get_entry(dev, pasid); 828 if (WARN_ON(!pte)) 829 return -EINVAL; 830 831 /* 832 * Caller must ensure PASID entry is not in use, i.e. not bind the 833 * same PASID to the same device twice. 834 */ 835 if (pasid_pte_is_present(pte)) 836 return -EBUSY; 837 838 pasid_clear_entry(pte); 839 840 /* Sanity checking performed by caller to make sure address 841 * width matching in two dimensions: 842 * 1. CPU vs. IOMMU 843 * 2. Guest vs. Host. 844 */ 845 switch (addr_width) { 846 #ifdef CONFIG_X86 847 case ADDR_WIDTH_5LEVEL: 848 if (!cpu_feature_enabled(X86_FEATURE_LA57) || 849 !cap_5lp_support(iommu->cap)) { 850 dev_err_ratelimited(dev, 851 "5-level paging not supported\n"); 852 return -EINVAL; 853 } 854 855 pasid_set_flpm(pte, 1); 856 break; 857 #endif 858 case ADDR_WIDTH_4LEVEL: 859 pasid_set_flpm(pte, 0); 860 break; 861 default: 862 dev_err_ratelimited(dev, "Invalid guest address width %d\n", 863 addr_width); 864 return -EINVAL; 865 } 866 867 /* First level PGD is in GPA, must be supported by the second level */ 868 if ((uintptr_t)gpgd > domain->max_addr) { 869 dev_err_ratelimited(dev, 870 "Guest PGD %lx not supported, max %llx\n", 871 (uintptr_t)gpgd, domain->max_addr); 872 return -EINVAL; 873 } 874 pasid_set_flptr(pte, (uintptr_t)gpgd); 875 876 ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data); 877 if (ret) 878 return ret; 879 880 /* Setup the second level based on the given domain */ 881 pgd = domain->pgd; 882 883 agaw = iommu_skip_agaw(domain, iommu, &pgd); 884 if (agaw < 0) { 885 dev_err_ratelimited(dev, "Invalid domain page table\n"); 886 return -EINVAL; 887 } 888 pgd_val = virt_to_phys(pgd); 889 pasid_set_slptr(pte, pgd_val); 890 pasid_set_fault_enable(pte); 891 892 did = domain->iommu_did[iommu->seq_id]; 893 pasid_set_domain_id(pte, did); 894 895 pasid_set_address_width(pte, agaw); 896 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 897 898 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_NESTED); 899 pasid_set_present(pte); 900 pasid_flush_caches(iommu, pte, pasid, did); 901 902 return ret; 903 } 904