1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/slab.h> 12 #include <linux/intel-svm.h> 13 #include <linux/rculist.h> 14 #include <linux/pci.h> 15 #include <linux/pci-ats.h> 16 #include <linux/dmar.h> 17 #include <linux/interrupt.h> 18 #include <linux/mm_types.h> 19 #include <linux/xarray.h> 20 #include <linux/ioasid.h> 21 #include <asm/page.h> 22 #include <asm/fpu/api.h> 23 24 #include "iommu.h" 25 #include "pasid.h" 26 #include "perf.h" 27 #include "../iommu-sva.h" 28 #include "trace.h" 29 30 static irqreturn_t prq_event_thread(int irq, void *d); 31 static void intel_svm_drain_prq(struct device *dev, u32 pasid); 32 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 33 34 static DEFINE_XARRAY_ALLOC(pasid_private_array); 35 static int pasid_private_add(ioasid_t pasid, void *priv) 36 { 37 return xa_alloc(&pasid_private_array, &pasid, priv, 38 XA_LIMIT(pasid, pasid), GFP_ATOMIC); 39 } 40 41 static void pasid_private_remove(ioasid_t pasid) 42 { 43 xa_erase(&pasid_private_array, pasid); 44 } 45 46 static void *pasid_private_find(ioasid_t pasid) 47 { 48 return xa_load(&pasid_private_array, pasid); 49 } 50 51 static struct intel_svm_dev * 52 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) 53 { 54 struct intel_svm_dev *sdev = NULL, *t; 55 56 rcu_read_lock(); 57 list_for_each_entry_rcu(t, &svm->devs, list) { 58 if (t->dev == dev) { 59 sdev = t; 60 break; 61 } 62 } 63 rcu_read_unlock(); 64 65 return sdev; 66 } 67 68 int intel_svm_enable_prq(struct intel_iommu *iommu) 69 { 70 struct iopf_queue *iopfq; 71 struct page *pages; 72 int irq, ret; 73 74 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 75 if (!pages) { 76 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 77 iommu->name); 78 return -ENOMEM; 79 } 80 iommu->prq = page_address(pages); 81 82 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 83 if (irq <= 0) { 84 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 85 iommu->name); 86 ret = -EINVAL; 87 goto free_prq; 88 } 89 iommu->pr_irq = irq; 90 91 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), 92 "dmar%d-iopfq", iommu->seq_id); 93 iopfq = iopf_queue_alloc(iommu->iopfq_name); 94 if (!iopfq) { 95 pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name); 96 ret = -ENOMEM; 97 goto free_hwirq; 98 } 99 iommu->iopf_queue = iopfq; 100 101 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 102 103 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 104 iommu->prq_name, iommu); 105 if (ret) { 106 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 107 iommu->name); 108 goto free_iopfq; 109 } 110 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 111 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 112 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 113 114 init_completion(&iommu->prq_complete); 115 116 return 0; 117 118 free_iopfq: 119 iopf_queue_free(iommu->iopf_queue); 120 iommu->iopf_queue = NULL; 121 free_hwirq: 122 dmar_free_hwirq(irq); 123 iommu->pr_irq = 0; 124 free_prq: 125 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 126 iommu->prq = NULL; 127 128 return ret; 129 } 130 131 int intel_svm_finish_prq(struct intel_iommu *iommu) 132 { 133 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 134 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 135 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 136 137 if (iommu->pr_irq) { 138 free_irq(iommu->pr_irq, iommu); 139 dmar_free_hwirq(iommu->pr_irq); 140 iommu->pr_irq = 0; 141 } 142 143 if (iommu->iopf_queue) { 144 iopf_queue_free(iommu->iopf_queue); 145 iommu->iopf_queue = NULL; 146 } 147 148 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 149 iommu->prq = NULL; 150 151 return 0; 152 } 153 154 void intel_svm_check(struct intel_iommu *iommu) 155 { 156 if (!pasid_supported(iommu)) 157 return; 158 159 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 160 !cap_fl1gp_support(iommu->cap)) { 161 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 162 iommu->name); 163 return; 164 } 165 166 if (cpu_feature_enabled(X86_FEATURE_LA57) && 167 !cap_fl5lp_support(iommu->cap)) { 168 pr_err("%s SVM disabled, incompatible paging mode\n", 169 iommu->name); 170 return; 171 } 172 173 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 174 } 175 176 static void __flush_svm_range_dev(struct intel_svm *svm, 177 struct intel_svm_dev *sdev, 178 unsigned long address, 179 unsigned long pages, int ih) 180 { 181 struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); 182 183 if (WARN_ON(!pages)) 184 return; 185 186 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); 187 if (info->ats_enabled) { 188 qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, 189 svm->pasid, sdev->qdep, address, 190 order_base_2(pages)); 191 quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), 192 svm->pasid, sdev->qdep); 193 } 194 } 195 196 static void intel_flush_svm_range_dev(struct intel_svm *svm, 197 struct intel_svm_dev *sdev, 198 unsigned long address, 199 unsigned long pages, int ih) 200 { 201 unsigned long shift = ilog2(__roundup_pow_of_two(pages)); 202 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); 203 unsigned long start = ALIGN_DOWN(address, align); 204 unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); 205 206 while (start < end) { 207 __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); 208 start += align; 209 } 210 } 211 212 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 213 unsigned long pages, int ih) 214 { 215 struct intel_svm_dev *sdev; 216 217 rcu_read_lock(); 218 list_for_each_entry_rcu(sdev, &svm->devs, list) 219 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 220 rcu_read_unlock(); 221 } 222 223 /* Pages have been freed at this point */ 224 static void intel_invalidate_range(struct mmu_notifier *mn, 225 struct mm_struct *mm, 226 unsigned long start, unsigned long end) 227 { 228 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 229 230 intel_flush_svm_range(svm, start, 231 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 232 } 233 234 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 235 { 236 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 237 struct intel_svm_dev *sdev; 238 239 /* This might end up being called from exit_mmap(), *before* the page 240 * tables are cleared. And __mmu_notifier_release() will delete us from 241 * the list of notifiers so that our invalidate_range() callback doesn't 242 * get called when the page tables are cleared. So we need to protect 243 * against hardware accessing those page tables. 244 * 245 * We do it by clearing the entry in the PASID table and then flushing 246 * the IOTLB and the PASID table caches. This might upset hardware; 247 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 248 * page) so that we end up taking a fault that the hardware really 249 * *has* to handle gracefully without affecting other processes. 250 */ 251 rcu_read_lock(); 252 list_for_each_entry_rcu(sdev, &svm->devs, list) 253 intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, 254 svm->pasid, true); 255 rcu_read_unlock(); 256 257 } 258 259 static const struct mmu_notifier_ops intel_mmuops = { 260 .release = intel_mm_release, 261 .invalidate_range = intel_invalidate_range, 262 }; 263 264 static DEFINE_MUTEX(pasid_mutex); 265 266 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 267 struct intel_svm **rsvm, 268 struct intel_svm_dev **rsdev) 269 { 270 struct intel_svm_dev *sdev = NULL; 271 struct intel_svm *svm; 272 273 /* The caller should hold the pasid_mutex lock */ 274 if (WARN_ON(!mutex_is_locked(&pasid_mutex))) 275 return -EINVAL; 276 277 if (pasid == INVALID_IOASID || pasid >= PASID_MAX) 278 return -EINVAL; 279 280 svm = pasid_private_find(pasid); 281 if (IS_ERR(svm)) 282 return PTR_ERR(svm); 283 284 if (!svm) 285 goto out; 286 287 /* 288 * If we found svm for the PASID, there must be at least one device 289 * bond. 290 */ 291 if (WARN_ON(list_empty(&svm->devs))) 292 return -EINVAL; 293 sdev = svm_lookup_device_by_dev(svm, dev); 294 295 out: 296 *rsvm = svm; 297 *rsdev = sdev; 298 299 return 0; 300 } 301 302 static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, 303 struct device *dev, 304 struct mm_struct *mm) 305 { 306 struct device_domain_info *info = dev_iommu_priv_get(dev); 307 struct intel_svm_dev *sdev; 308 struct intel_svm *svm; 309 unsigned long sflags; 310 int ret = 0; 311 312 svm = pasid_private_find(mm->pasid); 313 if (!svm) { 314 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 315 if (!svm) 316 return ERR_PTR(-ENOMEM); 317 318 svm->pasid = mm->pasid; 319 svm->mm = mm; 320 INIT_LIST_HEAD_RCU(&svm->devs); 321 322 svm->notifier.ops = &intel_mmuops; 323 ret = mmu_notifier_register(&svm->notifier, mm); 324 if (ret) { 325 kfree(svm); 326 return ERR_PTR(ret); 327 } 328 329 ret = pasid_private_add(svm->pasid, svm); 330 if (ret) { 331 mmu_notifier_unregister(&svm->notifier, mm); 332 kfree(svm); 333 return ERR_PTR(ret); 334 } 335 } 336 337 /* Find the matching device in svm list */ 338 sdev = svm_lookup_device_by_dev(svm, dev); 339 if (sdev) { 340 sdev->users++; 341 goto success; 342 } 343 344 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 345 if (!sdev) { 346 ret = -ENOMEM; 347 goto free_svm; 348 } 349 350 sdev->dev = dev; 351 sdev->iommu = iommu; 352 sdev->did = FLPT_DEFAULT_DID; 353 sdev->sid = PCI_DEVID(info->bus, info->devfn); 354 sdev->users = 1; 355 sdev->pasid = svm->pasid; 356 sdev->sva.dev = dev; 357 init_rcu_head(&sdev->rcu); 358 if (info->ats_enabled) { 359 sdev->dev_iotlb = 1; 360 sdev->qdep = info->ats_qdep; 361 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 362 sdev->qdep = 0; 363 } 364 365 /* Setup the pasid table: */ 366 sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 367 ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid, 368 FLPT_DEFAULT_DID, sflags); 369 if (ret) 370 goto free_sdev; 371 372 list_add_rcu(&sdev->list, &svm->devs); 373 success: 374 return &sdev->sva; 375 376 free_sdev: 377 kfree(sdev); 378 free_svm: 379 if (list_empty(&svm->devs)) { 380 mmu_notifier_unregister(&svm->notifier, mm); 381 pasid_private_remove(mm->pasid); 382 kfree(svm); 383 } 384 385 return ERR_PTR(ret); 386 } 387 388 /* Caller must hold pasid_mutex */ 389 static int intel_svm_unbind_mm(struct device *dev, u32 pasid) 390 { 391 struct intel_svm_dev *sdev; 392 struct intel_iommu *iommu; 393 struct intel_svm *svm; 394 struct mm_struct *mm; 395 int ret = -EINVAL; 396 397 iommu = device_to_iommu(dev, NULL, NULL); 398 if (!iommu) 399 goto out; 400 401 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 402 if (ret) 403 goto out; 404 mm = svm->mm; 405 406 if (sdev) { 407 sdev->users--; 408 if (!sdev->users) { 409 list_del_rcu(&sdev->list); 410 /* Flush the PASID cache and IOTLB for this device. 411 * Note that we do depend on the hardware *not* using 412 * the PASID any more. Just as we depend on other 413 * devices never using PASIDs that they have no right 414 * to use. We have a *shared* PASID table, because it's 415 * large and has to be physically contiguous. So it's 416 * hard to be as defensive as we might like. */ 417 intel_pasid_tear_down_entry(iommu, dev, 418 svm->pasid, false); 419 intel_svm_drain_prq(dev, svm->pasid); 420 kfree_rcu(sdev, rcu); 421 422 if (list_empty(&svm->devs)) { 423 if (svm->notifier.ops) 424 mmu_notifier_unregister(&svm->notifier, mm); 425 pasid_private_remove(svm->pasid); 426 /* We mandate that no page faults may be outstanding 427 * for the PASID when intel_svm_unbind_mm() is called. 428 * If that is not obeyed, subtle errors will happen. 429 * Let's make them less subtle... */ 430 memset(svm, 0x6b, sizeof(*svm)); 431 kfree(svm); 432 } 433 } 434 } 435 out: 436 return ret; 437 } 438 439 /* Page request queue descriptor */ 440 struct page_req_dsc { 441 union { 442 struct { 443 u64 type:8; 444 u64 pasid_present:1; 445 u64 priv_data_present:1; 446 u64 rsvd:6; 447 u64 rid:16; 448 u64 pasid:20; 449 u64 exe_req:1; 450 u64 pm_req:1; 451 u64 rsvd2:10; 452 }; 453 u64 qw_0; 454 }; 455 union { 456 struct { 457 u64 rd_req:1; 458 u64 wr_req:1; 459 u64 lpig:1; 460 u64 prg_index:9; 461 u64 addr:52; 462 }; 463 u64 qw_1; 464 }; 465 u64 priv_data[2]; 466 }; 467 468 static bool is_canonical_address(u64 addr) 469 { 470 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 471 long saddr = (long) addr; 472 473 return (((saddr << shift) >> shift) == saddr); 474 } 475 476 /** 477 * intel_svm_drain_prq - Drain page requests and responses for a pasid 478 * @dev: target device 479 * @pasid: pasid for draining 480 * 481 * Drain all pending page requests and responses related to @pasid in both 482 * software and hardware. This is supposed to be called after the device 483 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 484 * and DevTLB have been invalidated. 485 * 486 * It waits until all pending page requests for @pasid in the page fault 487 * queue are completed by the prq handling thread. Then follow the steps 488 * described in VT-d spec CH7.10 to drain all page requests and page 489 * responses pending in the hardware. 490 */ 491 static void intel_svm_drain_prq(struct device *dev, u32 pasid) 492 { 493 struct device_domain_info *info; 494 struct dmar_domain *domain; 495 struct intel_iommu *iommu; 496 struct qi_desc desc[3]; 497 struct pci_dev *pdev; 498 int head, tail; 499 u16 sid, did; 500 int qdep; 501 502 info = dev_iommu_priv_get(dev); 503 if (WARN_ON(!info || !dev_is_pci(dev))) 504 return; 505 506 if (!info->pri_enabled) 507 return; 508 509 iommu = info->iommu; 510 domain = info->domain; 511 pdev = to_pci_dev(dev); 512 sid = PCI_DEVID(info->bus, info->devfn); 513 did = domain_id_iommu(domain, iommu); 514 qdep = pci_ats_queue_depth(pdev); 515 516 /* 517 * Check and wait until all pending page requests in the queue are 518 * handled by the prq handling thread. 519 */ 520 prq_retry: 521 reinit_completion(&iommu->prq_complete); 522 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 523 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 524 while (head != tail) { 525 struct page_req_dsc *req; 526 527 req = &iommu->prq[head / sizeof(*req)]; 528 if (!req->pasid_present || req->pasid != pasid) { 529 head = (head + sizeof(*req)) & PRQ_RING_MASK; 530 continue; 531 } 532 533 wait_for_completion(&iommu->prq_complete); 534 goto prq_retry; 535 } 536 537 /* 538 * A work in IO page fault workqueue may try to lock pasid_mutex now. 539 * Holding pasid_mutex while waiting in iopf_queue_flush_dev() for 540 * all works in the workqueue to finish may cause deadlock. 541 * 542 * It's unnecessary to hold pasid_mutex in iopf_queue_flush_dev(). 543 * Unlock it to allow the works to be handled while waiting for 544 * them to finish. 545 */ 546 lockdep_assert_held(&pasid_mutex); 547 mutex_unlock(&pasid_mutex); 548 iopf_queue_flush_dev(dev); 549 mutex_lock(&pasid_mutex); 550 551 /* 552 * Perform steps described in VT-d spec CH7.10 to drain page 553 * requests and responses in hardware. 554 */ 555 memset(desc, 0, sizeof(desc)); 556 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 557 QI_IWD_FENCE | 558 QI_IWD_TYPE; 559 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 560 QI_EIOTLB_DID(did) | 561 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 562 QI_EIOTLB_TYPE; 563 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 564 QI_DEV_EIOTLB_SID(sid) | 565 QI_DEV_EIOTLB_QDEP(qdep) | 566 QI_DEIOTLB_TYPE | 567 QI_DEV_IOTLB_PFSID(info->pfsid); 568 qi_retry: 569 reinit_completion(&iommu->prq_complete); 570 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 571 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 572 wait_for_completion(&iommu->prq_complete); 573 goto qi_retry; 574 } 575 } 576 577 static int prq_to_iommu_prot(struct page_req_dsc *req) 578 { 579 int prot = 0; 580 581 if (req->rd_req) 582 prot |= IOMMU_FAULT_PERM_READ; 583 if (req->wr_req) 584 prot |= IOMMU_FAULT_PERM_WRITE; 585 if (req->exe_req) 586 prot |= IOMMU_FAULT_PERM_EXEC; 587 if (req->pm_req) 588 prot |= IOMMU_FAULT_PERM_PRIV; 589 590 return prot; 591 } 592 593 static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, 594 struct page_req_dsc *desc) 595 { 596 struct iommu_fault_event event; 597 598 if (!dev || !dev_is_pci(dev)) 599 return -ENODEV; 600 601 /* Fill in event data for device specific processing */ 602 memset(&event, 0, sizeof(struct iommu_fault_event)); 603 event.fault.type = IOMMU_FAULT_PAGE_REQ; 604 event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; 605 event.fault.prm.pasid = desc->pasid; 606 event.fault.prm.grpid = desc->prg_index; 607 event.fault.prm.perm = prq_to_iommu_prot(desc); 608 609 if (desc->lpig) 610 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 611 if (desc->pasid_present) { 612 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 613 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 614 } 615 if (desc->priv_data_present) { 616 /* 617 * Set last page in group bit if private data is present, 618 * page response is required as it does for LPIG. 619 * iommu_report_device_fault() doesn't understand this vendor 620 * specific requirement thus we set last_page as a workaround. 621 */ 622 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 623 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 624 event.fault.prm.private_data[0] = desc->priv_data[0]; 625 event.fault.prm.private_data[1] = desc->priv_data[1]; 626 } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { 627 /* 628 * If the private data fields are not used by hardware, use it 629 * to monitor the prq handle latency. 630 */ 631 event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); 632 } 633 634 return iommu_report_device_fault(dev, &event); 635 } 636 637 static void handle_bad_prq_event(struct intel_iommu *iommu, 638 struct page_req_dsc *req, int result) 639 { 640 struct qi_desc desc; 641 642 pr_err("%s: Invalid page request: %08llx %08llx\n", 643 iommu->name, ((unsigned long long *)req)[0], 644 ((unsigned long long *)req)[1]); 645 646 /* 647 * Per VT-d spec. v3.0 ch7.7, system software must 648 * respond with page group response if private data 649 * is present (PDP) or last page in group (LPIG) bit 650 * is set. This is an additional VT-d feature beyond 651 * PCI ATS spec. 652 */ 653 if (!req->lpig && !req->priv_data_present) 654 return; 655 656 desc.qw0 = QI_PGRP_PASID(req->pasid) | 657 QI_PGRP_DID(req->rid) | 658 QI_PGRP_PASID_P(req->pasid_present) | 659 QI_PGRP_PDP(req->priv_data_present) | 660 QI_PGRP_RESP_CODE(result) | 661 QI_PGRP_RESP_TYPE; 662 desc.qw1 = QI_PGRP_IDX(req->prg_index) | 663 QI_PGRP_LPIG(req->lpig); 664 665 if (req->priv_data_present) { 666 desc.qw2 = req->priv_data[0]; 667 desc.qw3 = req->priv_data[1]; 668 } else { 669 desc.qw2 = 0; 670 desc.qw3 = 0; 671 } 672 673 qi_submit_sync(iommu, &desc, 1, 0); 674 } 675 676 static irqreturn_t prq_event_thread(int irq, void *d) 677 { 678 struct intel_iommu *iommu = d; 679 struct page_req_dsc *req; 680 int head, tail, handled; 681 struct pci_dev *pdev; 682 u64 address; 683 684 /* 685 * Clear PPR bit before reading head/tail registers, to ensure that 686 * we get a new interrupt if needed. 687 */ 688 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 689 690 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 691 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 692 handled = (head != tail); 693 while (head != tail) { 694 req = &iommu->prq[head / sizeof(*req)]; 695 address = (u64)req->addr << VTD_PAGE_SHIFT; 696 697 if (unlikely(!req->pasid_present)) { 698 pr_err("IOMMU: %s: Page request without PASID\n", 699 iommu->name); 700 bad_req: 701 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 702 goto prq_advance; 703 } 704 705 if (unlikely(!is_canonical_address(address))) { 706 pr_err("IOMMU: %s: Address is not canonical\n", 707 iommu->name); 708 goto bad_req; 709 } 710 711 if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) { 712 pr_err("IOMMU: %s: Page request in Privilege Mode\n", 713 iommu->name); 714 goto bad_req; 715 } 716 717 if (unlikely(req->exe_req && req->rd_req)) { 718 pr_err("IOMMU: %s: Execution request not supported\n", 719 iommu->name); 720 goto bad_req; 721 } 722 723 /* Drop Stop Marker message. No need for a response. */ 724 if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) 725 goto prq_advance; 726 727 pdev = pci_get_domain_bus_and_slot(iommu->segment, 728 PCI_BUS_NUM(req->rid), 729 req->rid & 0xff); 730 /* 731 * If prq is to be handled outside iommu driver via receiver of 732 * the fault notifiers, we skip the page response here. 733 */ 734 if (!pdev) 735 goto bad_req; 736 737 if (intel_svm_prq_report(iommu, &pdev->dev, req)) 738 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 739 else 740 trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, 741 req->priv_data[0], req->priv_data[1], 742 iommu->prq_seq_number++); 743 pci_dev_put(pdev); 744 prq_advance: 745 head = (head + sizeof(*req)) & PRQ_RING_MASK; 746 } 747 748 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 749 750 /* 751 * Clear the page request overflow bit and wake up all threads that 752 * are waiting for the completion of this handling. 753 */ 754 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 755 pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", 756 iommu->name); 757 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 758 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 759 if (head == tail) { 760 iopf_queue_discard_partial(iommu->iopf_queue); 761 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 762 pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", 763 iommu->name); 764 } 765 } 766 767 if (!completion_done(&iommu->prq_complete)) 768 complete(&iommu->prq_complete); 769 770 return IRQ_RETVAL(handled); 771 } 772 773 int intel_svm_page_response(struct device *dev, 774 struct iommu_fault_event *evt, 775 struct iommu_page_response *msg) 776 { 777 struct iommu_fault_page_request *prm; 778 struct intel_iommu *iommu; 779 bool private_present; 780 bool pasid_present; 781 bool last_page; 782 u8 bus, devfn; 783 int ret = 0; 784 u16 sid; 785 786 if (!dev || !dev_is_pci(dev)) 787 return -ENODEV; 788 789 iommu = device_to_iommu(dev, &bus, &devfn); 790 if (!iommu) 791 return -ENODEV; 792 793 if (!msg || !evt) 794 return -EINVAL; 795 796 prm = &evt->fault.prm; 797 sid = PCI_DEVID(bus, devfn); 798 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 799 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 800 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 801 802 if (!pasid_present) { 803 ret = -EINVAL; 804 goto out; 805 } 806 807 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 808 ret = -EINVAL; 809 goto out; 810 } 811 812 /* 813 * Per VT-d spec. v3.0 ch7.7, system software must respond 814 * with page group response if private data is present (PDP) 815 * or last page in group (LPIG) bit is set. This is an 816 * additional VT-d requirement beyond PCI ATS spec. 817 */ 818 if (last_page || private_present) { 819 struct qi_desc desc; 820 821 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 822 QI_PGRP_PASID_P(pasid_present) | 823 QI_PGRP_PDP(private_present) | 824 QI_PGRP_RESP_CODE(msg->code) | 825 QI_PGRP_RESP_TYPE; 826 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 827 desc.qw2 = 0; 828 desc.qw3 = 0; 829 830 if (private_present) { 831 desc.qw2 = prm->private_data[0]; 832 desc.qw3 = prm->private_data[1]; 833 } else if (prm->private_data[0]) { 834 dmar_latency_update(iommu, DMAR_LATENCY_PRQ, 835 ktime_to_ns(ktime_get()) - prm->private_data[0]); 836 } 837 838 qi_submit_sync(iommu, &desc, 1, 0); 839 } 840 out: 841 return ret; 842 } 843 844 void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid) 845 { 846 mutex_lock(&pasid_mutex); 847 intel_svm_unbind_mm(dev, pasid); 848 mutex_unlock(&pasid_mutex); 849 } 850 851 static int intel_svm_set_dev_pasid(struct iommu_domain *domain, 852 struct device *dev, ioasid_t pasid) 853 { 854 struct device_domain_info *info = dev_iommu_priv_get(dev); 855 struct intel_iommu *iommu = info->iommu; 856 struct mm_struct *mm = domain->mm; 857 struct iommu_sva *sva; 858 int ret = 0; 859 860 mutex_lock(&pasid_mutex); 861 sva = intel_svm_bind_mm(iommu, dev, mm); 862 if (IS_ERR(sva)) 863 ret = PTR_ERR(sva); 864 mutex_unlock(&pasid_mutex); 865 866 return ret; 867 } 868 869 static void intel_svm_domain_free(struct iommu_domain *domain) 870 { 871 kfree(to_dmar_domain(domain)); 872 } 873 874 static const struct iommu_domain_ops intel_svm_domain_ops = { 875 .set_dev_pasid = intel_svm_set_dev_pasid, 876 .free = intel_svm_domain_free 877 }; 878 879 struct iommu_domain *intel_svm_domain_alloc(void) 880 { 881 struct dmar_domain *domain; 882 883 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 884 if (!domain) 885 return NULL; 886 domain->domain.ops = &intel_svm_domain_ops; 887 888 return &domain->domain; 889 } 890