1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/slab.h> 12 #include <linux/rculist.h> 13 #include <linux/pci.h> 14 #include <linux/pci-ats.h> 15 #include <linux/dmar.h> 16 #include <linux/interrupt.h> 17 #include <linux/mm_types.h> 18 #include <linux/xarray.h> 19 #include <asm/page.h> 20 #include <asm/fpu/api.h> 21 22 #include "iommu.h" 23 #include "pasid.h" 24 #include "perf.h" 25 #include "../iommu-sva.h" 26 #include "trace.h" 27 28 static irqreturn_t prq_event_thread(int irq, void *d); 29 static void intel_svm_drain_prq(struct device *dev, u32 pasid); 30 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 31 32 static DEFINE_XARRAY_ALLOC(pasid_private_array); 33 static int pasid_private_add(ioasid_t pasid, void *priv) 34 { 35 return xa_alloc(&pasid_private_array, &pasid, priv, 36 XA_LIMIT(pasid, pasid), GFP_ATOMIC); 37 } 38 39 static void pasid_private_remove(ioasid_t pasid) 40 { 41 xa_erase(&pasid_private_array, pasid); 42 } 43 44 static void *pasid_private_find(ioasid_t pasid) 45 { 46 return xa_load(&pasid_private_array, pasid); 47 } 48 49 static struct intel_svm_dev * 50 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) 51 { 52 struct intel_svm_dev *sdev = NULL, *t; 53 54 rcu_read_lock(); 55 list_for_each_entry_rcu(t, &svm->devs, list) { 56 if (t->dev == dev) { 57 sdev = t; 58 break; 59 } 60 } 61 rcu_read_unlock(); 62 63 return sdev; 64 } 65 66 int intel_svm_enable_prq(struct intel_iommu *iommu) 67 { 68 struct iopf_queue *iopfq; 69 struct page *pages; 70 int irq, ret; 71 72 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 73 if (!pages) { 74 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 75 iommu->name); 76 return -ENOMEM; 77 } 78 iommu->prq = page_address(pages); 79 80 irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); 81 if (irq <= 0) { 82 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 83 iommu->name); 84 ret = -EINVAL; 85 goto free_prq; 86 } 87 iommu->pr_irq = irq; 88 89 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), 90 "dmar%d-iopfq", iommu->seq_id); 91 iopfq = iopf_queue_alloc(iommu->iopfq_name); 92 if (!iopfq) { 93 pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name); 94 ret = -ENOMEM; 95 goto free_hwirq; 96 } 97 iommu->iopf_queue = iopfq; 98 99 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 100 101 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 102 iommu->prq_name, iommu); 103 if (ret) { 104 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 105 iommu->name); 106 goto free_iopfq; 107 } 108 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 109 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 110 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 111 112 init_completion(&iommu->prq_complete); 113 114 return 0; 115 116 free_iopfq: 117 iopf_queue_free(iommu->iopf_queue); 118 iommu->iopf_queue = NULL; 119 free_hwirq: 120 dmar_free_hwirq(irq); 121 iommu->pr_irq = 0; 122 free_prq: 123 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 124 iommu->prq = NULL; 125 126 return ret; 127 } 128 129 int intel_svm_finish_prq(struct intel_iommu *iommu) 130 { 131 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 132 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 133 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 134 135 if (iommu->pr_irq) { 136 free_irq(iommu->pr_irq, iommu); 137 dmar_free_hwirq(iommu->pr_irq); 138 iommu->pr_irq = 0; 139 } 140 141 if (iommu->iopf_queue) { 142 iopf_queue_free(iommu->iopf_queue); 143 iommu->iopf_queue = NULL; 144 } 145 146 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 147 iommu->prq = NULL; 148 149 return 0; 150 } 151 152 void intel_svm_check(struct intel_iommu *iommu) 153 { 154 if (!pasid_supported(iommu)) 155 return; 156 157 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 158 !cap_fl1gp_support(iommu->cap)) { 159 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 160 iommu->name); 161 return; 162 } 163 164 if (cpu_feature_enabled(X86_FEATURE_LA57) && 165 !cap_fl5lp_support(iommu->cap)) { 166 pr_err("%s SVM disabled, incompatible paging mode\n", 167 iommu->name); 168 return; 169 } 170 171 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 172 } 173 174 static void __flush_svm_range_dev(struct intel_svm *svm, 175 struct intel_svm_dev *sdev, 176 unsigned long address, 177 unsigned long pages, int ih) 178 { 179 struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); 180 181 if (WARN_ON(!pages)) 182 return; 183 184 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); 185 if (info->ats_enabled) { 186 qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, 187 svm->pasid, sdev->qdep, address, 188 order_base_2(pages)); 189 quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), 190 svm->pasid, sdev->qdep); 191 } 192 } 193 194 static void intel_flush_svm_range_dev(struct intel_svm *svm, 195 struct intel_svm_dev *sdev, 196 unsigned long address, 197 unsigned long pages, int ih) 198 { 199 unsigned long shift = ilog2(__roundup_pow_of_two(pages)); 200 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); 201 unsigned long start = ALIGN_DOWN(address, align); 202 unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); 203 204 while (start < end) { 205 __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); 206 start += align; 207 } 208 } 209 210 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 211 unsigned long pages, int ih) 212 { 213 struct intel_svm_dev *sdev; 214 215 rcu_read_lock(); 216 list_for_each_entry_rcu(sdev, &svm->devs, list) 217 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 218 rcu_read_unlock(); 219 } 220 221 /* Pages have been freed at this point */ 222 static void intel_invalidate_range(struct mmu_notifier *mn, 223 struct mm_struct *mm, 224 unsigned long start, unsigned long end) 225 { 226 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 227 228 intel_flush_svm_range(svm, start, 229 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 230 } 231 232 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 233 { 234 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 235 struct intel_svm_dev *sdev; 236 237 /* This might end up being called from exit_mmap(), *before* the page 238 * tables are cleared. And __mmu_notifier_release() will delete us from 239 * the list of notifiers so that our invalidate_range() callback doesn't 240 * get called when the page tables are cleared. So we need to protect 241 * against hardware accessing those page tables. 242 * 243 * We do it by clearing the entry in the PASID table and then flushing 244 * the IOTLB and the PASID table caches. This might upset hardware; 245 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 246 * page) so that we end up taking a fault that the hardware really 247 * *has* to handle gracefully without affecting other processes. 248 */ 249 rcu_read_lock(); 250 list_for_each_entry_rcu(sdev, &svm->devs, list) 251 intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, 252 svm->pasid, true); 253 rcu_read_unlock(); 254 255 } 256 257 static const struct mmu_notifier_ops intel_mmuops = { 258 .release = intel_mm_release, 259 .invalidate_range = intel_invalidate_range, 260 }; 261 262 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 263 struct intel_svm **rsvm, 264 struct intel_svm_dev **rsdev) 265 { 266 struct intel_svm_dev *sdev = NULL; 267 struct intel_svm *svm; 268 269 if (pasid == IOMMU_PASID_INVALID || pasid >= PASID_MAX) 270 return -EINVAL; 271 272 svm = pasid_private_find(pasid); 273 if (IS_ERR(svm)) 274 return PTR_ERR(svm); 275 276 if (!svm) 277 goto out; 278 279 /* 280 * If we found svm for the PASID, there must be at least one device 281 * bond. 282 */ 283 if (WARN_ON(list_empty(&svm->devs))) 284 return -EINVAL; 285 sdev = svm_lookup_device_by_dev(svm, dev); 286 287 out: 288 *rsvm = svm; 289 *rsdev = sdev; 290 291 return 0; 292 } 293 294 static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, 295 struct mm_struct *mm) 296 { 297 struct device_domain_info *info = dev_iommu_priv_get(dev); 298 struct intel_svm_dev *sdev; 299 struct intel_svm *svm; 300 unsigned long sflags; 301 int ret = 0; 302 303 svm = pasid_private_find(mm->pasid); 304 if (!svm) { 305 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 306 if (!svm) 307 return -ENOMEM; 308 309 svm->pasid = mm->pasid; 310 svm->mm = mm; 311 INIT_LIST_HEAD_RCU(&svm->devs); 312 313 svm->notifier.ops = &intel_mmuops; 314 ret = mmu_notifier_register(&svm->notifier, mm); 315 if (ret) { 316 kfree(svm); 317 return ret; 318 } 319 320 ret = pasid_private_add(svm->pasid, svm); 321 if (ret) { 322 mmu_notifier_unregister(&svm->notifier, mm); 323 kfree(svm); 324 return ret; 325 } 326 } 327 328 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 329 if (!sdev) { 330 ret = -ENOMEM; 331 goto free_svm; 332 } 333 334 sdev->dev = dev; 335 sdev->iommu = iommu; 336 sdev->did = FLPT_DEFAULT_DID; 337 sdev->sid = PCI_DEVID(info->bus, info->devfn); 338 init_rcu_head(&sdev->rcu); 339 if (info->ats_enabled) { 340 sdev->qdep = info->ats_qdep; 341 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 342 sdev->qdep = 0; 343 } 344 345 /* Setup the pasid table: */ 346 sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 347 ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid, 348 FLPT_DEFAULT_DID, sflags); 349 if (ret) 350 goto free_sdev; 351 352 list_add_rcu(&sdev->list, &svm->devs); 353 354 return 0; 355 356 free_sdev: 357 kfree(sdev); 358 free_svm: 359 if (list_empty(&svm->devs)) { 360 mmu_notifier_unregister(&svm->notifier, mm); 361 pasid_private_remove(mm->pasid); 362 kfree(svm); 363 } 364 365 return ret; 366 } 367 368 void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) 369 { 370 struct intel_svm_dev *sdev; 371 struct intel_iommu *iommu; 372 struct intel_svm *svm; 373 struct mm_struct *mm; 374 375 iommu = device_to_iommu(dev, NULL, NULL); 376 if (!iommu) 377 return; 378 379 if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev)) 380 return; 381 mm = svm->mm; 382 383 if (sdev) { 384 list_del_rcu(&sdev->list); 385 /* 386 * Flush the PASID cache and IOTLB for this device. 387 * Note that we do depend on the hardware *not* using 388 * the PASID any more. Just as we depend on other 389 * devices never using PASIDs that they have no right 390 * to use. We have a *shared* PASID table, because it's 391 * large and has to be physically contiguous. So it's 392 * hard to be as defensive as we might like. 393 */ 394 intel_pasid_tear_down_entry(iommu, dev, svm->pasid, false); 395 intel_svm_drain_prq(dev, svm->pasid); 396 kfree_rcu(sdev, rcu); 397 398 if (list_empty(&svm->devs)) { 399 if (svm->notifier.ops) 400 mmu_notifier_unregister(&svm->notifier, mm); 401 pasid_private_remove(svm->pasid); 402 /* 403 * We mandate that no page faults may be outstanding 404 * for the PASID when intel_svm_unbind_mm() is called. 405 * If that is not obeyed, subtle errors will happen. 406 * Let's make them less subtle... 407 */ 408 memset(svm, 0x6b, sizeof(*svm)); 409 kfree(svm); 410 } 411 } 412 } 413 414 /* Page request queue descriptor */ 415 struct page_req_dsc { 416 union { 417 struct { 418 u64 type:8; 419 u64 pasid_present:1; 420 u64 priv_data_present:1; 421 u64 rsvd:6; 422 u64 rid:16; 423 u64 pasid:20; 424 u64 exe_req:1; 425 u64 pm_req:1; 426 u64 rsvd2:10; 427 }; 428 u64 qw_0; 429 }; 430 union { 431 struct { 432 u64 rd_req:1; 433 u64 wr_req:1; 434 u64 lpig:1; 435 u64 prg_index:9; 436 u64 addr:52; 437 }; 438 u64 qw_1; 439 }; 440 u64 priv_data[2]; 441 }; 442 443 static bool is_canonical_address(u64 addr) 444 { 445 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 446 long saddr = (long) addr; 447 448 return (((saddr << shift) >> shift) == saddr); 449 } 450 451 /** 452 * intel_svm_drain_prq - Drain page requests and responses for a pasid 453 * @dev: target device 454 * @pasid: pasid for draining 455 * 456 * Drain all pending page requests and responses related to @pasid in both 457 * software and hardware. This is supposed to be called after the device 458 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 459 * and DevTLB have been invalidated. 460 * 461 * It waits until all pending page requests for @pasid in the page fault 462 * queue are completed by the prq handling thread. Then follow the steps 463 * described in VT-d spec CH7.10 to drain all page requests and page 464 * responses pending in the hardware. 465 */ 466 static void intel_svm_drain_prq(struct device *dev, u32 pasid) 467 { 468 struct device_domain_info *info; 469 struct dmar_domain *domain; 470 struct intel_iommu *iommu; 471 struct qi_desc desc[3]; 472 struct pci_dev *pdev; 473 int head, tail; 474 u16 sid, did; 475 int qdep; 476 477 info = dev_iommu_priv_get(dev); 478 if (WARN_ON(!info || !dev_is_pci(dev))) 479 return; 480 481 if (!info->pri_enabled) 482 return; 483 484 iommu = info->iommu; 485 domain = info->domain; 486 pdev = to_pci_dev(dev); 487 sid = PCI_DEVID(info->bus, info->devfn); 488 did = domain_id_iommu(domain, iommu); 489 qdep = pci_ats_queue_depth(pdev); 490 491 /* 492 * Check and wait until all pending page requests in the queue are 493 * handled by the prq handling thread. 494 */ 495 prq_retry: 496 reinit_completion(&iommu->prq_complete); 497 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 498 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 499 while (head != tail) { 500 struct page_req_dsc *req; 501 502 req = &iommu->prq[head / sizeof(*req)]; 503 if (!req->pasid_present || req->pasid != pasid) { 504 head = (head + sizeof(*req)) & PRQ_RING_MASK; 505 continue; 506 } 507 508 wait_for_completion(&iommu->prq_complete); 509 goto prq_retry; 510 } 511 512 iopf_queue_flush_dev(dev); 513 514 /* 515 * Perform steps described in VT-d spec CH7.10 to drain page 516 * requests and responses in hardware. 517 */ 518 memset(desc, 0, sizeof(desc)); 519 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 520 QI_IWD_FENCE | 521 QI_IWD_TYPE; 522 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 523 QI_EIOTLB_DID(did) | 524 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 525 QI_EIOTLB_TYPE; 526 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 527 QI_DEV_EIOTLB_SID(sid) | 528 QI_DEV_EIOTLB_QDEP(qdep) | 529 QI_DEIOTLB_TYPE | 530 QI_DEV_IOTLB_PFSID(info->pfsid); 531 qi_retry: 532 reinit_completion(&iommu->prq_complete); 533 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 534 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 535 wait_for_completion(&iommu->prq_complete); 536 goto qi_retry; 537 } 538 } 539 540 static int prq_to_iommu_prot(struct page_req_dsc *req) 541 { 542 int prot = 0; 543 544 if (req->rd_req) 545 prot |= IOMMU_FAULT_PERM_READ; 546 if (req->wr_req) 547 prot |= IOMMU_FAULT_PERM_WRITE; 548 if (req->exe_req) 549 prot |= IOMMU_FAULT_PERM_EXEC; 550 if (req->pm_req) 551 prot |= IOMMU_FAULT_PERM_PRIV; 552 553 return prot; 554 } 555 556 static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, 557 struct page_req_dsc *desc) 558 { 559 struct iommu_fault_event event; 560 561 if (!dev || !dev_is_pci(dev)) 562 return -ENODEV; 563 564 /* Fill in event data for device specific processing */ 565 memset(&event, 0, sizeof(struct iommu_fault_event)); 566 event.fault.type = IOMMU_FAULT_PAGE_REQ; 567 event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; 568 event.fault.prm.pasid = desc->pasid; 569 event.fault.prm.grpid = desc->prg_index; 570 event.fault.prm.perm = prq_to_iommu_prot(desc); 571 572 if (desc->lpig) 573 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 574 if (desc->pasid_present) { 575 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 576 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 577 } 578 if (desc->priv_data_present) { 579 /* 580 * Set last page in group bit if private data is present, 581 * page response is required as it does for LPIG. 582 * iommu_report_device_fault() doesn't understand this vendor 583 * specific requirement thus we set last_page as a workaround. 584 */ 585 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 586 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 587 event.fault.prm.private_data[0] = desc->priv_data[0]; 588 event.fault.prm.private_data[1] = desc->priv_data[1]; 589 } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { 590 /* 591 * If the private data fields are not used by hardware, use it 592 * to monitor the prq handle latency. 593 */ 594 event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); 595 } 596 597 return iommu_report_device_fault(dev, &event); 598 } 599 600 static void handle_bad_prq_event(struct intel_iommu *iommu, 601 struct page_req_dsc *req, int result) 602 { 603 struct qi_desc desc; 604 605 pr_err("%s: Invalid page request: %08llx %08llx\n", 606 iommu->name, ((unsigned long long *)req)[0], 607 ((unsigned long long *)req)[1]); 608 609 /* 610 * Per VT-d spec. v3.0 ch7.7, system software must 611 * respond with page group response if private data 612 * is present (PDP) or last page in group (LPIG) bit 613 * is set. This is an additional VT-d feature beyond 614 * PCI ATS spec. 615 */ 616 if (!req->lpig && !req->priv_data_present) 617 return; 618 619 desc.qw0 = QI_PGRP_PASID(req->pasid) | 620 QI_PGRP_DID(req->rid) | 621 QI_PGRP_PASID_P(req->pasid_present) | 622 QI_PGRP_PDP(req->priv_data_present) | 623 QI_PGRP_RESP_CODE(result) | 624 QI_PGRP_RESP_TYPE; 625 desc.qw1 = QI_PGRP_IDX(req->prg_index) | 626 QI_PGRP_LPIG(req->lpig); 627 628 if (req->priv_data_present) { 629 desc.qw2 = req->priv_data[0]; 630 desc.qw3 = req->priv_data[1]; 631 } else { 632 desc.qw2 = 0; 633 desc.qw3 = 0; 634 } 635 636 qi_submit_sync(iommu, &desc, 1, 0); 637 } 638 639 static irqreturn_t prq_event_thread(int irq, void *d) 640 { 641 struct intel_iommu *iommu = d; 642 struct page_req_dsc *req; 643 int head, tail, handled; 644 struct pci_dev *pdev; 645 u64 address; 646 647 /* 648 * Clear PPR bit before reading head/tail registers, to ensure that 649 * we get a new interrupt if needed. 650 */ 651 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 652 653 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 654 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 655 handled = (head != tail); 656 while (head != tail) { 657 req = &iommu->prq[head / sizeof(*req)]; 658 address = (u64)req->addr << VTD_PAGE_SHIFT; 659 660 if (unlikely(!req->pasid_present)) { 661 pr_err("IOMMU: %s: Page request without PASID\n", 662 iommu->name); 663 bad_req: 664 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 665 goto prq_advance; 666 } 667 668 if (unlikely(!is_canonical_address(address))) { 669 pr_err("IOMMU: %s: Address is not canonical\n", 670 iommu->name); 671 goto bad_req; 672 } 673 674 if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) { 675 pr_err("IOMMU: %s: Page request in Privilege Mode\n", 676 iommu->name); 677 goto bad_req; 678 } 679 680 if (unlikely(req->exe_req && req->rd_req)) { 681 pr_err("IOMMU: %s: Execution request not supported\n", 682 iommu->name); 683 goto bad_req; 684 } 685 686 /* Drop Stop Marker message. No need for a response. */ 687 if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) 688 goto prq_advance; 689 690 pdev = pci_get_domain_bus_and_slot(iommu->segment, 691 PCI_BUS_NUM(req->rid), 692 req->rid & 0xff); 693 /* 694 * If prq is to be handled outside iommu driver via receiver of 695 * the fault notifiers, we skip the page response here. 696 */ 697 if (!pdev) 698 goto bad_req; 699 700 if (intel_svm_prq_report(iommu, &pdev->dev, req)) 701 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 702 else 703 trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, 704 req->priv_data[0], req->priv_data[1], 705 iommu->prq_seq_number++); 706 pci_dev_put(pdev); 707 prq_advance: 708 head = (head + sizeof(*req)) & PRQ_RING_MASK; 709 } 710 711 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 712 713 /* 714 * Clear the page request overflow bit and wake up all threads that 715 * are waiting for the completion of this handling. 716 */ 717 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 718 pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", 719 iommu->name); 720 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 721 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 722 if (head == tail) { 723 iopf_queue_discard_partial(iommu->iopf_queue); 724 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 725 pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", 726 iommu->name); 727 } 728 } 729 730 if (!completion_done(&iommu->prq_complete)) 731 complete(&iommu->prq_complete); 732 733 return IRQ_RETVAL(handled); 734 } 735 736 int intel_svm_page_response(struct device *dev, 737 struct iommu_fault_event *evt, 738 struct iommu_page_response *msg) 739 { 740 struct iommu_fault_page_request *prm; 741 struct intel_iommu *iommu; 742 bool private_present; 743 bool pasid_present; 744 bool last_page; 745 u8 bus, devfn; 746 int ret = 0; 747 u16 sid; 748 749 if (!dev || !dev_is_pci(dev)) 750 return -ENODEV; 751 752 iommu = device_to_iommu(dev, &bus, &devfn); 753 if (!iommu) 754 return -ENODEV; 755 756 if (!msg || !evt) 757 return -EINVAL; 758 759 prm = &evt->fault.prm; 760 sid = PCI_DEVID(bus, devfn); 761 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 762 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 763 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 764 765 if (!pasid_present) { 766 ret = -EINVAL; 767 goto out; 768 } 769 770 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 771 ret = -EINVAL; 772 goto out; 773 } 774 775 /* 776 * Per VT-d spec. v3.0 ch7.7, system software must respond 777 * with page group response if private data is present (PDP) 778 * or last page in group (LPIG) bit is set. This is an 779 * additional VT-d requirement beyond PCI ATS spec. 780 */ 781 if (last_page || private_present) { 782 struct qi_desc desc; 783 784 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 785 QI_PGRP_PASID_P(pasid_present) | 786 QI_PGRP_PDP(private_present) | 787 QI_PGRP_RESP_CODE(msg->code) | 788 QI_PGRP_RESP_TYPE; 789 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 790 desc.qw2 = 0; 791 desc.qw3 = 0; 792 793 if (private_present) { 794 desc.qw2 = prm->private_data[0]; 795 desc.qw3 = prm->private_data[1]; 796 } else if (prm->private_data[0]) { 797 dmar_latency_update(iommu, DMAR_LATENCY_PRQ, 798 ktime_to_ns(ktime_get()) - prm->private_data[0]); 799 } 800 801 qi_submit_sync(iommu, &desc, 1, 0); 802 } 803 out: 804 return ret; 805 } 806 807 static int intel_svm_set_dev_pasid(struct iommu_domain *domain, 808 struct device *dev, ioasid_t pasid) 809 { 810 struct device_domain_info *info = dev_iommu_priv_get(dev); 811 struct intel_iommu *iommu = info->iommu; 812 struct mm_struct *mm = domain->mm; 813 814 return intel_svm_bind_mm(iommu, dev, mm); 815 } 816 817 static void intel_svm_domain_free(struct iommu_domain *domain) 818 { 819 kfree(to_dmar_domain(domain)); 820 } 821 822 static const struct iommu_domain_ops intel_svm_domain_ops = { 823 .set_dev_pasid = intel_svm_set_dev_pasid, 824 .free = intel_svm_domain_free 825 }; 826 827 struct iommu_domain *intel_svm_domain_alloc(void) 828 { 829 struct dmar_domain *domain; 830 831 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 832 if (!domain) 833 return NULL; 834 domain->domain.ops = &intel_svm_domain_ops; 835 836 return &domain->domain; 837 } 838