1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO: IOMMU DMA mapping support for Type1 IOMMU 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 * 12 * We arbitrarily define a Type1 IOMMU as one matching the below code. 13 * It could be called the x86 IOMMU as it's designed for AMD-Vi & Intel 14 * VT-d, but that makes it harder to re-use as theoretically anyone 15 * implementing a similar IOMMU could make use of this. We expect the 16 * IOMMU to support the IOMMU API and have few to no restrictions around 17 * the IOVA range that can be mapped. The Type1 IOMMU is currently 18 * optimized for relatively static mappings of a userspace process with 19 * userpsace pages pinned into memory. We also assume devices and IOMMU 20 * domains are PCI based as the IOMMU API is still centered around a 21 * device/bus interface rather than a group interface. 22 */ 23 24 #include <linux/compat.h> 25 #include <linux/device.h> 26 #include <linux/fs.h> 27 #include <linux/iommu.h> 28 #include <linux/module.h> 29 #include <linux/mm.h> 30 #include <linux/kthread.h> 31 #include <linux/rbtree.h> 32 #include <linux/sched/signal.h> 33 #include <linux/sched/mm.h> 34 #include <linux/slab.h> 35 #include <linux/uaccess.h> 36 #include <linux/vfio.h> 37 #include <linux/workqueue.h> 38 #include <linux/mdev.h> 39 #include <linux/notifier.h> 40 #include <linux/dma-iommu.h> 41 #include <linux/irqdomain.h> 42 43 #define DRIVER_VERSION "0.2" 44 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 45 #define DRIVER_DESC "Type1 IOMMU driver for VFIO" 46 47 static bool allow_unsafe_interrupts; 48 module_param_named(allow_unsafe_interrupts, 49 allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR); 50 MODULE_PARM_DESC(allow_unsafe_interrupts, 51 "Enable VFIO IOMMU support for on platforms without interrupt remapping support."); 52 53 static bool disable_hugepages; 54 module_param_named(disable_hugepages, 55 disable_hugepages, bool, S_IRUGO | S_IWUSR); 56 MODULE_PARM_DESC(disable_hugepages, 57 "Disable VFIO IOMMU support for IOMMU hugepages."); 58 59 static unsigned int dma_entry_limit __read_mostly = U16_MAX; 60 module_param_named(dma_entry_limit, dma_entry_limit, uint, 0644); 61 MODULE_PARM_DESC(dma_entry_limit, 62 "Maximum number of user DMA mappings per container (65535)."); 63 64 struct vfio_iommu { 65 struct list_head domain_list; 66 struct list_head iova_list; 67 struct vfio_domain *external_domain; /* domain for external user */ 68 struct mutex lock; 69 struct rb_root dma_list; 70 struct blocking_notifier_head notifier; 71 unsigned int dma_avail; 72 uint64_t pgsize_bitmap; 73 bool v2; 74 bool nesting; 75 bool dirty_page_tracking; 76 bool pinned_page_dirty_scope; 77 }; 78 79 struct vfio_domain { 80 struct iommu_domain *domain; 81 struct list_head next; 82 struct list_head group_list; 83 int prot; /* IOMMU_CACHE */ 84 bool fgsp; /* Fine-grained super pages */ 85 }; 86 87 struct vfio_dma { 88 struct rb_node node; 89 dma_addr_t iova; /* Device address */ 90 unsigned long vaddr; /* Process virtual addr */ 91 size_t size; /* Map size (bytes) */ 92 int prot; /* IOMMU_READ/WRITE */ 93 bool iommu_mapped; 94 bool lock_cap; /* capable(CAP_IPC_LOCK) */ 95 struct task_struct *task; 96 struct rb_root pfn_list; /* Ex-user pinned pfn list */ 97 unsigned long *bitmap; 98 }; 99 100 struct vfio_group { 101 struct iommu_group *iommu_group; 102 struct list_head next; 103 bool mdev_group; /* An mdev group */ 104 bool pinned_page_dirty_scope; 105 }; 106 107 struct vfio_iova { 108 struct list_head list; 109 dma_addr_t start; 110 dma_addr_t end; 111 }; 112 113 /* 114 * Guest RAM pinning working set or DMA target 115 */ 116 struct vfio_pfn { 117 struct rb_node node; 118 dma_addr_t iova; /* Device address */ 119 unsigned long pfn; /* Host pfn */ 120 unsigned int ref_count; 121 }; 122 123 struct vfio_regions { 124 struct list_head list; 125 dma_addr_t iova; 126 phys_addr_t phys; 127 size_t len; 128 }; 129 130 #define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \ 131 (!list_empty(&iommu->domain_list)) 132 133 #define DIRTY_BITMAP_BYTES(n) (ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE) 134 135 /* 136 * Input argument of number of bits to bitmap_set() is unsigned integer, which 137 * further casts to signed integer for unaligned multi-bit operation, 138 * __bitmap_set(). 139 * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte, 140 * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page 141 * system. 142 */ 143 #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) 144 #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) 145 146 static int put_pfn(unsigned long pfn, int prot); 147 148 static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, 149 struct iommu_group *iommu_group); 150 151 static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); 152 /* 153 * This code handles mapping and unmapping of user data buffers 154 * into DMA'ble space using the IOMMU 155 */ 156 157 static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, 158 dma_addr_t start, size_t size) 159 { 160 struct rb_node *node = iommu->dma_list.rb_node; 161 162 while (node) { 163 struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); 164 165 if (start + size <= dma->iova) 166 node = node->rb_left; 167 else if (start >= dma->iova + dma->size) 168 node = node->rb_right; 169 else 170 return dma; 171 } 172 173 return NULL; 174 } 175 176 static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) 177 { 178 struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; 179 struct vfio_dma *dma; 180 181 while (*link) { 182 parent = *link; 183 dma = rb_entry(parent, struct vfio_dma, node); 184 185 if (new->iova + new->size <= dma->iova) 186 link = &(*link)->rb_left; 187 else 188 link = &(*link)->rb_right; 189 } 190 191 rb_link_node(&new->node, parent, link); 192 rb_insert_color(&new->node, &iommu->dma_list); 193 } 194 195 static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) 196 { 197 rb_erase(&old->node, &iommu->dma_list); 198 } 199 200 201 static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) 202 { 203 uint64_t npages = dma->size / pgsize; 204 205 if (npages > DIRTY_BITMAP_PAGES_MAX) 206 return -EINVAL; 207 208 /* 209 * Allocate extra 64 bits that are used to calculate shift required for 210 * bitmap_shift_left() to manipulate and club unaligned number of pages 211 * in adjacent vfio_dma ranges. 212 */ 213 dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages) + sizeof(u64), 214 GFP_KERNEL); 215 if (!dma->bitmap) 216 return -ENOMEM; 217 218 return 0; 219 } 220 221 static void vfio_dma_bitmap_free(struct vfio_dma *dma) 222 { 223 kfree(dma->bitmap); 224 dma->bitmap = NULL; 225 } 226 227 static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) 228 { 229 struct rb_node *p; 230 unsigned long pgshift = __ffs(pgsize); 231 232 for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { 233 struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); 234 235 bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); 236 } 237 } 238 239 static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) 240 { 241 struct rb_node *n; 242 243 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 244 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 245 int ret; 246 247 ret = vfio_dma_bitmap_alloc(dma, pgsize); 248 if (ret) { 249 struct rb_node *p; 250 251 for (p = rb_prev(n); p; p = rb_prev(p)) { 252 struct vfio_dma *dma = rb_entry(n, 253 struct vfio_dma, node); 254 255 vfio_dma_bitmap_free(dma); 256 } 257 return ret; 258 } 259 vfio_dma_populate_bitmap(dma, pgsize); 260 } 261 return 0; 262 } 263 264 static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu) 265 { 266 struct rb_node *n; 267 268 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 269 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 270 271 vfio_dma_bitmap_free(dma); 272 } 273 } 274 275 /* 276 * Helper Functions for host iova-pfn list 277 */ 278 static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova) 279 { 280 struct vfio_pfn *vpfn; 281 struct rb_node *node = dma->pfn_list.rb_node; 282 283 while (node) { 284 vpfn = rb_entry(node, struct vfio_pfn, node); 285 286 if (iova < vpfn->iova) 287 node = node->rb_left; 288 else if (iova > vpfn->iova) 289 node = node->rb_right; 290 else 291 return vpfn; 292 } 293 return NULL; 294 } 295 296 static void vfio_link_pfn(struct vfio_dma *dma, 297 struct vfio_pfn *new) 298 { 299 struct rb_node **link, *parent = NULL; 300 struct vfio_pfn *vpfn; 301 302 link = &dma->pfn_list.rb_node; 303 while (*link) { 304 parent = *link; 305 vpfn = rb_entry(parent, struct vfio_pfn, node); 306 307 if (new->iova < vpfn->iova) 308 link = &(*link)->rb_left; 309 else 310 link = &(*link)->rb_right; 311 } 312 313 rb_link_node(&new->node, parent, link); 314 rb_insert_color(&new->node, &dma->pfn_list); 315 } 316 317 static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) 318 { 319 rb_erase(&old->node, &dma->pfn_list); 320 } 321 322 static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, 323 unsigned long pfn) 324 { 325 struct vfio_pfn *vpfn; 326 327 vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL); 328 if (!vpfn) 329 return -ENOMEM; 330 331 vpfn->iova = iova; 332 vpfn->pfn = pfn; 333 vpfn->ref_count = 1; 334 vfio_link_pfn(dma, vpfn); 335 return 0; 336 } 337 338 static void vfio_remove_from_pfn_list(struct vfio_dma *dma, 339 struct vfio_pfn *vpfn) 340 { 341 vfio_unlink_pfn(dma, vpfn); 342 kfree(vpfn); 343 } 344 345 static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma, 346 unsigned long iova) 347 { 348 struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova); 349 350 if (vpfn) 351 vpfn->ref_count++; 352 return vpfn; 353 } 354 355 static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) 356 { 357 int ret = 0; 358 359 vpfn->ref_count--; 360 if (!vpfn->ref_count) { 361 ret = put_pfn(vpfn->pfn, dma->prot); 362 vfio_remove_from_pfn_list(dma, vpfn); 363 } 364 return ret; 365 } 366 367 static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) 368 { 369 struct mm_struct *mm; 370 int ret; 371 372 if (!npage) 373 return 0; 374 375 mm = async ? get_task_mm(dma->task) : dma->task->mm; 376 if (!mm) 377 return -ESRCH; /* process exited */ 378 379 ret = mmap_write_lock_killable(mm); 380 if (!ret) { 381 ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task, 382 dma->lock_cap); 383 mmap_write_unlock(mm); 384 } 385 386 if (async) 387 mmput(mm); 388 389 return ret; 390 } 391 392 /* 393 * Some mappings aren't backed by a struct page, for example an mmap'd 394 * MMIO range for our own or another device. These use a different 395 * pfn conversion and shouldn't be tracked as locked pages. 396 * For compound pages, any driver that sets the reserved bit in head 397 * page needs to set the reserved bit in all subpages to be safe. 398 */ 399 static bool is_invalid_reserved_pfn(unsigned long pfn) 400 { 401 if (pfn_valid(pfn)) 402 return PageReserved(pfn_to_page(pfn)); 403 404 return true; 405 } 406 407 static int put_pfn(unsigned long pfn, int prot) 408 { 409 if (!is_invalid_reserved_pfn(pfn)) { 410 struct page *page = pfn_to_page(pfn); 411 412 unpin_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE); 413 return 1; 414 } 415 return 0; 416 } 417 418 static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, 419 unsigned long vaddr, unsigned long *pfn, 420 bool write_fault) 421 { 422 int ret; 423 424 ret = follow_pfn(vma, vaddr, pfn); 425 if (ret) { 426 bool unlocked = false; 427 428 ret = fixup_user_fault(mm, vaddr, 429 FAULT_FLAG_REMOTE | 430 (write_fault ? FAULT_FLAG_WRITE : 0), 431 &unlocked); 432 if (unlocked) 433 return -EAGAIN; 434 435 if (ret) 436 return ret; 437 438 ret = follow_pfn(vma, vaddr, pfn); 439 } 440 441 return ret; 442 } 443 444 static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, 445 int prot, unsigned long *pfn) 446 { 447 struct page *page[1]; 448 struct vm_area_struct *vma; 449 unsigned int flags = 0; 450 int ret; 451 452 if (prot & IOMMU_WRITE) 453 flags |= FOLL_WRITE; 454 455 mmap_read_lock(mm); 456 ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM, 457 page, NULL, NULL); 458 if (ret == 1) { 459 *pfn = page_to_pfn(page[0]); 460 ret = 0; 461 goto done; 462 } 463 464 vaddr = untagged_addr(vaddr); 465 466 retry: 467 vma = find_vma_intersection(mm, vaddr, vaddr + 1); 468 469 if (vma && vma->vm_flags & VM_PFNMAP) { 470 ret = follow_fault_pfn(vma, mm, vaddr, pfn, prot & IOMMU_WRITE); 471 if (ret == -EAGAIN) 472 goto retry; 473 474 if (!ret && !is_invalid_reserved_pfn(*pfn)) 475 ret = -EFAULT; 476 } 477 done: 478 mmap_read_unlock(mm); 479 return ret; 480 } 481 482 /* 483 * Attempt to pin pages. We really don't want to track all the pfns and 484 * the iommu can only map chunks of consecutive pfns anyway, so get the 485 * first page and all consecutive pages with the same locking. 486 */ 487 static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 488 long npage, unsigned long *pfn_base, 489 unsigned long limit) 490 { 491 unsigned long pfn = 0; 492 long ret, pinned = 0, lock_acct = 0; 493 bool rsvd; 494 dma_addr_t iova = vaddr - dma->vaddr + dma->iova; 495 496 /* This code path is only user initiated */ 497 if (!current->mm) 498 return -ENODEV; 499 500 ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); 501 if (ret) 502 return ret; 503 504 pinned++; 505 rsvd = is_invalid_reserved_pfn(*pfn_base); 506 507 /* 508 * Reserved pages aren't counted against the user, externally pinned 509 * pages are already counted against the user. 510 */ 511 if (!rsvd && !vfio_find_vpfn(dma, iova)) { 512 if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) { 513 put_pfn(*pfn_base, dma->prot); 514 pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, 515 limit << PAGE_SHIFT); 516 return -ENOMEM; 517 } 518 lock_acct++; 519 } 520 521 if (unlikely(disable_hugepages)) 522 goto out; 523 524 /* Lock all the consecutive pages from pfn_base */ 525 for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; 526 pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { 527 ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); 528 if (ret) 529 break; 530 531 if (pfn != *pfn_base + pinned || 532 rsvd != is_invalid_reserved_pfn(pfn)) { 533 put_pfn(pfn, dma->prot); 534 break; 535 } 536 537 if (!rsvd && !vfio_find_vpfn(dma, iova)) { 538 if (!dma->lock_cap && 539 current->mm->locked_vm + lock_acct + 1 > limit) { 540 put_pfn(pfn, dma->prot); 541 pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", 542 __func__, limit << PAGE_SHIFT); 543 ret = -ENOMEM; 544 goto unpin_out; 545 } 546 lock_acct++; 547 } 548 } 549 550 out: 551 ret = vfio_lock_acct(dma, lock_acct, false); 552 553 unpin_out: 554 if (ret) { 555 if (!rsvd) { 556 for (pfn = *pfn_base ; pinned ; pfn++, pinned--) 557 put_pfn(pfn, dma->prot); 558 } 559 560 return ret; 561 } 562 563 return pinned; 564 } 565 566 static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, 567 unsigned long pfn, long npage, 568 bool do_accounting) 569 { 570 long unlocked = 0, locked = 0; 571 long i; 572 573 for (i = 0; i < npage; i++, iova += PAGE_SIZE) { 574 if (put_pfn(pfn++, dma->prot)) { 575 unlocked++; 576 if (vfio_find_vpfn(dma, iova)) 577 locked++; 578 } 579 } 580 581 if (do_accounting) 582 vfio_lock_acct(dma, locked - unlocked, true); 583 584 return unlocked; 585 } 586 587 static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, 588 unsigned long *pfn_base, bool do_accounting) 589 { 590 struct mm_struct *mm; 591 int ret; 592 593 mm = get_task_mm(dma->task); 594 if (!mm) 595 return -ENODEV; 596 597 ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); 598 if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { 599 ret = vfio_lock_acct(dma, 1, true); 600 if (ret) { 601 put_pfn(*pfn_base, dma->prot); 602 if (ret == -ENOMEM) 603 pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK " 604 "(%ld) exceeded\n", __func__, 605 dma->task->comm, task_pid_nr(dma->task), 606 task_rlimit(dma->task, RLIMIT_MEMLOCK)); 607 } 608 } 609 610 mmput(mm); 611 return ret; 612 } 613 614 static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, 615 bool do_accounting) 616 { 617 int unlocked; 618 struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova); 619 620 if (!vpfn) 621 return 0; 622 623 unlocked = vfio_iova_put_vfio_pfn(dma, vpfn); 624 625 if (do_accounting) 626 vfio_lock_acct(dma, -unlocked, true); 627 628 return unlocked; 629 } 630 631 static int vfio_iommu_type1_pin_pages(void *iommu_data, 632 struct iommu_group *iommu_group, 633 unsigned long *user_pfn, 634 int npage, int prot, 635 unsigned long *phys_pfn) 636 { 637 struct vfio_iommu *iommu = iommu_data; 638 struct vfio_group *group; 639 int i, j, ret; 640 unsigned long remote_vaddr; 641 struct vfio_dma *dma; 642 bool do_accounting; 643 644 if (!iommu || !user_pfn || !phys_pfn) 645 return -EINVAL; 646 647 /* Supported for v2 version only */ 648 if (!iommu->v2) 649 return -EACCES; 650 651 mutex_lock(&iommu->lock); 652 653 /* Fail if notifier list is empty */ 654 if (!iommu->notifier.head) { 655 ret = -EINVAL; 656 goto pin_done; 657 } 658 659 /* 660 * If iommu capable domain exist in the container then all pages are 661 * already pinned and accounted. Accouting should be done if there is no 662 * iommu capable domain in the container. 663 */ 664 do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); 665 666 for (i = 0; i < npage; i++) { 667 dma_addr_t iova; 668 struct vfio_pfn *vpfn; 669 670 iova = user_pfn[i] << PAGE_SHIFT; 671 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 672 if (!dma) { 673 ret = -EINVAL; 674 goto pin_unwind; 675 } 676 677 if ((dma->prot & prot) != prot) { 678 ret = -EPERM; 679 goto pin_unwind; 680 } 681 682 vpfn = vfio_iova_get_vfio_pfn(dma, iova); 683 if (vpfn) { 684 phys_pfn[i] = vpfn->pfn; 685 continue; 686 } 687 688 remote_vaddr = dma->vaddr + (iova - dma->iova); 689 ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i], 690 do_accounting); 691 if (ret) 692 goto pin_unwind; 693 694 ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]); 695 if (ret) { 696 vfio_unpin_page_external(dma, iova, do_accounting); 697 goto pin_unwind; 698 } 699 700 if (iommu->dirty_page_tracking) { 701 unsigned long pgshift = __ffs(iommu->pgsize_bitmap); 702 703 /* 704 * Bitmap populated with the smallest supported page 705 * size 706 */ 707 bitmap_set(dma->bitmap, 708 (iova - dma->iova) >> pgshift, 1); 709 } 710 } 711 ret = i; 712 713 group = vfio_iommu_find_iommu_group(iommu, iommu_group); 714 if (!group->pinned_page_dirty_scope) { 715 group->pinned_page_dirty_scope = true; 716 update_pinned_page_dirty_scope(iommu); 717 } 718 719 goto pin_done; 720 721 pin_unwind: 722 phys_pfn[i] = 0; 723 for (j = 0; j < i; j++) { 724 dma_addr_t iova; 725 726 iova = user_pfn[j] << PAGE_SHIFT; 727 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 728 vfio_unpin_page_external(dma, iova, do_accounting); 729 phys_pfn[j] = 0; 730 } 731 pin_done: 732 mutex_unlock(&iommu->lock); 733 return ret; 734 } 735 736 static int vfio_iommu_type1_unpin_pages(void *iommu_data, 737 unsigned long *user_pfn, 738 int npage) 739 { 740 struct vfio_iommu *iommu = iommu_data; 741 bool do_accounting; 742 int i; 743 744 if (!iommu || !user_pfn) 745 return -EINVAL; 746 747 /* Supported for v2 version only */ 748 if (!iommu->v2) 749 return -EACCES; 750 751 mutex_lock(&iommu->lock); 752 753 do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); 754 for (i = 0; i < npage; i++) { 755 struct vfio_dma *dma; 756 dma_addr_t iova; 757 758 iova = user_pfn[i] << PAGE_SHIFT; 759 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 760 if (!dma) 761 goto unpin_exit; 762 vfio_unpin_page_external(dma, iova, do_accounting); 763 } 764 765 unpin_exit: 766 mutex_unlock(&iommu->lock); 767 return i > npage ? npage : (i > 0 ? i : -EINVAL); 768 } 769 770 static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, 771 struct list_head *regions, 772 struct iommu_iotlb_gather *iotlb_gather) 773 { 774 long unlocked = 0; 775 struct vfio_regions *entry, *next; 776 777 iommu_tlb_sync(domain->domain, iotlb_gather); 778 779 list_for_each_entry_safe(entry, next, regions, list) { 780 unlocked += vfio_unpin_pages_remote(dma, 781 entry->iova, 782 entry->phys >> PAGE_SHIFT, 783 entry->len >> PAGE_SHIFT, 784 false); 785 list_del(&entry->list); 786 kfree(entry); 787 } 788 789 cond_resched(); 790 791 return unlocked; 792 } 793 794 /* 795 * Generally, VFIO needs to unpin remote pages after each IOTLB flush. 796 * Therefore, when using IOTLB flush sync interface, VFIO need to keep track 797 * of these regions (currently using a list). 798 * 799 * This value specifies maximum number of regions for each IOTLB flush sync. 800 */ 801 #define VFIO_IOMMU_TLB_SYNC_MAX 512 802 803 static size_t unmap_unpin_fast(struct vfio_domain *domain, 804 struct vfio_dma *dma, dma_addr_t *iova, 805 size_t len, phys_addr_t phys, long *unlocked, 806 struct list_head *unmapped_list, 807 int *unmapped_cnt, 808 struct iommu_iotlb_gather *iotlb_gather) 809 { 810 size_t unmapped = 0; 811 struct vfio_regions *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 812 813 if (entry) { 814 unmapped = iommu_unmap_fast(domain->domain, *iova, len, 815 iotlb_gather); 816 817 if (!unmapped) { 818 kfree(entry); 819 } else { 820 entry->iova = *iova; 821 entry->phys = phys; 822 entry->len = unmapped; 823 list_add_tail(&entry->list, unmapped_list); 824 825 *iova += unmapped; 826 (*unmapped_cnt)++; 827 } 828 } 829 830 /* 831 * Sync if the number of fast-unmap regions hits the limit 832 * or in case of errors. 833 */ 834 if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX || !unmapped) { 835 *unlocked += vfio_sync_unpin(dma, domain, unmapped_list, 836 iotlb_gather); 837 *unmapped_cnt = 0; 838 } 839 840 return unmapped; 841 } 842 843 static size_t unmap_unpin_slow(struct vfio_domain *domain, 844 struct vfio_dma *dma, dma_addr_t *iova, 845 size_t len, phys_addr_t phys, 846 long *unlocked) 847 { 848 size_t unmapped = iommu_unmap(domain->domain, *iova, len); 849 850 if (unmapped) { 851 *unlocked += vfio_unpin_pages_remote(dma, *iova, 852 phys >> PAGE_SHIFT, 853 unmapped >> PAGE_SHIFT, 854 false); 855 *iova += unmapped; 856 cond_resched(); 857 } 858 return unmapped; 859 } 860 861 static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, 862 bool do_accounting) 863 { 864 dma_addr_t iova = dma->iova, end = dma->iova + dma->size; 865 struct vfio_domain *domain, *d; 866 LIST_HEAD(unmapped_region_list); 867 struct iommu_iotlb_gather iotlb_gather; 868 int unmapped_region_cnt = 0; 869 long unlocked = 0; 870 871 if (!dma->size) 872 return 0; 873 874 if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) 875 return 0; 876 877 /* 878 * We use the IOMMU to track the physical addresses, otherwise we'd 879 * need a much more complicated tracking system. Unfortunately that 880 * means we need to use one of the iommu domains to figure out the 881 * pfns to unpin. The rest need to be unmapped in advance so we have 882 * no iommu translations remaining when the pages are unpinned. 883 */ 884 domain = d = list_first_entry(&iommu->domain_list, 885 struct vfio_domain, next); 886 887 list_for_each_entry_continue(d, &iommu->domain_list, next) { 888 iommu_unmap(d->domain, dma->iova, dma->size); 889 cond_resched(); 890 } 891 892 iommu_iotlb_gather_init(&iotlb_gather); 893 while (iova < end) { 894 size_t unmapped, len; 895 phys_addr_t phys, next; 896 897 phys = iommu_iova_to_phys(domain->domain, iova); 898 if (WARN_ON(!phys)) { 899 iova += PAGE_SIZE; 900 continue; 901 } 902 903 /* 904 * To optimize for fewer iommu_unmap() calls, each of which 905 * may require hardware cache flushing, try to find the 906 * largest contiguous physical memory chunk to unmap. 907 */ 908 for (len = PAGE_SIZE; 909 !domain->fgsp && iova + len < end; len += PAGE_SIZE) { 910 next = iommu_iova_to_phys(domain->domain, iova + len); 911 if (next != phys + len) 912 break; 913 } 914 915 /* 916 * First, try to use fast unmap/unpin. In case of failure, 917 * switch to slow unmap/unpin path. 918 */ 919 unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys, 920 &unlocked, &unmapped_region_list, 921 &unmapped_region_cnt, 922 &iotlb_gather); 923 if (!unmapped) { 924 unmapped = unmap_unpin_slow(domain, dma, &iova, len, 925 phys, &unlocked); 926 if (WARN_ON(!unmapped)) 927 break; 928 } 929 } 930 931 dma->iommu_mapped = false; 932 933 if (unmapped_region_cnt) { 934 unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list, 935 &iotlb_gather); 936 } 937 938 if (do_accounting) { 939 vfio_lock_acct(dma, -unlocked, true); 940 return 0; 941 } 942 return unlocked; 943 } 944 945 static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) 946 { 947 vfio_unmap_unpin(iommu, dma, true); 948 vfio_unlink_dma(iommu, dma); 949 put_task_struct(dma->task); 950 vfio_dma_bitmap_free(dma); 951 kfree(dma); 952 iommu->dma_avail++; 953 } 954 955 static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) 956 { 957 struct vfio_domain *domain; 958 959 iommu->pgsize_bitmap = ULONG_MAX; 960 961 list_for_each_entry(domain, &iommu->domain_list, next) 962 iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; 963 964 /* 965 * In case the IOMMU supports page sizes smaller than PAGE_SIZE 966 * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. 967 * That way the user will be able to map/unmap buffers whose size/ 968 * start address is aligned with PAGE_SIZE. Pinning code uses that 969 * granularity while iommu driver can use the sub-PAGE_SIZE size 970 * to map the buffer. 971 */ 972 if (iommu->pgsize_bitmap & ~PAGE_MASK) { 973 iommu->pgsize_bitmap &= PAGE_MASK; 974 iommu->pgsize_bitmap |= PAGE_SIZE; 975 } 976 } 977 978 static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, 979 struct vfio_dma *dma, dma_addr_t base_iova, 980 size_t pgsize) 981 { 982 unsigned long pgshift = __ffs(pgsize); 983 unsigned long nbits = dma->size >> pgshift; 984 unsigned long bit_offset = (dma->iova - base_iova) >> pgshift; 985 unsigned long copy_offset = bit_offset / BITS_PER_LONG; 986 unsigned long shift = bit_offset % BITS_PER_LONG; 987 unsigned long leftover; 988 989 /* 990 * mark all pages dirty if any IOMMU capable device is not able 991 * to report dirty pages and all pages are pinned and mapped. 992 */ 993 if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) 994 bitmap_set(dma->bitmap, 0, nbits); 995 996 if (shift) { 997 bitmap_shift_left(dma->bitmap, dma->bitmap, shift, 998 nbits + shift); 999 1000 if (copy_from_user(&leftover, 1001 (void __user *)(bitmap + copy_offset), 1002 sizeof(leftover))) 1003 return -EFAULT; 1004 1005 bitmap_or(dma->bitmap, dma->bitmap, &leftover, shift); 1006 } 1007 1008 if (copy_to_user((void __user *)(bitmap + copy_offset), dma->bitmap, 1009 DIRTY_BITMAP_BYTES(nbits + shift))) 1010 return -EFAULT; 1011 1012 return 0; 1013 } 1014 1015 static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, 1016 dma_addr_t iova, size_t size, size_t pgsize) 1017 { 1018 struct vfio_dma *dma; 1019 struct rb_node *n; 1020 unsigned long pgshift = __ffs(pgsize); 1021 int ret; 1022 1023 /* 1024 * GET_BITMAP request must fully cover vfio_dma mappings. Multiple 1025 * vfio_dma mappings may be clubbed by specifying large ranges, but 1026 * there must not be any previous mappings bisected by the range. 1027 * An error will be returned if these conditions are not met. 1028 */ 1029 dma = vfio_find_dma(iommu, iova, 1); 1030 if (dma && dma->iova != iova) 1031 return -EINVAL; 1032 1033 dma = vfio_find_dma(iommu, iova + size - 1, 0); 1034 if (dma && dma->iova + dma->size != iova + size) 1035 return -EINVAL; 1036 1037 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 1038 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 1039 1040 if (dma->iova < iova) 1041 continue; 1042 1043 if (dma->iova > iova + size - 1) 1044 break; 1045 1046 ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize); 1047 if (ret) 1048 return ret; 1049 1050 /* 1051 * Re-populate bitmap to include all pinned pages which are 1052 * considered as dirty but exclude pages which are unpinned and 1053 * pages which are marked dirty by vfio_dma_rw() 1054 */ 1055 bitmap_clear(dma->bitmap, 0, dma->size >> pgshift); 1056 vfio_dma_populate_bitmap(dma, pgsize); 1057 } 1058 return 0; 1059 } 1060 1061 static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size) 1062 { 1063 if (!npages || !bitmap_size || (bitmap_size > DIRTY_BITMAP_SIZE_MAX) || 1064 (bitmap_size < DIRTY_BITMAP_BYTES(npages))) 1065 return -EINVAL; 1066 1067 return 0; 1068 } 1069 1070 static int vfio_dma_do_unmap(struct vfio_iommu *iommu, 1071 struct vfio_iommu_type1_dma_unmap *unmap, 1072 struct vfio_bitmap *bitmap) 1073 { 1074 struct vfio_dma *dma, *dma_last = NULL; 1075 size_t unmapped = 0, pgsize; 1076 int ret = 0, retries = 0; 1077 unsigned long pgshift; 1078 1079 mutex_lock(&iommu->lock); 1080 1081 pgshift = __ffs(iommu->pgsize_bitmap); 1082 pgsize = (size_t)1 << pgshift; 1083 1084 if (unmap->iova & (pgsize - 1)) { 1085 ret = -EINVAL; 1086 goto unlock; 1087 } 1088 1089 if (!unmap->size || unmap->size & (pgsize - 1)) { 1090 ret = -EINVAL; 1091 goto unlock; 1092 } 1093 1094 if (unmap->iova + unmap->size - 1 < unmap->iova || 1095 unmap->size > SIZE_MAX) { 1096 ret = -EINVAL; 1097 goto unlock; 1098 } 1099 1100 /* When dirty tracking is enabled, allow only min supported pgsize */ 1101 if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && 1102 (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { 1103 ret = -EINVAL; 1104 goto unlock; 1105 } 1106 1107 WARN_ON((pgsize - 1) & PAGE_MASK); 1108 again: 1109 /* 1110 * vfio-iommu-type1 (v1) - User mappings were coalesced together to 1111 * avoid tracking individual mappings. This means that the granularity 1112 * of the original mapping was lost and the user was allowed to attempt 1113 * to unmap any range. Depending on the contiguousness of physical 1114 * memory and page sizes supported by the IOMMU, arbitrary unmaps may 1115 * or may not have worked. We only guaranteed unmap granularity 1116 * matching the original mapping; even though it was untracked here, 1117 * the original mappings are reflected in IOMMU mappings. This 1118 * resulted in a couple unusual behaviors. First, if a range is not 1119 * able to be unmapped, ex. a set of 4k pages that was mapped as a 1120 * 2M hugepage into the IOMMU, the unmap ioctl returns success but with 1121 * a zero sized unmap. Also, if an unmap request overlaps the first 1122 * address of a hugepage, the IOMMU will unmap the entire hugepage. 1123 * This also returns success and the returned unmap size reflects the 1124 * actual size unmapped. 1125 * 1126 * We attempt to maintain compatibility with this "v1" interface, but 1127 * we take control out of the hands of the IOMMU. Therefore, an unmap 1128 * request offset from the beginning of the original mapping will 1129 * return success with zero sized unmap. And an unmap request covering 1130 * the first iova of mapping will unmap the entire range. 1131 * 1132 * The v2 version of this interface intends to be more deterministic. 1133 * Unmap requests must fully cover previous mappings. Multiple 1134 * mappings may still be unmaped by specifying large ranges, but there 1135 * must not be any previous mappings bisected by the range. An error 1136 * will be returned if these conditions are not met. The v2 interface 1137 * will only return success and a size of zero if there were no 1138 * mappings within the range. 1139 */ 1140 if (iommu->v2) { 1141 dma = vfio_find_dma(iommu, unmap->iova, 1); 1142 if (dma && dma->iova != unmap->iova) { 1143 ret = -EINVAL; 1144 goto unlock; 1145 } 1146 dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0); 1147 if (dma && dma->iova + dma->size != unmap->iova + unmap->size) { 1148 ret = -EINVAL; 1149 goto unlock; 1150 } 1151 } 1152 1153 while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) { 1154 if (!iommu->v2 && unmap->iova > dma->iova) 1155 break; 1156 /* 1157 * Task with same address space who mapped this iova range is 1158 * allowed to unmap the iova range. 1159 */ 1160 if (dma->task->mm != current->mm) 1161 break; 1162 1163 if (!RB_EMPTY_ROOT(&dma->pfn_list)) { 1164 struct vfio_iommu_type1_dma_unmap nb_unmap; 1165 1166 if (dma_last == dma) { 1167 BUG_ON(++retries > 10); 1168 } else { 1169 dma_last = dma; 1170 retries = 0; 1171 } 1172 1173 nb_unmap.iova = dma->iova; 1174 nb_unmap.size = dma->size; 1175 1176 /* 1177 * Notify anyone (mdev vendor drivers) to invalidate and 1178 * unmap iovas within the range we're about to unmap. 1179 * Vendor drivers MUST unpin pages in response to an 1180 * invalidation. 1181 */ 1182 mutex_unlock(&iommu->lock); 1183 blocking_notifier_call_chain(&iommu->notifier, 1184 VFIO_IOMMU_NOTIFY_DMA_UNMAP, 1185 &nb_unmap); 1186 mutex_lock(&iommu->lock); 1187 goto again; 1188 } 1189 1190 if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { 1191 ret = update_user_bitmap(bitmap->data, iommu, dma, 1192 unmap->iova, pgsize); 1193 if (ret) 1194 break; 1195 } 1196 1197 unmapped += dma->size; 1198 vfio_remove_dma(iommu, dma); 1199 } 1200 1201 unlock: 1202 mutex_unlock(&iommu->lock); 1203 1204 /* Report how much was unmapped */ 1205 unmap->size = unmapped; 1206 1207 return ret; 1208 } 1209 1210 static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, 1211 unsigned long pfn, long npage, int prot) 1212 { 1213 struct vfio_domain *d; 1214 int ret; 1215 1216 list_for_each_entry(d, &iommu->domain_list, next) { 1217 ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, 1218 npage << PAGE_SHIFT, prot | d->prot); 1219 if (ret) 1220 goto unwind; 1221 1222 cond_resched(); 1223 } 1224 1225 return 0; 1226 1227 unwind: 1228 list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) { 1229 iommu_unmap(d->domain, iova, npage << PAGE_SHIFT); 1230 cond_resched(); 1231 } 1232 1233 return ret; 1234 } 1235 1236 static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, 1237 size_t map_size) 1238 { 1239 dma_addr_t iova = dma->iova; 1240 unsigned long vaddr = dma->vaddr; 1241 size_t size = map_size; 1242 long npage; 1243 unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1244 int ret = 0; 1245 1246 while (size) { 1247 /* Pin a contiguous chunk of memory */ 1248 npage = vfio_pin_pages_remote(dma, vaddr + dma->size, 1249 size >> PAGE_SHIFT, &pfn, limit); 1250 if (npage <= 0) { 1251 WARN_ON(!npage); 1252 ret = (int)npage; 1253 break; 1254 } 1255 1256 /* Map it! */ 1257 ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, 1258 dma->prot); 1259 if (ret) { 1260 vfio_unpin_pages_remote(dma, iova + dma->size, pfn, 1261 npage, true); 1262 break; 1263 } 1264 1265 size -= npage << PAGE_SHIFT; 1266 dma->size += npage << PAGE_SHIFT; 1267 } 1268 1269 dma->iommu_mapped = true; 1270 1271 if (ret) 1272 vfio_remove_dma(iommu, dma); 1273 1274 return ret; 1275 } 1276 1277 /* 1278 * Check dma map request is within a valid iova range 1279 */ 1280 static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, 1281 dma_addr_t start, dma_addr_t end) 1282 { 1283 struct list_head *iova = &iommu->iova_list; 1284 struct vfio_iova *node; 1285 1286 list_for_each_entry(node, iova, list) { 1287 if (start >= node->start && end <= node->end) 1288 return true; 1289 } 1290 1291 /* 1292 * Check for list_empty() as well since a container with 1293 * a single mdev device will have an empty list. 1294 */ 1295 return list_empty(iova); 1296 } 1297 1298 static int vfio_dma_do_map(struct vfio_iommu *iommu, 1299 struct vfio_iommu_type1_dma_map *map) 1300 { 1301 dma_addr_t iova = map->iova; 1302 unsigned long vaddr = map->vaddr; 1303 size_t size = map->size; 1304 int ret = 0, prot = 0; 1305 size_t pgsize; 1306 struct vfio_dma *dma; 1307 1308 /* Verify that none of our __u64 fields overflow */ 1309 if (map->size != size || map->vaddr != vaddr || map->iova != iova) 1310 return -EINVAL; 1311 1312 /* READ/WRITE from device perspective */ 1313 if (map->flags & VFIO_DMA_MAP_FLAG_WRITE) 1314 prot |= IOMMU_WRITE; 1315 if (map->flags & VFIO_DMA_MAP_FLAG_READ) 1316 prot |= IOMMU_READ; 1317 1318 mutex_lock(&iommu->lock); 1319 1320 pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); 1321 1322 WARN_ON((pgsize - 1) & PAGE_MASK); 1323 1324 if (!prot || !size || (size | iova | vaddr) & (pgsize - 1)) { 1325 ret = -EINVAL; 1326 goto out_unlock; 1327 } 1328 1329 /* Don't allow IOVA or virtual address wrap */ 1330 if (iova + size - 1 < iova || vaddr + size - 1 < vaddr) { 1331 ret = -EINVAL; 1332 goto out_unlock; 1333 } 1334 1335 if (vfio_find_dma(iommu, iova, size)) { 1336 ret = -EEXIST; 1337 goto out_unlock; 1338 } 1339 1340 if (!iommu->dma_avail) { 1341 ret = -ENOSPC; 1342 goto out_unlock; 1343 } 1344 1345 if (!vfio_iommu_iova_dma_valid(iommu, iova, iova + size - 1)) { 1346 ret = -EINVAL; 1347 goto out_unlock; 1348 } 1349 1350 dma = kzalloc(sizeof(*dma), GFP_KERNEL); 1351 if (!dma) { 1352 ret = -ENOMEM; 1353 goto out_unlock; 1354 } 1355 1356 iommu->dma_avail--; 1357 dma->iova = iova; 1358 dma->vaddr = vaddr; 1359 dma->prot = prot; 1360 1361 /* 1362 * We need to be able to both add to a task's locked memory and test 1363 * against the locked memory limit and we need to be able to do both 1364 * outside of this call path as pinning can be asynchronous via the 1365 * external interfaces for mdev devices. RLIMIT_MEMLOCK requires a 1366 * task_struct and VM locked pages requires an mm_struct, however 1367 * holding an indefinite mm reference is not recommended, therefore we 1368 * only hold a reference to a task. We could hold a reference to 1369 * current, however QEMU uses this call path through vCPU threads, 1370 * which can be killed resulting in a NULL mm and failure in the unmap 1371 * path when called via a different thread. Avoid this problem by 1372 * using the group_leader as threads within the same group require 1373 * both CLONE_THREAD and CLONE_VM and will therefore use the same 1374 * mm_struct. 1375 * 1376 * Previously we also used the task for testing CAP_IPC_LOCK at the 1377 * time of pinning and accounting, however has_capability() makes use 1378 * of real_cred, a copy-on-write field, so we can't guarantee that it 1379 * matches group_leader, or in fact that it might not change by the 1380 * time it's evaluated. If a process were to call MAP_DMA with 1381 * CAP_IPC_LOCK but later drop it, it doesn't make sense that they 1382 * possibly see different results for an iommu_mapped vfio_dma vs 1383 * externally mapped. Therefore track CAP_IPC_LOCK in vfio_dma at the 1384 * time of calling MAP_DMA. 1385 */ 1386 get_task_struct(current->group_leader); 1387 dma->task = current->group_leader; 1388 dma->lock_cap = capable(CAP_IPC_LOCK); 1389 1390 dma->pfn_list = RB_ROOT; 1391 1392 /* Insert zero-sized and grow as we map chunks of it */ 1393 vfio_link_dma(iommu, dma); 1394 1395 /* Don't pin and map if container doesn't contain IOMMU capable domain*/ 1396 if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) 1397 dma->size = size; 1398 else 1399 ret = vfio_pin_map_dma(iommu, dma, size); 1400 1401 if (!ret && iommu->dirty_page_tracking) { 1402 ret = vfio_dma_bitmap_alloc(dma, pgsize); 1403 if (ret) 1404 vfio_remove_dma(iommu, dma); 1405 } 1406 1407 out_unlock: 1408 mutex_unlock(&iommu->lock); 1409 return ret; 1410 } 1411 1412 static int vfio_bus_type(struct device *dev, void *data) 1413 { 1414 struct bus_type **bus = data; 1415 1416 if (*bus && *bus != dev->bus) 1417 return -EINVAL; 1418 1419 *bus = dev->bus; 1420 1421 return 0; 1422 } 1423 1424 static int vfio_iommu_replay(struct vfio_iommu *iommu, 1425 struct vfio_domain *domain) 1426 { 1427 struct vfio_domain *d; 1428 struct rb_node *n; 1429 unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1430 int ret; 1431 1432 /* Arbitrarily pick the first domain in the list for lookups */ 1433 d = list_first_entry(&iommu->domain_list, struct vfio_domain, next); 1434 n = rb_first(&iommu->dma_list); 1435 1436 for (; n; n = rb_next(n)) { 1437 struct vfio_dma *dma; 1438 dma_addr_t iova; 1439 1440 dma = rb_entry(n, struct vfio_dma, node); 1441 iova = dma->iova; 1442 1443 while (iova < dma->iova + dma->size) { 1444 phys_addr_t phys; 1445 size_t size; 1446 1447 if (dma->iommu_mapped) { 1448 phys_addr_t p; 1449 dma_addr_t i; 1450 1451 phys = iommu_iova_to_phys(d->domain, iova); 1452 1453 if (WARN_ON(!phys)) { 1454 iova += PAGE_SIZE; 1455 continue; 1456 } 1457 1458 size = PAGE_SIZE; 1459 p = phys + size; 1460 i = iova + size; 1461 while (i < dma->iova + dma->size && 1462 p == iommu_iova_to_phys(d->domain, i)) { 1463 size += PAGE_SIZE; 1464 p += PAGE_SIZE; 1465 i += PAGE_SIZE; 1466 } 1467 } else { 1468 unsigned long pfn; 1469 unsigned long vaddr = dma->vaddr + 1470 (iova - dma->iova); 1471 size_t n = dma->iova + dma->size - iova; 1472 long npage; 1473 1474 npage = vfio_pin_pages_remote(dma, vaddr, 1475 n >> PAGE_SHIFT, 1476 &pfn, limit); 1477 if (npage <= 0) { 1478 WARN_ON(!npage); 1479 ret = (int)npage; 1480 return ret; 1481 } 1482 1483 phys = pfn << PAGE_SHIFT; 1484 size = npage << PAGE_SHIFT; 1485 } 1486 1487 ret = iommu_map(domain->domain, iova, phys, 1488 size, dma->prot | domain->prot); 1489 if (ret) 1490 return ret; 1491 1492 iova += size; 1493 } 1494 dma->iommu_mapped = true; 1495 } 1496 return 0; 1497 } 1498 1499 /* 1500 * We change our unmap behavior slightly depending on whether the IOMMU 1501 * supports fine-grained superpages. IOMMUs like AMD-Vi will use a superpage 1502 * for practically any contiguous power-of-two mapping we give it. This means 1503 * we don't need to look for contiguous chunks ourselves to make unmapping 1504 * more efficient. On IOMMUs with coarse-grained super pages, like Intel VT-d 1505 * with discrete 2M/1G/512G/1T superpages, identifying contiguous chunks 1506 * significantly boosts non-hugetlbfs mappings and doesn't seem to hurt when 1507 * hugetlbfs is in use. 1508 */ 1509 static void vfio_test_domain_fgsp(struct vfio_domain *domain) 1510 { 1511 struct page *pages; 1512 int ret, order = get_order(PAGE_SIZE * 2); 1513 1514 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1515 if (!pages) 1516 return; 1517 1518 ret = iommu_map(domain->domain, 0, page_to_phys(pages), PAGE_SIZE * 2, 1519 IOMMU_READ | IOMMU_WRITE | domain->prot); 1520 if (!ret) { 1521 size_t unmapped = iommu_unmap(domain->domain, 0, PAGE_SIZE); 1522 1523 if (unmapped == PAGE_SIZE) 1524 iommu_unmap(domain->domain, PAGE_SIZE, PAGE_SIZE); 1525 else 1526 domain->fgsp = true; 1527 } 1528 1529 __free_pages(pages, order); 1530 } 1531 1532 static struct vfio_group *find_iommu_group(struct vfio_domain *domain, 1533 struct iommu_group *iommu_group) 1534 { 1535 struct vfio_group *g; 1536 1537 list_for_each_entry(g, &domain->group_list, next) { 1538 if (g->iommu_group == iommu_group) 1539 return g; 1540 } 1541 1542 return NULL; 1543 } 1544 1545 static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, 1546 struct iommu_group *iommu_group) 1547 { 1548 struct vfio_domain *domain; 1549 struct vfio_group *group = NULL; 1550 1551 list_for_each_entry(domain, &iommu->domain_list, next) { 1552 group = find_iommu_group(domain, iommu_group); 1553 if (group) 1554 return group; 1555 } 1556 1557 if (iommu->external_domain) 1558 group = find_iommu_group(iommu->external_domain, iommu_group); 1559 1560 return group; 1561 } 1562 1563 static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) 1564 { 1565 struct vfio_domain *domain; 1566 struct vfio_group *group; 1567 1568 list_for_each_entry(domain, &iommu->domain_list, next) { 1569 list_for_each_entry(group, &domain->group_list, next) { 1570 if (!group->pinned_page_dirty_scope) { 1571 iommu->pinned_page_dirty_scope = false; 1572 return; 1573 } 1574 } 1575 } 1576 1577 if (iommu->external_domain) { 1578 domain = iommu->external_domain; 1579 list_for_each_entry(group, &domain->group_list, next) { 1580 if (!group->pinned_page_dirty_scope) { 1581 iommu->pinned_page_dirty_scope = false; 1582 return; 1583 } 1584 } 1585 } 1586 1587 iommu->pinned_page_dirty_scope = true; 1588 } 1589 1590 static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, 1591 phys_addr_t *base) 1592 { 1593 struct iommu_resv_region *region; 1594 bool ret = false; 1595 1596 list_for_each_entry(region, group_resv_regions, list) { 1597 /* 1598 * The presence of any 'real' MSI regions should take 1599 * precedence over the software-managed one if the 1600 * IOMMU driver happens to advertise both types. 1601 */ 1602 if (region->type == IOMMU_RESV_MSI) { 1603 ret = false; 1604 break; 1605 } 1606 1607 if (region->type == IOMMU_RESV_SW_MSI) { 1608 *base = region->start; 1609 ret = true; 1610 } 1611 } 1612 1613 return ret; 1614 } 1615 1616 static struct device *vfio_mdev_get_iommu_device(struct device *dev) 1617 { 1618 struct device *(*fn)(struct device *dev); 1619 struct device *iommu_device; 1620 1621 fn = symbol_get(mdev_get_iommu_device); 1622 if (fn) { 1623 iommu_device = fn(dev); 1624 symbol_put(mdev_get_iommu_device); 1625 1626 return iommu_device; 1627 } 1628 1629 return NULL; 1630 } 1631 1632 static int vfio_mdev_attach_domain(struct device *dev, void *data) 1633 { 1634 struct iommu_domain *domain = data; 1635 struct device *iommu_device; 1636 1637 iommu_device = vfio_mdev_get_iommu_device(dev); 1638 if (iommu_device) { 1639 if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX)) 1640 return iommu_aux_attach_device(domain, iommu_device); 1641 else 1642 return iommu_attach_device(domain, iommu_device); 1643 } 1644 1645 return -EINVAL; 1646 } 1647 1648 static int vfio_mdev_detach_domain(struct device *dev, void *data) 1649 { 1650 struct iommu_domain *domain = data; 1651 struct device *iommu_device; 1652 1653 iommu_device = vfio_mdev_get_iommu_device(dev); 1654 if (iommu_device) { 1655 if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX)) 1656 iommu_aux_detach_device(domain, iommu_device); 1657 else 1658 iommu_detach_device(domain, iommu_device); 1659 } 1660 1661 return 0; 1662 } 1663 1664 static int vfio_iommu_attach_group(struct vfio_domain *domain, 1665 struct vfio_group *group) 1666 { 1667 if (group->mdev_group) 1668 return iommu_group_for_each_dev(group->iommu_group, 1669 domain->domain, 1670 vfio_mdev_attach_domain); 1671 else 1672 return iommu_attach_group(domain->domain, group->iommu_group); 1673 } 1674 1675 static void vfio_iommu_detach_group(struct vfio_domain *domain, 1676 struct vfio_group *group) 1677 { 1678 if (group->mdev_group) 1679 iommu_group_for_each_dev(group->iommu_group, domain->domain, 1680 vfio_mdev_detach_domain); 1681 else 1682 iommu_detach_group(domain->domain, group->iommu_group); 1683 } 1684 1685 static bool vfio_bus_is_mdev(struct bus_type *bus) 1686 { 1687 struct bus_type *mdev_bus; 1688 bool ret = false; 1689 1690 mdev_bus = symbol_get(mdev_bus_type); 1691 if (mdev_bus) { 1692 ret = (bus == mdev_bus); 1693 symbol_put(mdev_bus_type); 1694 } 1695 1696 return ret; 1697 } 1698 1699 static int vfio_mdev_iommu_device(struct device *dev, void *data) 1700 { 1701 struct device **old = data, *new; 1702 1703 new = vfio_mdev_get_iommu_device(dev); 1704 if (!new || (*old && *old != new)) 1705 return -EINVAL; 1706 1707 *old = new; 1708 1709 return 0; 1710 } 1711 1712 /* 1713 * This is a helper function to insert an address range to iova list. 1714 * The list is initially created with a single entry corresponding to 1715 * the IOMMU domain geometry to which the device group is attached. 1716 * The list aperture gets modified when a new domain is added to the 1717 * container if the new aperture doesn't conflict with the current one 1718 * or with any existing dma mappings. The list is also modified to 1719 * exclude any reserved regions associated with the device group. 1720 */ 1721 static int vfio_iommu_iova_insert(struct list_head *head, 1722 dma_addr_t start, dma_addr_t end) 1723 { 1724 struct vfio_iova *region; 1725 1726 region = kmalloc(sizeof(*region), GFP_KERNEL); 1727 if (!region) 1728 return -ENOMEM; 1729 1730 INIT_LIST_HEAD(®ion->list); 1731 region->start = start; 1732 region->end = end; 1733 1734 list_add_tail(®ion->list, head); 1735 return 0; 1736 } 1737 1738 /* 1739 * Check the new iommu aperture conflicts with existing aper or with any 1740 * existing dma mappings. 1741 */ 1742 static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu, 1743 dma_addr_t start, dma_addr_t end) 1744 { 1745 struct vfio_iova *first, *last; 1746 struct list_head *iova = &iommu->iova_list; 1747 1748 if (list_empty(iova)) 1749 return false; 1750 1751 /* Disjoint sets, return conflict */ 1752 first = list_first_entry(iova, struct vfio_iova, list); 1753 last = list_last_entry(iova, struct vfio_iova, list); 1754 if (start > last->end || end < first->start) 1755 return true; 1756 1757 /* Check for any existing dma mappings below the new start */ 1758 if (start > first->start) { 1759 if (vfio_find_dma(iommu, first->start, start - first->start)) 1760 return true; 1761 } 1762 1763 /* Check for any existing dma mappings beyond the new end */ 1764 if (end < last->end) { 1765 if (vfio_find_dma(iommu, end + 1, last->end - end)) 1766 return true; 1767 } 1768 1769 return false; 1770 } 1771 1772 /* 1773 * Resize iommu iova aperture window. This is called only if the new 1774 * aperture has no conflict with existing aperture and dma mappings. 1775 */ 1776 static int vfio_iommu_aper_resize(struct list_head *iova, 1777 dma_addr_t start, dma_addr_t end) 1778 { 1779 struct vfio_iova *node, *next; 1780 1781 if (list_empty(iova)) 1782 return vfio_iommu_iova_insert(iova, start, end); 1783 1784 /* Adjust iova list start */ 1785 list_for_each_entry_safe(node, next, iova, list) { 1786 if (start < node->start) 1787 break; 1788 if (start >= node->start && start < node->end) { 1789 node->start = start; 1790 break; 1791 } 1792 /* Delete nodes before new start */ 1793 list_del(&node->list); 1794 kfree(node); 1795 } 1796 1797 /* Adjust iova list end */ 1798 list_for_each_entry_safe(node, next, iova, list) { 1799 if (end > node->end) 1800 continue; 1801 if (end > node->start && end <= node->end) { 1802 node->end = end; 1803 continue; 1804 } 1805 /* Delete nodes after new end */ 1806 list_del(&node->list); 1807 kfree(node); 1808 } 1809 1810 return 0; 1811 } 1812 1813 /* 1814 * Check reserved region conflicts with existing dma mappings 1815 */ 1816 static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu, 1817 struct list_head *resv_regions) 1818 { 1819 struct iommu_resv_region *region; 1820 1821 /* Check for conflict with existing dma mappings */ 1822 list_for_each_entry(region, resv_regions, list) { 1823 if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) 1824 continue; 1825 1826 if (vfio_find_dma(iommu, region->start, region->length)) 1827 return true; 1828 } 1829 1830 return false; 1831 } 1832 1833 /* 1834 * Check iova region overlap with reserved regions and 1835 * exclude them from the iommu iova range 1836 */ 1837 static int vfio_iommu_resv_exclude(struct list_head *iova, 1838 struct list_head *resv_regions) 1839 { 1840 struct iommu_resv_region *resv; 1841 struct vfio_iova *n, *next; 1842 1843 list_for_each_entry(resv, resv_regions, list) { 1844 phys_addr_t start, end; 1845 1846 if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE) 1847 continue; 1848 1849 start = resv->start; 1850 end = resv->start + resv->length - 1; 1851 1852 list_for_each_entry_safe(n, next, iova, list) { 1853 int ret = 0; 1854 1855 /* No overlap */ 1856 if (start > n->end || end < n->start) 1857 continue; 1858 /* 1859 * Insert a new node if current node overlaps with the 1860 * reserve region to exlude that from valid iova range. 1861 * Note that, new node is inserted before the current 1862 * node and finally the current node is deleted keeping 1863 * the list updated and sorted. 1864 */ 1865 if (start > n->start) 1866 ret = vfio_iommu_iova_insert(&n->list, n->start, 1867 start - 1); 1868 if (!ret && end < n->end) 1869 ret = vfio_iommu_iova_insert(&n->list, end + 1, 1870 n->end); 1871 if (ret) 1872 return ret; 1873 1874 list_del(&n->list); 1875 kfree(n); 1876 } 1877 } 1878 1879 if (list_empty(iova)) 1880 return -EINVAL; 1881 1882 return 0; 1883 } 1884 1885 static void vfio_iommu_resv_free(struct list_head *resv_regions) 1886 { 1887 struct iommu_resv_region *n, *next; 1888 1889 list_for_each_entry_safe(n, next, resv_regions, list) { 1890 list_del(&n->list); 1891 kfree(n); 1892 } 1893 } 1894 1895 static void vfio_iommu_iova_free(struct list_head *iova) 1896 { 1897 struct vfio_iova *n, *next; 1898 1899 list_for_each_entry_safe(n, next, iova, list) { 1900 list_del(&n->list); 1901 kfree(n); 1902 } 1903 } 1904 1905 static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu, 1906 struct list_head *iova_copy) 1907 { 1908 struct list_head *iova = &iommu->iova_list; 1909 struct vfio_iova *n; 1910 int ret; 1911 1912 list_for_each_entry(n, iova, list) { 1913 ret = vfio_iommu_iova_insert(iova_copy, n->start, n->end); 1914 if (ret) 1915 goto out_free; 1916 } 1917 1918 return 0; 1919 1920 out_free: 1921 vfio_iommu_iova_free(iova_copy); 1922 return ret; 1923 } 1924 1925 static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu, 1926 struct list_head *iova_copy) 1927 { 1928 struct list_head *iova = &iommu->iova_list; 1929 1930 vfio_iommu_iova_free(iova); 1931 1932 list_splice_tail(iova_copy, iova); 1933 } 1934 static int vfio_iommu_type1_attach_group(void *iommu_data, 1935 struct iommu_group *iommu_group) 1936 { 1937 struct vfio_iommu *iommu = iommu_data; 1938 struct vfio_group *group; 1939 struct vfio_domain *domain, *d; 1940 struct bus_type *bus = NULL; 1941 int ret; 1942 bool resv_msi, msi_remap; 1943 phys_addr_t resv_msi_base = 0; 1944 struct iommu_domain_geometry geo; 1945 LIST_HEAD(iova_copy); 1946 LIST_HEAD(group_resv_regions); 1947 1948 mutex_lock(&iommu->lock); 1949 1950 list_for_each_entry(d, &iommu->domain_list, next) { 1951 if (find_iommu_group(d, iommu_group)) { 1952 mutex_unlock(&iommu->lock); 1953 return -EINVAL; 1954 } 1955 } 1956 1957 if (iommu->external_domain) { 1958 if (find_iommu_group(iommu->external_domain, iommu_group)) { 1959 mutex_unlock(&iommu->lock); 1960 return -EINVAL; 1961 } 1962 } 1963 1964 group = kzalloc(sizeof(*group), GFP_KERNEL); 1965 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 1966 if (!group || !domain) { 1967 ret = -ENOMEM; 1968 goto out_free; 1969 } 1970 1971 group->iommu_group = iommu_group; 1972 1973 /* Determine bus_type in order to allocate a domain */ 1974 ret = iommu_group_for_each_dev(iommu_group, &bus, vfio_bus_type); 1975 if (ret) 1976 goto out_free; 1977 1978 if (vfio_bus_is_mdev(bus)) { 1979 struct device *iommu_device = NULL; 1980 1981 group->mdev_group = true; 1982 1983 /* Determine the isolation type */ 1984 ret = iommu_group_for_each_dev(iommu_group, &iommu_device, 1985 vfio_mdev_iommu_device); 1986 if (ret || !iommu_device) { 1987 if (!iommu->external_domain) { 1988 INIT_LIST_HEAD(&domain->group_list); 1989 iommu->external_domain = domain; 1990 vfio_update_pgsize_bitmap(iommu); 1991 } else { 1992 kfree(domain); 1993 } 1994 1995 list_add(&group->next, 1996 &iommu->external_domain->group_list); 1997 /* 1998 * Non-iommu backed group cannot dirty memory directly, 1999 * it can only use interfaces that provide dirty 2000 * tracking. 2001 * The iommu scope can only be promoted with the 2002 * addition of a dirty tracking group. 2003 */ 2004 group->pinned_page_dirty_scope = true; 2005 if (!iommu->pinned_page_dirty_scope) 2006 update_pinned_page_dirty_scope(iommu); 2007 mutex_unlock(&iommu->lock); 2008 2009 return 0; 2010 } 2011 2012 bus = iommu_device->bus; 2013 } 2014 2015 domain->domain = iommu_domain_alloc(bus); 2016 if (!domain->domain) { 2017 ret = -EIO; 2018 goto out_free; 2019 } 2020 2021 if (iommu->nesting) { 2022 int attr = 1; 2023 2024 ret = iommu_domain_set_attr(domain->domain, DOMAIN_ATTR_NESTING, 2025 &attr); 2026 if (ret) 2027 goto out_domain; 2028 } 2029 2030 ret = vfio_iommu_attach_group(domain, group); 2031 if (ret) 2032 goto out_domain; 2033 2034 /* Get aperture info */ 2035 iommu_domain_get_attr(domain->domain, DOMAIN_ATTR_GEOMETRY, &geo); 2036 2037 if (vfio_iommu_aper_conflict(iommu, geo.aperture_start, 2038 geo.aperture_end)) { 2039 ret = -EINVAL; 2040 goto out_detach; 2041 } 2042 2043 ret = iommu_get_group_resv_regions(iommu_group, &group_resv_regions); 2044 if (ret) 2045 goto out_detach; 2046 2047 if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) { 2048 ret = -EINVAL; 2049 goto out_detach; 2050 } 2051 2052 /* 2053 * We don't want to work on the original iova list as the list 2054 * gets modified and in case of failure we have to retain the 2055 * original list. Get a copy here. 2056 */ 2057 ret = vfio_iommu_iova_get_copy(iommu, &iova_copy); 2058 if (ret) 2059 goto out_detach; 2060 2061 ret = vfio_iommu_aper_resize(&iova_copy, geo.aperture_start, 2062 geo.aperture_end); 2063 if (ret) 2064 goto out_detach; 2065 2066 ret = vfio_iommu_resv_exclude(&iova_copy, &group_resv_regions); 2067 if (ret) 2068 goto out_detach; 2069 2070 resv_msi = vfio_iommu_has_sw_msi(&group_resv_regions, &resv_msi_base); 2071 2072 INIT_LIST_HEAD(&domain->group_list); 2073 list_add(&group->next, &domain->group_list); 2074 2075 msi_remap = irq_domain_check_msi_remap() || 2076 iommu_capable(bus, IOMMU_CAP_INTR_REMAP); 2077 2078 if (!allow_unsafe_interrupts && !msi_remap) { 2079 pr_warn("%s: No interrupt remapping support. Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n", 2080 __func__); 2081 ret = -EPERM; 2082 goto out_detach; 2083 } 2084 2085 if (iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) 2086 domain->prot |= IOMMU_CACHE; 2087 2088 /* 2089 * Try to match an existing compatible domain. We don't want to 2090 * preclude an IOMMU driver supporting multiple bus_types and being 2091 * able to include different bus_types in the same IOMMU domain, so 2092 * we test whether the domains use the same iommu_ops rather than 2093 * testing if they're on the same bus_type. 2094 */ 2095 list_for_each_entry(d, &iommu->domain_list, next) { 2096 if (d->domain->ops == domain->domain->ops && 2097 d->prot == domain->prot) { 2098 vfio_iommu_detach_group(domain, group); 2099 if (!vfio_iommu_attach_group(d, group)) { 2100 list_add(&group->next, &d->group_list); 2101 iommu_domain_free(domain->domain); 2102 kfree(domain); 2103 goto done; 2104 } 2105 2106 ret = vfio_iommu_attach_group(domain, group); 2107 if (ret) 2108 goto out_domain; 2109 } 2110 } 2111 2112 vfio_test_domain_fgsp(domain); 2113 2114 /* replay mappings on new domains */ 2115 ret = vfio_iommu_replay(iommu, domain); 2116 if (ret) 2117 goto out_detach; 2118 2119 if (resv_msi) { 2120 ret = iommu_get_msi_cookie(domain->domain, resv_msi_base); 2121 if (ret && ret != -ENODEV) 2122 goto out_detach; 2123 } 2124 2125 list_add(&domain->next, &iommu->domain_list); 2126 vfio_update_pgsize_bitmap(iommu); 2127 done: 2128 /* Delete the old one and insert new iova list */ 2129 vfio_iommu_iova_insert_copy(iommu, &iova_copy); 2130 2131 /* 2132 * An iommu backed group can dirty memory directly and therefore 2133 * demotes the iommu scope until it declares itself dirty tracking 2134 * capable via the page pinning interface. 2135 */ 2136 iommu->pinned_page_dirty_scope = false; 2137 mutex_unlock(&iommu->lock); 2138 vfio_iommu_resv_free(&group_resv_regions); 2139 2140 return 0; 2141 2142 out_detach: 2143 vfio_iommu_detach_group(domain, group); 2144 out_domain: 2145 iommu_domain_free(domain->domain); 2146 vfio_iommu_iova_free(&iova_copy); 2147 vfio_iommu_resv_free(&group_resv_regions); 2148 out_free: 2149 kfree(domain); 2150 kfree(group); 2151 mutex_unlock(&iommu->lock); 2152 return ret; 2153 } 2154 2155 static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu) 2156 { 2157 struct rb_node *node; 2158 2159 while ((node = rb_first(&iommu->dma_list))) 2160 vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node)); 2161 } 2162 2163 static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) 2164 { 2165 struct rb_node *n, *p; 2166 2167 n = rb_first(&iommu->dma_list); 2168 for (; n; n = rb_next(n)) { 2169 struct vfio_dma *dma; 2170 long locked = 0, unlocked = 0; 2171 2172 dma = rb_entry(n, struct vfio_dma, node); 2173 unlocked += vfio_unmap_unpin(iommu, dma, false); 2174 p = rb_first(&dma->pfn_list); 2175 for (; p; p = rb_next(p)) { 2176 struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, 2177 node); 2178 2179 if (!is_invalid_reserved_pfn(vpfn->pfn)) 2180 locked++; 2181 } 2182 vfio_lock_acct(dma, locked - unlocked, true); 2183 } 2184 } 2185 2186 static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu) 2187 { 2188 struct rb_node *n; 2189 2190 n = rb_first(&iommu->dma_list); 2191 for (; n; n = rb_next(n)) { 2192 struct vfio_dma *dma; 2193 2194 dma = rb_entry(n, struct vfio_dma, node); 2195 2196 if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list))) 2197 break; 2198 } 2199 /* mdev vendor driver must unregister notifier */ 2200 WARN_ON(iommu->notifier.head); 2201 } 2202 2203 /* 2204 * Called when a domain is removed in detach. It is possible that 2205 * the removed domain decided the iova aperture window. Modify the 2206 * iova aperture with the smallest window among existing domains. 2207 */ 2208 static void vfio_iommu_aper_expand(struct vfio_iommu *iommu, 2209 struct list_head *iova_copy) 2210 { 2211 struct vfio_domain *domain; 2212 struct iommu_domain_geometry geo; 2213 struct vfio_iova *node; 2214 dma_addr_t start = 0; 2215 dma_addr_t end = (dma_addr_t)~0; 2216 2217 if (list_empty(iova_copy)) 2218 return; 2219 2220 list_for_each_entry(domain, &iommu->domain_list, next) { 2221 iommu_domain_get_attr(domain->domain, DOMAIN_ATTR_GEOMETRY, 2222 &geo); 2223 if (geo.aperture_start > start) 2224 start = geo.aperture_start; 2225 if (geo.aperture_end < end) 2226 end = geo.aperture_end; 2227 } 2228 2229 /* Modify aperture limits. The new aper is either same or bigger */ 2230 node = list_first_entry(iova_copy, struct vfio_iova, list); 2231 node->start = start; 2232 node = list_last_entry(iova_copy, struct vfio_iova, list); 2233 node->end = end; 2234 } 2235 2236 /* 2237 * Called when a group is detached. The reserved regions for that 2238 * group can be part of valid iova now. But since reserved regions 2239 * may be duplicated among groups, populate the iova valid regions 2240 * list again. 2241 */ 2242 static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu, 2243 struct list_head *iova_copy) 2244 { 2245 struct vfio_domain *d; 2246 struct vfio_group *g; 2247 struct vfio_iova *node; 2248 dma_addr_t start, end; 2249 LIST_HEAD(resv_regions); 2250 int ret; 2251 2252 if (list_empty(iova_copy)) 2253 return -EINVAL; 2254 2255 list_for_each_entry(d, &iommu->domain_list, next) { 2256 list_for_each_entry(g, &d->group_list, next) { 2257 ret = iommu_get_group_resv_regions(g->iommu_group, 2258 &resv_regions); 2259 if (ret) 2260 goto done; 2261 } 2262 } 2263 2264 node = list_first_entry(iova_copy, struct vfio_iova, list); 2265 start = node->start; 2266 node = list_last_entry(iova_copy, struct vfio_iova, list); 2267 end = node->end; 2268 2269 /* purge the iova list and create new one */ 2270 vfio_iommu_iova_free(iova_copy); 2271 2272 ret = vfio_iommu_aper_resize(iova_copy, start, end); 2273 if (ret) 2274 goto done; 2275 2276 /* Exclude current reserved regions from iova ranges */ 2277 ret = vfio_iommu_resv_exclude(iova_copy, &resv_regions); 2278 done: 2279 vfio_iommu_resv_free(&resv_regions); 2280 return ret; 2281 } 2282 2283 static void vfio_iommu_type1_detach_group(void *iommu_data, 2284 struct iommu_group *iommu_group) 2285 { 2286 struct vfio_iommu *iommu = iommu_data; 2287 struct vfio_domain *domain; 2288 struct vfio_group *group; 2289 bool update_dirty_scope = false; 2290 LIST_HEAD(iova_copy); 2291 2292 mutex_lock(&iommu->lock); 2293 2294 if (iommu->external_domain) { 2295 group = find_iommu_group(iommu->external_domain, iommu_group); 2296 if (group) { 2297 update_dirty_scope = !group->pinned_page_dirty_scope; 2298 list_del(&group->next); 2299 kfree(group); 2300 2301 if (list_empty(&iommu->external_domain->group_list)) { 2302 vfio_sanity_check_pfn_list(iommu); 2303 2304 if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) 2305 vfio_iommu_unmap_unpin_all(iommu); 2306 2307 kfree(iommu->external_domain); 2308 iommu->external_domain = NULL; 2309 } 2310 goto detach_group_done; 2311 } 2312 } 2313 2314 /* 2315 * Get a copy of iova list. This will be used to update 2316 * and to replace the current one later. Please note that 2317 * we will leave the original list as it is if update fails. 2318 */ 2319 vfio_iommu_iova_get_copy(iommu, &iova_copy); 2320 2321 list_for_each_entry(domain, &iommu->domain_list, next) { 2322 group = find_iommu_group(domain, iommu_group); 2323 if (!group) 2324 continue; 2325 2326 vfio_iommu_detach_group(domain, group); 2327 update_dirty_scope = !group->pinned_page_dirty_scope; 2328 list_del(&group->next); 2329 kfree(group); 2330 /* 2331 * Group ownership provides privilege, if the group list is 2332 * empty, the domain goes away. If it's the last domain with 2333 * iommu and external domain doesn't exist, then all the 2334 * mappings go away too. If it's the last domain with iommu and 2335 * external domain exist, update accounting 2336 */ 2337 if (list_empty(&domain->group_list)) { 2338 if (list_is_singular(&iommu->domain_list)) { 2339 if (!iommu->external_domain) 2340 vfio_iommu_unmap_unpin_all(iommu); 2341 else 2342 vfio_iommu_unmap_unpin_reaccount(iommu); 2343 } 2344 iommu_domain_free(domain->domain); 2345 list_del(&domain->next); 2346 kfree(domain); 2347 vfio_iommu_aper_expand(iommu, &iova_copy); 2348 vfio_update_pgsize_bitmap(iommu); 2349 } 2350 break; 2351 } 2352 2353 if (!vfio_iommu_resv_refresh(iommu, &iova_copy)) 2354 vfio_iommu_iova_insert_copy(iommu, &iova_copy); 2355 else 2356 vfio_iommu_iova_free(&iova_copy); 2357 2358 detach_group_done: 2359 /* 2360 * Removal of a group without dirty tracking may allow the iommu scope 2361 * to be promoted. 2362 */ 2363 if (update_dirty_scope) 2364 update_pinned_page_dirty_scope(iommu); 2365 mutex_unlock(&iommu->lock); 2366 } 2367 2368 static void *vfio_iommu_type1_open(unsigned long arg) 2369 { 2370 struct vfio_iommu *iommu; 2371 2372 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); 2373 if (!iommu) 2374 return ERR_PTR(-ENOMEM); 2375 2376 switch (arg) { 2377 case VFIO_TYPE1_IOMMU: 2378 break; 2379 case VFIO_TYPE1_NESTING_IOMMU: 2380 iommu->nesting = true; 2381 /* fall through */ 2382 case VFIO_TYPE1v2_IOMMU: 2383 iommu->v2 = true; 2384 break; 2385 default: 2386 kfree(iommu); 2387 return ERR_PTR(-EINVAL); 2388 } 2389 2390 INIT_LIST_HEAD(&iommu->domain_list); 2391 INIT_LIST_HEAD(&iommu->iova_list); 2392 iommu->dma_list = RB_ROOT; 2393 iommu->dma_avail = dma_entry_limit; 2394 mutex_init(&iommu->lock); 2395 BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); 2396 2397 return iommu; 2398 } 2399 2400 static void vfio_release_domain(struct vfio_domain *domain, bool external) 2401 { 2402 struct vfio_group *group, *group_tmp; 2403 2404 list_for_each_entry_safe(group, group_tmp, 2405 &domain->group_list, next) { 2406 if (!external) 2407 vfio_iommu_detach_group(domain, group); 2408 list_del(&group->next); 2409 kfree(group); 2410 } 2411 2412 if (!external) 2413 iommu_domain_free(domain->domain); 2414 } 2415 2416 static void vfio_iommu_type1_release(void *iommu_data) 2417 { 2418 struct vfio_iommu *iommu = iommu_data; 2419 struct vfio_domain *domain, *domain_tmp; 2420 2421 if (iommu->external_domain) { 2422 vfio_release_domain(iommu->external_domain, true); 2423 vfio_sanity_check_pfn_list(iommu); 2424 kfree(iommu->external_domain); 2425 } 2426 2427 vfio_iommu_unmap_unpin_all(iommu); 2428 2429 list_for_each_entry_safe(domain, domain_tmp, 2430 &iommu->domain_list, next) { 2431 vfio_release_domain(domain, false); 2432 list_del(&domain->next); 2433 kfree(domain); 2434 } 2435 2436 vfio_iommu_iova_free(&iommu->iova_list); 2437 2438 kfree(iommu); 2439 } 2440 2441 static int vfio_domains_have_iommu_cache(struct vfio_iommu *iommu) 2442 { 2443 struct vfio_domain *domain; 2444 int ret = 1; 2445 2446 mutex_lock(&iommu->lock); 2447 list_for_each_entry(domain, &iommu->domain_list, next) { 2448 if (!(domain->prot & IOMMU_CACHE)) { 2449 ret = 0; 2450 break; 2451 } 2452 } 2453 mutex_unlock(&iommu->lock); 2454 2455 return ret; 2456 } 2457 2458 static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, 2459 unsigned long arg) 2460 { 2461 switch (arg) { 2462 case VFIO_TYPE1_IOMMU: 2463 case VFIO_TYPE1v2_IOMMU: 2464 case VFIO_TYPE1_NESTING_IOMMU: 2465 return 1; 2466 case VFIO_DMA_CC_IOMMU: 2467 if (!iommu) 2468 return 0; 2469 return vfio_domains_have_iommu_cache(iommu); 2470 default: 2471 return 0; 2472 } 2473 } 2474 2475 static int vfio_iommu_iova_add_cap(struct vfio_info_cap *caps, 2476 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas, 2477 size_t size) 2478 { 2479 struct vfio_info_cap_header *header; 2480 struct vfio_iommu_type1_info_cap_iova_range *iova_cap; 2481 2482 header = vfio_info_cap_add(caps, size, 2483 VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE, 1); 2484 if (IS_ERR(header)) 2485 return PTR_ERR(header); 2486 2487 iova_cap = container_of(header, 2488 struct vfio_iommu_type1_info_cap_iova_range, 2489 header); 2490 iova_cap->nr_iovas = cap_iovas->nr_iovas; 2491 memcpy(iova_cap->iova_ranges, cap_iovas->iova_ranges, 2492 cap_iovas->nr_iovas * sizeof(*cap_iovas->iova_ranges)); 2493 return 0; 2494 } 2495 2496 static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu, 2497 struct vfio_info_cap *caps) 2498 { 2499 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas; 2500 struct vfio_iova *iova; 2501 size_t size; 2502 int iovas = 0, i = 0, ret; 2503 2504 list_for_each_entry(iova, &iommu->iova_list, list) 2505 iovas++; 2506 2507 if (!iovas) { 2508 /* 2509 * Return 0 as a container with a single mdev device 2510 * will have an empty list 2511 */ 2512 return 0; 2513 } 2514 2515 size = sizeof(*cap_iovas) + (iovas * sizeof(*cap_iovas->iova_ranges)); 2516 2517 cap_iovas = kzalloc(size, GFP_KERNEL); 2518 if (!cap_iovas) 2519 return -ENOMEM; 2520 2521 cap_iovas->nr_iovas = iovas; 2522 2523 list_for_each_entry(iova, &iommu->iova_list, list) { 2524 cap_iovas->iova_ranges[i].start = iova->start; 2525 cap_iovas->iova_ranges[i].end = iova->end; 2526 i++; 2527 } 2528 2529 ret = vfio_iommu_iova_add_cap(caps, cap_iovas, size); 2530 2531 kfree(cap_iovas); 2532 return ret; 2533 } 2534 2535 static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, 2536 struct vfio_info_cap *caps) 2537 { 2538 struct vfio_iommu_type1_info_cap_migration cap_mig; 2539 2540 cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION; 2541 cap_mig.header.version = 1; 2542 2543 cap_mig.flags = 0; 2544 /* support minimum pgsize */ 2545 cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); 2546 cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; 2547 2548 return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); 2549 } 2550 2551 static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu, 2552 unsigned long arg) 2553 { 2554 struct vfio_iommu_type1_info info; 2555 unsigned long minsz; 2556 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 2557 unsigned long capsz; 2558 int ret; 2559 2560 minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes); 2561 2562 /* For backward compatibility, cannot require this */ 2563 capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); 2564 2565 if (copy_from_user(&info, (void __user *)arg, minsz)) 2566 return -EFAULT; 2567 2568 if (info.argsz < minsz) 2569 return -EINVAL; 2570 2571 if (info.argsz >= capsz) { 2572 minsz = capsz; 2573 info.cap_offset = 0; /* output, no-recopy necessary */ 2574 } 2575 2576 mutex_lock(&iommu->lock); 2577 info.flags = VFIO_IOMMU_INFO_PGSIZES; 2578 2579 info.iova_pgsizes = iommu->pgsize_bitmap; 2580 2581 ret = vfio_iommu_migration_build_caps(iommu, &caps); 2582 2583 if (!ret) 2584 ret = vfio_iommu_iova_build_caps(iommu, &caps); 2585 2586 mutex_unlock(&iommu->lock); 2587 2588 if (ret) 2589 return ret; 2590 2591 if (caps.size) { 2592 info.flags |= VFIO_IOMMU_INFO_CAPS; 2593 2594 if (info.argsz < sizeof(info) + caps.size) { 2595 info.argsz = sizeof(info) + caps.size; 2596 } else { 2597 vfio_info_cap_shift(&caps, sizeof(info)); 2598 if (copy_to_user((void __user *)arg + 2599 sizeof(info), caps.buf, 2600 caps.size)) { 2601 kfree(caps.buf); 2602 return -EFAULT; 2603 } 2604 info.cap_offset = sizeof(info); 2605 } 2606 2607 kfree(caps.buf); 2608 } 2609 2610 return copy_to_user((void __user *)arg, &info, minsz) ? 2611 -EFAULT : 0; 2612 } 2613 2614 static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, 2615 unsigned long arg) 2616 { 2617 struct vfio_iommu_type1_dma_map map; 2618 unsigned long minsz; 2619 uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; 2620 2621 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); 2622 2623 if (copy_from_user(&map, (void __user *)arg, minsz)) 2624 return -EFAULT; 2625 2626 if (map.argsz < minsz || map.flags & ~mask) 2627 return -EINVAL; 2628 2629 return vfio_dma_do_map(iommu, &map); 2630 } 2631 2632 static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, 2633 unsigned long arg) 2634 { 2635 struct vfio_iommu_type1_dma_unmap unmap; 2636 struct vfio_bitmap bitmap = { 0 }; 2637 unsigned long minsz; 2638 int ret; 2639 2640 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size); 2641 2642 if (copy_from_user(&unmap, (void __user *)arg, minsz)) 2643 return -EFAULT; 2644 2645 if (unmap.argsz < minsz || 2646 unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) 2647 return -EINVAL; 2648 2649 if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { 2650 unsigned long pgshift; 2651 2652 if (unmap.argsz < (minsz + sizeof(bitmap))) 2653 return -EINVAL; 2654 2655 if (copy_from_user(&bitmap, 2656 (void __user *)(arg + minsz), 2657 sizeof(bitmap))) 2658 return -EFAULT; 2659 2660 if (!access_ok((void __user *)bitmap.data, bitmap.size)) 2661 return -EINVAL; 2662 2663 pgshift = __ffs(bitmap.pgsize); 2664 ret = verify_bitmap_size(unmap.size >> pgshift, 2665 bitmap.size); 2666 if (ret) 2667 return ret; 2668 } 2669 2670 ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap); 2671 if (ret) 2672 return ret; 2673 2674 return copy_to_user((void __user *)arg, &unmap, minsz) ? 2675 -EFAULT : 0; 2676 } 2677 2678 static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, 2679 unsigned long arg) 2680 { 2681 struct vfio_iommu_type1_dirty_bitmap dirty; 2682 uint32_t mask = VFIO_IOMMU_DIRTY_PAGES_FLAG_START | 2683 VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP | 2684 VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; 2685 unsigned long minsz; 2686 int ret = 0; 2687 2688 if (!iommu->v2) 2689 return -EACCES; 2690 2691 minsz = offsetofend(struct vfio_iommu_type1_dirty_bitmap, flags); 2692 2693 if (copy_from_user(&dirty, (void __user *)arg, minsz)) 2694 return -EFAULT; 2695 2696 if (dirty.argsz < minsz || dirty.flags & ~mask) 2697 return -EINVAL; 2698 2699 /* only one flag should be set at a time */ 2700 if (__ffs(dirty.flags) != __fls(dirty.flags)) 2701 return -EINVAL; 2702 2703 if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_START) { 2704 size_t pgsize; 2705 2706 mutex_lock(&iommu->lock); 2707 pgsize = 1 << __ffs(iommu->pgsize_bitmap); 2708 if (!iommu->dirty_page_tracking) { 2709 ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); 2710 if (!ret) 2711 iommu->dirty_page_tracking = true; 2712 } 2713 mutex_unlock(&iommu->lock); 2714 return ret; 2715 } else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP) { 2716 mutex_lock(&iommu->lock); 2717 if (iommu->dirty_page_tracking) { 2718 iommu->dirty_page_tracking = false; 2719 vfio_dma_bitmap_free_all(iommu); 2720 } 2721 mutex_unlock(&iommu->lock); 2722 return 0; 2723 } else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP) { 2724 struct vfio_iommu_type1_dirty_bitmap_get range; 2725 unsigned long pgshift; 2726 size_t data_size = dirty.argsz - minsz; 2727 size_t iommu_pgsize; 2728 2729 if (!data_size || data_size < sizeof(range)) 2730 return -EINVAL; 2731 2732 if (copy_from_user(&range, (void __user *)(arg + minsz), 2733 sizeof(range))) 2734 return -EFAULT; 2735 2736 if (range.iova + range.size < range.iova) 2737 return -EINVAL; 2738 if (!access_ok((void __user *)range.bitmap.data, 2739 range.bitmap.size)) 2740 return -EINVAL; 2741 2742 pgshift = __ffs(range.bitmap.pgsize); 2743 ret = verify_bitmap_size(range.size >> pgshift, 2744 range.bitmap.size); 2745 if (ret) 2746 return ret; 2747 2748 mutex_lock(&iommu->lock); 2749 2750 iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); 2751 2752 /* allow only smallest supported pgsize */ 2753 if (range.bitmap.pgsize != iommu_pgsize) { 2754 ret = -EINVAL; 2755 goto out_unlock; 2756 } 2757 if (range.iova & (iommu_pgsize - 1)) { 2758 ret = -EINVAL; 2759 goto out_unlock; 2760 } 2761 if (!range.size || range.size & (iommu_pgsize - 1)) { 2762 ret = -EINVAL; 2763 goto out_unlock; 2764 } 2765 2766 if (iommu->dirty_page_tracking) 2767 ret = vfio_iova_dirty_bitmap(range.bitmap.data, 2768 iommu, range.iova, 2769 range.size, 2770 range.bitmap.pgsize); 2771 else 2772 ret = -EINVAL; 2773 out_unlock: 2774 mutex_unlock(&iommu->lock); 2775 2776 return ret; 2777 } 2778 2779 return -EINVAL; 2780 } 2781 2782 static long vfio_iommu_type1_ioctl(void *iommu_data, 2783 unsigned int cmd, unsigned long arg) 2784 { 2785 struct vfio_iommu *iommu = iommu_data; 2786 2787 switch (cmd) { 2788 case VFIO_CHECK_EXTENSION: 2789 return vfio_iommu_type1_check_extension(iommu, arg); 2790 case VFIO_IOMMU_GET_INFO: 2791 return vfio_iommu_type1_get_info(iommu, arg); 2792 case VFIO_IOMMU_MAP_DMA: 2793 return vfio_iommu_type1_map_dma(iommu, arg); 2794 case VFIO_IOMMU_UNMAP_DMA: 2795 return vfio_iommu_type1_unmap_dma(iommu, arg); 2796 case VFIO_IOMMU_DIRTY_PAGES: 2797 return vfio_iommu_type1_dirty_pages(iommu, arg); 2798 default: 2799 return -ENOTTY; 2800 } 2801 } 2802 2803 static int vfio_iommu_type1_register_notifier(void *iommu_data, 2804 unsigned long *events, 2805 struct notifier_block *nb) 2806 { 2807 struct vfio_iommu *iommu = iommu_data; 2808 2809 /* clear known events */ 2810 *events &= ~VFIO_IOMMU_NOTIFY_DMA_UNMAP; 2811 2812 /* refuse to register if still events remaining */ 2813 if (*events) 2814 return -EINVAL; 2815 2816 return blocking_notifier_chain_register(&iommu->notifier, nb); 2817 } 2818 2819 static int vfio_iommu_type1_unregister_notifier(void *iommu_data, 2820 struct notifier_block *nb) 2821 { 2822 struct vfio_iommu *iommu = iommu_data; 2823 2824 return blocking_notifier_chain_unregister(&iommu->notifier, nb); 2825 } 2826 2827 static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, 2828 dma_addr_t user_iova, void *data, 2829 size_t count, bool write, 2830 size_t *copied) 2831 { 2832 struct mm_struct *mm; 2833 unsigned long vaddr; 2834 struct vfio_dma *dma; 2835 bool kthread = current->mm == NULL; 2836 size_t offset; 2837 2838 *copied = 0; 2839 2840 dma = vfio_find_dma(iommu, user_iova, 1); 2841 if (!dma) 2842 return -EINVAL; 2843 2844 if ((write && !(dma->prot & IOMMU_WRITE)) || 2845 !(dma->prot & IOMMU_READ)) 2846 return -EPERM; 2847 2848 mm = get_task_mm(dma->task); 2849 2850 if (!mm) 2851 return -EPERM; 2852 2853 if (kthread) 2854 kthread_use_mm(mm); 2855 else if (current->mm != mm) 2856 goto out; 2857 2858 offset = user_iova - dma->iova; 2859 2860 if (count > dma->size - offset) 2861 count = dma->size - offset; 2862 2863 vaddr = dma->vaddr + offset; 2864 2865 if (write) { 2866 *copied = copy_to_user((void __user *)vaddr, data, 2867 count) ? 0 : count; 2868 if (*copied && iommu->dirty_page_tracking) { 2869 unsigned long pgshift = __ffs(iommu->pgsize_bitmap); 2870 /* 2871 * Bitmap populated with the smallest supported page 2872 * size 2873 */ 2874 bitmap_set(dma->bitmap, offset >> pgshift, 2875 *copied >> pgshift); 2876 } 2877 } else 2878 *copied = copy_from_user(data, (void __user *)vaddr, 2879 count) ? 0 : count; 2880 if (kthread) 2881 kthread_unuse_mm(mm); 2882 out: 2883 mmput(mm); 2884 return *copied ? 0 : -EFAULT; 2885 } 2886 2887 static int vfio_iommu_type1_dma_rw(void *iommu_data, dma_addr_t user_iova, 2888 void *data, size_t count, bool write) 2889 { 2890 struct vfio_iommu *iommu = iommu_data; 2891 int ret = 0; 2892 size_t done; 2893 2894 mutex_lock(&iommu->lock); 2895 while (count > 0) { 2896 ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data, 2897 count, write, &done); 2898 if (ret) 2899 break; 2900 2901 count -= done; 2902 data += done; 2903 user_iova += done; 2904 } 2905 2906 mutex_unlock(&iommu->lock); 2907 return ret; 2908 } 2909 2910 static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { 2911 .name = "vfio-iommu-type1", 2912 .owner = THIS_MODULE, 2913 .open = vfio_iommu_type1_open, 2914 .release = vfio_iommu_type1_release, 2915 .ioctl = vfio_iommu_type1_ioctl, 2916 .attach_group = vfio_iommu_type1_attach_group, 2917 .detach_group = vfio_iommu_type1_detach_group, 2918 .pin_pages = vfio_iommu_type1_pin_pages, 2919 .unpin_pages = vfio_iommu_type1_unpin_pages, 2920 .register_notifier = vfio_iommu_type1_register_notifier, 2921 .unregister_notifier = vfio_iommu_type1_unregister_notifier, 2922 .dma_rw = vfio_iommu_type1_dma_rw, 2923 }; 2924 2925 static int __init vfio_iommu_type1_init(void) 2926 { 2927 return vfio_register_iommu_driver(&vfio_iommu_driver_ops_type1); 2928 } 2929 2930 static void __exit vfio_iommu_type1_cleanup(void) 2931 { 2932 vfio_unregister_iommu_driver(&vfio_iommu_driver_ops_type1); 2933 } 2934 2935 module_init(vfio_iommu_type1_init); 2936 module_exit(vfio_iommu_type1_cleanup); 2937 2938 MODULE_VERSION(DRIVER_VERSION); 2939 MODULE_LICENSE("GPL v2"); 2940 MODULE_AUTHOR(DRIVER_AUTHOR); 2941 MODULE_DESCRIPTION(DRIVER_DESC); 2942