1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO: IOMMU DMA mapping support for Type1 IOMMU 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 * 12 * We arbitrarily define a Type1 IOMMU as one matching the below code. 13 * It could be called the x86 IOMMU as it's designed for AMD-Vi & Intel 14 * VT-d, but that makes it harder to re-use as theoretically anyone 15 * implementing a similar IOMMU could make use of this. We expect the 16 * IOMMU to support the IOMMU API and have few to no restrictions around 17 * the IOVA range that can be mapped. The Type1 IOMMU is currently 18 * optimized for relatively static mappings of a userspace process with 19 * userspace pages pinned into memory. We also assume devices and IOMMU 20 * domains are PCI based as the IOMMU API is still centered around a 21 * device/bus interface rather than a group interface. 22 */ 23 24 #include <linux/compat.h> 25 #include <linux/device.h> 26 #include <linux/fs.h> 27 #include <linux/highmem.h> 28 #include <linux/iommu.h> 29 #include <linux/module.h> 30 #include <linux/mm.h> 31 #include <linux/kthread.h> 32 #include <linux/rbtree.h> 33 #include <linux/sched/signal.h> 34 #include <linux/sched/mm.h> 35 #include <linux/slab.h> 36 #include <linux/uaccess.h> 37 #include <linux/vfio.h> 38 #include <linux/workqueue.h> 39 #include <linux/notifier.h> 40 #include <linux/irqdomain.h> 41 #include "vfio.h" 42 43 #define DRIVER_VERSION "0.2" 44 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 45 #define DRIVER_DESC "Type1 IOMMU driver for VFIO" 46 47 static bool allow_unsafe_interrupts; 48 module_param_named(allow_unsafe_interrupts, 49 allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR); 50 MODULE_PARM_DESC(allow_unsafe_interrupts, 51 "Enable VFIO IOMMU support for on platforms without interrupt remapping support."); 52 53 static bool disable_hugepages; 54 module_param_named(disable_hugepages, 55 disable_hugepages, bool, S_IRUGO | S_IWUSR); 56 MODULE_PARM_DESC(disable_hugepages, 57 "Disable VFIO IOMMU support for IOMMU hugepages."); 58 59 static unsigned int dma_entry_limit __read_mostly = U16_MAX; 60 module_param_named(dma_entry_limit, dma_entry_limit, uint, 0644); 61 MODULE_PARM_DESC(dma_entry_limit, 62 "Maximum number of user DMA mappings per container (65535)."); 63 64 struct vfio_iommu { 65 struct list_head domain_list; 66 struct list_head iova_list; 67 struct mutex lock; 68 struct rb_root dma_list; 69 struct list_head device_list; 70 struct mutex device_list_lock; 71 unsigned int dma_avail; 72 unsigned int vaddr_invalid_count; 73 uint64_t pgsize_bitmap; 74 uint64_t num_non_pinned_groups; 75 wait_queue_head_t vaddr_wait; 76 bool v2; 77 bool nesting; 78 bool dirty_page_tracking; 79 bool container_open; 80 struct list_head emulated_iommu_groups; 81 }; 82 83 struct vfio_domain { 84 struct iommu_domain *domain; 85 struct list_head next; 86 struct list_head group_list; 87 bool fgsp : 1; /* Fine-grained super pages */ 88 bool enforce_cache_coherency : 1; 89 }; 90 91 struct vfio_dma { 92 struct rb_node node; 93 dma_addr_t iova; /* Device address */ 94 unsigned long vaddr; /* Process virtual addr */ 95 size_t size; /* Map size (bytes) */ 96 int prot; /* IOMMU_READ/WRITE */ 97 bool iommu_mapped; 98 bool lock_cap; /* capable(CAP_IPC_LOCK) */ 99 bool vaddr_invalid; 100 struct task_struct *task; 101 struct rb_root pfn_list; /* Ex-user pinned pfn list */ 102 unsigned long *bitmap; 103 }; 104 105 struct vfio_batch { 106 struct page **pages; /* for pin_user_pages_remote */ 107 struct page *fallback_page; /* if pages alloc fails */ 108 int capacity; /* length of pages array */ 109 int size; /* of batch currently */ 110 int offset; /* of next entry in pages */ 111 }; 112 113 struct vfio_iommu_group { 114 struct iommu_group *iommu_group; 115 struct list_head next; 116 bool pinned_page_dirty_scope; 117 }; 118 119 struct vfio_iova { 120 struct list_head list; 121 dma_addr_t start; 122 dma_addr_t end; 123 }; 124 125 /* 126 * Guest RAM pinning working set or DMA target 127 */ 128 struct vfio_pfn { 129 struct rb_node node; 130 dma_addr_t iova; /* Device address */ 131 unsigned long pfn; /* Host pfn */ 132 unsigned int ref_count; 133 }; 134 135 struct vfio_regions { 136 struct list_head list; 137 dma_addr_t iova; 138 phys_addr_t phys; 139 size_t len; 140 }; 141 142 #define DIRTY_BITMAP_BYTES(n) (ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE) 143 144 /* 145 * Input argument of number of bits to bitmap_set() is unsigned integer, which 146 * further casts to signed integer for unaligned multi-bit operation, 147 * __bitmap_set(). 148 * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte, 149 * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page 150 * system. 151 */ 152 #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) 153 #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) 154 155 #define WAITED 1 156 157 static int put_pfn(unsigned long pfn, int prot); 158 159 static struct vfio_iommu_group* 160 vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, 161 struct iommu_group *iommu_group); 162 163 /* 164 * This code handles mapping and unmapping of user data buffers 165 * into DMA'ble space using the IOMMU 166 */ 167 168 static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, 169 dma_addr_t start, size_t size) 170 { 171 struct rb_node *node = iommu->dma_list.rb_node; 172 173 while (node) { 174 struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); 175 176 if (start + size <= dma->iova) 177 node = node->rb_left; 178 else if (start >= dma->iova + dma->size) 179 node = node->rb_right; 180 else 181 return dma; 182 } 183 184 return NULL; 185 } 186 187 static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu, 188 dma_addr_t start, u64 size) 189 { 190 struct rb_node *res = NULL; 191 struct rb_node *node = iommu->dma_list.rb_node; 192 struct vfio_dma *dma_res = NULL; 193 194 while (node) { 195 struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); 196 197 if (start < dma->iova + dma->size) { 198 res = node; 199 dma_res = dma; 200 if (start >= dma->iova) 201 break; 202 node = node->rb_left; 203 } else { 204 node = node->rb_right; 205 } 206 } 207 if (res && size && dma_res->iova >= start + size) 208 res = NULL; 209 return res; 210 } 211 212 static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) 213 { 214 struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; 215 struct vfio_dma *dma; 216 217 while (*link) { 218 parent = *link; 219 dma = rb_entry(parent, struct vfio_dma, node); 220 221 if (new->iova + new->size <= dma->iova) 222 link = &(*link)->rb_left; 223 else 224 link = &(*link)->rb_right; 225 } 226 227 rb_link_node(&new->node, parent, link); 228 rb_insert_color(&new->node, &iommu->dma_list); 229 } 230 231 static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) 232 { 233 rb_erase(&old->node, &iommu->dma_list); 234 } 235 236 237 static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) 238 { 239 uint64_t npages = dma->size / pgsize; 240 241 if (npages > DIRTY_BITMAP_PAGES_MAX) 242 return -EINVAL; 243 244 /* 245 * Allocate extra 64 bits that are used to calculate shift required for 246 * bitmap_shift_left() to manipulate and club unaligned number of pages 247 * in adjacent vfio_dma ranges. 248 */ 249 dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages) + sizeof(u64), 250 GFP_KERNEL); 251 if (!dma->bitmap) 252 return -ENOMEM; 253 254 return 0; 255 } 256 257 static void vfio_dma_bitmap_free(struct vfio_dma *dma) 258 { 259 kvfree(dma->bitmap); 260 dma->bitmap = NULL; 261 } 262 263 static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) 264 { 265 struct rb_node *p; 266 unsigned long pgshift = __ffs(pgsize); 267 268 for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { 269 struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); 270 271 bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); 272 } 273 } 274 275 static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) 276 { 277 struct rb_node *n; 278 unsigned long pgshift = __ffs(iommu->pgsize_bitmap); 279 280 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 281 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 282 283 bitmap_set(dma->bitmap, 0, dma->size >> pgshift); 284 } 285 } 286 287 static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) 288 { 289 struct rb_node *n; 290 291 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 292 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 293 int ret; 294 295 ret = vfio_dma_bitmap_alloc(dma, pgsize); 296 if (ret) { 297 struct rb_node *p; 298 299 for (p = rb_prev(n); p; p = rb_prev(p)) { 300 struct vfio_dma *dma = rb_entry(n, 301 struct vfio_dma, node); 302 303 vfio_dma_bitmap_free(dma); 304 } 305 return ret; 306 } 307 vfio_dma_populate_bitmap(dma, pgsize); 308 } 309 return 0; 310 } 311 312 static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu) 313 { 314 struct rb_node *n; 315 316 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 317 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 318 319 vfio_dma_bitmap_free(dma); 320 } 321 } 322 323 /* 324 * Helper Functions for host iova-pfn list 325 */ 326 static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova) 327 { 328 struct vfio_pfn *vpfn; 329 struct rb_node *node = dma->pfn_list.rb_node; 330 331 while (node) { 332 vpfn = rb_entry(node, struct vfio_pfn, node); 333 334 if (iova < vpfn->iova) 335 node = node->rb_left; 336 else if (iova > vpfn->iova) 337 node = node->rb_right; 338 else 339 return vpfn; 340 } 341 return NULL; 342 } 343 344 static void vfio_link_pfn(struct vfio_dma *dma, 345 struct vfio_pfn *new) 346 { 347 struct rb_node **link, *parent = NULL; 348 struct vfio_pfn *vpfn; 349 350 link = &dma->pfn_list.rb_node; 351 while (*link) { 352 parent = *link; 353 vpfn = rb_entry(parent, struct vfio_pfn, node); 354 355 if (new->iova < vpfn->iova) 356 link = &(*link)->rb_left; 357 else 358 link = &(*link)->rb_right; 359 } 360 361 rb_link_node(&new->node, parent, link); 362 rb_insert_color(&new->node, &dma->pfn_list); 363 } 364 365 static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) 366 { 367 rb_erase(&old->node, &dma->pfn_list); 368 } 369 370 static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, 371 unsigned long pfn) 372 { 373 struct vfio_pfn *vpfn; 374 375 vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL); 376 if (!vpfn) 377 return -ENOMEM; 378 379 vpfn->iova = iova; 380 vpfn->pfn = pfn; 381 vpfn->ref_count = 1; 382 vfio_link_pfn(dma, vpfn); 383 return 0; 384 } 385 386 static void vfio_remove_from_pfn_list(struct vfio_dma *dma, 387 struct vfio_pfn *vpfn) 388 { 389 vfio_unlink_pfn(dma, vpfn); 390 kfree(vpfn); 391 } 392 393 static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma, 394 unsigned long iova) 395 { 396 struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova); 397 398 if (vpfn) 399 vpfn->ref_count++; 400 return vpfn; 401 } 402 403 static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) 404 { 405 int ret = 0; 406 407 vpfn->ref_count--; 408 if (!vpfn->ref_count) { 409 ret = put_pfn(vpfn->pfn, dma->prot); 410 vfio_remove_from_pfn_list(dma, vpfn); 411 } 412 return ret; 413 } 414 415 static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) 416 { 417 struct mm_struct *mm; 418 int ret; 419 420 if (!npage) 421 return 0; 422 423 mm = async ? get_task_mm(dma->task) : dma->task->mm; 424 if (!mm) 425 return -ESRCH; /* process exited */ 426 427 ret = mmap_write_lock_killable(mm); 428 if (!ret) { 429 ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task, 430 dma->lock_cap); 431 mmap_write_unlock(mm); 432 } 433 434 if (async) 435 mmput(mm); 436 437 return ret; 438 } 439 440 /* 441 * Some mappings aren't backed by a struct page, for example an mmap'd 442 * MMIO range for our own or another device. These use a different 443 * pfn conversion and shouldn't be tracked as locked pages. 444 * For compound pages, any driver that sets the reserved bit in head 445 * page needs to set the reserved bit in all subpages to be safe. 446 */ 447 static bool is_invalid_reserved_pfn(unsigned long pfn) 448 { 449 if (pfn_valid(pfn)) 450 return PageReserved(pfn_to_page(pfn)); 451 452 return true; 453 } 454 455 static int put_pfn(unsigned long pfn, int prot) 456 { 457 if (!is_invalid_reserved_pfn(pfn)) { 458 struct page *page = pfn_to_page(pfn); 459 460 unpin_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE); 461 return 1; 462 } 463 return 0; 464 } 465 466 #define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *)) 467 468 static void vfio_batch_init(struct vfio_batch *batch) 469 { 470 batch->size = 0; 471 batch->offset = 0; 472 473 if (unlikely(disable_hugepages)) 474 goto fallback; 475 476 batch->pages = (struct page **) __get_free_page(GFP_KERNEL); 477 if (!batch->pages) 478 goto fallback; 479 480 batch->capacity = VFIO_BATCH_MAX_CAPACITY; 481 return; 482 483 fallback: 484 batch->pages = &batch->fallback_page; 485 batch->capacity = 1; 486 } 487 488 static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma) 489 { 490 while (batch->size) { 491 unsigned long pfn = page_to_pfn(batch->pages[batch->offset]); 492 493 put_pfn(pfn, dma->prot); 494 batch->offset++; 495 batch->size--; 496 } 497 } 498 499 static void vfio_batch_fini(struct vfio_batch *batch) 500 { 501 if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) 502 free_page((unsigned long)batch->pages); 503 } 504 505 static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, 506 unsigned long vaddr, unsigned long *pfn, 507 bool write_fault) 508 { 509 pte_t *ptep; 510 spinlock_t *ptl; 511 int ret; 512 513 ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); 514 if (ret) { 515 bool unlocked = false; 516 517 ret = fixup_user_fault(mm, vaddr, 518 FAULT_FLAG_REMOTE | 519 (write_fault ? FAULT_FLAG_WRITE : 0), 520 &unlocked); 521 if (unlocked) 522 return -EAGAIN; 523 524 if (ret) 525 return ret; 526 527 ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); 528 if (ret) 529 return ret; 530 } 531 532 if (write_fault && !pte_write(*ptep)) 533 ret = -EFAULT; 534 else 535 *pfn = pte_pfn(*ptep); 536 537 pte_unmap_unlock(ptep, ptl); 538 return ret; 539 } 540 541 /* 542 * Returns the positive number of pfns successfully obtained or a negative 543 * error code. 544 */ 545 static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, 546 long npages, int prot, unsigned long *pfn, 547 struct page **pages) 548 { 549 struct vm_area_struct *vma; 550 unsigned int flags = 0; 551 int ret; 552 553 if (prot & IOMMU_WRITE) 554 flags |= FOLL_WRITE; 555 556 mmap_read_lock(mm); 557 ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM, 558 pages, NULL, NULL); 559 if (ret > 0) { 560 int i; 561 562 /* 563 * The zero page is always resident, we don't need to pin it 564 * and it falls into our invalid/reserved test so we don't 565 * unpin in put_pfn(). Unpin all zero pages in the batch here. 566 */ 567 for (i = 0 ; i < ret; i++) { 568 if (unlikely(is_zero_pfn(page_to_pfn(pages[i])))) 569 unpin_user_page(pages[i]); 570 } 571 572 *pfn = page_to_pfn(pages[0]); 573 goto done; 574 } 575 576 vaddr = untagged_addr(vaddr); 577 578 retry: 579 vma = vma_lookup(mm, vaddr); 580 581 if (vma && vma->vm_flags & VM_PFNMAP) { 582 ret = follow_fault_pfn(vma, mm, vaddr, pfn, prot & IOMMU_WRITE); 583 if (ret == -EAGAIN) 584 goto retry; 585 586 if (!ret) { 587 if (is_invalid_reserved_pfn(*pfn)) 588 ret = 1; 589 else 590 ret = -EFAULT; 591 } 592 } 593 done: 594 mmap_read_unlock(mm); 595 return ret; 596 } 597 598 static int vfio_wait(struct vfio_iommu *iommu) 599 { 600 DEFINE_WAIT(wait); 601 602 prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE); 603 mutex_unlock(&iommu->lock); 604 schedule(); 605 mutex_lock(&iommu->lock); 606 finish_wait(&iommu->vaddr_wait, &wait); 607 if (kthread_should_stop() || !iommu->container_open || 608 fatal_signal_pending(current)) { 609 return -EFAULT; 610 } 611 return WAITED; 612 } 613 614 /* 615 * Find dma struct and wait for its vaddr to be valid. iommu lock is dropped 616 * if the task waits, but is re-locked on return. Return result in *dma_p. 617 * Return 0 on success with no waiting, WAITED on success if waited, and -errno 618 * on error. 619 */ 620 static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start, 621 size_t size, struct vfio_dma **dma_p) 622 { 623 int ret = 0; 624 625 do { 626 *dma_p = vfio_find_dma(iommu, start, size); 627 if (!*dma_p) 628 return -EINVAL; 629 else if (!(*dma_p)->vaddr_invalid) 630 return ret; 631 else 632 ret = vfio_wait(iommu); 633 } while (ret == WAITED); 634 635 return ret; 636 } 637 638 /* 639 * Wait for all vaddr in the dma_list to become valid. iommu lock is dropped 640 * if the task waits, but is re-locked on return. Return 0 on success with no 641 * waiting, WAITED on success if waited, and -errno on error. 642 */ 643 static int vfio_wait_all_valid(struct vfio_iommu *iommu) 644 { 645 int ret = 0; 646 647 while (iommu->vaddr_invalid_count && ret >= 0) 648 ret = vfio_wait(iommu); 649 650 return ret; 651 } 652 653 /* 654 * Attempt to pin pages. We really don't want to track all the pfns and 655 * the iommu can only map chunks of consecutive pfns anyway, so get the 656 * first page and all consecutive pages with the same locking. 657 */ 658 static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, 659 long npage, unsigned long *pfn_base, 660 unsigned long limit, struct vfio_batch *batch) 661 { 662 unsigned long pfn; 663 struct mm_struct *mm = current->mm; 664 long ret, pinned = 0, lock_acct = 0; 665 bool rsvd; 666 dma_addr_t iova = vaddr - dma->vaddr + dma->iova; 667 668 /* This code path is only user initiated */ 669 if (!mm) 670 return -ENODEV; 671 672 if (batch->size) { 673 /* Leftover pages in batch from an earlier call. */ 674 *pfn_base = page_to_pfn(batch->pages[batch->offset]); 675 pfn = *pfn_base; 676 rsvd = is_invalid_reserved_pfn(*pfn_base); 677 } else { 678 *pfn_base = 0; 679 } 680 681 while (npage) { 682 if (!batch->size) { 683 /* Empty batch, so refill it. */ 684 long req_pages = min_t(long, npage, batch->capacity); 685 686 ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot, 687 &pfn, batch->pages); 688 if (ret < 0) 689 goto unpin_out; 690 691 batch->size = ret; 692 batch->offset = 0; 693 694 if (!*pfn_base) { 695 *pfn_base = pfn; 696 rsvd = is_invalid_reserved_pfn(*pfn_base); 697 } 698 } 699 700 /* 701 * pfn is preset for the first iteration of this inner loop and 702 * updated at the end to handle a VM_PFNMAP pfn. In that case, 703 * batch->pages isn't valid (there's no struct page), so allow 704 * batch->pages to be touched only when there's more than one 705 * pfn to check, which guarantees the pfns are from a 706 * !VM_PFNMAP vma. 707 */ 708 while (true) { 709 if (pfn != *pfn_base + pinned || 710 rsvd != is_invalid_reserved_pfn(pfn)) 711 goto out; 712 713 /* 714 * Reserved pages aren't counted against the user, 715 * externally pinned pages are already counted against 716 * the user. 717 */ 718 if (!rsvd && !vfio_find_vpfn(dma, iova)) { 719 if (!dma->lock_cap && 720 mm->locked_vm + lock_acct + 1 > limit) { 721 pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", 722 __func__, limit << PAGE_SHIFT); 723 ret = -ENOMEM; 724 goto unpin_out; 725 } 726 lock_acct++; 727 } 728 729 pinned++; 730 npage--; 731 vaddr += PAGE_SIZE; 732 iova += PAGE_SIZE; 733 batch->offset++; 734 batch->size--; 735 736 if (!batch->size) 737 break; 738 739 pfn = page_to_pfn(batch->pages[batch->offset]); 740 } 741 742 if (unlikely(disable_hugepages)) 743 break; 744 } 745 746 out: 747 ret = vfio_lock_acct(dma, lock_acct, false); 748 749 unpin_out: 750 if (batch->size == 1 && !batch->offset) { 751 /* May be a VM_PFNMAP pfn, which the batch can't remember. */ 752 put_pfn(pfn, dma->prot); 753 batch->size = 0; 754 } 755 756 if (ret < 0) { 757 if (pinned && !rsvd) { 758 for (pfn = *pfn_base ; pinned ; pfn++, pinned--) 759 put_pfn(pfn, dma->prot); 760 } 761 vfio_batch_unpin(batch, dma); 762 763 return ret; 764 } 765 766 return pinned; 767 } 768 769 static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, 770 unsigned long pfn, long npage, 771 bool do_accounting) 772 { 773 long unlocked = 0, locked = 0; 774 long i; 775 776 for (i = 0; i < npage; i++, iova += PAGE_SIZE) { 777 if (put_pfn(pfn++, dma->prot)) { 778 unlocked++; 779 if (vfio_find_vpfn(dma, iova)) 780 locked++; 781 } 782 } 783 784 if (do_accounting) 785 vfio_lock_acct(dma, locked - unlocked, true); 786 787 return unlocked; 788 } 789 790 static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, 791 unsigned long *pfn_base, bool do_accounting) 792 { 793 struct page *pages[1]; 794 struct mm_struct *mm; 795 int ret; 796 797 mm = get_task_mm(dma->task); 798 if (!mm) 799 return -ENODEV; 800 801 ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages); 802 if (ret != 1) 803 goto out; 804 805 ret = 0; 806 807 if (do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { 808 ret = vfio_lock_acct(dma, 1, true); 809 if (ret) { 810 put_pfn(*pfn_base, dma->prot); 811 if (ret == -ENOMEM) 812 pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK " 813 "(%ld) exceeded\n", __func__, 814 dma->task->comm, task_pid_nr(dma->task), 815 task_rlimit(dma->task, RLIMIT_MEMLOCK)); 816 } 817 } 818 819 out: 820 mmput(mm); 821 return ret; 822 } 823 824 static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, 825 bool do_accounting) 826 { 827 int unlocked; 828 struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova); 829 830 if (!vpfn) 831 return 0; 832 833 unlocked = vfio_iova_put_vfio_pfn(dma, vpfn); 834 835 if (do_accounting) 836 vfio_lock_acct(dma, -unlocked, true); 837 838 return unlocked; 839 } 840 841 static int vfio_iommu_type1_pin_pages(void *iommu_data, 842 struct iommu_group *iommu_group, 843 dma_addr_t user_iova, 844 int npage, int prot, 845 struct page **pages) 846 { 847 struct vfio_iommu *iommu = iommu_data; 848 struct vfio_iommu_group *group; 849 int i, j, ret; 850 unsigned long remote_vaddr; 851 struct vfio_dma *dma; 852 bool do_accounting; 853 dma_addr_t iova; 854 855 if (!iommu || !pages) 856 return -EINVAL; 857 858 /* Supported for v2 version only */ 859 if (!iommu->v2) 860 return -EACCES; 861 862 mutex_lock(&iommu->lock); 863 864 /* 865 * Wait for all necessary vaddr's to be valid so they can be used in 866 * the main loop without dropping the lock, to avoid racing vs unmap. 867 */ 868 again: 869 if (iommu->vaddr_invalid_count) { 870 for (i = 0; i < npage; i++) { 871 iova = user_iova + PAGE_SIZE * i; 872 ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma); 873 if (ret < 0) 874 goto pin_done; 875 if (ret == WAITED) 876 goto again; 877 } 878 } 879 880 /* Fail if no dma_umap notifier is registered */ 881 if (list_empty(&iommu->device_list)) { 882 ret = -EINVAL; 883 goto pin_done; 884 } 885 886 /* 887 * If iommu capable domain exist in the container then all pages are 888 * already pinned and accounted. Accounting should be done if there is no 889 * iommu capable domain in the container. 890 */ 891 do_accounting = list_empty(&iommu->domain_list); 892 893 for (i = 0; i < npage; i++) { 894 unsigned long phys_pfn; 895 struct vfio_pfn *vpfn; 896 897 iova = user_iova + PAGE_SIZE * i; 898 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 899 if (!dma) { 900 ret = -EINVAL; 901 goto pin_unwind; 902 } 903 904 if ((dma->prot & prot) != prot) { 905 ret = -EPERM; 906 goto pin_unwind; 907 } 908 909 vpfn = vfio_iova_get_vfio_pfn(dma, iova); 910 if (vpfn) { 911 pages[i] = pfn_to_page(vpfn->pfn); 912 continue; 913 } 914 915 remote_vaddr = dma->vaddr + (iova - dma->iova); 916 ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, 917 do_accounting); 918 if (ret) 919 goto pin_unwind; 920 921 ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); 922 if (ret) { 923 if (put_pfn(phys_pfn, dma->prot) && do_accounting) 924 vfio_lock_acct(dma, -1, true); 925 goto pin_unwind; 926 } 927 928 pages[i] = pfn_to_page(phys_pfn); 929 930 if (iommu->dirty_page_tracking) { 931 unsigned long pgshift = __ffs(iommu->pgsize_bitmap); 932 933 /* 934 * Bitmap populated with the smallest supported page 935 * size 936 */ 937 bitmap_set(dma->bitmap, 938 (iova - dma->iova) >> pgshift, 1); 939 } 940 } 941 ret = i; 942 943 group = vfio_iommu_find_iommu_group(iommu, iommu_group); 944 if (!group->pinned_page_dirty_scope) { 945 group->pinned_page_dirty_scope = true; 946 iommu->num_non_pinned_groups--; 947 } 948 949 goto pin_done; 950 951 pin_unwind: 952 pages[i] = NULL; 953 for (j = 0; j < i; j++) { 954 dma_addr_t iova; 955 956 iova = user_iova + PAGE_SIZE * j; 957 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 958 vfio_unpin_page_external(dma, iova, do_accounting); 959 pages[j] = NULL; 960 } 961 pin_done: 962 mutex_unlock(&iommu->lock); 963 return ret; 964 } 965 966 static void vfio_iommu_type1_unpin_pages(void *iommu_data, 967 dma_addr_t user_iova, int npage) 968 { 969 struct vfio_iommu *iommu = iommu_data; 970 bool do_accounting; 971 int i; 972 973 /* Supported for v2 version only */ 974 if (WARN_ON(!iommu->v2)) 975 return; 976 977 mutex_lock(&iommu->lock); 978 979 do_accounting = list_empty(&iommu->domain_list); 980 for (i = 0; i < npage; i++) { 981 dma_addr_t iova = user_iova + PAGE_SIZE * i; 982 struct vfio_dma *dma; 983 984 dma = vfio_find_dma(iommu, iova, PAGE_SIZE); 985 if (!dma) 986 break; 987 988 vfio_unpin_page_external(dma, iova, do_accounting); 989 } 990 991 mutex_unlock(&iommu->lock); 992 993 WARN_ON(i != npage); 994 } 995 996 static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, 997 struct list_head *regions, 998 struct iommu_iotlb_gather *iotlb_gather) 999 { 1000 long unlocked = 0; 1001 struct vfio_regions *entry, *next; 1002 1003 iommu_iotlb_sync(domain->domain, iotlb_gather); 1004 1005 list_for_each_entry_safe(entry, next, regions, list) { 1006 unlocked += vfio_unpin_pages_remote(dma, 1007 entry->iova, 1008 entry->phys >> PAGE_SHIFT, 1009 entry->len >> PAGE_SHIFT, 1010 false); 1011 list_del(&entry->list); 1012 kfree(entry); 1013 } 1014 1015 cond_resched(); 1016 1017 return unlocked; 1018 } 1019 1020 /* 1021 * Generally, VFIO needs to unpin remote pages after each IOTLB flush. 1022 * Therefore, when using IOTLB flush sync interface, VFIO need to keep track 1023 * of these regions (currently using a list). 1024 * 1025 * This value specifies maximum number of regions for each IOTLB flush sync. 1026 */ 1027 #define VFIO_IOMMU_TLB_SYNC_MAX 512 1028 1029 static size_t unmap_unpin_fast(struct vfio_domain *domain, 1030 struct vfio_dma *dma, dma_addr_t *iova, 1031 size_t len, phys_addr_t phys, long *unlocked, 1032 struct list_head *unmapped_list, 1033 int *unmapped_cnt, 1034 struct iommu_iotlb_gather *iotlb_gather) 1035 { 1036 size_t unmapped = 0; 1037 struct vfio_regions *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1038 1039 if (entry) { 1040 unmapped = iommu_unmap_fast(domain->domain, *iova, len, 1041 iotlb_gather); 1042 1043 if (!unmapped) { 1044 kfree(entry); 1045 } else { 1046 entry->iova = *iova; 1047 entry->phys = phys; 1048 entry->len = unmapped; 1049 list_add_tail(&entry->list, unmapped_list); 1050 1051 *iova += unmapped; 1052 (*unmapped_cnt)++; 1053 } 1054 } 1055 1056 /* 1057 * Sync if the number of fast-unmap regions hits the limit 1058 * or in case of errors. 1059 */ 1060 if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX || !unmapped) { 1061 *unlocked += vfio_sync_unpin(dma, domain, unmapped_list, 1062 iotlb_gather); 1063 *unmapped_cnt = 0; 1064 } 1065 1066 return unmapped; 1067 } 1068 1069 static size_t unmap_unpin_slow(struct vfio_domain *domain, 1070 struct vfio_dma *dma, dma_addr_t *iova, 1071 size_t len, phys_addr_t phys, 1072 long *unlocked) 1073 { 1074 size_t unmapped = iommu_unmap(domain->domain, *iova, len); 1075 1076 if (unmapped) { 1077 *unlocked += vfio_unpin_pages_remote(dma, *iova, 1078 phys >> PAGE_SHIFT, 1079 unmapped >> PAGE_SHIFT, 1080 false); 1081 *iova += unmapped; 1082 cond_resched(); 1083 } 1084 return unmapped; 1085 } 1086 1087 static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, 1088 bool do_accounting) 1089 { 1090 dma_addr_t iova = dma->iova, end = dma->iova + dma->size; 1091 struct vfio_domain *domain, *d; 1092 LIST_HEAD(unmapped_region_list); 1093 struct iommu_iotlb_gather iotlb_gather; 1094 int unmapped_region_cnt = 0; 1095 long unlocked = 0; 1096 1097 if (!dma->size) 1098 return 0; 1099 1100 if (list_empty(&iommu->domain_list)) 1101 return 0; 1102 1103 /* 1104 * We use the IOMMU to track the physical addresses, otherwise we'd 1105 * need a much more complicated tracking system. Unfortunately that 1106 * means we need to use one of the iommu domains to figure out the 1107 * pfns to unpin. The rest need to be unmapped in advance so we have 1108 * no iommu translations remaining when the pages are unpinned. 1109 */ 1110 domain = d = list_first_entry(&iommu->domain_list, 1111 struct vfio_domain, next); 1112 1113 list_for_each_entry_continue(d, &iommu->domain_list, next) { 1114 iommu_unmap(d->domain, dma->iova, dma->size); 1115 cond_resched(); 1116 } 1117 1118 iommu_iotlb_gather_init(&iotlb_gather); 1119 while (iova < end) { 1120 size_t unmapped, len; 1121 phys_addr_t phys, next; 1122 1123 phys = iommu_iova_to_phys(domain->domain, iova); 1124 if (WARN_ON(!phys)) { 1125 iova += PAGE_SIZE; 1126 continue; 1127 } 1128 1129 /* 1130 * To optimize for fewer iommu_unmap() calls, each of which 1131 * may require hardware cache flushing, try to find the 1132 * largest contiguous physical memory chunk to unmap. 1133 */ 1134 for (len = PAGE_SIZE; 1135 !domain->fgsp && iova + len < end; len += PAGE_SIZE) { 1136 next = iommu_iova_to_phys(domain->domain, iova + len); 1137 if (next != phys + len) 1138 break; 1139 } 1140 1141 /* 1142 * First, try to use fast unmap/unpin. In case of failure, 1143 * switch to slow unmap/unpin path. 1144 */ 1145 unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys, 1146 &unlocked, &unmapped_region_list, 1147 &unmapped_region_cnt, 1148 &iotlb_gather); 1149 if (!unmapped) { 1150 unmapped = unmap_unpin_slow(domain, dma, &iova, len, 1151 phys, &unlocked); 1152 if (WARN_ON(!unmapped)) 1153 break; 1154 } 1155 } 1156 1157 dma->iommu_mapped = false; 1158 1159 if (unmapped_region_cnt) { 1160 unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list, 1161 &iotlb_gather); 1162 } 1163 1164 if (do_accounting) { 1165 vfio_lock_acct(dma, -unlocked, true); 1166 return 0; 1167 } 1168 return unlocked; 1169 } 1170 1171 static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) 1172 { 1173 WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); 1174 vfio_unmap_unpin(iommu, dma, true); 1175 vfio_unlink_dma(iommu, dma); 1176 put_task_struct(dma->task); 1177 vfio_dma_bitmap_free(dma); 1178 if (dma->vaddr_invalid) { 1179 iommu->vaddr_invalid_count--; 1180 wake_up_all(&iommu->vaddr_wait); 1181 } 1182 kfree(dma); 1183 iommu->dma_avail++; 1184 } 1185 1186 static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) 1187 { 1188 struct vfio_domain *domain; 1189 1190 iommu->pgsize_bitmap = ULONG_MAX; 1191 1192 list_for_each_entry(domain, &iommu->domain_list, next) 1193 iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; 1194 1195 /* 1196 * In case the IOMMU supports page sizes smaller than PAGE_SIZE 1197 * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. 1198 * That way the user will be able to map/unmap buffers whose size/ 1199 * start address is aligned with PAGE_SIZE. Pinning code uses that 1200 * granularity while iommu driver can use the sub-PAGE_SIZE size 1201 * to map the buffer. 1202 */ 1203 if (iommu->pgsize_bitmap & ~PAGE_MASK) { 1204 iommu->pgsize_bitmap &= PAGE_MASK; 1205 iommu->pgsize_bitmap |= PAGE_SIZE; 1206 } 1207 } 1208 1209 static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, 1210 struct vfio_dma *dma, dma_addr_t base_iova, 1211 size_t pgsize) 1212 { 1213 unsigned long pgshift = __ffs(pgsize); 1214 unsigned long nbits = dma->size >> pgshift; 1215 unsigned long bit_offset = (dma->iova - base_iova) >> pgshift; 1216 unsigned long copy_offset = bit_offset / BITS_PER_LONG; 1217 unsigned long shift = bit_offset % BITS_PER_LONG; 1218 unsigned long leftover; 1219 1220 /* 1221 * mark all pages dirty if any IOMMU capable device is not able 1222 * to report dirty pages and all pages are pinned and mapped. 1223 */ 1224 if (iommu->num_non_pinned_groups && dma->iommu_mapped) 1225 bitmap_set(dma->bitmap, 0, nbits); 1226 1227 if (shift) { 1228 bitmap_shift_left(dma->bitmap, dma->bitmap, shift, 1229 nbits + shift); 1230 1231 if (copy_from_user(&leftover, 1232 (void __user *)(bitmap + copy_offset), 1233 sizeof(leftover))) 1234 return -EFAULT; 1235 1236 bitmap_or(dma->bitmap, dma->bitmap, &leftover, shift); 1237 } 1238 1239 if (copy_to_user((void __user *)(bitmap + copy_offset), dma->bitmap, 1240 DIRTY_BITMAP_BYTES(nbits + shift))) 1241 return -EFAULT; 1242 1243 return 0; 1244 } 1245 1246 static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, 1247 dma_addr_t iova, size_t size, size_t pgsize) 1248 { 1249 struct vfio_dma *dma; 1250 struct rb_node *n; 1251 unsigned long pgshift = __ffs(pgsize); 1252 int ret; 1253 1254 /* 1255 * GET_BITMAP request must fully cover vfio_dma mappings. Multiple 1256 * vfio_dma mappings may be clubbed by specifying large ranges, but 1257 * there must not be any previous mappings bisected by the range. 1258 * An error will be returned if these conditions are not met. 1259 */ 1260 dma = vfio_find_dma(iommu, iova, 1); 1261 if (dma && dma->iova != iova) 1262 return -EINVAL; 1263 1264 dma = vfio_find_dma(iommu, iova + size - 1, 0); 1265 if (dma && dma->iova + dma->size != iova + size) 1266 return -EINVAL; 1267 1268 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 1269 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 1270 1271 if (dma->iova < iova) 1272 continue; 1273 1274 if (dma->iova > iova + size - 1) 1275 break; 1276 1277 ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize); 1278 if (ret) 1279 return ret; 1280 1281 /* 1282 * Re-populate bitmap to include all pinned pages which are 1283 * considered as dirty but exclude pages which are unpinned and 1284 * pages which are marked dirty by vfio_dma_rw() 1285 */ 1286 bitmap_clear(dma->bitmap, 0, dma->size >> pgshift); 1287 vfio_dma_populate_bitmap(dma, pgsize); 1288 } 1289 return 0; 1290 } 1291 1292 static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size) 1293 { 1294 if (!npages || !bitmap_size || (bitmap_size > DIRTY_BITMAP_SIZE_MAX) || 1295 (bitmap_size < DIRTY_BITMAP_BYTES(npages))) 1296 return -EINVAL; 1297 1298 return 0; 1299 } 1300 1301 /* 1302 * Notify VFIO drivers using vfio_register_emulated_iommu_dev() to invalidate 1303 * and unmap iovas within the range we're about to unmap. Drivers MUST unpin 1304 * pages in response to an invalidation. 1305 */ 1306 static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, 1307 struct vfio_dma *dma) 1308 { 1309 struct vfio_device *device; 1310 1311 if (list_empty(&iommu->device_list)) 1312 return; 1313 1314 /* 1315 * The device is expected to call vfio_unpin_pages() for any IOVA it has 1316 * pinned within the range. Since vfio_unpin_pages() will eventually 1317 * call back down to this code and try to obtain the iommu->lock we must 1318 * drop it. 1319 */ 1320 mutex_lock(&iommu->device_list_lock); 1321 mutex_unlock(&iommu->lock); 1322 1323 list_for_each_entry(device, &iommu->device_list, iommu_entry) 1324 device->ops->dma_unmap(device, dma->iova, dma->size); 1325 1326 mutex_unlock(&iommu->device_list_lock); 1327 mutex_lock(&iommu->lock); 1328 } 1329 1330 static int vfio_dma_do_unmap(struct vfio_iommu *iommu, 1331 struct vfio_iommu_type1_dma_unmap *unmap, 1332 struct vfio_bitmap *bitmap) 1333 { 1334 struct vfio_dma *dma, *dma_last = NULL; 1335 size_t unmapped = 0, pgsize; 1336 int ret = -EINVAL, retries = 0; 1337 unsigned long pgshift; 1338 dma_addr_t iova = unmap->iova; 1339 u64 size = unmap->size; 1340 bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; 1341 bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR; 1342 struct rb_node *n, *first_n; 1343 1344 mutex_lock(&iommu->lock); 1345 1346 pgshift = __ffs(iommu->pgsize_bitmap); 1347 pgsize = (size_t)1 << pgshift; 1348 1349 if (iova & (pgsize - 1)) 1350 goto unlock; 1351 1352 if (unmap_all) { 1353 if (iova || size) 1354 goto unlock; 1355 size = U64_MAX; 1356 } else if (!size || size & (pgsize - 1) || 1357 iova + size - 1 < iova || size > SIZE_MAX) { 1358 goto unlock; 1359 } 1360 1361 /* When dirty tracking is enabled, allow only min supported pgsize */ 1362 if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && 1363 (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { 1364 goto unlock; 1365 } 1366 1367 WARN_ON((pgsize - 1) & PAGE_MASK); 1368 again: 1369 /* 1370 * vfio-iommu-type1 (v1) - User mappings were coalesced together to 1371 * avoid tracking individual mappings. This means that the granularity 1372 * of the original mapping was lost and the user was allowed to attempt 1373 * to unmap any range. Depending on the contiguousness of physical 1374 * memory and page sizes supported by the IOMMU, arbitrary unmaps may 1375 * or may not have worked. We only guaranteed unmap granularity 1376 * matching the original mapping; even though it was untracked here, 1377 * the original mappings are reflected in IOMMU mappings. This 1378 * resulted in a couple unusual behaviors. First, if a range is not 1379 * able to be unmapped, ex. a set of 4k pages that was mapped as a 1380 * 2M hugepage into the IOMMU, the unmap ioctl returns success but with 1381 * a zero sized unmap. Also, if an unmap request overlaps the first 1382 * address of a hugepage, the IOMMU will unmap the entire hugepage. 1383 * This also returns success and the returned unmap size reflects the 1384 * actual size unmapped. 1385 * 1386 * We attempt to maintain compatibility with this "v1" interface, but 1387 * we take control out of the hands of the IOMMU. Therefore, an unmap 1388 * request offset from the beginning of the original mapping will 1389 * return success with zero sized unmap. And an unmap request covering 1390 * the first iova of mapping will unmap the entire range. 1391 * 1392 * The v2 version of this interface intends to be more deterministic. 1393 * Unmap requests must fully cover previous mappings. Multiple 1394 * mappings may still be unmaped by specifying large ranges, but there 1395 * must not be any previous mappings bisected by the range. An error 1396 * will be returned if these conditions are not met. The v2 interface 1397 * will only return success and a size of zero if there were no 1398 * mappings within the range. 1399 */ 1400 if (iommu->v2 && !unmap_all) { 1401 dma = vfio_find_dma(iommu, iova, 1); 1402 if (dma && dma->iova != iova) 1403 goto unlock; 1404 1405 dma = vfio_find_dma(iommu, iova + size - 1, 0); 1406 if (dma && dma->iova + dma->size != iova + size) 1407 goto unlock; 1408 } 1409 1410 ret = 0; 1411 n = first_n = vfio_find_dma_first_node(iommu, iova, size); 1412 1413 while (n) { 1414 dma = rb_entry(n, struct vfio_dma, node); 1415 if (dma->iova >= iova + size) 1416 break; 1417 1418 if (!iommu->v2 && iova > dma->iova) 1419 break; 1420 1421 if (invalidate_vaddr) { 1422 if (dma->vaddr_invalid) { 1423 struct rb_node *last_n = n; 1424 1425 for (n = first_n; n != last_n; n = rb_next(n)) { 1426 dma = rb_entry(n, 1427 struct vfio_dma, node); 1428 dma->vaddr_invalid = false; 1429 iommu->vaddr_invalid_count--; 1430 } 1431 ret = -EINVAL; 1432 unmapped = 0; 1433 break; 1434 } 1435 dma->vaddr_invalid = true; 1436 iommu->vaddr_invalid_count++; 1437 unmapped += dma->size; 1438 n = rb_next(n); 1439 continue; 1440 } 1441 1442 if (!RB_EMPTY_ROOT(&dma->pfn_list)) { 1443 if (dma_last == dma) { 1444 BUG_ON(++retries > 10); 1445 } else { 1446 dma_last = dma; 1447 retries = 0; 1448 } 1449 1450 vfio_notify_dma_unmap(iommu, dma); 1451 goto again; 1452 } 1453 1454 if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { 1455 ret = update_user_bitmap(bitmap->data, iommu, dma, 1456 iova, pgsize); 1457 if (ret) 1458 break; 1459 } 1460 1461 unmapped += dma->size; 1462 n = rb_next(n); 1463 vfio_remove_dma(iommu, dma); 1464 } 1465 1466 unlock: 1467 mutex_unlock(&iommu->lock); 1468 1469 /* Report how much was unmapped */ 1470 unmap->size = unmapped; 1471 1472 return ret; 1473 } 1474 1475 static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, 1476 unsigned long pfn, long npage, int prot) 1477 { 1478 struct vfio_domain *d; 1479 int ret; 1480 1481 list_for_each_entry(d, &iommu->domain_list, next) { 1482 ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, 1483 npage << PAGE_SHIFT, prot | IOMMU_CACHE); 1484 if (ret) 1485 goto unwind; 1486 1487 cond_resched(); 1488 } 1489 1490 return 0; 1491 1492 unwind: 1493 list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) { 1494 iommu_unmap(d->domain, iova, npage << PAGE_SHIFT); 1495 cond_resched(); 1496 } 1497 1498 return ret; 1499 } 1500 1501 static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, 1502 size_t map_size) 1503 { 1504 dma_addr_t iova = dma->iova; 1505 unsigned long vaddr = dma->vaddr; 1506 struct vfio_batch batch; 1507 size_t size = map_size; 1508 long npage; 1509 unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1510 int ret = 0; 1511 1512 vfio_batch_init(&batch); 1513 1514 while (size) { 1515 /* Pin a contiguous chunk of memory */ 1516 npage = vfio_pin_pages_remote(dma, vaddr + dma->size, 1517 size >> PAGE_SHIFT, &pfn, limit, 1518 &batch); 1519 if (npage <= 0) { 1520 WARN_ON(!npage); 1521 ret = (int)npage; 1522 break; 1523 } 1524 1525 /* Map it! */ 1526 ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, 1527 dma->prot); 1528 if (ret) { 1529 vfio_unpin_pages_remote(dma, iova + dma->size, pfn, 1530 npage, true); 1531 vfio_batch_unpin(&batch, dma); 1532 break; 1533 } 1534 1535 size -= npage << PAGE_SHIFT; 1536 dma->size += npage << PAGE_SHIFT; 1537 } 1538 1539 vfio_batch_fini(&batch); 1540 dma->iommu_mapped = true; 1541 1542 if (ret) 1543 vfio_remove_dma(iommu, dma); 1544 1545 return ret; 1546 } 1547 1548 /* 1549 * Check dma map request is within a valid iova range 1550 */ 1551 static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, 1552 dma_addr_t start, dma_addr_t end) 1553 { 1554 struct list_head *iova = &iommu->iova_list; 1555 struct vfio_iova *node; 1556 1557 list_for_each_entry(node, iova, list) { 1558 if (start >= node->start && end <= node->end) 1559 return true; 1560 } 1561 1562 /* 1563 * Check for list_empty() as well since a container with 1564 * a single mdev device will have an empty list. 1565 */ 1566 return list_empty(iova); 1567 } 1568 1569 static int vfio_dma_do_map(struct vfio_iommu *iommu, 1570 struct vfio_iommu_type1_dma_map *map) 1571 { 1572 bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR; 1573 dma_addr_t iova = map->iova; 1574 unsigned long vaddr = map->vaddr; 1575 size_t size = map->size; 1576 int ret = 0, prot = 0; 1577 size_t pgsize; 1578 struct vfio_dma *dma; 1579 1580 /* Verify that none of our __u64 fields overflow */ 1581 if (map->size != size || map->vaddr != vaddr || map->iova != iova) 1582 return -EINVAL; 1583 1584 /* READ/WRITE from device perspective */ 1585 if (map->flags & VFIO_DMA_MAP_FLAG_WRITE) 1586 prot |= IOMMU_WRITE; 1587 if (map->flags & VFIO_DMA_MAP_FLAG_READ) 1588 prot |= IOMMU_READ; 1589 1590 if ((prot && set_vaddr) || (!prot && !set_vaddr)) 1591 return -EINVAL; 1592 1593 mutex_lock(&iommu->lock); 1594 1595 pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); 1596 1597 WARN_ON((pgsize - 1) & PAGE_MASK); 1598 1599 if (!size || (size | iova | vaddr) & (pgsize - 1)) { 1600 ret = -EINVAL; 1601 goto out_unlock; 1602 } 1603 1604 /* Don't allow IOVA or virtual address wrap */ 1605 if (iova + size - 1 < iova || vaddr + size - 1 < vaddr) { 1606 ret = -EINVAL; 1607 goto out_unlock; 1608 } 1609 1610 dma = vfio_find_dma(iommu, iova, size); 1611 if (set_vaddr) { 1612 if (!dma) { 1613 ret = -ENOENT; 1614 } else if (!dma->vaddr_invalid || dma->iova != iova || 1615 dma->size != size) { 1616 ret = -EINVAL; 1617 } else { 1618 dma->vaddr = vaddr; 1619 dma->vaddr_invalid = false; 1620 iommu->vaddr_invalid_count--; 1621 wake_up_all(&iommu->vaddr_wait); 1622 } 1623 goto out_unlock; 1624 } else if (dma) { 1625 ret = -EEXIST; 1626 goto out_unlock; 1627 } 1628 1629 if (!iommu->dma_avail) { 1630 ret = -ENOSPC; 1631 goto out_unlock; 1632 } 1633 1634 if (!vfio_iommu_iova_dma_valid(iommu, iova, iova + size - 1)) { 1635 ret = -EINVAL; 1636 goto out_unlock; 1637 } 1638 1639 dma = kzalloc(sizeof(*dma), GFP_KERNEL); 1640 if (!dma) { 1641 ret = -ENOMEM; 1642 goto out_unlock; 1643 } 1644 1645 iommu->dma_avail--; 1646 dma->iova = iova; 1647 dma->vaddr = vaddr; 1648 dma->prot = prot; 1649 1650 /* 1651 * We need to be able to both add to a task's locked memory and test 1652 * against the locked memory limit and we need to be able to do both 1653 * outside of this call path as pinning can be asynchronous via the 1654 * external interfaces for mdev devices. RLIMIT_MEMLOCK requires a 1655 * task_struct and VM locked pages requires an mm_struct, however 1656 * holding an indefinite mm reference is not recommended, therefore we 1657 * only hold a reference to a task. We could hold a reference to 1658 * current, however QEMU uses this call path through vCPU threads, 1659 * which can be killed resulting in a NULL mm and failure in the unmap 1660 * path when called via a different thread. Avoid this problem by 1661 * using the group_leader as threads within the same group require 1662 * both CLONE_THREAD and CLONE_VM and will therefore use the same 1663 * mm_struct. 1664 * 1665 * Previously we also used the task for testing CAP_IPC_LOCK at the 1666 * time of pinning and accounting, however has_capability() makes use 1667 * of real_cred, a copy-on-write field, so we can't guarantee that it 1668 * matches group_leader, or in fact that it might not change by the 1669 * time it's evaluated. If a process were to call MAP_DMA with 1670 * CAP_IPC_LOCK but later drop it, it doesn't make sense that they 1671 * possibly see different results for an iommu_mapped vfio_dma vs 1672 * externally mapped. Therefore track CAP_IPC_LOCK in vfio_dma at the 1673 * time of calling MAP_DMA. 1674 */ 1675 get_task_struct(current->group_leader); 1676 dma->task = current->group_leader; 1677 dma->lock_cap = capable(CAP_IPC_LOCK); 1678 1679 dma->pfn_list = RB_ROOT; 1680 1681 /* Insert zero-sized and grow as we map chunks of it */ 1682 vfio_link_dma(iommu, dma); 1683 1684 /* Don't pin and map if container doesn't contain IOMMU capable domain*/ 1685 if (list_empty(&iommu->domain_list)) 1686 dma->size = size; 1687 else 1688 ret = vfio_pin_map_dma(iommu, dma, size); 1689 1690 if (!ret && iommu->dirty_page_tracking) { 1691 ret = vfio_dma_bitmap_alloc(dma, pgsize); 1692 if (ret) 1693 vfio_remove_dma(iommu, dma); 1694 } 1695 1696 out_unlock: 1697 mutex_unlock(&iommu->lock); 1698 return ret; 1699 } 1700 1701 static int vfio_iommu_replay(struct vfio_iommu *iommu, 1702 struct vfio_domain *domain) 1703 { 1704 struct vfio_batch batch; 1705 struct vfio_domain *d = NULL; 1706 struct rb_node *n; 1707 unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1708 int ret; 1709 1710 ret = vfio_wait_all_valid(iommu); 1711 if (ret < 0) 1712 return ret; 1713 1714 /* Arbitrarily pick the first domain in the list for lookups */ 1715 if (!list_empty(&iommu->domain_list)) 1716 d = list_first_entry(&iommu->domain_list, 1717 struct vfio_domain, next); 1718 1719 vfio_batch_init(&batch); 1720 1721 n = rb_first(&iommu->dma_list); 1722 1723 for (; n; n = rb_next(n)) { 1724 struct vfio_dma *dma; 1725 dma_addr_t iova; 1726 1727 dma = rb_entry(n, struct vfio_dma, node); 1728 iova = dma->iova; 1729 1730 while (iova < dma->iova + dma->size) { 1731 phys_addr_t phys; 1732 size_t size; 1733 1734 if (dma->iommu_mapped) { 1735 phys_addr_t p; 1736 dma_addr_t i; 1737 1738 if (WARN_ON(!d)) { /* mapped w/o a domain?! */ 1739 ret = -EINVAL; 1740 goto unwind; 1741 } 1742 1743 phys = iommu_iova_to_phys(d->domain, iova); 1744 1745 if (WARN_ON(!phys)) { 1746 iova += PAGE_SIZE; 1747 continue; 1748 } 1749 1750 size = PAGE_SIZE; 1751 p = phys + size; 1752 i = iova + size; 1753 while (i < dma->iova + dma->size && 1754 p == iommu_iova_to_phys(d->domain, i)) { 1755 size += PAGE_SIZE; 1756 p += PAGE_SIZE; 1757 i += PAGE_SIZE; 1758 } 1759 } else { 1760 unsigned long pfn; 1761 unsigned long vaddr = dma->vaddr + 1762 (iova - dma->iova); 1763 size_t n = dma->iova + dma->size - iova; 1764 long npage; 1765 1766 npage = vfio_pin_pages_remote(dma, vaddr, 1767 n >> PAGE_SHIFT, 1768 &pfn, limit, 1769 &batch); 1770 if (npage <= 0) { 1771 WARN_ON(!npage); 1772 ret = (int)npage; 1773 goto unwind; 1774 } 1775 1776 phys = pfn << PAGE_SHIFT; 1777 size = npage << PAGE_SHIFT; 1778 } 1779 1780 ret = iommu_map(domain->domain, iova, phys, 1781 size, dma->prot | IOMMU_CACHE); 1782 if (ret) { 1783 if (!dma->iommu_mapped) { 1784 vfio_unpin_pages_remote(dma, iova, 1785 phys >> PAGE_SHIFT, 1786 size >> PAGE_SHIFT, 1787 true); 1788 vfio_batch_unpin(&batch, dma); 1789 } 1790 goto unwind; 1791 } 1792 1793 iova += size; 1794 } 1795 } 1796 1797 /* All dmas are now mapped, defer to second tree walk for unwind */ 1798 for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { 1799 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 1800 1801 dma->iommu_mapped = true; 1802 } 1803 1804 vfio_batch_fini(&batch); 1805 return 0; 1806 1807 unwind: 1808 for (; n; n = rb_prev(n)) { 1809 struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); 1810 dma_addr_t iova; 1811 1812 if (dma->iommu_mapped) { 1813 iommu_unmap(domain->domain, dma->iova, dma->size); 1814 continue; 1815 } 1816 1817 iova = dma->iova; 1818 while (iova < dma->iova + dma->size) { 1819 phys_addr_t phys, p; 1820 size_t size; 1821 dma_addr_t i; 1822 1823 phys = iommu_iova_to_phys(domain->domain, iova); 1824 if (!phys) { 1825 iova += PAGE_SIZE; 1826 continue; 1827 } 1828 1829 size = PAGE_SIZE; 1830 p = phys + size; 1831 i = iova + size; 1832 while (i < dma->iova + dma->size && 1833 p == iommu_iova_to_phys(domain->domain, i)) { 1834 size += PAGE_SIZE; 1835 p += PAGE_SIZE; 1836 i += PAGE_SIZE; 1837 } 1838 1839 iommu_unmap(domain->domain, iova, size); 1840 vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT, 1841 size >> PAGE_SHIFT, true); 1842 } 1843 } 1844 1845 vfio_batch_fini(&batch); 1846 return ret; 1847 } 1848 1849 /* 1850 * We change our unmap behavior slightly depending on whether the IOMMU 1851 * supports fine-grained superpages. IOMMUs like AMD-Vi will use a superpage 1852 * for practically any contiguous power-of-two mapping we give it. This means 1853 * we don't need to look for contiguous chunks ourselves to make unmapping 1854 * more efficient. On IOMMUs with coarse-grained super pages, like Intel VT-d 1855 * with discrete 2M/1G/512G/1T superpages, identifying contiguous chunks 1856 * significantly boosts non-hugetlbfs mappings and doesn't seem to hurt when 1857 * hugetlbfs is in use. 1858 */ 1859 static void vfio_test_domain_fgsp(struct vfio_domain *domain) 1860 { 1861 struct page *pages; 1862 int ret, order = get_order(PAGE_SIZE * 2); 1863 1864 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1865 if (!pages) 1866 return; 1867 1868 ret = iommu_map(domain->domain, 0, page_to_phys(pages), PAGE_SIZE * 2, 1869 IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); 1870 if (!ret) { 1871 size_t unmapped = iommu_unmap(domain->domain, 0, PAGE_SIZE); 1872 1873 if (unmapped == PAGE_SIZE) 1874 iommu_unmap(domain->domain, PAGE_SIZE, PAGE_SIZE); 1875 else 1876 domain->fgsp = true; 1877 } 1878 1879 __free_pages(pages, order); 1880 } 1881 1882 static struct vfio_iommu_group *find_iommu_group(struct vfio_domain *domain, 1883 struct iommu_group *iommu_group) 1884 { 1885 struct vfio_iommu_group *g; 1886 1887 list_for_each_entry(g, &domain->group_list, next) { 1888 if (g->iommu_group == iommu_group) 1889 return g; 1890 } 1891 1892 return NULL; 1893 } 1894 1895 static struct vfio_iommu_group* 1896 vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, 1897 struct iommu_group *iommu_group) 1898 { 1899 struct vfio_iommu_group *group; 1900 struct vfio_domain *domain; 1901 1902 list_for_each_entry(domain, &iommu->domain_list, next) { 1903 group = find_iommu_group(domain, iommu_group); 1904 if (group) 1905 return group; 1906 } 1907 1908 list_for_each_entry(group, &iommu->emulated_iommu_groups, next) 1909 if (group->iommu_group == iommu_group) 1910 return group; 1911 return NULL; 1912 } 1913 1914 static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, 1915 phys_addr_t *base) 1916 { 1917 struct iommu_resv_region *region; 1918 bool ret = false; 1919 1920 list_for_each_entry(region, group_resv_regions, list) { 1921 /* 1922 * The presence of any 'real' MSI regions should take 1923 * precedence over the software-managed one if the 1924 * IOMMU driver happens to advertise both types. 1925 */ 1926 if (region->type == IOMMU_RESV_MSI) { 1927 ret = false; 1928 break; 1929 } 1930 1931 if (region->type == IOMMU_RESV_SW_MSI) { 1932 *base = region->start; 1933 ret = true; 1934 } 1935 } 1936 1937 return ret; 1938 } 1939 1940 /* 1941 * This is a helper function to insert an address range to iova list. 1942 * The list is initially created with a single entry corresponding to 1943 * the IOMMU domain geometry to which the device group is attached. 1944 * The list aperture gets modified when a new domain is added to the 1945 * container if the new aperture doesn't conflict with the current one 1946 * or with any existing dma mappings. The list is also modified to 1947 * exclude any reserved regions associated with the device group. 1948 */ 1949 static int vfio_iommu_iova_insert(struct list_head *head, 1950 dma_addr_t start, dma_addr_t end) 1951 { 1952 struct vfio_iova *region; 1953 1954 region = kmalloc(sizeof(*region), GFP_KERNEL); 1955 if (!region) 1956 return -ENOMEM; 1957 1958 INIT_LIST_HEAD(®ion->list); 1959 region->start = start; 1960 region->end = end; 1961 1962 list_add_tail(®ion->list, head); 1963 return 0; 1964 } 1965 1966 /* 1967 * Check the new iommu aperture conflicts with existing aper or with any 1968 * existing dma mappings. 1969 */ 1970 static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu, 1971 dma_addr_t start, dma_addr_t end) 1972 { 1973 struct vfio_iova *first, *last; 1974 struct list_head *iova = &iommu->iova_list; 1975 1976 if (list_empty(iova)) 1977 return false; 1978 1979 /* Disjoint sets, return conflict */ 1980 first = list_first_entry(iova, struct vfio_iova, list); 1981 last = list_last_entry(iova, struct vfio_iova, list); 1982 if (start > last->end || end < first->start) 1983 return true; 1984 1985 /* Check for any existing dma mappings below the new start */ 1986 if (start > first->start) { 1987 if (vfio_find_dma(iommu, first->start, start - first->start)) 1988 return true; 1989 } 1990 1991 /* Check for any existing dma mappings beyond the new end */ 1992 if (end < last->end) { 1993 if (vfio_find_dma(iommu, end + 1, last->end - end)) 1994 return true; 1995 } 1996 1997 return false; 1998 } 1999 2000 /* 2001 * Resize iommu iova aperture window. This is called only if the new 2002 * aperture has no conflict with existing aperture and dma mappings. 2003 */ 2004 static int vfio_iommu_aper_resize(struct list_head *iova, 2005 dma_addr_t start, dma_addr_t end) 2006 { 2007 struct vfio_iova *node, *next; 2008 2009 if (list_empty(iova)) 2010 return vfio_iommu_iova_insert(iova, start, end); 2011 2012 /* Adjust iova list start */ 2013 list_for_each_entry_safe(node, next, iova, list) { 2014 if (start < node->start) 2015 break; 2016 if (start >= node->start && start < node->end) { 2017 node->start = start; 2018 break; 2019 } 2020 /* Delete nodes before new start */ 2021 list_del(&node->list); 2022 kfree(node); 2023 } 2024 2025 /* Adjust iova list end */ 2026 list_for_each_entry_safe(node, next, iova, list) { 2027 if (end > node->end) 2028 continue; 2029 if (end > node->start && end <= node->end) { 2030 node->end = end; 2031 continue; 2032 } 2033 /* Delete nodes after new end */ 2034 list_del(&node->list); 2035 kfree(node); 2036 } 2037 2038 return 0; 2039 } 2040 2041 /* 2042 * Check reserved region conflicts with existing dma mappings 2043 */ 2044 static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu, 2045 struct list_head *resv_regions) 2046 { 2047 struct iommu_resv_region *region; 2048 2049 /* Check for conflict with existing dma mappings */ 2050 list_for_each_entry(region, resv_regions, list) { 2051 if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) 2052 continue; 2053 2054 if (vfio_find_dma(iommu, region->start, region->length)) 2055 return true; 2056 } 2057 2058 return false; 2059 } 2060 2061 /* 2062 * Check iova region overlap with reserved regions and 2063 * exclude them from the iommu iova range 2064 */ 2065 static int vfio_iommu_resv_exclude(struct list_head *iova, 2066 struct list_head *resv_regions) 2067 { 2068 struct iommu_resv_region *resv; 2069 struct vfio_iova *n, *next; 2070 2071 list_for_each_entry(resv, resv_regions, list) { 2072 phys_addr_t start, end; 2073 2074 if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE) 2075 continue; 2076 2077 start = resv->start; 2078 end = resv->start + resv->length - 1; 2079 2080 list_for_each_entry_safe(n, next, iova, list) { 2081 int ret = 0; 2082 2083 /* No overlap */ 2084 if (start > n->end || end < n->start) 2085 continue; 2086 /* 2087 * Insert a new node if current node overlaps with the 2088 * reserve region to exclude that from valid iova range. 2089 * Note that, new node is inserted before the current 2090 * node and finally the current node is deleted keeping 2091 * the list updated and sorted. 2092 */ 2093 if (start > n->start) 2094 ret = vfio_iommu_iova_insert(&n->list, n->start, 2095 start - 1); 2096 if (!ret && end < n->end) 2097 ret = vfio_iommu_iova_insert(&n->list, end + 1, 2098 n->end); 2099 if (ret) 2100 return ret; 2101 2102 list_del(&n->list); 2103 kfree(n); 2104 } 2105 } 2106 2107 if (list_empty(iova)) 2108 return -EINVAL; 2109 2110 return 0; 2111 } 2112 2113 static void vfio_iommu_resv_free(struct list_head *resv_regions) 2114 { 2115 struct iommu_resv_region *n, *next; 2116 2117 list_for_each_entry_safe(n, next, resv_regions, list) { 2118 list_del(&n->list); 2119 kfree(n); 2120 } 2121 } 2122 2123 static void vfio_iommu_iova_free(struct list_head *iova) 2124 { 2125 struct vfio_iova *n, *next; 2126 2127 list_for_each_entry_safe(n, next, iova, list) { 2128 list_del(&n->list); 2129 kfree(n); 2130 } 2131 } 2132 2133 static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu, 2134 struct list_head *iova_copy) 2135 { 2136 struct list_head *iova = &iommu->iova_list; 2137 struct vfio_iova *n; 2138 int ret; 2139 2140 list_for_each_entry(n, iova, list) { 2141 ret = vfio_iommu_iova_insert(iova_copy, n->start, n->end); 2142 if (ret) 2143 goto out_free; 2144 } 2145 2146 return 0; 2147 2148 out_free: 2149 vfio_iommu_iova_free(iova_copy); 2150 return ret; 2151 } 2152 2153 static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu, 2154 struct list_head *iova_copy) 2155 { 2156 struct list_head *iova = &iommu->iova_list; 2157 2158 vfio_iommu_iova_free(iova); 2159 2160 list_splice_tail(iova_copy, iova); 2161 } 2162 2163 /* Redundantly walks non-present capabilities to simplify caller */ 2164 static int vfio_iommu_device_capable(struct device *dev, void *data) 2165 { 2166 return device_iommu_capable(dev, (enum iommu_cap)data); 2167 } 2168 2169 static int vfio_iommu_domain_alloc(struct device *dev, void *data) 2170 { 2171 struct iommu_domain **domain = data; 2172 2173 *domain = iommu_domain_alloc(dev->bus); 2174 return 1; /* Don't iterate */ 2175 } 2176 2177 static int vfio_iommu_type1_attach_group(void *iommu_data, 2178 struct iommu_group *iommu_group, enum vfio_group_type type) 2179 { 2180 struct vfio_iommu *iommu = iommu_data; 2181 struct vfio_iommu_group *group; 2182 struct vfio_domain *domain, *d; 2183 bool resv_msi, msi_remap; 2184 phys_addr_t resv_msi_base = 0; 2185 struct iommu_domain_geometry *geo; 2186 LIST_HEAD(iova_copy); 2187 LIST_HEAD(group_resv_regions); 2188 int ret = -EINVAL; 2189 2190 mutex_lock(&iommu->lock); 2191 2192 /* Check for duplicates */ 2193 if (vfio_iommu_find_iommu_group(iommu, iommu_group)) 2194 goto out_unlock; 2195 2196 ret = -ENOMEM; 2197 group = kzalloc(sizeof(*group), GFP_KERNEL); 2198 if (!group) 2199 goto out_unlock; 2200 group->iommu_group = iommu_group; 2201 2202 if (type == VFIO_EMULATED_IOMMU) { 2203 list_add(&group->next, &iommu->emulated_iommu_groups); 2204 /* 2205 * An emulated IOMMU group cannot dirty memory directly, it can 2206 * only use interfaces that provide dirty tracking. 2207 * The iommu scope can only be promoted with the addition of a 2208 * dirty tracking group. 2209 */ 2210 group->pinned_page_dirty_scope = true; 2211 ret = 0; 2212 goto out_unlock; 2213 } 2214 2215 ret = -ENOMEM; 2216 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 2217 if (!domain) 2218 goto out_free_group; 2219 2220 /* 2221 * Going via the iommu_group iterator avoids races, and trivially gives 2222 * us a representative device for the IOMMU API call. We don't actually 2223 * want to iterate beyond the first device (if any). 2224 */ 2225 ret = -EIO; 2226 iommu_group_for_each_dev(iommu_group, &domain->domain, 2227 vfio_iommu_domain_alloc); 2228 if (!domain->domain) 2229 goto out_free_domain; 2230 2231 if (iommu->nesting) { 2232 ret = iommu_enable_nesting(domain->domain); 2233 if (ret) 2234 goto out_domain; 2235 } 2236 2237 ret = iommu_attach_group(domain->domain, group->iommu_group); 2238 if (ret) 2239 goto out_domain; 2240 2241 /* Get aperture info */ 2242 geo = &domain->domain->geometry; 2243 if (vfio_iommu_aper_conflict(iommu, geo->aperture_start, 2244 geo->aperture_end)) { 2245 ret = -EINVAL; 2246 goto out_detach; 2247 } 2248 2249 ret = iommu_get_group_resv_regions(iommu_group, &group_resv_regions); 2250 if (ret) 2251 goto out_detach; 2252 2253 if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) { 2254 ret = -EINVAL; 2255 goto out_detach; 2256 } 2257 2258 /* 2259 * We don't want to work on the original iova list as the list 2260 * gets modified and in case of failure we have to retain the 2261 * original list. Get a copy here. 2262 */ 2263 ret = vfio_iommu_iova_get_copy(iommu, &iova_copy); 2264 if (ret) 2265 goto out_detach; 2266 2267 ret = vfio_iommu_aper_resize(&iova_copy, geo->aperture_start, 2268 geo->aperture_end); 2269 if (ret) 2270 goto out_detach; 2271 2272 ret = vfio_iommu_resv_exclude(&iova_copy, &group_resv_regions); 2273 if (ret) 2274 goto out_detach; 2275 2276 resv_msi = vfio_iommu_has_sw_msi(&group_resv_regions, &resv_msi_base); 2277 2278 INIT_LIST_HEAD(&domain->group_list); 2279 list_add(&group->next, &domain->group_list); 2280 2281 msi_remap = irq_domain_check_msi_remap() || 2282 iommu_group_for_each_dev(iommu_group, (void *)IOMMU_CAP_INTR_REMAP, 2283 vfio_iommu_device_capable); 2284 2285 if (!allow_unsafe_interrupts && !msi_remap) { 2286 pr_warn("%s: No interrupt remapping support. Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n", 2287 __func__); 2288 ret = -EPERM; 2289 goto out_detach; 2290 } 2291 2292 /* 2293 * If the IOMMU can block non-coherent operations (ie PCIe TLPs with 2294 * no-snoop set) then VFIO always turns this feature on because on Intel 2295 * platforms it optimizes KVM to disable wbinvd emulation. 2296 */ 2297 if (domain->domain->ops->enforce_cache_coherency) 2298 domain->enforce_cache_coherency = 2299 domain->domain->ops->enforce_cache_coherency( 2300 domain->domain); 2301 2302 /* 2303 * Try to match an existing compatible domain. We don't want to 2304 * preclude an IOMMU driver supporting multiple bus_types and being 2305 * able to include different bus_types in the same IOMMU domain, so 2306 * we test whether the domains use the same iommu_ops rather than 2307 * testing if they're on the same bus_type. 2308 */ 2309 list_for_each_entry(d, &iommu->domain_list, next) { 2310 if (d->domain->ops == domain->domain->ops && 2311 d->enforce_cache_coherency == 2312 domain->enforce_cache_coherency) { 2313 iommu_detach_group(domain->domain, group->iommu_group); 2314 if (!iommu_attach_group(d->domain, 2315 group->iommu_group)) { 2316 list_add(&group->next, &d->group_list); 2317 iommu_domain_free(domain->domain); 2318 kfree(domain); 2319 goto done; 2320 } 2321 2322 ret = iommu_attach_group(domain->domain, 2323 group->iommu_group); 2324 if (ret) 2325 goto out_domain; 2326 } 2327 } 2328 2329 vfio_test_domain_fgsp(domain); 2330 2331 /* replay mappings on new domains */ 2332 ret = vfio_iommu_replay(iommu, domain); 2333 if (ret) 2334 goto out_detach; 2335 2336 if (resv_msi) { 2337 ret = iommu_get_msi_cookie(domain->domain, resv_msi_base); 2338 if (ret && ret != -ENODEV) 2339 goto out_detach; 2340 } 2341 2342 list_add(&domain->next, &iommu->domain_list); 2343 vfio_update_pgsize_bitmap(iommu); 2344 done: 2345 /* Delete the old one and insert new iova list */ 2346 vfio_iommu_iova_insert_copy(iommu, &iova_copy); 2347 2348 /* 2349 * An iommu backed group can dirty memory directly and therefore 2350 * demotes the iommu scope until it declares itself dirty tracking 2351 * capable via the page pinning interface. 2352 */ 2353 iommu->num_non_pinned_groups++; 2354 mutex_unlock(&iommu->lock); 2355 vfio_iommu_resv_free(&group_resv_regions); 2356 2357 return 0; 2358 2359 out_detach: 2360 iommu_detach_group(domain->domain, group->iommu_group); 2361 out_domain: 2362 iommu_domain_free(domain->domain); 2363 vfio_iommu_iova_free(&iova_copy); 2364 vfio_iommu_resv_free(&group_resv_regions); 2365 out_free_domain: 2366 kfree(domain); 2367 out_free_group: 2368 kfree(group); 2369 out_unlock: 2370 mutex_unlock(&iommu->lock); 2371 return ret; 2372 } 2373 2374 static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu) 2375 { 2376 struct rb_node *node; 2377 2378 while ((node = rb_first(&iommu->dma_list))) 2379 vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node)); 2380 } 2381 2382 static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) 2383 { 2384 struct rb_node *n, *p; 2385 2386 n = rb_first(&iommu->dma_list); 2387 for (; n; n = rb_next(n)) { 2388 struct vfio_dma *dma; 2389 long locked = 0, unlocked = 0; 2390 2391 dma = rb_entry(n, struct vfio_dma, node); 2392 unlocked += vfio_unmap_unpin(iommu, dma, false); 2393 p = rb_first(&dma->pfn_list); 2394 for (; p; p = rb_next(p)) { 2395 struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, 2396 node); 2397 2398 if (!is_invalid_reserved_pfn(vpfn->pfn)) 2399 locked++; 2400 } 2401 vfio_lock_acct(dma, locked - unlocked, true); 2402 } 2403 } 2404 2405 /* 2406 * Called when a domain is removed in detach. It is possible that 2407 * the removed domain decided the iova aperture window. Modify the 2408 * iova aperture with the smallest window among existing domains. 2409 */ 2410 static void vfio_iommu_aper_expand(struct vfio_iommu *iommu, 2411 struct list_head *iova_copy) 2412 { 2413 struct vfio_domain *domain; 2414 struct vfio_iova *node; 2415 dma_addr_t start = 0; 2416 dma_addr_t end = (dma_addr_t)~0; 2417 2418 if (list_empty(iova_copy)) 2419 return; 2420 2421 list_for_each_entry(domain, &iommu->domain_list, next) { 2422 struct iommu_domain_geometry *geo = &domain->domain->geometry; 2423 2424 if (geo->aperture_start > start) 2425 start = geo->aperture_start; 2426 if (geo->aperture_end < end) 2427 end = geo->aperture_end; 2428 } 2429 2430 /* Modify aperture limits. The new aper is either same or bigger */ 2431 node = list_first_entry(iova_copy, struct vfio_iova, list); 2432 node->start = start; 2433 node = list_last_entry(iova_copy, struct vfio_iova, list); 2434 node->end = end; 2435 } 2436 2437 /* 2438 * Called when a group is detached. The reserved regions for that 2439 * group can be part of valid iova now. But since reserved regions 2440 * may be duplicated among groups, populate the iova valid regions 2441 * list again. 2442 */ 2443 static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu, 2444 struct list_head *iova_copy) 2445 { 2446 struct vfio_domain *d; 2447 struct vfio_iommu_group *g; 2448 struct vfio_iova *node; 2449 dma_addr_t start, end; 2450 LIST_HEAD(resv_regions); 2451 int ret; 2452 2453 if (list_empty(iova_copy)) 2454 return -EINVAL; 2455 2456 list_for_each_entry(d, &iommu->domain_list, next) { 2457 list_for_each_entry(g, &d->group_list, next) { 2458 ret = iommu_get_group_resv_regions(g->iommu_group, 2459 &resv_regions); 2460 if (ret) 2461 goto done; 2462 } 2463 } 2464 2465 node = list_first_entry(iova_copy, struct vfio_iova, list); 2466 start = node->start; 2467 node = list_last_entry(iova_copy, struct vfio_iova, list); 2468 end = node->end; 2469 2470 /* purge the iova list and create new one */ 2471 vfio_iommu_iova_free(iova_copy); 2472 2473 ret = vfio_iommu_aper_resize(iova_copy, start, end); 2474 if (ret) 2475 goto done; 2476 2477 /* Exclude current reserved regions from iova ranges */ 2478 ret = vfio_iommu_resv_exclude(iova_copy, &resv_regions); 2479 done: 2480 vfio_iommu_resv_free(&resv_regions); 2481 return ret; 2482 } 2483 2484 static void vfio_iommu_type1_detach_group(void *iommu_data, 2485 struct iommu_group *iommu_group) 2486 { 2487 struct vfio_iommu *iommu = iommu_data; 2488 struct vfio_domain *domain; 2489 struct vfio_iommu_group *group; 2490 bool update_dirty_scope = false; 2491 LIST_HEAD(iova_copy); 2492 2493 mutex_lock(&iommu->lock); 2494 list_for_each_entry(group, &iommu->emulated_iommu_groups, next) { 2495 if (group->iommu_group != iommu_group) 2496 continue; 2497 update_dirty_scope = !group->pinned_page_dirty_scope; 2498 list_del(&group->next); 2499 kfree(group); 2500 2501 if (list_empty(&iommu->emulated_iommu_groups) && 2502 list_empty(&iommu->domain_list)) { 2503 WARN_ON(!list_empty(&iommu->device_list)); 2504 vfio_iommu_unmap_unpin_all(iommu); 2505 } 2506 goto detach_group_done; 2507 } 2508 2509 /* 2510 * Get a copy of iova list. This will be used to update 2511 * and to replace the current one later. Please note that 2512 * we will leave the original list as it is if update fails. 2513 */ 2514 vfio_iommu_iova_get_copy(iommu, &iova_copy); 2515 2516 list_for_each_entry(domain, &iommu->domain_list, next) { 2517 group = find_iommu_group(domain, iommu_group); 2518 if (!group) 2519 continue; 2520 2521 iommu_detach_group(domain->domain, group->iommu_group); 2522 update_dirty_scope = !group->pinned_page_dirty_scope; 2523 list_del(&group->next); 2524 kfree(group); 2525 /* 2526 * Group ownership provides privilege, if the group list is 2527 * empty, the domain goes away. If it's the last domain with 2528 * iommu and external domain doesn't exist, then all the 2529 * mappings go away too. If it's the last domain with iommu and 2530 * external domain exist, update accounting 2531 */ 2532 if (list_empty(&domain->group_list)) { 2533 if (list_is_singular(&iommu->domain_list)) { 2534 if (list_empty(&iommu->emulated_iommu_groups)) { 2535 WARN_ON(!list_empty( 2536 &iommu->device_list)); 2537 vfio_iommu_unmap_unpin_all(iommu); 2538 } else { 2539 vfio_iommu_unmap_unpin_reaccount(iommu); 2540 } 2541 } 2542 iommu_domain_free(domain->domain); 2543 list_del(&domain->next); 2544 kfree(domain); 2545 vfio_iommu_aper_expand(iommu, &iova_copy); 2546 vfio_update_pgsize_bitmap(iommu); 2547 } 2548 break; 2549 } 2550 2551 if (!vfio_iommu_resv_refresh(iommu, &iova_copy)) 2552 vfio_iommu_iova_insert_copy(iommu, &iova_copy); 2553 else 2554 vfio_iommu_iova_free(&iova_copy); 2555 2556 detach_group_done: 2557 /* 2558 * Removal of a group without dirty tracking may allow the iommu scope 2559 * to be promoted. 2560 */ 2561 if (update_dirty_scope) { 2562 iommu->num_non_pinned_groups--; 2563 if (iommu->dirty_page_tracking) 2564 vfio_iommu_populate_bitmap_full(iommu); 2565 } 2566 mutex_unlock(&iommu->lock); 2567 } 2568 2569 static void *vfio_iommu_type1_open(unsigned long arg) 2570 { 2571 struct vfio_iommu *iommu; 2572 2573 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); 2574 if (!iommu) 2575 return ERR_PTR(-ENOMEM); 2576 2577 switch (arg) { 2578 case VFIO_TYPE1_IOMMU: 2579 break; 2580 case VFIO_TYPE1_NESTING_IOMMU: 2581 iommu->nesting = true; 2582 fallthrough; 2583 case VFIO_TYPE1v2_IOMMU: 2584 iommu->v2 = true; 2585 break; 2586 default: 2587 kfree(iommu); 2588 return ERR_PTR(-EINVAL); 2589 } 2590 2591 INIT_LIST_HEAD(&iommu->domain_list); 2592 INIT_LIST_HEAD(&iommu->iova_list); 2593 iommu->dma_list = RB_ROOT; 2594 iommu->dma_avail = dma_entry_limit; 2595 iommu->container_open = true; 2596 mutex_init(&iommu->lock); 2597 mutex_init(&iommu->device_list_lock); 2598 INIT_LIST_HEAD(&iommu->device_list); 2599 init_waitqueue_head(&iommu->vaddr_wait); 2600 iommu->pgsize_bitmap = PAGE_MASK; 2601 INIT_LIST_HEAD(&iommu->emulated_iommu_groups); 2602 2603 return iommu; 2604 } 2605 2606 static void vfio_release_domain(struct vfio_domain *domain) 2607 { 2608 struct vfio_iommu_group *group, *group_tmp; 2609 2610 list_for_each_entry_safe(group, group_tmp, 2611 &domain->group_list, next) { 2612 iommu_detach_group(domain->domain, group->iommu_group); 2613 list_del(&group->next); 2614 kfree(group); 2615 } 2616 2617 iommu_domain_free(domain->domain); 2618 } 2619 2620 static void vfio_iommu_type1_release(void *iommu_data) 2621 { 2622 struct vfio_iommu *iommu = iommu_data; 2623 struct vfio_domain *domain, *domain_tmp; 2624 struct vfio_iommu_group *group, *next_group; 2625 2626 list_for_each_entry_safe(group, next_group, 2627 &iommu->emulated_iommu_groups, next) { 2628 list_del(&group->next); 2629 kfree(group); 2630 } 2631 2632 vfio_iommu_unmap_unpin_all(iommu); 2633 2634 list_for_each_entry_safe(domain, domain_tmp, 2635 &iommu->domain_list, next) { 2636 vfio_release_domain(domain); 2637 list_del(&domain->next); 2638 kfree(domain); 2639 } 2640 2641 vfio_iommu_iova_free(&iommu->iova_list); 2642 2643 kfree(iommu); 2644 } 2645 2646 static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu) 2647 { 2648 struct vfio_domain *domain; 2649 int ret = 1; 2650 2651 mutex_lock(&iommu->lock); 2652 list_for_each_entry(domain, &iommu->domain_list, next) { 2653 if (!(domain->enforce_cache_coherency)) { 2654 ret = 0; 2655 break; 2656 } 2657 } 2658 mutex_unlock(&iommu->lock); 2659 2660 return ret; 2661 } 2662 2663 static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, 2664 unsigned long arg) 2665 { 2666 switch (arg) { 2667 case VFIO_TYPE1_IOMMU: 2668 case VFIO_TYPE1v2_IOMMU: 2669 case VFIO_TYPE1_NESTING_IOMMU: 2670 case VFIO_UNMAP_ALL: 2671 case VFIO_UPDATE_VADDR: 2672 return 1; 2673 case VFIO_DMA_CC_IOMMU: 2674 if (!iommu) 2675 return 0; 2676 return vfio_domains_have_enforce_cache_coherency(iommu); 2677 default: 2678 return 0; 2679 } 2680 } 2681 2682 static int vfio_iommu_iova_add_cap(struct vfio_info_cap *caps, 2683 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas, 2684 size_t size) 2685 { 2686 struct vfio_info_cap_header *header; 2687 struct vfio_iommu_type1_info_cap_iova_range *iova_cap; 2688 2689 header = vfio_info_cap_add(caps, size, 2690 VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE, 1); 2691 if (IS_ERR(header)) 2692 return PTR_ERR(header); 2693 2694 iova_cap = container_of(header, 2695 struct vfio_iommu_type1_info_cap_iova_range, 2696 header); 2697 iova_cap->nr_iovas = cap_iovas->nr_iovas; 2698 memcpy(iova_cap->iova_ranges, cap_iovas->iova_ranges, 2699 cap_iovas->nr_iovas * sizeof(*cap_iovas->iova_ranges)); 2700 return 0; 2701 } 2702 2703 static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu, 2704 struct vfio_info_cap *caps) 2705 { 2706 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas; 2707 struct vfio_iova *iova; 2708 size_t size; 2709 int iovas = 0, i = 0, ret; 2710 2711 list_for_each_entry(iova, &iommu->iova_list, list) 2712 iovas++; 2713 2714 if (!iovas) { 2715 /* 2716 * Return 0 as a container with a single mdev device 2717 * will have an empty list 2718 */ 2719 return 0; 2720 } 2721 2722 size = struct_size(cap_iovas, iova_ranges, iovas); 2723 2724 cap_iovas = kzalloc(size, GFP_KERNEL); 2725 if (!cap_iovas) 2726 return -ENOMEM; 2727 2728 cap_iovas->nr_iovas = iovas; 2729 2730 list_for_each_entry(iova, &iommu->iova_list, list) { 2731 cap_iovas->iova_ranges[i].start = iova->start; 2732 cap_iovas->iova_ranges[i].end = iova->end; 2733 i++; 2734 } 2735 2736 ret = vfio_iommu_iova_add_cap(caps, cap_iovas, size); 2737 2738 kfree(cap_iovas); 2739 return ret; 2740 } 2741 2742 static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, 2743 struct vfio_info_cap *caps) 2744 { 2745 struct vfio_iommu_type1_info_cap_migration cap_mig; 2746 2747 cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION; 2748 cap_mig.header.version = 1; 2749 2750 cap_mig.flags = 0; 2751 /* support minimum pgsize */ 2752 cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); 2753 cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; 2754 2755 return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); 2756 } 2757 2758 static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu, 2759 struct vfio_info_cap *caps) 2760 { 2761 struct vfio_iommu_type1_info_dma_avail cap_dma_avail; 2762 2763 cap_dma_avail.header.id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL; 2764 cap_dma_avail.header.version = 1; 2765 2766 cap_dma_avail.avail = iommu->dma_avail; 2767 2768 return vfio_info_add_capability(caps, &cap_dma_avail.header, 2769 sizeof(cap_dma_avail)); 2770 } 2771 2772 static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu, 2773 unsigned long arg) 2774 { 2775 struct vfio_iommu_type1_info info; 2776 unsigned long minsz; 2777 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 2778 unsigned long capsz; 2779 int ret; 2780 2781 minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes); 2782 2783 /* For backward compatibility, cannot require this */ 2784 capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); 2785 2786 if (copy_from_user(&info, (void __user *)arg, minsz)) 2787 return -EFAULT; 2788 2789 if (info.argsz < minsz) 2790 return -EINVAL; 2791 2792 if (info.argsz >= capsz) { 2793 minsz = capsz; 2794 info.cap_offset = 0; /* output, no-recopy necessary */ 2795 } 2796 2797 mutex_lock(&iommu->lock); 2798 info.flags = VFIO_IOMMU_INFO_PGSIZES; 2799 2800 info.iova_pgsizes = iommu->pgsize_bitmap; 2801 2802 ret = vfio_iommu_migration_build_caps(iommu, &caps); 2803 2804 if (!ret) 2805 ret = vfio_iommu_dma_avail_build_caps(iommu, &caps); 2806 2807 if (!ret) 2808 ret = vfio_iommu_iova_build_caps(iommu, &caps); 2809 2810 mutex_unlock(&iommu->lock); 2811 2812 if (ret) 2813 return ret; 2814 2815 if (caps.size) { 2816 info.flags |= VFIO_IOMMU_INFO_CAPS; 2817 2818 if (info.argsz < sizeof(info) + caps.size) { 2819 info.argsz = sizeof(info) + caps.size; 2820 } else { 2821 vfio_info_cap_shift(&caps, sizeof(info)); 2822 if (copy_to_user((void __user *)arg + 2823 sizeof(info), caps.buf, 2824 caps.size)) { 2825 kfree(caps.buf); 2826 return -EFAULT; 2827 } 2828 info.cap_offset = sizeof(info); 2829 } 2830 2831 kfree(caps.buf); 2832 } 2833 2834 return copy_to_user((void __user *)arg, &info, minsz) ? 2835 -EFAULT : 0; 2836 } 2837 2838 static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, 2839 unsigned long arg) 2840 { 2841 struct vfio_iommu_type1_dma_map map; 2842 unsigned long minsz; 2843 uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE | 2844 VFIO_DMA_MAP_FLAG_VADDR; 2845 2846 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); 2847 2848 if (copy_from_user(&map, (void __user *)arg, minsz)) 2849 return -EFAULT; 2850 2851 if (map.argsz < minsz || map.flags & ~mask) 2852 return -EINVAL; 2853 2854 return vfio_dma_do_map(iommu, &map); 2855 } 2856 2857 static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, 2858 unsigned long arg) 2859 { 2860 struct vfio_iommu_type1_dma_unmap unmap; 2861 struct vfio_bitmap bitmap = { 0 }; 2862 uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP | 2863 VFIO_DMA_UNMAP_FLAG_VADDR | 2864 VFIO_DMA_UNMAP_FLAG_ALL; 2865 unsigned long minsz; 2866 int ret; 2867 2868 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size); 2869 2870 if (copy_from_user(&unmap, (void __user *)arg, minsz)) 2871 return -EFAULT; 2872 2873 if (unmap.argsz < minsz || unmap.flags & ~mask) 2874 return -EINVAL; 2875 2876 if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && 2877 (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL | 2878 VFIO_DMA_UNMAP_FLAG_VADDR))) 2879 return -EINVAL; 2880 2881 if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { 2882 unsigned long pgshift; 2883 2884 if (unmap.argsz < (minsz + sizeof(bitmap))) 2885 return -EINVAL; 2886 2887 if (copy_from_user(&bitmap, 2888 (void __user *)(arg + minsz), 2889 sizeof(bitmap))) 2890 return -EFAULT; 2891 2892 if (!access_ok((void __user *)bitmap.data, bitmap.size)) 2893 return -EINVAL; 2894 2895 pgshift = __ffs(bitmap.pgsize); 2896 ret = verify_bitmap_size(unmap.size >> pgshift, 2897 bitmap.size); 2898 if (ret) 2899 return ret; 2900 } 2901 2902 ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap); 2903 if (ret) 2904 return ret; 2905 2906 return copy_to_user((void __user *)arg, &unmap, minsz) ? 2907 -EFAULT : 0; 2908 } 2909 2910 static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, 2911 unsigned long arg) 2912 { 2913 struct vfio_iommu_type1_dirty_bitmap dirty; 2914 uint32_t mask = VFIO_IOMMU_DIRTY_PAGES_FLAG_START | 2915 VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP | 2916 VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; 2917 unsigned long minsz; 2918 int ret = 0; 2919 2920 if (!iommu->v2) 2921 return -EACCES; 2922 2923 minsz = offsetofend(struct vfio_iommu_type1_dirty_bitmap, flags); 2924 2925 if (copy_from_user(&dirty, (void __user *)arg, minsz)) 2926 return -EFAULT; 2927 2928 if (dirty.argsz < minsz || dirty.flags & ~mask) 2929 return -EINVAL; 2930 2931 /* only one flag should be set at a time */ 2932 if (__ffs(dirty.flags) != __fls(dirty.flags)) 2933 return -EINVAL; 2934 2935 if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_START) { 2936 size_t pgsize; 2937 2938 mutex_lock(&iommu->lock); 2939 pgsize = 1 << __ffs(iommu->pgsize_bitmap); 2940 if (!iommu->dirty_page_tracking) { 2941 ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); 2942 if (!ret) 2943 iommu->dirty_page_tracking = true; 2944 } 2945 mutex_unlock(&iommu->lock); 2946 return ret; 2947 } else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP) { 2948 mutex_lock(&iommu->lock); 2949 if (iommu->dirty_page_tracking) { 2950 iommu->dirty_page_tracking = false; 2951 vfio_dma_bitmap_free_all(iommu); 2952 } 2953 mutex_unlock(&iommu->lock); 2954 return 0; 2955 } else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP) { 2956 struct vfio_iommu_type1_dirty_bitmap_get range; 2957 unsigned long pgshift; 2958 size_t data_size = dirty.argsz - minsz; 2959 size_t iommu_pgsize; 2960 2961 if (!data_size || data_size < sizeof(range)) 2962 return -EINVAL; 2963 2964 if (copy_from_user(&range, (void __user *)(arg + minsz), 2965 sizeof(range))) 2966 return -EFAULT; 2967 2968 if (range.iova + range.size < range.iova) 2969 return -EINVAL; 2970 if (!access_ok((void __user *)range.bitmap.data, 2971 range.bitmap.size)) 2972 return -EINVAL; 2973 2974 pgshift = __ffs(range.bitmap.pgsize); 2975 ret = verify_bitmap_size(range.size >> pgshift, 2976 range.bitmap.size); 2977 if (ret) 2978 return ret; 2979 2980 mutex_lock(&iommu->lock); 2981 2982 iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); 2983 2984 /* allow only smallest supported pgsize */ 2985 if (range.bitmap.pgsize != iommu_pgsize) { 2986 ret = -EINVAL; 2987 goto out_unlock; 2988 } 2989 if (range.iova & (iommu_pgsize - 1)) { 2990 ret = -EINVAL; 2991 goto out_unlock; 2992 } 2993 if (!range.size || range.size & (iommu_pgsize - 1)) { 2994 ret = -EINVAL; 2995 goto out_unlock; 2996 } 2997 2998 if (iommu->dirty_page_tracking) 2999 ret = vfio_iova_dirty_bitmap(range.bitmap.data, 3000 iommu, range.iova, 3001 range.size, 3002 range.bitmap.pgsize); 3003 else 3004 ret = -EINVAL; 3005 out_unlock: 3006 mutex_unlock(&iommu->lock); 3007 3008 return ret; 3009 } 3010 3011 return -EINVAL; 3012 } 3013 3014 static long vfio_iommu_type1_ioctl(void *iommu_data, 3015 unsigned int cmd, unsigned long arg) 3016 { 3017 struct vfio_iommu *iommu = iommu_data; 3018 3019 switch (cmd) { 3020 case VFIO_CHECK_EXTENSION: 3021 return vfio_iommu_type1_check_extension(iommu, arg); 3022 case VFIO_IOMMU_GET_INFO: 3023 return vfio_iommu_type1_get_info(iommu, arg); 3024 case VFIO_IOMMU_MAP_DMA: 3025 return vfio_iommu_type1_map_dma(iommu, arg); 3026 case VFIO_IOMMU_UNMAP_DMA: 3027 return vfio_iommu_type1_unmap_dma(iommu, arg); 3028 case VFIO_IOMMU_DIRTY_PAGES: 3029 return vfio_iommu_type1_dirty_pages(iommu, arg); 3030 default: 3031 return -ENOTTY; 3032 } 3033 } 3034 3035 static void vfio_iommu_type1_register_device(void *iommu_data, 3036 struct vfio_device *vdev) 3037 { 3038 struct vfio_iommu *iommu = iommu_data; 3039 3040 if (!vdev->ops->dma_unmap) 3041 return; 3042 3043 /* 3044 * list_empty(&iommu->device_list) is tested under the iommu->lock while 3045 * iteration for dma_unmap must be done under the device_list_lock. 3046 * Holding both locks here allows avoiding the device_list_lock in 3047 * several fast paths. See vfio_notify_dma_unmap() 3048 */ 3049 mutex_lock(&iommu->lock); 3050 mutex_lock(&iommu->device_list_lock); 3051 list_add(&vdev->iommu_entry, &iommu->device_list); 3052 mutex_unlock(&iommu->device_list_lock); 3053 mutex_unlock(&iommu->lock); 3054 } 3055 3056 static void vfio_iommu_type1_unregister_device(void *iommu_data, 3057 struct vfio_device *vdev) 3058 { 3059 struct vfio_iommu *iommu = iommu_data; 3060 3061 if (!vdev->ops->dma_unmap) 3062 return; 3063 3064 mutex_lock(&iommu->lock); 3065 mutex_lock(&iommu->device_list_lock); 3066 list_del(&vdev->iommu_entry); 3067 mutex_unlock(&iommu->device_list_lock); 3068 mutex_unlock(&iommu->lock); 3069 } 3070 3071 static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, 3072 dma_addr_t user_iova, void *data, 3073 size_t count, bool write, 3074 size_t *copied) 3075 { 3076 struct mm_struct *mm; 3077 unsigned long vaddr; 3078 struct vfio_dma *dma; 3079 bool kthread = current->mm == NULL; 3080 size_t offset; 3081 int ret; 3082 3083 *copied = 0; 3084 3085 ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma); 3086 if (ret < 0) 3087 return ret; 3088 3089 if ((write && !(dma->prot & IOMMU_WRITE)) || 3090 !(dma->prot & IOMMU_READ)) 3091 return -EPERM; 3092 3093 mm = get_task_mm(dma->task); 3094 3095 if (!mm) 3096 return -EPERM; 3097 3098 if (kthread) 3099 kthread_use_mm(mm); 3100 else if (current->mm != mm) 3101 goto out; 3102 3103 offset = user_iova - dma->iova; 3104 3105 if (count > dma->size - offset) 3106 count = dma->size - offset; 3107 3108 vaddr = dma->vaddr + offset; 3109 3110 if (write) { 3111 *copied = copy_to_user((void __user *)vaddr, data, 3112 count) ? 0 : count; 3113 if (*copied && iommu->dirty_page_tracking) { 3114 unsigned long pgshift = __ffs(iommu->pgsize_bitmap); 3115 /* 3116 * Bitmap populated with the smallest supported page 3117 * size 3118 */ 3119 bitmap_set(dma->bitmap, offset >> pgshift, 3120 ((offset + *copied - 1) >> pgshift) - 3121 (offset >> pgshift) + 1); 3122 } 3123 } else 3124 *copied = copy_from_user(data, (void __user *)vaddr, 3125 count) ? 0 : count; 3126 if (kthread) 3127 kthread_unuse_mm(mm); 3128 out: 3129 mmput(mm); 3130 return *copied ? 0 : -EFAULT; 3131 } 3132 3133 static int vfio_iommu_type1_dma_rw(void *iommu_data, dma_addr_t user_iova, 3134 void *data, size_t count, bool write) 3135 { 3136 struct vfio_iommu *iommu = iommu_data; 3137 int ret = 0; 3138 size_t done; 3139 3140 mutex_lock(&iommu->lock); 3141 while (count > 0) { 3142 ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data, 3143 count, write, &done); 3144 if (ret) 3145 break; 3146 3147 count -= done; 3148 data += done; 3149 user_iova += done; 3150 } 3151 3152 mutex_unlock(&iommu->lock); 3153 return ret; 3154 } 3155 3156 static struct iommu_domain * 3157 vfio_iommu_type1_group_iommu_domain(void *iommu_data, 3158 struct iommu_group *iommu_group) 3159 { 3160 struct iommu_domain *domain = ERR_PTR(-ENODEV); 3161 struct vfio_iommu *iommu = iommu_data; 3162 struct vfio_domain *d; 3163 3164 if (!iommu || !iommu_group) 3165 return ERR_PTR(-EINVAL); 3166 3167 mutex_lock(&iommu->lock); 3168 list_for_each_entry(d, &iommu->domain_list, next) { 3169 if (find_iommu_group(d, iommu_group)) { 3170 domain = d->domain; 3171 break; 3172 } 3173 } 3174 mutex_unlock(&iommu->lock); 3175 3176 return domain; 3177 } 3178 3179 static void vfio_iommu_type1_notify(void *iommu_data, 3180 enum vfio_iommu_notify_type event) 3181 { 3182 struct vfio_iommu *iommu = iommu_data; 3183 3184 if (event != VFIO_IOMMU_CONTAINER_CLOSE) 3185 return; 3186 mutex_lock(&iommu->lock); 3187 iommu->container_open = false; 3188 mutex_unlock(&iommu->lock); 3189 wake_up_all(&iommu->vaddr_wait); 3190 } 3191 3192 static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { 3193 .name = "vfio-iommu-type1", 3194 .owner = THIS_MODULE, 3195 .open = vfio_iommu_type1_open, 3196 .release = vfio_iommu_type1_release, 3197 .ioctl = vfio_iommu_type1_ioctl, 3198 .attach_group = vfio_iommu_type1_attach_group, 3199 .detach_group = vfio_iommu_type1_detach_group, 3200 .pin_pages = vfio_iommu_type1_pin_pages, 3201 .unpin_pages = vfio_iommu_type1_unpin_pages, 3202 .register_device = vfio_iommu_type1_register_device, 3203 .unregister_device = vfio_iommu_type1_unregister_device, 3204 .dma_rw = vfio_iommu_type1_dma_rw, 3205 .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, 3206 .notify = vfio_iommu_type1_notify, 3207 }; 3208 3209 static int __init vfio_iommu_type1_init(void) 3210 { 3211 return vfio_register_iommu_driver(&vfio_iommu_driver_ops_type1); 3212 } 3213 3214 static void __exit vfio_iommu_type1_cleanup(void) 3215 { 3216 vfio_unregister_iommu_driver(&vfio_iommu_driver_ops_type1); 3217 } 3218 3219 module_init(vfio_iommu_type1_init); 3220 module_exit(vfio_iommu_type1_cleanup); 3221 3222 MODULE_VERSION(DRIVER_VERSION); 3223 MODULE_LICENSE("GPL v2"); 3224 MODULE_AUTHOR(DRIVER_AUTHOR); 3225 MODULE_DESCRIPTION(DRIVER_DESC); 3226