1 /* 2 * Copyright (c) 2014 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <linux/sched.h> 35 #include <linux/sched/mm.h> 36 #include <linux/sched/task.h> 37 #include <linux/pid.h> 38 #include <linux/slab.h> 39 #include <linux/export.h> 40 #include <linux/vmalloc.h> 41 #include <linux/hugetlb.h> 42 #include <linux/interval_tree_generic.h> 43 44 #include <rdma/ib_verbs.h> 45 #include <rdma/ib_umem.h> 46 #include <rdma/ib_umem_odp.h> 47 48 /* 49 * The ib_umem list keeps track of memory regions for which the HW 50 * device request to receive notification when the related memory 51 * mapping is changed. 52 * 53 * ib_umem_lock protects the list. 54 */ 55 56 static u64 node_start(struct umem_odp_node *n) 57 { 58 struct ib_umem_odp *umem_odp = 59 container_of(n, struct ib_umem_odp, interval_tree); 60 61 return ib_umem_start(umem_odp->umem); 62 } 63 64 /* Note that the representation of the intervals in the interval tree 65 * considers the ending point as contained in the interval, while the 66 * function ib_umem_end returns the first address which is not contained 67 * in the umem. 68 */ 69 static u64 node_last(struct umem_odp_node *n) 70 { 71 struct ib_umem_odp *umem_odp = 72 container_of(n, struct ib_umem_odp, interval_tree); 73 74 return ib_umem_end(umem_odp->umem) - 1; 75 } 76 77 INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last, 78 node_start, node_last, static, rbt_ib_umem) 79 80 static void ib_umem_notifier_start_account(struct ib_umem *item) 81 { 82 mutex_lock(&item->odp_data->umem_mutex); 83 84 /* Only update private counters for this umem if it has them. 85 * Otherwise skip it. All page faults will be delayed for this umem. */ 86 if (item->odp_data->mn_counters_active) { 87 int notifiers_count = item->odp_data->notifiers_count++; 88 89 if (notifiers_count == 0) 90 /* Initialize the completion object for waiting on 91 * notifiers. Since notifier_count is zero, no one 92 * should be waiting right now. */ 93 reinit_completion(&item->odp_data->notifier_completion); 94 } 95 mutex_unlock(&item->odp_data->umem_mutex); 96 } 97 98 static void ib_umem_notifier_end_account(struct ib_umem *item) 99 { 100 mutex_lock(&item->odp_data->umem_mutex); 101 102 /* Only update private counters for this umem if it has them. 103 * Otherwise skip it. All page faults will be delayed for this umem. */ 104 if (item->odp_data->mn_counters_active) { 105 /* 106 * This sequence increase will notify the QP page fault that 107 * the page that is going to be mapped in the spte could have 108 * been freed. 109 */ 110 ++item->odp_data->notifiers_seq; 111 if (--item->odp_data->notifiers_count == 0) 112 complete_all(&item->odp_data->notifier_completion); 113 } 114 mutex_unlock(&item->odp_data->umem_mutex); 115 } 116 117 /* Account for a new mmu notifier in an ib_ucontext. */ 118 static void ib_ucontext_notifier_start_account(struct ib_ucontext *context) 119 { 120 atomic_inc(&context->notifier_count); 121 } 122 123 /* Account for a terminating mmu notifier in an ib_ucontext. 124 * 125 * Must be called with the ib_ucontext->umem_rwsem semaphore unlocked, since 126 * the function takes the semaphore itself. */ 127 static void ib_ucontext_notifier_end_account(struct ib_ucontext *context) 128 { 129 int zero_notifiers = atomic_dec_and_test(&context->notifier_count); 130 131 if (zero_notifiers && 132 !list_empty(&context->no_private_counters)) { 133 /* No currently running mmu notifiers. Now is the chance to 134 * add private accounting to all previously added umems. */ 135 struct ib_umem_odp *odp_data, *next; 136 137 /* Prevent concurrent mmu notifiers from working on the 138 * no_private_counters list. */ 139 down_write(&context->umem_rwsem); 140 141 /* Read the notifier_count again, with the umem_rwsem 142 * semaphore taken for write. */ 143 if (!atomic_read(&context->notifier_count)) { 144 list_for_each_entry_safe(odp_data, next, 145 &context->no_private_counters, 146 no_private_counters) { 147 mutex_lock(&odp_data->umem_mutex); 148 odp_data->mn_counters_active = true; 149 list_del(&odp_data->no_private_counters); 150 complete_all(&odp_data->notifier_completion); 151 mutex_unlock(&odp_data->umem_mutex); 152 } 153 } 154 155 up_write(&context->umem_rwsem); 156 } 157 } 158 159 static int ib_umem_notifier_release_trampoline(struct ib_umem *item, u64 start, 160 u64 end, void *cookie) { 161 /* 162 * Increase the number of notifiers running, to 163 * prevent any further fault handling on this MR. 164 */ 165 ib_umem_notifier_start_account(item); 166 item->odp_data->dying = 1; 167 /* Make sure that the fact the umem is dying is out before we release 168 * all pending page faults. */ 169 smp_wmb(); 170 complete_all(&item->odp_data->notifier_completion); 171 item->context->invalidate_range(item, ib_umem_start(item), 172 ib_umem_end(item)); 173 return 0; 174 } 175 176 static void ib_umem_notifier_release(struct mmu_notifier *mn, 177 struct mm_struct *mm) 178 { 179 struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn); 180 181 if (!context->invalidate_range) 182 return; 183 184 ib_ucontext_notifier_start_account(context); 185 down_read(&context->umem_rwsem); 186 rbt_ib_umem_for_each_in_range(&context->umem_tree, 0, 187 ULLONG_MAX, 188 ib_umem_notifier_release_trampoline, 189 true, 190 NULL); 191 up_read(&context->umem_rwsem); 192 } 193 194 static int invalidate_page_trampoline(struct ib_umem *item, u64 start, 195 u64 end, void *cookie) 196 { 197 ib_umem_notifier_start_account(item); 198 item->context->invalidate_range(item, start, start + PAGE_SIZE); 199 ib_umem_notifier_end_account(item); 200 return 0; 201 } 202 203 static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start, 204 u64 end, void *cookie) 205 { 206 ib_umem_notifier_start_account(item); 207 item->context->invalidate_range(item, start, end); 208 return 0; 209 } 210 211 static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn, 212 struct mm_struct *mm, 213 unsigned long start, 214 unsigned long end, 215 bool blockable) 216 { 217 struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn); 218 int ret; 219 220 if (!context->invalidate_range) 221 return 0; 222 223 if (blockable) 224 down_read(&context->umem_rwsem); 225 else if (!down_read_trylock(&context->umem_rwsem)) 226 return -EAGAIN; 227 228 ib_ucontext_notifier_start_account(context); 229 ret = rbt_ib_umem_for_each_in_range(&context->umem_tree, start, 230 end, 231 invalidate_range_start_trampoline, 232 blockable, NULL); 233 up_read(&context->umem_rwsem); 234 235 return ret; 236 } 237 238 static int invalidate_range_end_trampoline(struct ib_umem *item, u64 start, 239 u64 end, void *cookie) 240 { 241 ib_umem_notifier_end_account(item); 242 return 0; 243 } 244 245 static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn, 246 struct mm_struct *mm, 247 unsigned long start, 248 unsigned long end) 249 { 250 struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn); 251 252 if (!context->invalidate_range) 253 return; 254 255 /* 256 * TODO: we currently bail out if there is any sleepable work to be done 257 * in ib_umem_notifier_invalidate_range_start so we shouldn't really block 258 * here. But this is ugly and fragile. 259 */ 260 down_read(&context->umem_rwsem); 261 rbt_ib_umem_for_each_in_range(&context->umem_tree, start, 262 end, 263 invalidate_range_end_trampoline, true, NULL); 264 up_read(&context->umem_rwsem); 265 ib_ucontext_notifier_end_account(context); 266 } 267 268 static const struct mmu_notifier_ops ib_umem_notifiers = { 269 .release = ib_umem_notifier_release, 270 .invalidate_range_start = ib_umem_notifier_invalidate_range_start, 271 .invalidate_range_end = ib_umem_notifier_invalidate_range_end, 272 }; 273 274 struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context, 275 unsigned long addr, 276 size_t size) 277 { 278 struct ib_umem *umem; 279 struct ib_umem_odp *odp_data; 280 int pages = size >> PAGE_SHIFT; 281 int ret; 282 283 umem = kzalloc(sizeof(*umem), GFP_KERNEL); 284 if (!umem) 285 return ERR_PTR(-ENOMEM); 286 287 umem->context = context; 288 umem->length = size; 289 umem->address = addr; 290 umem->page_shift = PAGE_SHIFT; 291 umem->writable = 1; 292 293 odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL); 294 if (!odp_data) { 295 ret = -ENOMEM; 296 goto out_umem; 297 } 298 odp_data->umem = umem; 299 300 mutex_init(&odp_data->umem_mutex); 301 init_completion(&odp_data->notifier_completion); 302 303 odp_data->page_list = 304 vzalloc(array_size(pages, sizeof(*odp_data->page_list))); 305 if (!odp_data->page_list) { 306 ret = -ENOMEM; 307 goto out_odp_data; 308 } 309 310 odp_data->dma_list = 311 vzalloc(array_size(pages, sizeof(*odp_data->dma_list))); 312 if (!odp_data->dma_list) { 313 ret = -ENOMEM; 314 goto out_page_list; 315 } 316 317 down_write(&context->umem_rwsem); 318 context->odp_mrs_count++; 319 rbt_ib_umem_insert(&odp_data->interval_tree, &context->umem_tree); 320 if (likely(!atomic_read(&context->notifier_count))) 321 odp_data->mn_counters_active = true; 322 else 323 list_add(&odp_data->no_private_counters, 324 &context->no_private_counters); 325 up_write(&context->umem_rwsem); 326 327 umem->odp_data = odp_data; 328 329 return umem; 330 331 out_page_list: 332 vfree(odp_data->page_list); 333 out_odp_data: 334 kfree(odp_data); 335 out_umem: 336 kfree(umem); 337 return ERR_PTR(ret); 338 } 339 EXPORT_SYMBOL(ib_alloc_odp_umem); 340 341 int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem, 342 int access) 343 { 344 int ret_val; 345 struct pid *our_pid; 346 struct mm_struct *mm = get_task_mm(current); 347 348 if (!mm) 349 return -EINVAL; 350 351 if (access & IB_ACCESS_HUGETLB) { 352 struct vm_area_struct *vma; 353 struct hstate *h; 354 355 down_read(&mm->mmap_sem); 356 vma = find_vma(mm, ib_umem_start(umem)); 357 if (!vma || !is_vm_hugetlb_page(vma)) { 358 up_read(&mm->mmap_sem); 359 return -EINVAL; 360 } 361 h = hstate_vma(vma); 362 umem->page_shift = huge_page_shift(h); 363 up_read(&mm->mmap_sem); 364 umem->hugetlb = 1; 365 } else { 366 umem->hugetlb = 0; 367 } 368 369 /* Prevent creating ODP MRs in child processes */ 370 rcu_read_lock(); 371 our_pid = get_task_pid(current->group_leader, PIDTYPE_PID); 372 rcu_read_unlock(); 373 put_pid(our_pid); 374 if (context->tgid != our_pid) { 375 ret_val = -EINVAL; 376 goto out_mm; 377 } 378 379 umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL); 380 if (!umem->odp_data) { 381 ret_val = -ENOMEM; 382 goto out_mm; 383 } 384 umem->odp_data->umem = umem; 385 386 mutex_init(&umem->odp_data->umem_mutex); 387 388 init_completion(&umem->odp_data->notifier_completion); 389 390 if (ib_umem_num_pages(umem)) { 391 umem->odp_data->page_list = 392 vzalloc(array_size(sizeof(*umem->odp_data->page_list), 393 ib_umem_num_pages(umem))); 394 if (!umem->odp_data->page_list) { 395 ret_val = -ENOMEM; 396 goto out_odp_data; 397 } 398 399 umem->odp_data->dma_list = 400 vzalloc(array_size(sizeof(*umem->odp_data->dma_list), 401 ib_umem_num_pages(umem))); 402 if (!umem->odp_data->dma_list) { 403 ret_val = -ENOMEM; 404 goto out_page_list; 405 } 406 } 407 408 /* 409 * When using MMU notifiers, we will get a 410 * notification before the "current" task (and MM) is 411 * destroyed. We use the umem_rwsem semaphore to synchronize. 412 */ 413 down_write(&context->umem_rwsem); 414 context->odp_mrs_count++; 415 if (likely(ib_umem_start(umem) != ib_umem_end(umem))) 416 rbt_ib_umem_insert(&umem->odp_data->interval_tree, 417 &context->umem_tree); 418 if (likely(!atomic_read(&context->notifier_count)) || 419 context->odp_mrs_count == 1) 420 umem->odp_data->mn_counters_active = true; 421 else 422 list_add(&umem->odp_data->no_private_counters, 423 &context->no_private_counters); 424 downgrade_write(&context->umem_rwsem); 425 426 if (context->odp_mrs_count == 1) { 427 /* 428 * Note that at this point, no MMU notifier is running 429 * for this context! 430 */ 431 atomic_set(&context->notifier_count, 0); 432 INIT_HLIST_NODE(&context->mn.hlist); 433 context->mn.ops = &ib_umem_notifiers; 434 /* 435 * Lock-dep detects a false positive for mmap_sem vs. 436 * umem_rwsem, due to not grasping downgrade_write correctly. 437 */ 438 lockdep_off(); 439 ret_val = mmu_notifier_register(&context->mn, mm); 440 lockdep_on(); 441 if (ret_val) { 442 pr_err("Failed to register mmu_notifier %d\n", ret_val); 443 ret_val = -EBUSY; 444 goto out_mutex; 445 } 446 } 447 448 up_read(&context->umem_rwsem); 449 450 /* 451 * Note that doing an mmput can cause a notifier for the relevant mm. 452 * If the notifier is called while we hold the umem_rwsem, this will 453 * cause a deadlock. Therefore, we release the reference only after we 454 * released the semaphore. 455 */ 456 mmput(mm); 457 return 0; 458 459 out_mutex: 460 up_read(&context->umem_rwsem); 461 vfree(umem->odp_data->dma_list); 462 out_page_list: 463 vfree(umem->odp_data->page_list); 464 out_odp_data: 465 kfree(umem->odp_data); 466 out_mm: 467 mmput(mm); 468 return ret_val; 469 } 470 471 void ib_umem_odp_release(struct ib_umem *umem) 472 { 473 struct ib_ucontext *context = umem->context; 474 475 /* 476 * Ensure that no more pages are mapped in the umem. 477 * 478 * It is the driver's responsibility to ensure, before calling us, 479 * that the hardware will not attempt to access the MR any more. 480 */ 481 ib_umem_odp_unmap_dma_pages(umem, ib_umem_start(umem), 482 ib_umem_end(umem)); 483 484 down_write(&context->umem_rwsem); 485 if (likely(ib_umem_start(umem) != ib_umem_end(umem))) 486 rbt_ib_umem_remove(&umem->odp_data->interval_tree, 487 &context->umem_tree); 488 context->odp_mrs_count--; 489 if (!umem->odp_data->mn_counters_active) { 490 list_del(&umem->odp_data->no_private_counters); 491 complete_all(&umem->odp_data->notifier_completion); 492 } 493 494 /* 495 * Downgrade the lock to a read lock. This ensures that the notifiers 496 * (who lock the mutex for reading) will be able to finish, and we 497 * will be able to enventually obtain the mmu notifiers SRCU. Note 498 * that since we are doing it atomically, no other user could register 499 * and unregister while we do the check. 500 */ 501 downgrade_write(&context->umem_rwsem); 502 if (!context->odp_mrs_count) { 503 struct task_struct *owning_process = NULL; 504 struct mm_struct *owning_mm = NULL; 505 506 owning_process = get_pid_task(context->tgid, 507 PIDTYPE_PID); 508 if (owning_process == NULL) 509 /* 510 * The process is already dead, notifier were removed 511 * already. 512 */ 513 goto out; 514 515 owning_mm = get_task_mm(owning_process); 516 if (owning_mm == NULL) 517 /* 518 * The process' mm is already dead, notifier were 519 * removed already. 520 */ 521 goto out_put_task; 522 mmu_notifier_unregister(&context->mn, owning_mm); 523 524 mmput(owning_mm); 525 526 out_put_task: 527 put_task_struct(owning_process); 528 } 529 out: 530 up_read(&context->umem_rwsem); 531 532 vfree(umem->odp_data->dma_list); 533 vfree(umem->odp_data->page_list); 534 kfree(umem->odp_data); 535 kfree(umem); 536 } 537 538 /* 539 * Map for DMA and insert a single page into the on-demand paging page tables. 540 * 541 * @umem: the umem to insert the page to. 542 * @page_index: index in the umem to add the page to. 543 * @page: the page struct to map and add. 544 * @access_mask: access permissions needed for this page. 545 * @current_seq: sequence number for synchronization with invalidations. 546 * the sequence number is taken from 547 * umem->odp_data->notifiers_seq. 548 * 549 * The function returns -EFAULT if the DMA mapping operation fails. It returns 550 * -EAGAIN if a concurrent invalidation prevents us from updating the page. 551 * 552 * The page is released via put_page even if the operation failed. For 553 * on-demand pinning, the page is released whenever it isn't stored in the 554 * umem. 555 */ 556 static int ib_umem_odp_map_dma_single_page( 557 struct ib_umem *umem, 558 int page_index, 559 struct page *page, 560 u64 access_mask, 561 unsigned long current_seq) 562 { 563 struct ib_device *dev = umem->context->device; 564 dma_addr_t dma_addr; 565 int stored_page = 0; 566 int remove_existing_mapping = 0; 567 int ret = 0; 568 569 /* 570 * Note: we avoid writing if seq is different from the initial seq, to 571 * handle case of a racing notifier. This check also allows us to bail 572 * early if we have a notifier running in parallel with us. 573 */ 574 if (ib_umem_mmu_notifier_retry(umem, current_seq)) { 575 ret = -EAGAIN; 576 goto out; 577 } 578 if (!(umem->odp_data->dma_list[page_index])) { 579 dma_addr = ib_dma_map_page(dev, 580 page, 581 0, BIT(umem->page_shift), 582 DMA_BIDIRECTIONAL); 583 if (ib_dma_mapping_error(dev, dma_addr)) { 584 ret = -EFAULT; 585 goto out; 586 } 587 umem->odp_data->dma_list[page_index] = dma_addr | access_mask; 588 umem->odp_data->page_list[page_index] = page; 589 umem->npages++; 590 stored_page = 1; 591 } else if (umem->odp_data->page_list[page_index] == page) { 592 umem->odp_data->dma_list[page_index] |= access_mask; 593 } else { 594 pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n", 595 umem->odp_data->page_list[page_index], page); 596 /* Better remove the mapping now, to prevent any further 597 * damage. */ 598 remove_existing_mapping = 1; 599 } 600 601 out: 602 /* On Demand Paging - avoid pinning the page */ 603 if (umem->context->invalidate_range || !stored_page) 604 put_page(page); 605 606 if (remove_existing_mapping && umem->context->invalidate_range) { 607 invalidate_page_trampoline( 608 umem, 609 ib_umem_start(umem) + (page_index >> umem->page_shift), 610 ib_umem_start(umem) + ((page_index + 1) >> 611 umem->page_shift), 612 NULL); 613 ret = -EAGAIN; 614 } 615 616 return ret; 617 } 618 619 /** 620 * ib_umem_odp_map_dma_pages - Pin and DMA map userspace memory in an ODP MR. 621 * 622 * Pins the range of pages passed in the argument, and maps them to 623 * DMA addresses. The DMA addresses of the mapped pages is updated in 624 * umem->odp_data->dma_list. 625 * 626 * Returns the number of pages mapped in success, negative error code 627 * for failure. 628 * An -EAGAIN error code is returned when a concurrent mmu notifier prevents 629 * the function from completing its task. 630 * An -ENOENT error code indicates that userspace process is being terminated 631 * and mm was already destroyed. 632 * @umem: the umem to map and pin 633 * @user_virt: the address from which we need to map. 634 * @bcnt: the minimal number of bytes to pin and map. The mapping might be 635 * bigger due to alignment, and may also be smaller in case of an error 636 * pinning or mapping a page. The actual pages mapped is returned in 637 * the return value. 638 * @access_mask: bit mask of the requested access permissions for the given 639 * range. 640 * @current_seq: the MMU notifiers sequance value for synchronization with 641 * invalidations. the sequance number is read from 642 * umem->odp_data->notifiers_seq before calling this function 643 */ 644 int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt, 645 u64 access_mask, unsigned long current_seq) 646 { 647 struct task_struct *owning_process = NULL; 648 struct mm_struct *owning_mm = NULL; 649 struct page **local_page_list = NULL; 650 u64 page_mask, off; 651 int j, k, ret = 0, start_idx, npages = 0, page_shift; 652 unsigned int flags = 0; 653 phys_addr_t p = 0; 654 655 if (access_mask == 0) 656 return -EINVAL; 657 658 if (user_virt < ib_umem_start(umem) || 659 user_virt + bcnt > ib_umem_end(umem)) 660 return -EFAULT; 661 662 local_page_list = (struct page **)__get_free_page(GFP_KERNEL); 663 if (!local_page_list) 664 return -ENOMEM; 665 666 page_shift = umem->page_shift; 667 page_mask = ~(BIT(page_shift) - 1); 668 off = user_virt & (~page_mask); 669 user_virt = user_virt & page_mask; 670 bcnt += off; /* Charge for the first page offset as well. */ 671 672 owning_process = get_pid_task(umem->context->tgid, PIDTYPE_PID); 673 if (owning_process == NULL) { 674 ret = -EINVAL; 675 goto out_no_task; 676 } 677 678 owning_mm = get_task_mm(owning_process); 679 if (owning_mm == NULL) { 680 ret = -ENOENT; 681 goto out_put_task; 682 } 683 684 if (access_mask & ODP_WRITE_ALLOWED_BIT) 685 flags |= FOLL_WRITE; 686 687 start_idx = (user_virt - ib_umem_start(umem)) >> page_shift; 688 k = start_idx; 689 690 while (bcnt > 0) { 691 const size_t gup_num_pages = min_t(size_t, 692 (bcnt + BIT(page_shift) - 1) >> page_shift, 693 PAGE_SIZE / sizeof(struct page *)); 694 695 down_read(&owning_mm->mmap_sem); 696 /* 697 * Note: this might result in redundent page getting. We can 698 * avoid this by checking dma_list to be 0 before calling 699 * get_user_pages. However, this make the code much more 700 * complex (and doesn't gain us much performance in most use 701 * cases). 702 */ 703 npages = get_user_pages_remote(owning_process, owning_mm, 704 user_virt, gup_num_pages, 705 flags, local_page_list, NULL, NULL); 706 up_read(&owning_mm->mmap_sem); 707 708 if (npages < 0) 709 break; 710 711 bcnt -= min_t(size_t, npages << PAGE_SHIFT, bcnt); 712 mutex_lock(&umem->odp_data->umem_mutex); 713 for (j = 0; j < npages; j++, user_virt += PAGE_SIZE) { 714 if (user_virt & ~page_mask) { 715 p += PAGE_SIZE; 716 if (page_to_phys(local_page_list[j]) != p) { 717 ret = -EFAULT; 718 break; 719 } 720 put_page(local_page_list[j]); 721 continue; 722 } 723 724 ret = ib_umem_odp_map_dma_single_page( 725 umem, k, local_page_list[j], 726 access_mask, current_seq); 727 if (ret < 0) 728 break; 729 730 p = page_to_phys(local_page_list[j]); 731 k++; 732 } 733 mutex_unlock(&umem->odp_data->umem_mutex); 734 735 if (ret < 0) { 736 /* Release left over pages when handling errors. */ 737 for (++j; j < npages; ++j) 738 put_page(local_page_list[j]); 739 break; 740 } 741 } 742 743 if (ret >= 0) { 744 if (npages < 0 && k == start_idx) 745 ret = npages; 746 else 747 ret = k - start_idx; 748 } 749 750 mmput(owning_mm); 751 out_put_task: 752 put_task_struct(owning_process); 753 out_no_task: 754 free_page((unsigned long)local_page_list); 755 return ret; 756 } 757 EXPORT_SYMBOL(ib_umem_odp_map_dma_pages); 758 759 void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 virt, 760 u64 bound) 761 { 762 int idx; 763 u64 addr; 764 struct ib_device *dev = umem->context->device; 765 766 virt = max_t(u64, virt, ib_umem_start(umem)); 767 bound = min_t(u64, bound, ib_umem_end(umem)); 768 /* Note that during the run of this function, the 769 * notifiers_count of the MR is > 0, preventing any racing 770 * faults from completion. We might be racing with other 771 * invalidations, so we must make sure we free each page only 772 * once. */ 773 mutex_lock(&umem->odp_data->umem_mutex); 774 for (addr = virt; addr < bound; addr += BIT(umem->page_shift)) { 775 idx = (addr - ib_umem_start(umem)) >> umem->page_shift; 776 if (umem->odp_data->page_list[idx]) { 777 struct page *page = umem->odp_data->page_list[idx]; 778 dma_addr_t dma = umem->odp_data->dma_list[idx]; 779 dma_addr_t dma_addr = dma & ODP_DMA_ADDR_MASK; 780 781 WARN_ON(!dma_addr); 782 783 ib_dma_unmap_page(dev, dma_addr, PAGE_SIZE, 784 DMA_BIDIRECTIONAL); 785 if (dma & ODP_WRITE_ALLOWED_BIT) { 786 struct page *head_page = compound_head(page); 787 /* 788 * set_page_dirty prefers being called with 789 * the page lock. However, MMU notifiers are 790 * called sometimes with and sometimes without 791 * the lock. We rely on the umem_mutex instead 792 * to prevent other mmu notifiers from 793 * continuing and allowing the page mapping to 794 * be removed. 795 */ 796 set_page_dirty(head_page); 797 } 798 /* on demand pinning support */ 799 if (!umem->context->invalidate_range) 800 put_page(page); 801 umem->odp_data->page_list[idx] = NULL; 802 umem->odp_data->dma_list[idx] = 0; 803 umem->npages--; 804 } 805 } 806 mutex_unlock(&umem->odp_data->umem_mutex); 807 } 808 EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages); 809 810 /* @last is not a part of the interval. See comment for function 811 * node_last. 812 */ 813 int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root, 814 u64 start, u64 last, 815 umem_call_back cb, 816 bool blockable, 817 void *cookie) 818 { 819 int ret_val = 0; 820 struct umem_odp_node *node, *next; 821 struct ib_umem_odp *umem; 822 823 if (unlikely(start == last)) 824 return ret_val; 825 826 for (node = rbt_ib_umem_iter_first(root, start, last - 1); 827 node; node = next) { 828 /* TODO move the blockable decision up to the callback */ 829 if (!blockable) 830 return -EAGAIN; 831 next = rbt_ib_umem_iter_next(node, start, last - 1); 832 umem = container_of(node, struct ib_umem_odp, interval_tree); 833 ret_val = cb(umem->umem, start, last, cookie) || ret_val; 834 } 835 836 return ret_val; 837 } 838 EXPORT_SYMBOL(rbt_ib_umem_for_each_in_range); 839 840 struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root, 841 u64 addr, u64 length) 842 { 843 struct umem_odp_node *node; 844 845 node = rbt_ib_umem_iter_first(root, addr, addr + length - 1); 846 if (node) 847 return container_of(node, struct ib_umem_odp, interval_tree); 848 return NULL; 849 850 } 851 EXPORT_SYMBOL(rbt_ib_umem_lookup); 852