1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem.h> 34 #include <rdma/ib_umem_odp.h> 35 #include <linux/kernel.h> 36 37 #include "mlx5_ib.h" 38 #include "cmd.h" 39 40 #define MAX_PREFETCH_LEN (4*1024*1024U) 41 42 /* Timeout in ms to wait for an active mmu notifier to complete when handling 43 * a pagefault. */ 44 #define MMU_NOTIFIER_TIMEOUT 1000 45 46 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 47 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 48 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 49 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 50 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 51 52 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 53 54 static u64 mlx5_imr_ksm_entries; 55 56 static int check_parent(struct ib_umem_odp *odp, 57 struct mlx5_ib_mr *parent) 58 { 59 struct mlx5_ib_mr *mr = odp->private; 60 61 return mr && mr->parent == parent && !odp->dying; 62 } 63 64 struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr) 65 { 66 if (WARN_ON(!mr || !mr->umem || !mr->umem->is_odp)) 67 return NULL; 68 69 return to_ib_umem_odp(mr->umem)->per_mm; 70 } 71 72 static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp) 73 { 74 struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent; 75 struct ib_ucontext_per_mm *per_mm = odp->per_mm; 76 struct rb_node *rb; 77 78 down_read(&per_mm->umem_rwsem); 79 while (1) { 80 rb = rb_next(&odp->interval_tree.rb); 81 if (!rb) 82 goto not_found; 83 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 84 if (check_parent(odp, parent)) 85 goto end; 86 } 87 not_found: 88 odp = NULL; 89 end: 90 up_read(&per_mm->umem_rwsem); 91 return odp; 92 } 93 94 static struct ib_umem_odp *odp_lookup(u64 start, u64 length, 95 struct mlx5_ib_mr *parent) 96 { 97 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent); 98 struct ib_umem_odp *odp; 99 struct rb_node *rb; 100 101 down_read(&per_mm->umem_rwsem); 102 odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length); 103 if (!odp) 104 goto end; 105 106 while (1) { 107 if (check_parent(odp, parent)) 108 goto end; 109 rb = rb_next(&odp->interval_tree.rb); 110 if (!rb) 111 goto not_found; 112 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 113 if (ib_umem_start(&odp->umem) > start + length) 114 goto not_found; 115 } 116 not_found: 117 odp = NULL; 118 end: 119 up_read(&per_mm->umem_rwsem); 120 return odp; 121 } 122 123 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, 124 size_t nentries, struct mlx5_ib_mr *mr, int flags) 125 { 126 struct ib_pd *pd = mr->ibmr.pd; 127 struct mlx5_ib_dev *dev = to_mdev(pd->device); 128 struct ib_umem_odp *odp; 129 unsigned long va; 130 int i; 131 132 if (flags & MLX5_IB_UPD_XLT_ZAP) { 133 for (i = 0; i < nentries; i++, pklm++) { 134 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 135 pklm->key = cpu_to_be32(dev->null_mkey); 136 pklm->va = 0; 137 } 138 return; 139 } 140 141 odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE, 142 nentries * MLX5_IMR_MTT_SIZE, mr); 143 144 for (i = 0; i < nentries; i++, pklm++) { 145 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 146 va = (offset + i) * MLX5_IMR_MTT_SIZE; 147 if (odp && odp->umem.address == va) { 148 struct mlx5_ib_mr *mtt = odp->private; 149 150 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 151 odp = odp_next(odp); 152 } else { 153 pklm->key = cpu_to_be32(dev->null_mkey); 154 } 155 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n", 156 i, va, be32_to_cpu(pklm->key)); 157 } 158 } 159 160 static void mr_leaf_free_action(struct work_struct *work) 161 { 162 struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); 163 int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT; 164 struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; 165 166 mr->parent = NULL; 167 synchronize_srcu(&mr->dev->mr_srcu); 168 169 ib_umem_release(&odp->umem); 170 if (imr->live) 171 mlx5_ib_update_xlt(imr, idx, 1, 0, 172 MLX5_IB_UPD_XLT_INDIRECT | 173 MLX5_IB_UPD_XLT_ATOMIC); 174 mlx5_mr_cache_free(mr->dev, mr); 175 176 if (atomic_dec_and_test(&imr->num_leaf_free)) 177 wake_up(&imr->q_leaf_free); 178 } 179 180 void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, 181 unsigned long end) 182 { 183 struct mlx5_ib_mr *mr; 184 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 185 sizeof(struct mlx5_mtt)) - 1; 186 u64 idx = 0, blk_start_idx = 0; 187 struct ib_umem *umem; 188 int in_block = 0; 189 u64 addr; 190 191 if (!umem_odp) { 192 pr_err("invalidation called on NULL umem or non-ODP umem\n"); 193 return; 194 } 195 umem = &umem_odp->umem; 196 197 mr = umem_odp->private; 198 199 if (!mr || !mr->ibmr.pd) 200 return; 201 202 start = max_t(u64, ib_umem_start(umem), start); 203 end = min_t(u64, ib_umem_end(umem), end); 204 205 /* 206 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 207 * while we are doing the invalidation, no page fault will attempt to 208 * overwrite the same MTTs. Concurent invalidations might race us, 209 * but they will write 0s as well, so no difference in the end result. 210 */ 211 212 for (addr = start; addr < end; addr += BIT(umem->page_shift)) { 213 idx = (addr - ib_umem_start(umem)) >> umem->page_shift; 214 /* 215 * Strive to write the MTTs in chunks, but avoid overwriting 216 * non-existing MTTs. The huristic here can be improved to 217 * estimate the cost of another UMR vs. the cost of bigger 218 * UMR. 219 */ 220 if (umem_odp->dma_list[idx] & 221 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 222 if (!in_block) { 223 blk_start_idx = idx; 224 in_block = 1; 225 } 226 } else { 227 u64 umr_offset = idx & umr_block_mask; 228 229 if (in_block && umr_offset == 0) { 230 mlx5_ib_update_xlt(mr, blk_start_idx, 231 idx - blk_start_idx, 0, 232 MLX5_IB_UPD_XLT_ZAP | 233 MLX5_IB_UPD_XLT_ATOMIC); 234 in_block = 0; 235 } 236 } 237 } 238 if (in_block) 239 mlx5_ib_update_xlt(mr, blk_start_idx, 240 idx - blk_start_idx + 1, 0, 241 MLX5_IB_UPD_XLT_ZAP | 242 MLX5_IB_UPD_XLT_ATOMIC); 243 /* 244 * We are now sure that the device will not access the 245 * memory. We can safely unmap it, and mark it as dirty if 246 * needed. 247 */ 248 249 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 250 251 if (unlikely(!umem->npages && mr->parent && 252 !umem_odp->dying)) { 253 WRITE_ONCE(umem_odp->dying, 1); 254 atomic_inc(&mr->parent->num_leaf_free); 255 schedule_work(&umem_odp->work); 256 } 257 } 258 259 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 260 { 261 struct ib_odp_caps *caps = &dev->odp_caps; 262 263 memset(caps, 0, sizeof(*caps)); 264 265 if (!MLX5_CAP_GEN(dev->mdev, pg)) 266 return; 267 268 caps->general_caps = IB_ODP_SUPPORT; 269 270 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 271 dev->odp_max_size = U64_MAX; 272 else 273 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 274 275 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 276 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 277 278 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 279 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 280 281 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 282 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 283 284 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 285 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 286 287 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 288 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 289 290 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 291 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 292 293 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 294 MLX5_CAP_GEN(dev->mdev, null_mkey) && 295 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 296 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 297 298 return; 299 } 300 301 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 302 struct mlx5_pagefault *pfault, 303 int error) 304 { 305 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 306 pfault->wqe.wq_num : pfault->token; 307 int ret = mlx5_core_page_fault_resume(dev->mdev, 308 pfault->token, 309 wq_num, 310 pfault->type, 311 error); 312 if (ret) 313 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x\n", 314 wq_num); 315 } 316 317 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd, 318 struct ib_umem *umem, 319 bool ksm, int access_flags) 320 { 321 struct mlx5_ib_dev *dev = to_mdev(pd->device); 322 struct mlx5_ib_mr *mr; 323 int err; 324 325 mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY : 326 MLX5_IMR_MTT_CACHE_ENTRY); 327 328 if (IS_ERR(mr)) 329 return mr; 330 331 mr->ibmr.pd = pd; 332 333 mr->dev = dev; 334 mr->access_flags = access_flags; 335 mr->mmkey.iova = 0; 336 mr->umem = umem; 337 338 if (ksm) { 339 err = mlx5_ib_update_xlt(mr, 0, 340 mlx5_imr_ksm_entries, 341 MLX5_KSM_PAGE_SHIFT, 342 MLX5_IB_UPD_XLT_INDIRECT | 343 MLX5_IB_UPD_XLT_ZAP | 344 MLX5_IB_UPD_XLT_ENABLE); 345 346 } else { 347 err = mlx5_ib_update_xlt(mr, 0, 348 MLX5_IMR_MTT_ENTRIES, 349 PAGE_SHIFT, 350 MLX5_IB_UPD_XLT_ZAP | 351 MLX5_IB_UPD_XLT_ENABLE | 352 MLX5_IB_UPD_XLT_ATOMIC); 353 } 354 355 if (err) 356 goto fail; 357 358 mr->ibmr.lkey = mr->mmkey.key; 359 mr->ibmr.rkey = mr->mmkey.key; 360 361 mr->live = 1; 362 363 mlx5_ib_dbg(dev, "key %x dev %p mr %p\n", 364 mr->mmkey.key, dev->mdev, mr); 365 366 return mr; 367 368 fail: 369 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 370 mlx5_mr_cache_free(dev, mr); 371 372 return ERR_PTR(err); 373 } 374 375 static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr, 376 u64 io_virt, size_t bcnt) 377 { 378 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device); 379 struct ib_umem_odp *odp, *result = NULL; 380 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 381 u64 addr = io_virt & MLX5_IMR_MTT_MASK; 382 int nentries = 0, start_idx = 0, ret; 383 struct mlx5_ib_mr *mtt; 384 385 mutex_lock(&odp_mr->umem_mutex); 386 odp = odp_lookup(addr, 1, mr); 387 388 mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n", 389 io_virt, bcnt, addr, odp); 390 391 next_mr: 392 if (likely(odp)) { 393 if (nentries) 394 nentries++; 395 } else { 396 odp = ib_alloc_odp_umem(odp_mr->per_mm, addr, 397 MLX5_IMR_MTT_SIZE); 398 if (IS_ERR(odp)) { 399 mutex_unlock(&odp_mr->umem_mutex); 400 return ERR_CAST(odp); 401 } 402 403 mtt = implicit_mr_alloc(mr->ibmr.pd, &odp->umem, 0, 404 mr->access_flags); 405 if (IS_ERR(mtt)) { 406 mutex_unlock(&odp_mr->umem_mutex); 407 ib_umem_release(&odp->umem); 408 return ERR_CAST(mtt); 409 } 410 411 odp->private = mtt; 412 mtt->umem = &odp->umem; 413 mtt->mmkey.iova = addr; 414 mtt->parent = mr; 415 INIT_WORK(&odp->work, mr_leaf_free_action); 416 417 if (!nentries) 418 start_idx = addr >> MLX5_IMR_MTT_SHIFT; 419 nentries++; 420 } 421 422 /* Return first odp if region not covered by single one */ 423 if (likely(!result)) 424 result = odp; 425 426 addr += MLX5_IMR_MTT_SIZE; 427 if (unlikely(addr < io_virt + bcnt)) { 428 odp = odp_next(odp); 429 if (odp && odp->umem.address != addr) 430 odp = NULL; 431 goto next_mr; 432 } 433 434 if (unlikely(nentries)) { 435 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0, 436 MLX5_IB_UPD_XLT_INDIRECT | 437 MLX5_IB_UPD_XLT_ATOMIC); 438 if (ret) { 439 mlx5_ib_err(dev, "Failed to update PAS\n"); 440 result = ERR_PTR(ret); 441 } 442 } 443 444 mutex_unlock(&odp_mr->umem_mutex); 445 return result; 446 } 447 448 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 449 int access_flags) 450 { 451 struct ib_ucontext *ctx = pd->ibpd.uobject->context; 452 struct mlx5_ib_mr *imr; 453 struct ib_umem *umem; 454 455 umem = ib_umem_get(ctx, 0, 0, IB_ACCESS_ON_DEMAND, 0); 456 if (IS_ERR(umem)) 457 return ERR_CAST(umem); 458 459 imr = implicit_mr_alloc(&pd->ibpd, umem, 1, access_flags); 460 if (IS_ERR(imr)) { 461 ib_umem_release(umem); 462 return ERR_CAST(imr); 463 } 464 465 imr->umem = umem; 466 init_waitqueue_head(&imr->q_leaf_free); 467 atomic_set(&imr->num_leaf_free, 0); 468 469 return imr; 470 } 471 472 static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end, 473 void *cookie) 474 { 475 struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie; 476 struct ib_umem *umem = &umem_odp->umem; 477 478 if (mr->parent != imr) 479 return 0; 480 481 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem), 482 ib_umem_end(umem)); 483 484 if (umem_odp->dying) 485 return 0; 486 487 WRITE_ONCE(umem_odp->dying, 1); 488 atomic_inc(&imr->num_leaf_free); 489 schedule_work(&umem_odp->work); 490 491 return 0; 492 } 493 494 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 495 { 496 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr); 497 498 down_read(&per_mm->umem_rwsem); 499 rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, 0, ULLONG_MAX, 500 mr_leaf_free, true, imr); 501 up_read(&per_mm->umem_rwsem); 502 503 wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); 504 } 505 506 static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 507 u64 io_virt, size_t bcnt, u32 *bytes_mapped) 508 { 509 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 510 u64 access_mask = ODP_READ_ALLOWED_BIT; 511 int npages = 0, page_shift, np; 512 u64 start_idx, page_mask; 513 struct ib_umem_odp *odp; 514 int current_seq; 515 size_t size; 516 int ret; 517 518 if (!odp_mr->page_list) { 519 odp = implicit_mr_get_data(mr, io_virt, bcnt); 520 521 if (IS_ERR(odp)) 522 return PTR_ERR(odp); 523 mr = odp->private; 524 525 } else { 526 odp = odp_mr; 527 } 528 529 next_mr: 530 size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt); 531 532 page_shift = mr->umem->page_shift; 533 page_mask = ~(BIT(page_shift) - 1); 534 start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; 535 536 if (mr->umem->writable) 537 access_mask |= ODP_WRITE_ALLOWED_BIT; 538 539 current_seq = READ_ONCE(odp->notifiers_seq); 540 /* 541 * Ensure the sequence number is valid for some time before we call 542 * gup. 543 */ 544 smp_rmb(); 545 546 ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr->umem), io_virt, size, 547 access_mask, current_seq); 548 549 if (ret < 0) 550 goto out; 551 552 np = ret; 553 554 mutex_lock(&odp->umem_mutex); 555 if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr->umem), 556 current_seq)) { 557 /* 558 * No need to check whether the MTTs really belong to 559 * this MR, since ib_umem_odp_map_dma_pages already 560 * checks this. 561 */ 562 ret = mlx5_ib_update_xlt(mr, start_idx, np, 563 page_shift, MLX5_IB_UPD_XLT_ATOMIC); 564 } else { 565 ret = -EAGAIN; 566 } 567 mutex_unlock(&odp->umem_mutex); 568 569 if (ret < 0) { 570 if (ret != -EAGAIN) 571 mlx5_ib_err(dev, "Failed to update mkey page tables\n"); 572 goto out; 573 } 574 575 if (bytes_mapped) { 576 u32 new_mappings = (np << page_shift) - 577 (io_virt - round_down(io_virt, 1 << page_shift)); 578 *bytes_mapped += min_t(u32, new_mappings, size); 579 } 580 581 npages += np << (page_shift - PAGE_SHIFT); 582 bcnt -= size; 583 584 if (unlikely(bcnt)) { 585 struct ib_umem_odp *next; 586 587 io_virt += size; 588 next = odp_next(odp); 589 if (unlikely(!next || next->umem.address != io_virt)) { 590 mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", 591 io_virt, next); 592 return -EAGAIN; 593 } 594 odp = next; 595 mr = odp->private; 596 goto next_mr; 597 } 598 599 return npages; 600 601 out: 602 if (ret == -EAGAIN) { 603 if (mr->parent || !odp->dying) { 604 unsigned long timeout = 605 msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT); 606 607 if (!wait_for_completion_timeout( 608 &odp->notifier_completion, 609 timeout)) { 610 mlx5_ib_warn(dev, "timeout waiting for mmu notifier. seq %d against %d\n", 611 current_seq, odp->notifiers_seq); 612 } 613 } else { 614 /* The MR is being killed, kill the QP as well. */ 615 ret = -EFAULT; 616 } 617 } 618 619 return ret; 620 } 621 622 struct pf_frame { 623 struct pf_frame *next; 624 u32 key; 625 u64 io_virt; 626 size_t bcnt; 627 int depth; 628 }; 629 630 /* 631 * Handle a single data segment in a page-fault WQE or RDMA region. 632 * 633 * Returns number of OS pages retrieved on success. The caller may continue to 634 * the next data segment. 635 * Can return the following error codes: 636 * -EAGAIN to designate a temporary error. The caller will abort handling the 637 * page fault and resolve it. 638 * -EFAULT when there's an error mapping the requested pages. The caller will 639 * abort the page fault handling. 640 */ 641 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 642 u32 key, u64 io_virt, size_t bcnt, 643 u32 *bytes_committed, 644 u32 *bytes_mapped) 645 { 646 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 647 struct pf_frame *head = NULL, *frame; 648 struct mlx5_core_mkey *mmkey; 649 struct mlx5_ib_mw *mw; 650 struct mlx5_ib_mr *mr; 651 struct mlx5_klm *pklm; 652 u32 *out = NULL; 653 size_t offset; 654 655 srcu_key = srcu_read_lock(&dev->mr_srcu); 656 657 io_virt += *bytes_committed; 658 bcnt -= *bytes_committed; 659 660 next_mr: 661 mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key)); 662 if (!mmkey || mmkey->key != key) { 663 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 664 ret = -EFAULT; 665 goto srcu_unlock; 666 } 667 668 switch (mmkey->type) { 669 case MLX5_MKEY_MR: 670 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 671 if (!mr->live || !mr->ibmr.pd) { 672 mlx5_ib_dbg(dev, "got dead MR\n"); 673 ret = -EFAULT; 674 goto srcu_unlock; 675 } 676 677 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped); 678 if (ret < 0) 679 goto srcu_unlock; 680 681 npages += ret; 682 ret = 0; 683 break; 684 685 case MLX5_MKEY_MW: 686 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 687 688 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 689 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 690 ret = -EFAULT; 691 goto srcu_unlock; 692 } 693 694 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 695 sizeof(*pklm) * (mw->ndescs - 2); 696 697 if (outlen > cur_outlen) { 698 kfree(out); 699 out = kzalloc(outlen, GFP_KERNEL); 700 if (!out) { 701 ret = -ENOMEM; 702 goto srcu_unlock; 703 } 704 cur_outlen = outlen; 705 } 706 707 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 708 bsf0_klm0_pas_mtt0_1); 709 710 ret = mlx5_core_query_mkey(dev->mdev, &mw->mmkey, out, outlen); 711 if (ret) 712 goto srcu_unlock; 713 714 offset = io_virt - MLX5_GET64(query_mkey_out, out, 715 memory_key_mkey_entry.start_addr); 716 717 for (i = 0; bcnt && i < mw->ndescs; i++, pklm++) { 718 if (offset >= be32_to_cpu(pklm->bcount)) { 719 offset -= be32_to_cpu(pklm->bcount); 720 continue; 721 } 722 723 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 724 if (!frame) { 725 ret = -ENOMEM; 726 goto srcu_unlock; 727 } 728 729 frame->key = be32_to_cpu(pklm->key); 730 frame->io_virt = be64_to_cpu(pklm->va) + offset; 731 frame->bcnt = min_t(size_t, bcnt, 732 be32_to_cpu(pklm->bcount) - offset); 733 frame->depth = depth + 1; 734 frame->next = head; 735 head = frame; 736 737 bcnt -= frame->bcnt; 738 } 739 break; 740 741 default: 742 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 743 ret = -EFAULT; 744 goto srcu_unlock; 745 } 746 747 if (head) { 748 frame = head; 749 head = frame->next; 750 751 key = frame->key; 752 io_virt = frame->io_virt; 753 bcnt = frame->bcnt; 754 depth = frame->depth; 755 kfree(frame); 756 757 goto next_mr; 758 } 759 760 srcu_unlock: 761 while (head) { 762 frame = head; 763 head = frame->next; 764 kfree(frame); 765 } 766 kfree(out); 767 768 srcu_read_unlock(&dev->mr_srcu, srcu_key); 769 *bytes_committed = 0; 770 return ret ? ret : npages; 771 } 772 773 /** 774 * Parse a series of data segments for page fault handling. 775 * 776 * @qp the QP on which the fault occurred. 777 * @pfault contains page fault information. 778 * @wqe points at the first data segment in the WQE. 779 * @wqe_end points after the end of the WQE. 780 * @bytes_mapped receives the number of bytes that the function was able to 781 * map. This allows the caller to decide intelligently whether 782 * enough memory was mapped to resolve the page fault 783 * successfully (e.g. enough for the next MTU, or the entire 784 * WQE). 785 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 786 * the committed bytes). 787 * 788 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 789 * negative error code. 790 */ 791 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 792 struct mlx5_pagefault *pfault, 793 struct mlx5_ib_qp *qp, void *wqe, 794 void *wqe_end, u32 *bytes_mapped, 795 u32 *total_wqe_bytes, int receive_queue) 796 { 797 int ret = 0, npages = 0; 798 u64 io_virt; 799 u32 key; 800 u32 byte_count; 801 size_t bcnt; 802 int inline_segment; 803 804 /* Skip SRQ next-WQE segment. */ 805 if (receive_queue && qp->ibqp.srq) 806 wqe += sizeof(struct mlx5_wqe_srq_next_seg); 807 808 if (bytes_mapped) 809 *bytes_mapped = 0; 810 if (total_wqe_bytes) 811 *total_wqe_bytes = 0; 812 813 while (wqe < wqe_end) { 814 struct mlx5_wqe_data_seg *dseg = wqe; 815 816 io_virt = be64_to_cpu(dseg->addr); 817 key = be32_to_cpu(dseg->lkey); 818 byte_count = be32_to_cpu(dseg->byte_count); 819 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 820 bcnt = byte_count & ~MLX5_INLINE_SEG; 821 822 if (inline_segment) { 823 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 824 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 825 16); 826 } else { 827 wqe += sizeof(*dseg); 828 } 829 830 /* receive WQE end of sg list. */ 831 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 832 io_virt == 0) 833 break; 834 835 if (!inline_segment && total_wqe_bytes) { 836 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 837 pfault->bytes_committed); 838 } 839 840 /* A zero length data segment designates a length of 2GB. */ 841 if (bcnt == 0) 842 bcnt = 1U << 31; 843 844 if (inline_segment || bcnt <= pfault->bytes_committed) { 845 pfault->bytes_committed -= 846 min_t(size_t, bcnt, 847 pfault->bytes_committed); 848 continue; 849 } 850 851 ret = pagefault_single_data_segment(dev, key, io_virt, bcnt, 852 &pfault->bytes_committed, 853 bytes_mapped); 854 if (ret < 0) 855 break; 856 npages += ret; 857 } 858 859 return ret < 0 ? ret : npages; 860 } 861 862 static const u32 mlx5_ib_odp_opcode_cap[] = { 863 [MLX5_OPCODE_SEND] = IB_ODP_SUPPORT_SEND, 864 [MLX5_OPCODE_SEND_IMM] = IB_ODP_SUPPORT_SEND, 865 [MLX5_OPCODE_SEND_INVAL] = IB_ODP_SUPPORT_SEND, 866 [MLX5_OPCODE_RDMA_WRITE] = IB_ODP_SUPPORT_WRITE, 867 [MLX5_OPCODE_RDMA_WRITE_IMM] = IB_ODP_SUPPORT_WRITE, 868 [MLX5_OPCODE_RDMA_READ] = IB_ODP_SUPPORT_READ, 869 [MLX5_OPCODE_ATOMIC_CS] = IB_ODP_SUPPORT_ATOMIC, 870 [MLX5_OPCODE_ATOMIC_FA] = IB_ODP_SUPPORT_ATOMIC, 871 }; 872 873 /* 874 * Parse initiator WQE. Advances the wqe pointer to point at the 875 * scatter-gather list, and set wqe_end to the end of the WQE. 876 */ 877 static int mlx5_ib_mr_initiator_pfault_handler( 878 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 879 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 880 { 881 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 882 u16 wqe_index = pfault->wqe.wqe_index; 883 u32 transport_caps; 884 struct mlx5_base_av *av; 885 unsigned ds, opcode; 886 #if defined(DEBUG) 887 u32 ctrl_wqe_index, ctrl_qpn; 888 #endif 889 u32 qpn = qp->trans_qp.base.mqp.qpn; 890 891 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 892 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 893 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 894 ds, wqe_length); 895 return -EFAULT; 896 } 897 898 if (ds == 0) { 899 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 900 wqe_index, qpn); 901 return -EFAULT; 902 } 903 904 #if defined(DEBUG) 905 ctrl_wqe_index = (be32_to_cpu(ctrl->opmod_idx_opcode) & 906 MLX5_WQE_CTRL_WQE_INDEX_MASK) >> 907 MLX5_WQE_CTRL_WQE_INDEX_SHIFT; 908 if (wqe_index != ctrl_wqe_index) { 909 mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n", 910 wqe_index, qpn, 911 ctrl_wqe_index); 912 return -EFAULT; 913 } 914 915 ctrl_qpn = (be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_QPN_MASK) >> 916 MLX5_WQE_CTRL_QPN_SHIFT; 917 if (qpn != ctrl_qpn) { 918 mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n", 919 wqe_index, qpn, 920 ctrl_qpn); 921 return -EFAULT; 922 } 923 #endif /* DEBUG */ 924 925 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 926 *wqe += sizeof(*ctrl); 927 928 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 929 MLX5_WQE_CTRL_OPCODE_MASK; 930 931 switch (qp->ibqp.qp_type) { 932 case IB_QPT_RC: 933 transport_caps = dev->odp_caps.per_transport_caps.rc_odp_caps; 934 break; 935 case IB_QPT_UD: 936 transport_caps = dev->odp_caps.per_transport_caps.ud_odp_caps; 937 break; 938 default: 939 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport 0x%x\n", 940 qp->ibqp.qp_type); 941 return -EFAULT; 942 } 943 944 if (unlikely(opcode >= ARRAY_SIZE(mlx5_ib_odp_opcode_cap) || 945 !(transport_caps & mlx5_ib_odp_opcode_cap[opcode]))) { 946 mlx5_ib_err(dev, "ODP fault on QP of an unsupported opcode 0x%x\n", 947 opcode); 948 return -EFAULT; 949 } 950 951 if (qp->ibqp.qp_type != IB_QPT_RC) { 952 av = *wqe; 953 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 954 *wqe += sizeof(struct mlx5_av); 955 else 956 *wqe += sizeof(struct mlx5_base_av); 957 } 958 959 switch (opcode) { 960 case MLX5_OPCODE_RDMA_WRITE: 961 case MLX5_OPCODE_RDMA_WRITE_IMM: 962 case MLX5_OPCODE_RDMA_READ: 963 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 964 break; 965 case MLX5_OPCODE_ATOMIC_CS: 966 case MLX5_OPCODE_ATOMIC_FA: 967 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 968 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 969 break; 970 } 971 972 return 0; 973 } 974 975 /* 976 * Parse responder WQE. Advances the wqe pointer to point at the 977 * scatter-gather list, and set wqe_end to the end of the WQE. 978 */ 979 static int mlx5_ib_mr_responder_pfault_handler( 980 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 981 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 982 { 983 struct mlx5_ib_wq *wq = &qp->rq; 984 int wqe_size = 1 << wq->wqe_shift; 985 986 if (qp->ibqp.srq) { 987 mlx5_ib_err(dev, "ODP fault on SRQ is not supported\n"); 988 return -EFAULT; 989 } 990 991 if (qp->wq_sig) { 992 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 993 return -EFAULT; 994 } 995 996 if (wqe_size > wqe_length) { 997 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 998 return -EFAULT; 999 } 1000 1001 switch (qp->ibqp.qp_type) { 1002 case IB_QPT_RC: 1003 if (!(dev->odp_caps.per_transport_caps.rc_odp_caps & 1004 IB_ODP_SUPPORT_RECV)) 1005 goto invalid_transport_or_opcode; 1006 break; 1007 default: 1008 invalid_transport_or_opcode: 1009 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport. transport: 0x%x\n", 1010 qp->ibqp.qp_type); 1011 return -EFAULT; 1012 } 1013 1014 *wqe_end = *wqe + wqe_size; 1015 1016 return 0; 1017 } 1018 1019 static struct mlx5_ib_qp *mlx5_ib_odp_find_qp(struct mlx5_ib_dev *dev, 1020 u32 wq_num) 1021 { 1022 struct mlx5_core_qp *mqp = __mlx5_qp_lookup(dev->mdev, wq_num); 1023 1024 if (!mqp) { 1025 mlx5_ib_err(dev, "QPN 0x%6x not found\n", wq_num); 1026 return NULL; 1027 } 1028 1029 return to_mibqp(mqp); 1030 } 1031 1032 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1033 struct mlx5_pagefault *pfault) 1034 { 1035 int ret; 1036 void *wqe, *wqe_end; 1037 u32 bytes_mapped, total_wqe_bytes; 1038 char *buffer = NULL; 1039 int resume_with_error = 1; 1040 u16 wqe_index = pfault->wqe.wqe_index; 1041 int requestor = pfault->type & MLX5_PFAULT_REQUESTOR; 1042 struct mlx5_ib_qp *qp; 1043 1044 buffer = (char *)__get_free_page(GFP_KERNEL); 1045 if (!buffer) { 1046 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1047 goto resolve_page_fault; 1048 } 1049 1050 qp = mlx5_ib_odp_find_qp(dev, pfault->wqe.wq_num); 1051 if (!qp) 1052 goto resolve_page_fault; 1053 1054 ret = mlx5_ib_read_user_wqe(qp, requestor, wqe_index, buffer, 1055 PAGE_SIZE, &qp->trans_qp.base); 1056 if (ret < 0) { 1057 mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n", 1058 ret, wqe_index, pfault->token); 1059 goto resolve_page_fault; 1060 } 1061 1062 wqe = buffer; 1063 if (requestor) 1064 ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp, &wqe, 1065 &wqe_end, ret); 1066 else 1067 ret = mlx5_ib_mr_responder_pfault_handler(dev, pfault, qp, &wqe, 1068 &wqe_end, ret); 1069 if (ret < 0) 1070 goto resolve_page_fault; 1071 1072 if (wqe >= wqe_end) { 1073 mlx5_ib_err(dev, "ODP fault on invalid WQE.\n"); 1074 goto resolve_page_fault; 1075 } 1076 1077 ret = pagefault_data_segments(dev, pfault, qp, wqe, wqe_end, 1078 &bytes_mapped, &total_wqe_bytes, 1079 !requestor); 1080 if (ret == -EAGAIN) { 1081 resume_with_error = 0; 1082 goto resolve_page_fault; 1083 } else if (ret < 0 || total_wqe_bytes > bytes_mapped) { 1084 goto resolve_page_fault; 1085 } 1086 1087 resume_with_error = 0; 1088 resolve_page_fault: 1089 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1090 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1091 pfault->wqe.wq_num, resume_with_error, 1092 pfault->type); 1093 free_page((unsigned long)buffer); 1094 } 1095 1096 static int pages_in_range(u64 address, u32 length) 1097 { 1098 return (ALIGN(address + length, PAGE_SIZE) - 1099 (address & PAGE_MASK)) >> PAGE_SHIFT; 1100 } 1101 1102 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1103 struct mlx5_pagefault *pfault) 1104 { 1105 u64 address; 1106 u32 length; 1107 u32 prefetch_len = pfault->bytes_committed; 1108 int prefetch_activated = 0; 1109 u32 rkey = pfault->rdma.r_key; 1110 int ret; 1111 1112 /* The RDMA responder handler handles the page fault in two parts. 1113 * First it brings the necessary pages for the current packet 1114 * (and uses the pfault context), and then (after resuming the QP) 1115 * prefetches more pages. The second operation cannot use the pfault 1116 * context and therefore uses the dummy_pfault context allocated on 1117 * the stack */ 1118 pfault->rdma.rdma_va += pfault->bytes_committed; 1119 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1120 pfault->rdma.rdma_op_len); 1121 pfault->bytes_committed = 0; 1122 1123 address = pfault->rdma.rdma_va; 1124 length = pfault->rdma.rdma_op_len; 1125 1126 /* For some operations, the hardware cannot tell the exact message 1127 * length, and in those cases it reports zero. Use prefetch 1128 * logic. */ 1129 if (length == 0) { 1130 prefetch_activated = 1; 1131 length = pfault->rdma.packet_size; 1132 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1133 } 1134 1135 ret = pagefault_single_data_segment(dev, rkey, address, length, 1136 &pfault->bytes_committed, NULL); 1137 if (ret == -EAGAIN) { 1138 /* We're racing with an invalidation, don't prefetch */ 1139 prefetch_activated = 0; 1140 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1141 mlx5_ib_page_fault_resume(dev, pfault, 1); 1142 if (ret != -ENOENT) 1143 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1144 ret, pfault->token, pfault->type); 1145 return; 1146 } 1147 1148 mlx5_ib_page_fault_resume(dev, pfault, 0); 1149 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1150 pfault->token, pfault->type, 1151 prefetch_activated); 1152 1153 /* At this point, there might be a new pagefault already arriving in 1154 * the eq, switch to the dummy pagefault for the rest of the 1155 * processing. We're still OK with the objects being alive as the 1156 * work-queue is being fenced. */ 1157 1158 if (prefetch_activated) { 1159 u32 bytes_committed = 0; 1160 1161 ret = pagefault_single_data_segment(dev, rkey, address, 1162 prefetch_len, 1163 &bytes_committed, NULL); 1164 if (ret < 0 && ret != -EAGAIN) { 1165 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1166 ret, pfault->token, address, prefetch_len); 1167 } 1168 } 1169 } 1170 1171 void mlx5_ib_pfault(struct mlx5_core_dev *mdev, void *context, 1172 struct mlx5_pagefault *pfault) 1173 { 1174 struct mlx5_ib_dev *dev = context; 1175 u8 event_subtype = pfault->event_subtype; 1176 1177 switch (event_subtype) { 1178 case MLX5_PFAULT_SUBTYPE_WQE: 1179 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1180 break; 1181 case MLX5_PFAULT_SUBTYPE_RDMA: 1182 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1183 break; 1184 default: 1185 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1186 event_subtype); 1187 mlx5_ib_page_fault_resume(dev, pfault, 1); 1188 } 1189 } 1190 1191 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1192 { 1193 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1194 return; 1195 1196 switch (ent->order - 2) { 1197 case MLX5_IMR_MTT_CACHE_ENTRY: 1198 ent->page = PAGE_SHIFT; 1199 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1200 sizeof(struct mlx5_mtt) / 1201 MLX5_IB_UMR_OCTOWORD; 1202 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1203 ent->limit = 0; 1204 break; 1205 1206 case MLX5_IMR_KSM_CACHE_ENTRY: 1207 ent->page = MLX5_KSM_PAGE_SHIFT; 1208 ent->xlt = mlx5_imr_ksm_entries * 1209 sizeof(struct mlx5_klm) / 1210 MLX5_IB_UMR_OCTOWORD; 1211 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1212 ent->limit = 0; 1213 break; 1214 } 1215 } 1216 1217 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1218 { 1219 int ret; 1220 1221 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1222 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1223 if (ret) { 1224 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1225 return ret; 1226 } 1227 } 1228 1229 return 0; 1230 } 1231 1232 int mlx5_ib_odp_init(void) 1233 { 1234 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1235 MLX5_IMR_MTT_BITS); 1236 1237 return 0; 1238 } 1239 1240