1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem.h> 34 #include <rdma/ib_umem_odp.h> 35 #include <linux/kernel.h> 36 37 #include "mlx5_ib.h" 38 #include "cmd.h" 39 40 #include <linux/mlx5/eq.h> 41 42 /* Contains the details of a pagefault. */ 43 struct mlx5_pagefault { 44 u32 bytes_committed; 45 u32 token; 46 u8 event_subtype; 47 u8 type; 48 union { 49 /* Initiator or send message responder pagefault details. */ 50 struct { 51 /* Received packet size, only valid for responders. */ 52 u32 packet_size; 53 /* 54 * Number of resource holding WQE, depends on type. 55 */ 56 u32 wq_num; 57 /* 58 * WQE index. Refers to either the send queue or 59 * receive queue, according to event_subtype. 60 */ 61 u16 wqe_index; 62 } wqe; 63 /* RDMA responder pagefault details */ 64 struct { 65 u32 r_key; 66 /* 67 * Received packet size, minimal size page fault 68 * resolution required for forward progress. 69 */ 70 u32 packet_size; 71 u32 rdma_op_len; 72 u64 rdma_va; 73 } rdma; 74 }; 75 76 struct mlx5_ib_pf_eq *eq; 77 struct work_struct work; 78 }; 79 80 #define MAX_PREFETCH_LEN (4*1024*1024U) 81 82 /* Timeout in ms to wait for an active mmu notifier to complete when handling 83 * a pagefault. */ 84 #define MMU_NOTIFIER_TIMEOUT 1000 85 86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 91 92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 93 94 static u64 mlx5_imr_ksm_entries; 95 96 static int check_parent(struct ib_umem_odp *odp, 97 struct mlx5_ib_mr *parent) 98 { 99 struct mlx5_ib_mr *mr = odp->private; 100 101 return mr && mr->parent == parent && !odp->dying; 102 } 103 104 static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr) 105 { 106 if (WARN_ON(!mr || !is_odp_mr(mr))) 107 return NULL; 108 109 return to_ib_umem_odp(mr->umem)->per_mm; 110 } 111 112 static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp) 113 { 114 struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent; 115 struct ib_ucontext_per_mm *per_mm = odp->per_mm; 116 struct rb_node *rb; 117 118 down_read(&per_mm->umem_rwsem); 119 while (1) { 120 rb = rb_next(&odp->interval_tree.rb); 121 if (!rb) 122 goto not_found; 123 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 124 if (check_parent(odp, parent)) 125 goto end; 126 } 127 not_found: 128 odp = NULL; 129 end: 130 up_read(&per_mm->umem_rwsem); 131 return odp; 132 } 133 134 static struct ib_umem_odp *odp_lookup(u64 start, u64 length, 135 struct mlx5_ib_mr *parent) 136 { 137 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent); 138 struct ib_umem_odp *odp; 139 struct rb_node *rb; 140 141 down_read(&per_mm->umem_rwsem); 142 odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length); 143 if (!odp) 144 goto end; 145 146 while (1) { 147 if (check_parent(odp, parent)) 148 goto end; 149 rb = rb_next(&odp->interval_tree.rb); 150 if (!rb) 151 goto not_found; 152 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 153 if (ib_umem_start(&odp->umem) > start + length) 154 goto not_found; 155 } 156 not_found: 157 odp = NULL; 158 end: 159 up_read(&per_mm->umem_rwsem); 160 return odp; 161 } 162 163 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, 164 size_t nentries, struct mlx5_ib_mr *mr, int flags) 165 { 166 struct ib_pd *pd = mr->ibmr.pd; 167 struct mlx5_ib_dev *dev = to_mdev(pd->device); 168 struct ib_umem_odp *odp; 169 unsigned long va; 170 int i; 171 172 if (flags & MLX5_IB_UPD_XLT_ZAP) { 173 for (i = 0; i < nentries; i++, pklm++) { 174 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 175 pklm->key = cpu_to_be32(dev->null_mkey); 176 pklm->va = 0; 177 } 178 return; 179 } 180 181 odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE, 182 nentries * MLX5_IMR_MTT_SIZE, mr); 183 184 for (i = 0; i < nentries; i++, pklm++) { 185 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 186 va = (offset + i) * MLX5_IMR_MTT_SIZE; 187 if (odp && odp->umem.address == va) { 188 struct mlx5_ib_mr *mtt = odp->private; 189 190 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 191 odp = odp_next(odp); 192 } else { 193 pklm->key = cpu_to_be32(dev->null_mkey); 194 } 195 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n", 196 i, va, be32_to_cpu(pklm->key)); 197 } 198 } 199 200 static void mr_leaf_free_action(struct work_struct *work) 201 { 202 struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); 203 int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT; 204 struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; 205 206 mr->parent = NULL; 207 synchronize_srcu(&mr->dev->mr_srcu); 208 209 ib_umem_release(&odp->umem); 210 if (imr->live) 211 mlx5_ib_update_xlt(imr, idx, 1, 0, 212 MLX5_IB_UPD_XLT_INDIRECT | 213 MLX5_IB_UPD_XLT_ATOMIC); 214 mlx5_mr_cache_free(mr->dev, mr); 215 216 if (atomic_dec_and_test(&imr->num_leaf_free)) 217 wake_up(&imr->q_leaf_free); 218 } 219 220 void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, 221 unsigned long end) 222 { 223 struct mlx5_ib_mr *mr; 224 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 225 sizeof(struct mlx5_mtt)) - 1; 226 u64 idx = 0, blk_start_idx = 0; 227 struct ib_umem *umem; 228 int in_block = 0; 229 u64 addr; 230 231 if (!umem_odp) { 232 pr_err("invalidation called on NULL umem or non-ODP umem\n"); 233 return; 234 } 235 umem = &umem_odp->umem; 236 237 mr = umem_odp->private; 238 239 if (!mr || !mr->ibmr.pd) 240 return; 241 242 start = max_t(u64, ib_umem_start(umem), start); 243 end = min_t(u64, ib_umem_end(umem), end); 244 245 /* 246 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 247 * while we are doing the invalidation, no page fault will attempt to 248 * overwrite the same MTTs. Concurent invalidations might race us, 249 * but they will write 0s as well, so no difference in the end result. 250 */ 251 252 for (addr = start; addr < end; addr += BIT(umem->page_shift)) { 253 idx = (addr - ib_umem_start(umem)) >> umem->page_shift; 254 /* 255 * Strive to write the MTTs in chunks, but avoid overwriting 256 * non-existing MTTs. The huristic here can be improved to 257 * estimate the cost of another UMR vs. the cost of bigger 258 * UMR. 259 */ 260 if (umem_odp->dma_list[idx] & 261 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 262 if (!in_block) { 263 blk_start_idx = idx; 264 in_block = 1; 265 } 266 } else { 267 u64 umr_offset = idx & umr_block_mask; 268 269 if (in_block && umr_offset == 0) { 270 mlx5_ib_update_xlt(mr, blk_start_idx, 271 idx - blk_start_idx, 0, 272 MLX5_IB_UPD_XLT_ZAP | 273 MLX5_IB_UPD_XLT_ATOMIC); 274 in_block = 0; 275 } 276 } 277 } 278 if (in_block) 279 mlx5_ib_update_xlt(mr, blk_start_idx, 280 idx - blk_start_idx + 1, 0, 281 MLX5_IB_UPD_XLT_ZAP | 282 MLX5_IB_UPD_XLT_ATOMIC); 283 /* 284 * We are now sure that the device will not access the 285 * memory. We can safely unmap it, and mark it as dirty if 286 * needed. 287 */ 288 289 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 290 291 if (unlikely(!umem->npages && mr->parent && 292 !umem_odp->dying)) { 293 WRITE_ONCE(umem_odp->dying, 1); 294 atomic_inc(&mr->parent->num_leaf_free); 295 schedule_work(&umem_odp->work); 296 } 297 } 298 299 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 300 { 301 struct ib_odp_caps *caps = &dev->odp_caps; 302 303 memset(caps, 0, sizeof(*caps)); 304 305 if (!MLX5_CAP_GEN(dev->mdev, pg)) 306 return; 307 308 caps->general_caps = IB_ODP_SUPPORT; 309 310 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 311 dev->odp_max_size = U64_MAX; 312 else 313 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 314 315 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 316 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 317 318 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive)) 319 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 320 321 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 322 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 323 324 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 325 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 326 327 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 328 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 329 330 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 331 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 332 333 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 334 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 335 336 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive)) 337 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 338 339 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send)) 340 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 341 342 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive)) 343 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 344 345 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write)) 346 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 347 348 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read)) 349 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 350 351 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic)) 352 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 353 354 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive)) 355 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 356 357 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 358 MLX5_CAP_GEN(dev->mdev, null_mkey) && 359 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 360 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 361 362 return; 363 } 364 365 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 366 struct mlx5_pagefault *pfault, 367 int error) 368 { 369 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 370 pfault->wqe.wq_num : pfault->token; 371 u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { }; 372 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = { }; 373 int err; 374 375 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 376 MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type); 377 MLX5_SET(page_fault_resume_in, in, token, pfault->token); 378 MLX5_SET(page_fault_resume_in, in, wq_number, wq_num); 379 MLX5_SET(page_fault_resume_in, in, error, !!error); 380 381 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 382 if (err) 383 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 384 wq_num, err); 385 } 386 387 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd, 388 struct ib_umem *umem, 389 bool ksm, int access_flags) 390 { 391 struct mlx5_ib_dev *dev = to_mdev(pd->device); 392 struct mlx5_ib_mr *mr; 393 int err; 394 395 mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY : 396 MLX5_IMR_MTT_CACHE_ENTRY); 397 398 if (IS_ERR(mr)) 399 return mr; 400 401 mr->ibmr.pd = pd; 402 403 mr->dev = dev; 404 mr->access_flags = access_flags; 405 mr->mmkey.iova = 0; 406 mr->umem = umem; 407 408 if (ksm) { 409 err = mlx5_ib_update_xlt(mr, 0, 410 mlx5_imr_ksm_entries, 411 MLX5_KSM_PAGE_SHIFT, 412 MLX5_IB_UPD_XLT_INDIRECT | 413 MLX5_IB_UPD_XLT_ZAP | 414 MLX5_IB_UPD_XLT_ENABLE); 415 416 } else { 417 err = mlx5_ib_update_xlt(mr, 0, 418 MLX5_IMR_MTT_ENTRIES, 419 PAGE_SHIFT, 420 MLX5_IB_UPD_XLT_ZAP | 421 MLX5_IB_UPD_XLT_ENABLE | 422 MLX5_IB_UPD_XLT_ATOMIC); 423 } 424 425 if (err) 426 goto fail; 427 428 mr->ibmr.lkey = mr->mmkey.key; 429 mr->ibmr.rkey = mr->mmkey.key; 430 431 mr->live = 1; 432 433 mlx5_ib_dbg(dev, "key %x dev %p mr %p\n", 434 mr->mmkey.key, dev->mdev, mr); 435 436 return mr; 437 438 fail: 439 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 440 mlx5_mr_cache_free(dev, mr); 441 442 return ERR_PTR(err); 443 } 444 445 static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr, 446 u64 io_virt, size_t bcnt) 447 { 448 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device); 449 struct ib_umem_odp *odp, *result = NULL; 450 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 451 u64 addr = io_virt & MLX5_IMR_MTT_MASK; 452 int nentries = 0, start_idx = 0, ret; 453 struct mlx5_ib_mr *mtt; 454 455 mutex_lock(&odp_mr->umem_mutex); 456 odp = odp_lookup(addr, 1, mr); 457 458 mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n", 459 io_virt, bcnt, addr, odp); 460 461 next_mr: 462 if (likely(odp)) { 463 if (nentries) 464 nentries++; 465 } else { 466 odp = ib_alloc_odp_umem(odp_mr, addr, 467 MLX5_IMR_MTT_SIZE); 468 if (IS_ERR(odp)) { 469 mutex_unlock(&odp_mr->umem_mutex); 470 return ERR_CAST(odp); 471 } 472 473 mtt = implicit_mr_alloc(mr->ibmr.pd, &odp->umem, 0, 474 mr->access_flags); 475 if (IS_ERR(mtt)) { 476 mutex_unlock(&odp_mr->umem_mutex); 477 ib_umem_release(&odp->umem); 478 return ERR_CAST(mtt); 479 } 480 481 odp->private = mtt; 482 mtt->umem = &odp->umem; 483 mtt->mmkey.iova = addr; 484 mtt->parent = mr; 485 INIT_WORK(&odp->work, mr_leaf_free_action); 486 487 if (!nentries) 488 start_idx = addr >> MLX5_IMR_MTT_SHIFT; 489 nentries++; 490 } 491 492 /* Return first odp if region not covered by single one */ 493 if (likely(!result)) 494 result = odp; 495 496 addr += MLX5_IMR_MTT_SIZE; 497 if (unlikely(addr < io_virt + bcnt)) { 498 odp = odp_next(odp); 499 if (odp && odp->umem.address != addr) 500 odp = NULL; 501 goto next_mr; 502 } 503 504 if (unlikely(nentries)) { 505 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0, 506 MLX5_IB_UPD_XLT_INDIRECT | 507 MLX5_IB_UPD_XLT_ATOMIC); 508 if (ret) { 509 mlx5_ib_err(dev, "Failed to update PAS\n"); 510 result = ERR_PTR(ret); 511 } 512 } 513 514 mutex_unlock(&odp_mr->umem_mutex); 515 return result; 516 } 517 518 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 519 struct ib_udata *udata, 520 int access_flags) 521 { 522 struct mlx5_ib_mr *imr; 523 struct ib_umem *umem; 524 525 umem = ib_umem_get(udata, 0, 0, access_flags, 0); 526 if (IS_ERR(umem)) 527 return ERR_CAST(umem); 528 529 imr = implicit_mr_alloc(&pd->ibpd, umem, 1, access_flags); 530 if (IS_ERR(imr)) { 531 ib_umem_release(umem); 532 return ERR_CAST(imr); 533 } 534 535 imr->umem = umem; 536 init_waitqueue_head(&imr->q_leaf_free); 537 atomic_set(&imr->num_leaf_free, 0); 538 atomic_set(&imr->num_pending_prefetch, 0); 539 540 return imr; 541 } 542 543 static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end, 544 void *cookie) 545 { 546 struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie; 547 struct ib_umem *umem = &umem_odp->umem; 548 549 if (mr->parent != imr) 550 return 0; 551 552 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem), 553 ib_umem_end(umem)); 554 555 if (umem_odp->dying) 556 return 0; 557 558 WRITE_ONCE(umem_odp->dying, 1); 559 atomic_inc(&imr->num_leaf_free); 560 schedule_work(&umem_odp->work); 561 562 return 0; 563 } 564 565 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 566 { 567 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr); 568 569 down_read(&per_mm->umem_rwsem); 570 rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, 0, ULLONG_MAX, 571 mr_leaf_free, true, imr); 572 up_read(&per_mm->umem_rwsem); 573 574 wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); 575 } 576 577 #define MLX5_PF_FLAGS_PREFETCH BIT(0) 578 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 579 static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 580 u64 io_virt, size_t bcnt, u32 *bytes_mapped, 581 u32 flags) 582 { 583 int npages = 0, current_seq, page_shift, ret, np; 584 bool implicit = false; 585 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 586 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 587 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 588 u64 access_mask = ODP_READ_ALLOWED_BIT; 589 u64 start_idx, page_mask; 590 struct ib_umem_odp *odp; 591 size_t size; 592 593 if (!odp_mr->page_list) { 594 odp = implicit_mr_get_data(mr, io_virt, bcnt); 595 596 if (IS_ERR(odp)) 597 return PTR_ERR(odp); 598 mr = odp->private; 599 implicit = true; 600 } else { 601 odp = odp_mr; 602 } 603 604 next_mr: 605 size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt); 606 607 page_shift = mr->umem->page_shift; 608 page_mask = ~(BIT(page_shift) - 1); 609 start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; 610 611 if (prefetch && !downgrade && !mr->umem->writable) { 612 /* prefetch with write-access must 613 * be supported by the MR 614 */ 615 ret = -EINVAL; 616 goto out; 617 } 618 619 if (mr->umem->writable && !downgrade) 620 access_mask |= ODP_WRITE_ALLOWED_BIT; 621 622 current_seq = READ_ONCE(odp->notifiers_seq); 623 /* 624 * Ensure the sequence number is valid for some time before we call 625 * gup. 626 */ 627 smp_rmb(); 628 629 ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr->umem), io_virt, size, 630 access_mask, current_seq); 631 632 if (ret < 0) 633 goto out; 634 635 np = ret; 636 637 mutex_lock(&odp->umem_mutex); 638 if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr->umem), 639 current_seq)) { 640 /* 641 * No need to check whether the MTTs really belong to 642 * this MR, since ib_umem_odp_map_dma_pages already 643 * checks this. 644 */ 645 ret = mlx5_ib_update_xlt(mr, start_idx, np, 646 page_shift, MLX5_IB_UPD_XLT_ATOMIC); 647 } else { 648 ret = -EAGAIN; 649 } 650 mutex_unlock(&odp->umem_mutex); 651 652 if (ret < 0) { 653 if (ret != -EAGAIN) 654 mlx5_ib_err(dev, "Failed to update mkey page tables\n"); 655 goto out; 656 } 657 658 if (bytes_mapped) { 659 u32 new_mappings = (np << page_shift) - 660 (io_virt - round_down(io_virt, 1 << page_shift)); 661 *bytes_mapped += min_t(u32, new_mappings, size); 662 } 663 664 npages += np << (page_shift - PAGE_SHIFT); 665 bcnt -= size; 666 667 if (unlikely(bcnt)) { 668 struct ib_umem_odp *next; 669 670 io_virt += size; 671 next = odp_next(odp); 672 if (unlikely(!next || next->umem.address != io_virt)) { 673 mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", 674 io_virt, next); 675 return -EAGAIN; 676 } 677 odp = next; 678 mr = odp->private; 679 goto next_mr; 680 } 681 682 return npages; 683 684 out: 685 if (ret == -EAGAIN) { 686 if (implicit || !odp->dying) { 687 unsigned long timeout = 688 msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT); 689 690 if (!wait_for_completion_timeout( 691 &odp->notifier_completion, 692 timeout)) { 693 mlx5_ib_warn(dev, "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", 694 current_seq, odp->notifiers_seq, odp->notifiers_count); 695 } 696 } else { 697 /* The MR is being killed, kill the QP as well. */ 698 ret = -EFAULT; 699 } 700 } 701 702 return ret; 703 } 704 705 struct pf_frame { 706 struct pf_frame *next; 707 u32 key; 708 u64 io_virt; 709 size_t bcnt; 710 int depth; 711 }; 712 713 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey) 714 { 715 struct mlx5_ib_mw *mw; 716 struct mlx5_ib_devx_mr *devx_mr; 717 718 if (mmkey->type == MLX5_MKEY_MW) { 719 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 720 return mw->ndescs; 721 } 722 723 devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr, 724 mmkey); 725 return devx_mr->ndescs; 726 } 727 728 /* 729 * Handle a single data segment in a page-fault WQE or RDMA region. 730 * 731 * Returns number of OS pages retrieved on success. The caller may continue to 732 * the next data segment. 733 * Can return the following error codes: 734 * -EAGAIN to designate a temporary error. The caller will abort handling the 735 * page fault and resolve it. 736 * -EFAULT when there's an error mapping the requested pages. The caller will 737 * abort the page fault handling. 738 */ 739 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 740 struct ib_pd *pd, u32 key, 741 u64 io_virt, size_t bcnt, 742 u32 *bytes_committed, 743 u32 *bytes_mapped, u32 flags) 744 { 745 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 746 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 747 struct pf_frame *head = NULL, *frame; 748 struct mlx5_core_mkey *mmkey; 749 struct mlx5_ib_mr *mr; 750 struct mlx5_klm *pklm; 751 u32 *out = NULL; 752 size_t offset; 753 int ndescs; 754 755 srcu_key = srcu_read_lock(&dev->mr_srcu); 756 757 io_virt += *bytes_committed; 758 bcnt -= *bytes_committed; 759 760 next_mr: 761 mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key)); 762 if (!mmkey || mmkey->key != key) { 763 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 764 ret = -EFAULT; 765 goto srcu_unlock; 766 } 767 768 if (prefetch && mmkey->type != MLX5_MKEY_MR) { 769 mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n"); 770 ret = -EINVAL; 771 goto srcu_unlock; 772 } 773 774 switch (mmkey->type) { 775 case MLX5_MKEY_MR: 776 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 777 if (!mr->live || !mr->ibmr.pd) { 778 mlx5_ib_dbg(dev, "got dead MR\n"); 779 ret = -EFAULT; 780 goto srcu_unlock; 781 } 782 783 if (prefetch) { 784 if (!is_odp_mr(mr) || 785 mr->ibmr.pd != pd) { 786 mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n", 787 is_odp_mr(mr) ? "MR is not ODP" : 788 "PD is not of the MR"); 789 ret = -EINVAL; 790 goto srcu_unlock; 791 } 792 } 793 794 if (!is_odp_mr(mr)) { 795 mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 796 key); 797 if (bytes_mapped) 798 *bytes_mapped += bcnt; 799 ret = 0; 800 goto srcu_unlock; 801 } 802 803 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags); 804 if (ret < 0) 805 goto srcu_unlock; 806 807 npages += ret; 808 ret = 0; 809 break; 810 811 case MLX5_MKEY_MW: 812 case MLX5_MKEY_INDIRECT_DEVX: 813 ndescs = get_indirect_num_descs(mmkey); 814 815 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 816 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 817 ret = -EFAULT; 818 goto srcu_unlock; 819 } 820 821 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 822 sizeof(*pklm) * (ndescs - 2); 823 824 if (outlen > cur_outlen) { 825 kfree(out); 826 out = kzalloc(outlen, GFP_KERNEL); 827 if (!out) { 828 ret = -ENOMEM; 829 goto srcu_unlock; 830 } 831 cur_outlen = outlen; 832 } 833 834 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 835 bsf0_klm0_pas_mtt0_1); 836 837 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); 838 if (ret) 839 goto srcu_unlock; 840 841 offset = io_virt - MLX5_GET64(query_mkey_out, out, 842 memory_key_mkey_entry.start_addr); 843 844 for (i = 0; bcnt && i < ndescs; i++, pklm++) { 845 if (offset >= be32_to_cpu(pklm->bcount)) { 846 offset -= be32_to_cpu(pklm->bcount); 847 continue; 848 } 849 850 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 851 if (!frame) { 852 ret = -ENOMEM; 853 goto srcu_unlock; 854 } 855 856 frame->key = be32_to_cpu(pklm->key); 857 frame->io_virt = be64_to_cpu(pklm->va) + offset; 858 frame->bcnt = min_t(size_t, bcnt, 859 be32_to_cpu(pklm->bcount) - offset); 860 frame->depth = depth + 1; 861 frame->next = head; 862 head = frame; 863 864 bcnt -= frame->bcnt; 865 offset = 0; 866 } 867 break; 868 869 default: 870 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 871 ret = -EFAULT; 872 goto srcu_unlock; 873 } 874 875 if (head) { 876 frame = head; 877 head = frame->next; 878 879 key = frame->key; 880 io_virt = frame->io_virt; 881 bcnt = frame->bcnt; 882 depth = frame->depth; 883 kfree(frame); 884 885 goto next_mr; 886 } 887 888 srcu_unlock: 889 while (head) { 890 frame = head; 891 head = frame->next; 892 kfree(frame); 893 } 894 kfree(out); 895 896 srcu_read_unlock(&dev->mr_srcu, srcu_key); 897 *bytes_committed = 0; 898 return ret ? ret : npages; 899 } 900 901 /** 902 * Parse a series of data segments for page fault handling. 903 * 904 * @pfault contains page fault information. 905 * @wqe points at the first data segment in the WQE. 906 * @wqe_end points after the end of the WQE. 907 * @bytes_mapped receives the number of bytes that the function was able to 908 * map. This allows the caller to decide intelligently whether 909 * enough memory was mapped to resolve the page fault 910 * successfully (e.g. enough for the next MTU, or the entire 911 * WQE). 912 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 913 * the committed bytes). 914 * 915 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 916 * negative error code. 917 */ 918 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 919 struct mlx5_pagefault *pfault, 920 void *wqe, 921 void *wqe_end, u32 *bytes_mapped, 922 u32 *total_wqe_bytes, int receive_queue) 923 { 924 int ret = 0, npages = 0; 925 u64 io_virt; 926 u32 key; 927 u32 byte_count; 928 size_t bcnt; 929 int inline_segment; 930 931 if (bytes_mapped) 932 *bytes_mapped = 0; 933 if (total_wqe_bytes) 934 *total_wqe_bytes = 0; 935 936 while (wqe < wqe_end) { 937 struct mlx5_wqe_data_seg *dseg = wqe; 938 939 io_virt = be64_to_cpu(dseg->addr); 940 key = be32_to_cpu(dseg->lkey); 941 byte_count = be32_to_cpu(dseg->byte_count); 942 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 943 bcnt = byte_count & ~MLX5_INLINE_SEG; 944 945 if (inline_segment) { 946 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 947 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 948 16); 949 } else { 950 wqe += sizeof(*dseg); 951 } 952 953 /* receive WQE end of sg list. */ 954 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 955 io_virt == 0) 956 break; 957 958 if (!inline_segment && total_wqe_bytes) { 959 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 960 pfault->bytes_committed); 961 } 962 963 /* A zero length data segment designates a length of 2GB. */ 964 if (bcnt == 0) 965 bcnt = 1U << 31; 966 967 if (inline_segment || bcnt <= pfault->bytes_committed) { 968 pfault->bytes_committed -= 969 min_t(size_t, bcnt, 970 pfault->bytes_committed); 971 continue; 972 } 973 974 ret = pagefault_single_data_segment(dev, NULL, key, 975 io_virt, bcnt, 976 &pfault->bytes_committed, 977 bytes_mapped, 0); 978 if (ret < 0) 979 break; 980 npages += ret; 981 } 982 983 return ret < 0 ? ret : npages; 984 } 985 986 static const u32 mlx5_ib_odp_opcode_cap[] = { 987 [MLX5_OPCODE_SEND] = IB_ODP_SUPPORT_SEND, 988 [MLX5_OPCODE_SEND_IMM] = IB_ODP_SUPPORT_SEND, 989 [MLX5_OPCODE_SEND_INVAL] = IB_ODP_SUPPORT_SEND, 990 [MLX5_OPCODE_RDMA_WRITE] = IB_ODP_SUPPORT_WRITE, 991 [MLX5_OPCODE_RDMA_WRITE_IMM] = IB_ODP_SUPPORT_WRITE, 992 [MLX5_OPCODE_RDMA_READ] = IB_ODP_SUPPORT_READ, 993 [MLX5_OPCODE_ATOMIC_CS] = IB_ODP_SUPPORT_ATOMIC, 994 [MLX5_OPCODE_ATOMIC_FA] = IB_ODP_SUPPORT_ATOMIC, 995 }; 996 997 /* 998 * Parse initiator WQE. Advances the wqe pointer to point at the 999 * scatter-gather list, and set wqe_end to the end of the WQE. 1000 */ 1001 static int mlx5_ib_mr_initiator_pfault_handler( 1002 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1003 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1004 { 1005 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1006 u16 wqe_index = pfault->wqe.wqe_index; 1007 u32 transport_caps; 1008 struct mlx5_base_av *av; 1009 unsigned ds, opcode; 1010 #if defined(DEBUG) 1011 u32 ctrl_wqe_index, ctrl_qpn; 1012 #endif 1013 u32 qpn = qp->trans_qp.base.mqp.qpn; 1014 1015 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1016 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1017 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1018 ds, wqe_length); 1019 return -EFAULT; 1020 } 1021 1022 if (ds == 0) { 1023 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1024 wqe_index, qpn); 1025 return -EFAULT; 1026 } 1027 1028 #if defined(DEBUG) 1029 ctrl_wqe_index = (be32_to_cpu(ctrl->opmod_idx_opcode) & 1030 MLX5_WQE_CTRL_WQE_INDEX_MASK) >> 1031 MLX5_WQE_CTRL_WQE_INDEX_SHIFT; 1032 if (wqe_index != ctrl_wqe_index) { 1033 mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n", 1034 wqe_index, qpn, 1035 ctrl_wqe_index); 1036 return -EFAULT; 1037 } 1038 1039 ctrl_qpn = (be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_QPN_MASK) >> 1040 MLX5_WQE_CTRL_QPN_SHIFT; 1041 if (qpn != ctrl_qpn) { 1042 mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n", 1043 wqe_index, qpn, 1044 ctrl_qpn); 1045 return -EFAULT; 1046 } 1047 #endif /* DEBUG */ 1048 1049 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1050 *wqe += sizeof(*ctrl); 1051 1052 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1053 MLX5_WQE_CTRL_OPCODE_MASK; 1054 1055 switch (qp->ibqp.qp_type) { 1056 case IB_QPT_XRC_INI: 1057 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1058 transport_caps = dev->odp_caps.per_transport_caps.xrc_odp_caps; 1059 break; 1060 case IB_QPT_RC: 1061 transport_caps = dev->odp_caps.per_transport_caps.rc_odp_caps; 1062 break; 1063 case IB_QPT_UD: 1064 transport_caps = dev->odp_caps.per_transport_caps.ud_odp_caps; 1065 break; 1066 default: 1067 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport 0x%x\n", 1068 qp->ibqp.qp_type); 1069 return -EFAULT; 1070 } 1071 1072 if (unlikely(opcode >= ARRAY_SIZE(mlx5_ib_odp_opcode_cap) || 1073 !(transport_caps & mlx5_ib_odp_opcode_cap[opcode]))) { 1074 mlx5_ib_err(dev, "ODP fault on QP of an unsupported opcode 0x%x\n", 1075 opcode); 1076 return -EFAULT; 1077 } 1078 1079 if (qp->ibqp.qp_type == IB_QPT_UD) { 1080 av = *wqe; 1081 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1082 *wqe += sizeof(struct mlx5_av); 1083 else 1084 *wqe += sizeof(struct mlx5_base_av); 1085 } 1086 1087 switch (opcode) { 1088 case MLX5_OPCODE_RDMA_WRITE: 1089 case MLX5_OPCODE_RDMA_WRITE_IMM: 1090 case MLX5_OPCODE_RDMA_READ: 1091 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1092 break; 1093 case MLX5_OPCODE_ATOMIC_CS: 1094 case MLX5_OPCODE_ATOMIC_FA: 1095 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1096 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1097 break; 1098 } 1099 1100 return 0; 1101 } 1102 1103 /* 1104 * Parse responder WQE and set wqe_end to the end of the WQE. 1105 */ 1106 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1107 struct mlx5_ib_srq *srq, 1108 void **wqe, void **wqe_end, 1109 int wqe_length) 1110 { 1111 int wqe_size = 1 << srq->msrq.wqe_shift; 1112 1113 if (wqe_size > wqe_length) { 1114 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1115 return -EFAULT; 1116 } 1117 1118 *wqe_end = *wqe + wqe_size; 1119 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1120 1121 return 0; 1122 } 1123 1124 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1125 struct mlx5_ib_qp *qp, 1126 void *wqe, void **wqe_end, 1127 int wqe_length) 1128 { 1129 struct mlx5_ib_wq *wq = &qp->rq; 1130 int wqe_size = 1 << wq->wqe_shift; 1131 1132 if (qp->wq_sig) { 1133 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1134 return -EFAULT; 1135 } 1136 1137 if (wqe_size > wqe_length) { 1138 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1139 return -EFAULT; 1140 } 1141 1142 switch (qp->ibqp.qp_type) { 1143 case IB_QPT_RC: 1144 if (!(dev->odp_caps.per_transport_caps.rc_odp_caps & 1145 IB_ODP_SUPPORT_RECV)) 1146 goto invalid_transport_or_opcode; 1147 break; 1148 default: 1149 invalid_transport_or_opcode: 1150 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport. transport: 0x%x\n", 1151 qp->ibqp.qp_type); 1152 return -EFAULT; 1153 } 1154 1155 *wqe_end = wqe + wqe_size; 1156 1157 return 0; 1158 } 1159 1160 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1161 u32 wq_num, int pf_type) 1162 { 1163 struct mlx5_core_rsc_common *common = NULL; 1164 struct mlx5_core_srq *srq; 1165 1166 switch (pf_type) { 1167 case MLX5_WQE_PF_TYPE_RMP: 1168 srq = mlx5_cmd_get_srq(dev, wq_num); 1169 if (srq) 1170 common = &srq->common; 1171 break; 1172 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1173 case MLX5_WQE_PF_TYPE_RESP: 1174 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1175 common = mlx5_core_res_hold(dev->mdev, wq_num, MLX5_RES_QP); 1176 break; 1177 default: 1178 break; 1179 } 1180 1181 return common; 1182 } 1183 1184 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1185 { 1186 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1187 1188 return to_mibqp(mqp); 1189 } 1190 1191 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1192 { 1193 struct mlx5_core_srq *msrq = 1194 container_of(res, struct mlx5_core_srq, common); 1195 1196 return to_mibsrq(msrq); 1197 } 1198 1199 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1200 struct mlx5_pagefault *pfault) 1201 { 1202 int ret; 1203 void *wqe, *wqe_end; 1204 u32 bytes_mapped, total_wqe_bytes; 1205 char *buffer = NULL; 1206 int resume_with_error = 1; 1207 u16 wqe_index = pfault->wqe.wqe_index; 1208 int requestor = pfault->type & MLX5_PFAULT_REQUESTOR; 1209 struct mlx5_core_rsc_common *res = NULL; 1210 struct mlx5_ib_qp *qp = NULL; 1211 struct mlx5_ib_srq *srq = NULL; 1212 size_t bytes_copied; 1213 1214 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1215 if (!res) { 1216 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1217 return; 1218 } 1219 1220 switch (res->res) { 1221 case MLX5_RES_QP: 1222 qp = res_to_qp(res); 1223 break; 1224 case MLX5_RES_SRQ: 1225 case MLX5_RES_XSRQ: 1226 srq = res_to_srq(res); 1227 break; 1228 default: 1229 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", pfault->type); 1230 goto resolve_page_fault; 1231 } 1232 1233 buffer = (char *)__get_free_page(GFP_KERNEL); 1234 if (!buffer) { 1235 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1236 goto resolve_page_fault; 1237 } 1238 1239 if (qp) { 1240 if (requestor) { 1241 ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, 1242 buffer, PAGE_SIZE, 1243 &bytes_copied); 1244 } else { 1245 ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, 1246 buffer, PAGE_SIZE, 1247 &bytes_copied); 1248 } 1249 } else { 1250 ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, 1251 buffer, PAGE_SIZE, 1252 &bytes_copied); 1253 } 1254 1255 if (ret) { 1256 mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n", 1257 ret, wqe_index, pfault->token); 1258 goto resolve_page_fault; 1259 } 1260 1261 wqe = buffer; 1262 if (requestor) 1263 ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp, 1264 &wqe, &wqe_end, 1265 bytes_copied); 1266 else if (qp) 1267 ret = mlx5_ib_mr_responder_pfault_handler_rq(dev, qp, 1268 wqe, &wqe_end, 1269 bytes_copied); 1270 else 1271 ret = mlx5_ib_mr_responder_pfault_handler_srq(dev, srq, 1272 &wqe, &wqe_end, 1273 bytes_copied); 1274 1275 if (ret < 0) 1276 goto resolve_page_fault; 1277 1278 if (wqe >= wqe_end) { 1279 mlx5_ib_err(dev, "ODP fault on invalid WQE.\n"); 1280 goto resolve_page_fault; 1281 } 1282 1283 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, 1284 &bytes_mapped, &total_wqe_bytes, 1285 !requestor); 1286 if (ret == -EAGAIN) { 1287 resume_with_error = 0; 1288 goto resolve_page_fault; 1289 } else if (ret < 0 || total_wqe_bytes > bytes_mapped) { 1290 goto resolve_page_fault; 1291 } 1292 1293 resume_with_error = 0; 1294 resolve_page_fault: 1295 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1296 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1297 pfault->wqe.wq_num, resume_with_error, 1298 pfault->type); 1299 mlx5_core_res_put(res); 1300 free_page((unsigned long)buffer); 1301 } 1302 1303 static int pages_in_range(u64 address, u32 length) 1304 { 1305 return (ALIGN(address + length, PAGE_SIZE) - 1306 (address & PAGE_MASK)) >> PAGE_SHIFT; 1307 } 1308 1309 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1310 struct mlx5_pagefault *pfault) 1311 { 1312 u64 address; 1313 u32 length; 1314 u32 prefetch_len = pfault->bytes_committed; 1315 int prefetch_activated = 0; 1316 u32 rkey = pfault->rdma.r_key; 1317 int ret; 1318 1319 /* The RDMA responder handler handles the page fault in two parts. 1320 * First it brings the necessary pages for the current packet 1321 * (and uses the pfault context), and then (after resuming the QP) 1322 * prefetches more pages. The second operation cannot use the pfault 1323 * context and therefore uses the dummy_pfault context allocated on 1324 * the stack */ 1325 pfault->rdma.rdma_va += pfault->bytes_committed; 1326 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1327 pfault->rdma.rdma_op_len); 1328 pfault->bytes_committed = 0; 1329 1330 address = pfault->rdma.rdma_va; 1331 length = pfault->rdma.rdma_op_len; 1332 1333 /* For some operations, the hardware cannot tell the exact message 1334 * length, and in those cases it reports zero. Use prefetch 1335 * logic. */ 1336 if (length == 0) { 1337 prefetch_activated = 1; 1338 length = pfault->rdma.packet_size; 1339 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1340 } 1341 1342 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1343 &pfault->bytes_committed, NULL, 1344 0); 1345 if (ret == -EAGAIN) { 1346 /* We're racing with an invalidation, don't prefetch */ 1347 prefetch_activated = 0; 1348 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1349 mlx5_ib_page_fault_resume(dev, pfault, 1); 1350 if (ret != -ENOENT) 1351 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1352 ret, pfault->token, pfault->type); 1353 return; 1354 } 1355 1356 mlx5_ib_page_fault_resume(dev, pfault, 0); 1357 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1358 pfault->token, pfault->type, 1359 prefetch_activated); 1360 1361 /* At this point, there might be a new pagefault already arriving in 1362 * the eq, switch to the dummy pagefault for the rest of the 1363 * processing. We're still OK with the objects being alive as the 1364 * work-queue is being fenced. */ 1365 1366 if (prefetch_activated) { 1367 u32 bytes_committed = 0; 1368 1369 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1370 prefetch_len, 1371 &bytes_committed, NULL, 1372 0); 1373 if (ret < 0 && ret != -EAGAIN) { 1374 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1375 ret, pfault->token, address, prefetch_len); 1376 } 1377 } 1378 } 1379 1380 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1381 { 1382 u8 event_subtype = pfault->event_subtype; 1383 1384 switch (event_subtype) { 1385 case MLX5_PFAULT_SUBTYPE_WQE: 1386 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1387 break; 1388 case MLX5_PFAULT_SUBTYPE_RDMA: 1389 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1390 break; 1391 default: 1392 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1393 event_subtype); 1394 mlx5_ib_page_fault_resume(dev, pfault, 1); 1395 } 1396 } 1397 1398 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1399 { 1400 struct mlx5_pagefault *pfault = container_of(work, 1401 struct mlx5_pagefault, 1402 work); 1403 struct mlx5_ib_pf_eq *eq = pfault->eq; 1404 1405 mlx5_ib_pfault(eq->dev, pfault); 1406 mempool_free(pfault, eq->pool); 1407 } 1408 1409 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1410 { 1411 struct mlx5_eqe_page_fault *pf_eqe; 1412 struct mlx5_pagefault *pfault; 1413 struct mlx5_eqe *eqe; 1414 int cc = 0; 1415 1416 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1417 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1418 if (!pfault) { 1419 schedule_work(&eq->work); 1420 break; 1421 } 1422 1423 pf_eqe = &eqe->data.page_fault; 1424 pfault->event_subtype = eqe->sub_type; 1425 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); 1426 1427 mlx5_ib_dbg(eq->dev, 1428 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", 1429 eqe->sub_type, pfault->bytes_committed); 1430 1431 switch (eqe->sub_type) { 1432 case MLX5_PFAULT_SUBTYPE_RDMA: 1433 /* RDMA based event */ 1434 pfault->type = 1435 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1436 pfault->token = 1437 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1438 MLX5_24BIT_MASK; 1439 pfault->rdma.r_key = 1440 be32_to_cpu(pf_eqe->rdma.r_key); 1441 pfault->rdma.packet_size = 1442 be16_to_cpu(pf_eqe->rdma.packet_length); 1443 pfault->rdma.rdma_op_len = 1444 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1445 pfault->rdma.rdma_va = 1446 be64_to_cpu(pf_eqe->rdma.rdma_va); 1447 mlx5_ib_dbg(eq->dev, 1448 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", 1449 pfault->type, pfault->token, 1450 pfault->rdma.r_key); 1451 mlx5_ib_dbg(eq->dev, 1452 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1453 pfault->rdma.rdma_op_len, 1454 pfault->rdma.rdma_va); 1455 break; 1456 1457 case MLX5_PFAULT_SUBTYPE_WQE: 1458 /* WQE based event */ 1459 pfault->type = 1460 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1461 pfault->token = 1462 be32_to_cpu(pf_eqe->wqe.token); 1463 pfault->wqe.wq_num = 1464 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1465 MLX5_24BIT_MASK; 1466 pfault->wqe.wqe_index = 1467 be16_to_cpu(pf_eqe->wqe.wqe_index); 1468 pfault->wqe.packet_size = 1469 be16_to_cpu(pf_eqe->wqe.packet_length); 1470 mlx5_ib_dbg(eq->dev, 1471 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1472 pfault->type, pfault->token, 1473 pfault->wqe.wq_num, 1474 pfault->wqe.wqe_index); 1475 break; 1476 1477 default: 1478 mlx5_ib_warn(eq->dev, 1479 "Unsupported page fault event sub-type: 0x%02hhx\n", 1480 eqe->sub_type); 1481 /* Unsupported page faults should still be 1482 * resolved by the page fault handler 1483 */ 1484 } 1485 1486 pfault->eq = eq; 1487 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1488 queue_work(eq->wq, &pfault->work); 1489 1490 cc = mlx5_eq_update_cc(eq->core, ++cc); 1491 } 1492 1493 mlx5_eq_update_ci(eq->core, cc, 1); 1494 } 1495 1496 static irqreturn_t mlx5_ib_eq_pf_int(int irq, void *eq_ptr) 1497 { 1498 struct mlx5_ib_pf_eq *eq = eq_ptr; 1499 unsigned long flags; 1500 1501 if (spin_trylock_irqsave(&eq->lock, flags)) { 1502 mlx5_ib_eq_pf_process(eq); 1503 spin_unlock_irqrestore(&eq->lock, flags); 1504 } else { 1505 schedule_work(&eq->work); 1506 } 1507 1508 return IRQ_HANDLED; 1509 } 1510 1511 /* mempool_refill() was proposed but unfortunately wasn't accepted 1512 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1513 * Cheap workaround. 1514 */ 1515 static void mempool_refill(mempool_t *pool) 1516 { 1517 while (pool->curr_nr < pool->min_nr) 1518 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1519 } 1520 1521 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1522 { 1523 struct mlx5_ib_pf_eq *eq = 1524 container_of(work, struct mlx5_ib_pf_eq, work); 1525 1526 mempool_refill(eq->pool); 1527 1528 spin_lock_irq(&eq->lock); 1529 mlx5_ib_eq_pf_process(eq); 1530 spin_unlock_irq(&eq->lock); 1531 } 1532 1533 enum { 1534 MLX5_IB_NUM_PF_EQE = 0x1000, 1535 MLX5_IB_NUM_PF_DRAIN = 64, 1536 }; 1537 1538 static int 1539 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1540 { 1541 struct mlx5_eq_param param = {}; 1542 int err; 1543 1544 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1545 spin_lock_init(&eq->lock); 1546 eq->dev = dev; 1547 1548 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1549 sizeof(struct mlx5_pagefault)); 1550 if (!eq->pool) 1551 return -ENOMEM; 1552 1553 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1554 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1555 MLX5_NUM_CMD_EQE); 1556 if (!eq->wq) { 1557 err = -ENOMEM; 1558 goto err_mempool; 1559 } 1560 1561 param = (struct mlx5_eq_param) { 1562 .index = MLX5_EQ_PFAULT_IDX, 1563 .mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT, 1564 .nent = MLX5_IB_NUM_PF_EQE, 1565 .context = eq, 1566 .handler = mlx5_ib_eq_pf_int 1567 }; 1568 eq->core = mlx5_eq_create_generic(dev->mdev, "mlx5_ib_page_fault_eq", ¶m); 1569 if (IS_ERR(eq->core)) { 1570 err = PTR_ERR(eq->core); 1571 goto err_wq; 1572 } 1573 1574 return 0; 1575 err_wq: 1576 destroy_workqueue(eq->wq); 1577 err_mempool: 1578 mempool_destroy(eq->pool); 1579 return err; 1580 } 1581 1582 static int 1583 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1584 { 1585 int err; 1586 1587 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1588 cancel_work_sync(&eq->work); 1589 destroy_workqueue(eq->wq); 1590 mempool_destroy(eq->pool); 1591 1592 return err; 1593 } 1594 1595 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1596 { 1597 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1598 return; 1599 1600 switch (ent->order - 2) { 1601 case MLX5_IMR_MTT_CACHE_ENTRY: 1602 ent->page = PAGE_SHIFT; 1603 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1604 sizeof(struct mlx5_mtt) / 1605 MLX5_IB_UMR_OCTOWORD; 1606 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1607 ent->limit = 0; 1608 break; 1609 1610 case MLX5_IMR_KSM_CACHE_ENTRY: 1611 ent->page = MLX5_KSM_PAGE_SHIFT; 1612 ent->xlt = mlx5_imr_ksm_entries * 1613 sizeof(struct mlx5_klm) / 1614 MLX5_IB_UMR_OCTOWORD; 1615 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1616 ent->limit = 0; 1617 break; 1618 } 1619 } 1620 1621 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1622 .advise_mr = mlx5_ib_advise_mr, 1623 }; 1624 1625 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1626 { 1627 int ret = 0; 1628 1629 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT) 1630 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1631 1632 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1633 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1634 if (ret) { 1635 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1636 return ret; 1637 } 1638 } 1639 1640 if (!MLX5_CAP_GEN(dev->mdev, pg)) 1641 return ret; 1642 1643 ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1644 1645 return ret; 1646 } 1647 1648 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1649 { 1650 if (!MLX5_CAP_GEN(dev->mdev, pg)) 1651 return; 1652 1653 mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1654 } 1655 1656 int mlx5_ib_odp_init(void) 1657 { 1658 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1659 MLX5_IMR_MTT_BITS); 1660 1661 return 0; 1662 } 1663 1664 struct prefetch_mr_work { 1665 struct work_struct work; 1666 struct ib_pd *pd; 1667 u32 pf_flags; 1668 u32 num_sge; 1669 struct ib_sge sg_list[0]; 1670 }; 1671 1672 static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, 1673 struct ib_sge *sg_list, u32 num_sge, 1674 u32 from) 1675 { 1676 u32 i; 1677 int srcu_key; 1678 1679 srcu_key = srcu_read_lock(&dev->mr_srcu); 1680 1681 for (i = from; i < num_sge; ++i) { 1682 struct mlx5_core_mkey *mmkey; 1683 struct mlx5_ib_mr *mr; 1684 1685 mmkey = __mlx5_mr_lookup(dev->mdev, 1686 mlx5_base_mkey(sg_list[i].lkey)); 1687 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1688 atomic_dec(&mr->num_pending_prefetch); 1689 } 1690 1691 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1692 } 1693 1694 static bool num_pending_prefetch_inc(struct ib_pd *pd, 1695 struct ib_sge *sg_list, u32 num_sge) 1696 { 1697 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1698 bool ret = true; 1699 u32 i; 1700 1701 for (i = 0; i < num_sge; ++i) { 1702 struct mlx5_core_mkey *mmkey; 1703 struct mlx5_ib_mr *mr; 1704 1705 mmkey = __mlx5_mr_lookup(dev->mdev, 1706 mlx5_base_mkey(sg_list[i].lkey)); 1707 if (!mmkey || mmkey->key != sg_list[i].lkey) { 1708 ret = false; 1709 break; 1710 } 1711 1712 if (mmkey->type != MLX5_MKEY_MR) { 1713 ret = false; 1714 break; 1715 } 1716 1717 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1718 1719 if (mr->ibmr.pd != pd) { 1720 ret = false; 1721 break; 1722 } 1723 1724 if (!mr->live) { 1725 ret = false; 1726 break; 1727 } 1728 1729 atomic_inc(&mr->num_pending_prefetch); 1730 } 1731 1732 if (!ret) 1733 num_pending_prefetch_dec(dev, sg_list, i, 0); 1734 1735 return ret; 1736 } 1737 1738 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags, 1739 struct ib_sge *sg_list, u32 num_sge) 1740 { 1741 u32 i; 1742 int ret = 0; 1743 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1744 1745 for (i = 0; i < num_sge; ++i) { 1746 struct ib_sge *sg = &sg_list[i]; 1747 int bytes_committed = 0; 1748 1749 ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr, 1750 sg->length, 1751 &bytes_committed, NULL, 1752 pf_flags); 1753 if (ret < 0) 1754 break; 1755 } 1756 1757 return ret < 0 ? ret : 0; 1758 } 1759 1760 static void mlx5_ib_prefetch_mr_work(struct work_struct *work) 1761 { 1762 struct prefetch_mr_work *w = 1763 container_of(work, struct prefetch_mr_work, work); 1764 1765 if (ib_device_try_get(w->pd->device)) { 1766 mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list, 1767 w->num_sge); 1768 ib_device_put(w->pd->device); 1769 } 1770 1771 num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list, 1772 w->num_sge, 0); 1773 kfree(w); 1774 } 1775 1776 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1777 enum ib_uverbs_advise_mr_advice advice, 1778 u32 flags, struct ib_sge *sg_list, u32 num_sge) 1779 { 1780 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1781 u32 pf_flags = MLX5_PF_FLAGS_PREFETCH; 1782 struct prefetch_mr_work *work; 1783 bool valid_req; 1784 int srcu_key; 1785 1786 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 1787 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 1788 1789 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 1790 return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list, 1791 num_sge); 1792 1793 work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL); 1794 if (!work) 1795 return -ENOMEM; 1796 1797 memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge)); 1798 1799 /* It is guaranteed that the pd when work is executed is the pd when 1800 * work was queued since pd can't be destroyed while it holds MRs and 1801 * destroying a MR leads to flushing the workquque 1802 */ 1803 work->pd = pd; 1804 work->pf_flags = pf_flags; 1805 work->num_sge = num_sge; 1806 1807 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 1808 1809 srcu_key = srcu_read_lock(&dev->mr_srcu); 1810 1811 valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge); 1812 if (valid_req) 1813 queue_work(system_unbound_wq, &work->work); 1814 else 1815 kfree(work); 1816 1817 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1818 1819 return valid_req ? 0 : -EINVAL; 1820 } 1821