1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem.h> 34 #include <rdma/ib_umem_odp.h> 35 #include <linux/kernel.h> 36 37 #include "mlx5_ib.h" 38 #include "cmd.h" 39 40 #include <linux/mlx5/eq.h> 41 42 /* Contains the details of a pagefault. */ 43 struct mlx5_pagefault { 44 u32 bytes_committed; 45 u32 token; 46 u8 event_subtype; 47 u8 type; 48 union { 49 /* Initiator or send message responder pagefault details. */ 50 struct { 51 /* Received packet size, only valid for responders. */ 52 u32 packet_size; 53 /* 54 * Number of resource holding WQE, depends on type. 55 */ 56 u32 wq_num; 57 /* 58 * WQE index. Refers to either the send queue or 59 * receive queue, according to event_subtype. 60 */ 61 u16 wqe_index; 62 } wqe; 63 /* RDMA responder pagefault details */ 64 struct { 65 u32 r_key; 66 /* 67 * Received packet size, minimal size page fault 68 * resolution required for forward progress. 69 */ 70 u32 packet_size; 71 u32 rdma_op_len; 72 u64 rdma_va; 73 } rdma; 74 }; 75 76 struct mlx5_ib_pf_eq *eq; 77 struct work_struct work; 78 }; 79 80 #define MAX_PREFETCH_LEN (4*1024*1024U) 81 82 /* Timeout in ms to wait for an active mmu notifier to complete when handling 83 * a pagefault. */ 84 #define MMU_NOTIFIER_TIMEOUT 1000 85 86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 91 92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 93 94 static u64 mlx5_imr_ksm_entries; 95 96 static int check_parent(struct ib_umem_odp *odp, 97 struct mlx5_ib_mr *parent) 98 { 99 struct mlx5_ib_mr *mr = odp->private; 100 101 return mr && mr->parent == parent && !odp->dying; 102 } 103 104 static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr) 105 { 106 if (WARN_ON(!mr || !is_odp_mr(mr))) 107 return NULL; 108 109 return to_ib_umem_odp(mr->umem)->per_mm; 110 } 111 112 static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp) 113 { 114 struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent; 115 struct ib_ucontext_per_mm *per_mm = odp->per_mm; 116 struct rb_node *rb; 117 118 down_read(&per_mm->umem_rwsem); 119 while (1) { 120 rb = rb_next(&odp->interval_tree.rb); 121 if (!rb) 122 goto not_found; 123 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 124 if (check_parent(odp, parent)) 125 goto end; 126 } 127 not_found: 128 odp = NULL; 129 end: 130 up_read(&per_mm->umem_rwsem); 131 return odp; 132 } 133 134 static struct ib_umem_odp *odp_lookup(u64 start, u64 length, 135 struct mlx5_ib_mr *parent) 136 { 137 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent); 138 struct ib_umem_odp *odp; 139 struct rb_node *rb; 140 141 down_read(&per_mm->umem_rwsem); 142 odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length); 143 if (!odp) 144 goto end; 145 146 while (1) { 147 if (check_parent(odp, parent)) 148 goto end; 149 rb = rb_next(&odp->interval_tree.rb); 150 if (!rb) 151 goto not_found; 152 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 153 if (ib_umem_start(&odp->umem) > start + length) 154 goto not_found; 155 } 156 not_found: 157 odp = NULL; 158 end: 159 up_read(&per_mm->umem_rwsem); 160 return odp; 161 } 162 163 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, 164 size_t nentries, struct mlx5_ib_mr *mr, int flags) 165 { 166 struct ib_pd *pd = mr->ibmr.pd; 167 struct mlx5_ib_dev *dev = to_mdev(pd->device); 168 struct ib_umem_odp *odp; 169 unsigned long va; 170 int i; 171 172 if (flags & MLX5_IB_UPD_XLT_ZAP) { 173 for (i = 0; i < nentries; i++, pklm++) { 174 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 175 pklm->key = cpu_to_be32(dev->null_mkey); 176 pklm->va = 0; 177 } 178 return; 179 } 180 181 odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE, 182 nentries * MLX5_IMR_MTT_SIZE, mr); 183 184 for (i = 0; i < nentries; i++, pklm++) { 185 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 186 va = (offset + i) * MLX5_IMR_MTT_SIZE; 187 if (odp && odp->umem.address == va) { 188 struct mlx5_ib_mr *mtt = odp->private; 189 190 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 191 odp = odp_next(odp); 192 } else { 193 pklm->key = cpu_to_be32(dev->null_mkey); 194 } 195 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n", 196 i, va, be32_to_cpu(pklm->key)); 197 } 198 } 199 200 static void mr_leaf_free_action(struct work_struct *work) 201 { 202 struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); 203 int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT; 204 struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; 205 206 mr->parent = NULL; 207 synchronize_srcu(&mr->dev->mr_srcu); 208 209 ib_umem_release(&odp->umem); 210 if (imr->live) 211 mlx5_ib_update_xlt(imr, idx, 1, 0, 212 MLX5_IB_UPD_XLT_INDIRECT | 213 MLX5_IB_UPD_XLT_ATOMIC); 214 mlx5_mr_cache_free(mr->dev, mr); 215 216 if (atomic_dec_and_test(&imr->num_leaf_free)) 217 wake_up(&imr->q_leaf_free); 218 } 219 220 void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, 221 unsigned long end) 222 { 223 struct mlx5_ib_mr *mr; 224 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 225 sizeof(struct mlx5_mtt)) - 1; 226 u64 idx = 0, blk_start_idx = 0; 227 struct ib_umem *umem; 228 int in_block = 0; 229 u64 addr; 230 231 if (!umem_odp) { 232 pr_err("invalidation called on NULL umem or non-ODP umem\n"); 233 return; 234 } 235 umem = &umem_odp->umem; 236 237 mr = umem_odp->private; 238 239 if (!mr || !mr->ibmr.pd) 240 return; 241 242 start = max_t(u64, ib_umem_start(umem), start); 243 end = min_t(u64, ib_umem_end(umem), end); 244 245 /* 246 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 247 * while we are doing the invalidation, no page fault will attempt to 248 * overwrite the same MTTs. Concurent invalidations might race us, 249 * but they will write 0s as well, so no difference in the end result. 250 */ 251 252 for (addr = start; addr < end; addr += BIT(umem->page_shift)) { 253 idx = (addr - ib_umem_start(umem)) >> umem->page_shift; 254 /* 255 * Strive to write the MTTs in chunks, but avoid overwriting 256 * non-existing MTTs. The huristic here can be improved to 257 * estimate the cost of another UMR vs. the cost of bigger 258 * UMR. 259 */ 260 if (umem_odp->dma_list[idx] & 261 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 262 if (!in_block) { 263 blk_start_idx = idx; 264 in_block = 1; 265 } 266 } else { 267 u64 umr_offset = idx & umr_block_mask; 268 269 if (in_block && umr_offset == 0) { 270 mlx5_ib_update_xlt(mr, blk_start_idx, 271 idx - blk_start_idx, 0, 272 MLX5_IB_UPD_XLT_ZAP | 273 MLX5_IB_UPD_XLT_ATOMIC); 274 in_block = 0; 275 } 276 } 277 } 278 if (in_block) 279 mlx5_ib_update_xlt(mr, blk_start_idx, 280 idx - blk_start_idx + 1, 0, 281 MLX5_IB_UPD_XLT_ZAP | 282 MLX5_IB_UPD_XLT_ATOMIC); 283 /* 284 * We are now sure that the device will not access the 285 * memory. We can safely unmap it, and mark it as dirty if 286 * needed. 287 */ 288 289 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 290 291 if (unlikely(!umem->npages && mr->parent && 292 !umem_odp->dying)) { 293 WRITE_ONCE(umem_odp->dying, 1); 294 atomic_inc(&mr->parent->num_leaf_free); 295 schedule_work(&umem_odp->work); 296 } 297 } 298 299 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 300 { 301 struct ib_odp_caps *caps = &dev->odp_caps; 302 303 memset(caps, 0, sizeof(*caps)); 304 305 if (!MLX5_CAP_GEN(dev->mdev, pg)) 306 return; 307 308 caps->general_caps = IB_ODP_SUPPORT; 309 310 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 311 dev->odp_max_size = U64_MAX; 312 else 313 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 314 315 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 316 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 317 318 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive)) 319 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 320 321 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 322 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 323 324 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 325 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 326 327 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 328 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 329 330 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 331 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 332 333 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 334 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 335 336 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive)) 337 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 338 339 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send)) 340 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 341 342 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive)) 343 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 344 345 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write)) 346 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 347 348 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read)) 349 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 350 351 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic)) 352 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 353 354 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive)) 355 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 356 357 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 358 MLX5_CAP_GEN(dev->mdev, null_mkey) && 359 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 360 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 361 362 return; 363 } 364 365 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 366 struct mlx5_pagefault *pfault, 367 int error) 368 { 369 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 370 pfault->wqe.wq_num : pfault->token; 371 u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { }; 372 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = { }; 373 int err; 374 375 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 376 MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type); 377 MLX5_SET(page_fault_resume_in, in, token, pfault->token); 378 MLX5_SET(page_fault_resume_in, in, wq_number, wq_num); 379 MLX5_SET(page_fault_resume_in, in, error, !!error); 380 381 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 382 if (err) 383 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 384 wq_num, err); 385 } 386 387 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd, 388 struct ib_umem *umem, 389 bool ksm, int access_flags) 390 { 391 struct mlx5_ib_dev *dev = to_mdev(pd->device); 392 struct mlx5_ib_mr *mr; 393 int err; 394 395 mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY : 396 MLX5_IMR_MTT_CACHE_ENTRY); 397 398 if (IS_ERR(mr)) 399 return mr; 400 401 mr->ibmr.pd = pd; 402 403 mr->dev = dev; 404 mr->access_flags = access_flags; 405 mr->mmkey.iova = 0; 406 mr->umem = umem; 407 408 if (ksm) { 409 err = mlx5_ib_update_xlt(mr, 0, 410 mlx5_imr_ksm_entries, 411 MLX5_KSM_PAGE_SHIFT, 412 MLX5_IB_UPD_XLT_INDIRECT | 413 MLX5_IB_UPD_XLT_ZAP | 414 MLX5_IB_UPD_XLT_ENABLE); 415 416 } else { 417 err = mlx5_ib_update_xlt(mr, 0, 418 MLX5_IMR_MTT_ENTRIES, 419 PAGE_SHIFT, 420 MLX5_IB_UPD_XLT_ZAP | 421 MLX5_IB_UPD_XLT_ENABLE | 422 MLX5_IB_UPD_XLT_ATOMIC); 423 } 424 425 if (err) 426 goto fail; 427 428 mr->ibmr.lkey = mr->mmkey.key; 429 mr->ibmr.rkey = mr->mmkey.key; 430 431 mr->live = 1; 432 433 mlx5_ib_dbg(dev, "key %x dev %p mr %p\n", 434 mr->mmkey.key, dev->mdev, mr); 435 436 return mr; 437 438 fail: 439 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 440 mlx5_mr_cache_free(dev, mr); 441 442 return ERR_PTR(err); 443 } 444 445 static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr, 446 u64 io_virt, size_t bcnt) 447 { 448 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device); 449 struct ib_umem_odp *odp, *result = NULL; 450 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 451 u64 addr = io_virt & MLX5_IMR_MTT_MASK; 452 int nentries = 0, start_idx = 0, ret; 453 struct mlx5_ib_mr *mtt; 454 455 mutex_lock(&odp_mr->umem_mutex); 456 odp = odp_lookup(addr, 1, mr); 457 458 mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n", 459 io_virt, bcnt, addr, odp); 460 461 next_mr: 462 if (likely(odp)) { 463 if (nentries) 464 nentries++; 465 } else { 466 odp = ib_alloc_odp_umem(odp_mr, addr, 467 MLX5_IMR_MTT_SIZE); 468 if (IS_ERR(odp)) { 469 mutex_unlock(&odp_mr->umem_mutex); 470 return ERR_CAST(odp); 471 } 472 473 mtt = implicit_mr_alloc(mr->ibmr.pd, &odp->umem, 0, 474 mr->access_flags); 475 if (IS_ERR(mtt)) { 476 mutex_unlock(&odp_mr->umem_mutex); 477 ib_umem_release(&odp->umem); 478 return ERR_CAST(mtt); 479 } 480 481 odp->private = mtt; 482 mtt->umem = &odp->umem; 483 mtt->mmkey.iova = addr; 484 mtt->parent = mr; 485 INIT_WORK(&odp->work, mr_leaf_free_action); 486 487 if (!nentries) 488 start_idx = addr >> MLX5_IMR_MTT_SHIFT; 489 nentries++; 490 } 491 492 /* Return first odp if region not covered by single one */ 493 if (likely(!result)) 494 result = odp; 495 496 addr += MLX5_IMR_MTT_SIZE; 497 if (unlikely(addr < io_virt + bcnt)) { 498 odp = odp_next(odp); 499 if (odp && odp->umem.address != addr) 500 odp = NULL; 501 goto next_mr; 502 } 503 504 if (unlikely(nentries)) { 505 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0, 506 MLX5_IB_UPD_XLT_INDIRECT | 507 MLX5_IB_UPD_XLT_ATOMIC); 508 if (ret) { 509 mlx5_ib_err(dev, "Failed to update PAS\n"); 510 result = ERR_PTR(ret); 511 } 512 } 513 514 mutex_unlock(&odp_mr->umem_mutex); 515 return result; 516 } 517 518 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 519 struct ib_udata *udata, 520 int access_flags) 521 { 522 struct mlx5_ib_mr *imr; 523 struct ib_umem *umem; 524 525 umem = ib_umem_get(udata, 0, 0, access_flags, 0); 526 if (IS_ERR(umem)) 527 return ERR_CAST(umem); 528 529 imr = implicit_mr_alloc(&pd->ibpd, umem, 1, access_flags); 530 if (IS_ERR(imr)) { 531 ib_umem_release(umem); 532 return ERR_CAST(imr); 533 } 534 535 imr->umem = umem; 536 init_waitqueue_head(&imr->q_leaf_free); 537 atomic_set(&imr->num_leaf_free, 0); 538 atomic_set(&imr->num_pending_prefetch, 0); 539 540 return imr; 541 } 542 543 static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end, 544 void *cookie) 545 { 546 struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie; 547 struct ib_umem *umem = &umem_odp->umem; 548 549 if (mr->parent != imr) 550 return 0; 551 552 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem), 553 ib_umem_end(umem)); 554 555 if (umem_odp->dying) 556 return 0; 557 558 WRITE_ONCE(umem_odp->dying, 1); 559 atomic_inc(&imr->num_leaf_free); 560 schedule_work(&umem_odp->work); 561 562 return 0; 563 } 564 565 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 566 { 567 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr); 568 569 down_read(&per_mm->umem_rwsem); 570 rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, 0, ULLONG_MAX, 571 mr_leaf_free, true, imr); 572 up_read(&per_mm->umem_rwsem); 573 574 wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); 575 } 576 577 #define MLX5_PF_FLAGS_PREFETCH BIT(0) 578 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 579 static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 580 u64 io_virt, size_t bcnt, u32 *bytes_mapped, 581 u32 flags) 582 { 583 int npages = 0, current_seq, page_shift, ret, np; 584 bool implicit = false; 585 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 586 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 587 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 588 u64 access_mask; 589 u64 start_idx, page_mask; 590 struct ib_umem_odp *odp; 591 size_t size; 592 593 if (!odp_mr->page_list) { 594 odp = implicit_mr_get_data(mr, io_virt, bcnt); 595 596 if (IS_ERR(odp)) 597 return PTR_ERR(odp); 598 mr = odp->private; 599 implicit = true; 600 } else { 601 odp = odp_mr; 602 } 603 604 next_mr: 605 size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt); 606 607 page_shift = mr->umem->page_shift; 608 page_mask = ~(BIT(page_shift) - 1); 609 start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; 610 access_mask = ODP_READ_ALLOWED_BIT; 611 612 if (prefetch && !downgrade && !mr->umem->writable) { 613 /* prefetch with write-access must 614 * be supported by the MR 615 */ 616 ret = -EINVAL; 617 goto out; 618 } 619 620 if (mr->umem->writable && !downgrade) 621 access_mask |= ODP_WRITE_ALLOWED_BIT; 622 623 current_seq = READ_ONCE(odp->notifiers_seq); 624 /* 625 * Ensure the sequence number is valid for some time before we call 626 * gup. 627 */ 628 smp_rmb(); 629 630 ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr->umem), io_virt, size, 631 access_mask, current_seq); 632 633 if (ret < 0) 634 goto out; 635 636 np = ret; 637 638 mutex_lock(&odp->umem_mutex); 639 if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr->umem), 640 current_seq)) { 641 /* 642 * No need to check whether the MTTs really belong to 643 * this MR, since ib_umem_odp_map_dma_pages already 644 * checks this. 645 */ 646 ret = mlx5_ib_update_xlt(mr, start_idx, np, 647 page_shift, MLX5_IB_UPD_XLT_ATOMIC); 648 } else { 649 ret = -EAGAIN; 650 } 651 mutex_unlock(&odp->umem_mutex); 652 653 if (ret < 0) { 654 if (ret != -EAGAIN) 655 mlx5_ib_err(dev, "Failed to update mkey page tables\n"); 656 goto out; 657 } 658 659 if (bytes_mapped) { 660 u32 new_mappings = (np << page_shift) - 661 (io_virt - round_down(io_virt, 1 << page_shift)); 662 *bytes_mapped += min_t(u32, new_mappings, size); 663 } 664 665 npages += np << (page_shift - PAGE_SHIFT); 666 bcnt -= size; 667 668 if (unlikely(bcnt)) { 669 struct ib_umem_odp *next; 670 671 io_virt += size; 672 next = odp_next(odp); 673 if (unlikely(!next || next->umem.address != io_virt)) { 674 mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", 675 io_virt, next); 676 return -EAGAIN; 677 } 678 odp = next; 679 mr = odp->private; 680 goto next_mr; 681 } 682 683 return npages; 684 685 out: 686 if (ret == -EAGAIN) { 687 if (implicit || !odp->dying) { 688 unsigned long timeout = 689 msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT); 690 691 if (!wait_for_completion_timeout( 692 &odp->notifier_completion, 693 timeout)) { 694 mlx5_ib_warn(dev, "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", 695 current_seq, odp->notifiers_seq, odp->notifiers_count); 696 } 697 } else { 698 /* The MR is being killed, kill the QP as well. */ 699 ret = -EFAULT; 700 } 701 } 702 703 return ret; 704 } 705 706 struct pf_frame { 707 struct pf_frame *next; 708 u32 key; 709 u64 io_virt; 710 size_t bcnt; 711 int depth; 712 }; 713 714 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey) 715 { 716 struct mlx5_ib_mw *mw; 717 struct mlx5_ib_devx_mr *devx_mr; 718 719 if (mmkey->type == MLX5_MKEY_MW) { 720 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 721 return mw->ndescs; 722 } 723 724 devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr, 725 mmkey); 726 return devx_mr->ndescs; 727 } 728 729 /* 730 * Handle a single data segment in a page-fault WQE or RDMA region. 731 * 732 * Returns number of OS pages retrieved on success. The caller may continue to 733 * the next data segment. 734 * Can return the following error codes: 735 * -EAGAIN to designate a temporary error. The caller will abort handling the 736 * page fault and resolve it. 737 * -EFAULT when there's an error mapping the requested pages. The caller will 738 * abort the page fault handling. 739 */ 740 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 741 struct ib_pd *pd, u32 key, 742 u64 io_virt, size_t bcnt, 743 u32 *bytes_committed, 744 u32 *bytes_mapped, u32 flags) 745 { 746 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 747 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 748 struct pf_frame *head = NULL, *frame; 749 struct mlx5_core_mkey *mmkey; 750 struct mlx5_ib_mr *mr; 751 struct mlx5_klm *pklm; 752 u32 *out = NULL; 753 size_t offset; 754 int ndescs; 755 756 srcu_key = srcu_read_lock(&dev->mr_srcu); 757 758 io_virt += *bytes_committed; 759 bcnt -= *bytes_committed; 760 761 next_mr: 762 mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key)); 763 if (!mmkey || mmkey->key != key) { 764 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 765 ret = -EFAULT; 766 goto srcu_unlock; 767 } 768 769 if (prefetch && mmkey->type != MLX5_MKEY_MR) { 770 mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n"); 771 ret = -EINVAL; 772 goto srcu_unlock; 773 } 774 775 switch (mmkey->type) { 776 case MLX5_MKEY_MR: 777 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 778 if (!mr->live || !mr->ibmr.pd) { 779 mlx5_ib_dbg(dev, "got dead MR\n"); 780 ret = -EFAULT; 781 goto srcu_unlock; 782 } 783 784 if (prefetch) { 785 if (!is_odp_mr(mr) || 786 mr->ibmr.pd != pd) { 787 mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n", 788 is_odp_mr(mr) ? "MR is not ODP" : 789 "PD is not of the MR"); 790 ret = -EINVAL; 791 goto srcu_unlock; 792 } 793 } 794 795 if (!is_odp_mr(mr)) { 796 mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 797 key); 798 if (bytes_mapped) 799 *bytes_mapped += bcnt; 800 ret = 0; 801 goto srcu_unlock; 802 } 803 804 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags); 805 if (ret < 0) 806 goto srcu_unlock; 807 808 npages += ret; 809 ret = 0; 810 break; 811 812 case MLX5_MKEY_MW: 813 case MLX5_MKEY_INDIRECT_DEVX: 814 ndescs = get_indirect_num_descs(mmkey); 815 816 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 817 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 818 ret = -EFAULT; 819 goto srcu_unlock; 820 } 821 822 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 823 sizeof(*pklm) * (ndescs - 2); 824 825 if (outlen > cur_outlen) { 826 kfree(out); 827 out = kzalloc(outlen, GFP_KERNEL); 828 if (!out) { 829 ret = -ENOMEM; 830 goto srcu_unlock; 831 } 832 cur_outlen = outlen; 833 } 834 835 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 836 bsf0_klm0_pas_mtt0_1); 837 838 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); 839 if (ret) 840 goto srcu_unlock; 841 842 offset = io_virt - MLX5_GET64(query_mkey_out, out, 843 memory_key_mkey_entry.start_addr); 844 845 for (i = 0; bcnt && i < ndescs; i++, pklm++) { 846 if (offset >= be32_to_cpu(pklm->bcount)) { 847 offset -= be32_to_cpu(pklm->bcount); 848 continue; 849 } 850 851 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 852 if (!frame) { 853 ret = -ENOMEM; 854 goto srcu_unlock; 855 } 856 857 frame->key = be32_to_cpu(pklm->key); 858 frame->io_virt = be64_to_cpu(pklm->va) + offset; 859 frame->bcnt = min_t(size_t, bcnt, 860 be32_to_cpu(pklm->bcount) - offset); 861 frame->depth = depth + 1; 862 frame->next = head; 863 head = frame; 864 865 bcnt -= frame->bcnt; 866 offset = 0; 867 } 868 break; 869 870 default: 871 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 872 ret = -EFAULT; 873 goto srcu_unlock; 874 } 875 876 if (head) { 877 frame = head; 878 head = frame->next; 879 880 key = frame->key; 881 io_virt = frame->io_virt; 882 bcnt = frame->bcnt; 883 depth = frame->depth; 884 kfree(frame); 885 886 goto next_mr; 887 } 888 889 srcu_unlock: 890 while (head) { 891 frame = head; 892 head = frame->next; 893 kfree(frame); 894 } 895 kfree(out); 896 897 srcu_read_unlock(&dev->mr_srcu, srcu_key); 898 *bytes_committed = 0; 899 return ret ? ret : npages; 900 } 901 902 /** 903 * Parse a series of data segments for page fault handling. 904 * 905 * @pfault contains page fault information. 906 * @wqe points at the first data segment in the WQE. 907 * @wqe_end points after the end of the WQE. 908 * @bytes_mapped receives the number of bytes that the function was able to 909 * map. This allows the caller to decide intelligently whether 910 * enough memory was mapped to resolve the page fault 911 * successfully (e.g. enough for the next MTU, or the entire 912 * WQE). 913 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 914 * the committed bytes). 915 * 916 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 917 * negative error code. 918 */ 919 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 920 struct mlx5_pagefault *pfault, 921 void *wqe, 922 void *wqe_end, u32 *bytes_mapped, 923 u32 *total_wqe_bytes, int receive_queue) 924 { 925 int ret = 0, npages = 0; 926 u64 io_virt; 927 u32 key; 928 u32 byte_count; 929 size_t bcnt; 930 int inline_segment; 931 932 if (bytes_mapped) 933 *bytes_mapped = 0; 934 if (total_wqe_bytes) 935 *total_wqe_bytes = 0; 936 937 while (wqe < wqe_end) { 938 struct mlx5_wqe_data_seg *dseg = wqe; 939 940 io_virt = be64_to_cpu(dseg->addr); 941 key = be32_to_cpu(dseg->lkey); 942 byte_count = be32_to_cpu(dseg->byte_count); 943 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 944 bcnt = byte_count & ~MLX5_INLINE_SEG; 945 946 if (inline_segment) { 947 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 948 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 949 16); 950 } else { 951 wqe += sizeof(*dseg); 952 } 953 954 /* receive WQE end of sg list. */ 955 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 956 io_virt == 0) 957 break; 958 959 if (!inline_segment && total_wqe_bytes) { 960 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 961 pfault->bytes_committed); 962 } 963 964 /* A zero length data segment designates a length of 2GB. */ 965 if (bcnt == 0) 966 bcnt = 1U << 31; 967 968 if (inline_segment || bcnt <= pfault->bytes_committed) { 969 pfault->bytes_committed -= 970 min_t(size_t, bcnt, 971 pfault->bytes_committed); 972 continue; 973 } 974 975 ret = pagefault_single_data_segment(dev, NULL, key, 976 io_virt, bcnt, 977 &pfault->bytes_committed, 978 bytes_mapped, 0); 979 if (ret < 0) 980 break; 981 npages += ret; 982 } 983 984 return ret < 0 ? ret : npages; 985 } 986 987 static const u32 mlx5_ib_odp_opcode_cap[] = { 988 [MLX5_OPCODE_SEND] = IB_ODP_SUPPORT_SEND, 989 [MLX5_OPCODE_SEND_IMM] = IB_ODP_SUPPORT_SEND, 990 [MLX5_OPCODE_SEND_INVAL] = IB_ODP_SUPPORT_SEND, 991 [MLX5_OPCODE_RDMA_WRITE] = IB_ODP_SUPPORT_WRITE, 992 [MLX5_OPCODE_RDMA_WRITE_IMM] = IB_ODP_SUPPORT_WRITE, 993 [MLX5_OPCODE_RDMA_READ] = IB_ODP_SUPPORT_READ, 994 [MLX5_OPCODE_ATOMIC_CS] = IB_ODP_SUPPORT_ATOMIC, 995 [MLX5_OPCODE_ATOMIC_FA] = IB_ODP_SUPPORT_ATOMIC, 996 }; 997 998 /* 999 * Parse initiator WQE. Advances the wqe pointer to point at the 1000 * scatter-gather list, and set wqe_end to the end of the WQE. 1001 */ 1002 static int mlx5_ib_mr_initiator_pfault_handler( 1003 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1004 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1005 { 1006 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1007 u16 wqe_index = pfault->wqe.wqe_index; 1008 u32 transport_caps; 1009 struct mlx5_base_av *av; 1010 unsigned ds, opcode; 1011 #if defined(DEBUG) 1012 u32 ctrl_wqe_index, ctrl_qpn; 1013 #endif 1014 u32 qpn = qp->trans_qp.base.mqp.qpn; 1015 1016 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1017 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1018 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1019 ds, wqe_length); 1020 return -EFAULT; 1021 } 1022 1023 if (ds == 0) { 1024 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1025 wqe_index, qpn); 1026 return -EFAULT; 1027 } 1028 1029 #if defined(DEBUG) 1030 ctrl_wqe_index = (be32_to_cpu(ctrl->opmod_idx_opcode) & 1031 MLX5_WQE_CTRL_WQE_INDEX_MASK) >> 1032 MLX5_WQE_CTRL_WQE_INDEX_SHIFT; 1033 if (wqe_index != ctrl_wqe_index) { 1034 mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n", 1035 wqe_index, qpn, 1036 ctrl_wqe_index); 1037 return -EFAULT; 1038 } 1039 1040 ctrl_qpn = (be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_QPN_MASK) >> 1041 MLX5_WQE_CTRL_QPN_SHIFT; 1042 if (qpn != ctrl_qpn) { 1043 mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n", 1044 wqe_index, qpn, 1045 ctrl_qpn); 1046 return -EFAULT; 1047 } 1048 #endif /* DEBUG */ 1049 1050 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1051 *wqe += sizeof(*ctrl); 1052 1053 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1054 MLX5_WQE_CTRL_OPCODE_MASK; 1055 1056 switch (qp->ibqp.qp_type) { 1057 case IB_QPT_XRC_INI: 1058 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1059 transport_caps = dev->odp_caps.per_transport_caps.xrc_odp_caps; 1060 break; 1061 case IB_QPT_RC: 1062 transport_caps = dev->odp_caps.per_transport_caps.rc_odp_caps; 1063 break; 1064 case IB_QPT_UD: 1065 transport_caps = dev->odp_caps.per_transport_caps.ud_odp_caps; 1066 break; 1067 default: 1068 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport 0x%x\n", 1069 qp->ibqp.qp_type); 1070 return -EFAULT; 1071 } 1072 1073 if (unlikely(opcode >= ARRAY_SIZE(mlx5_ib_odp_opcode_cap) || 1074 !(transport_caps & mlx5_ib_odp_opcode_cap[opcode]))) { 1075 mlx5_ib_err(dev, "ODP fault on QP of an unsupported opcode 0x%x\n", 1076 opcode); 1077 return -EFAULT; 1078 } 1079 1080 if (qp->ibqp.qp_type == IB_QPT_UD) { 1081 av = *wqe; 1082 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1083 *wqe += sizeof(struct mlx5_av); 1084 else 1085 *wqe += sizeof(struct mlx5_base_av); 1086 } 1087 1088 switch (opcode) { 1089 case MLX5_OPCODE_RDMA_WRITE: 1090 case MLX5_OPCODE_RDMA_WRITE_IMM: 1091 case MLX5_OPCODE_RDMA_READ: 1092 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1093 break; 1094 case MLX5_OPCODE_ATOMIC_CS: 1095 case MLX5_OPCODE_ATOMIC_FA: 1096 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1097 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1098 break; 1099 } 1100 1101 return 0; 1102 } 1103 1104 /* 1105 * Parse responder WQE and set wqe_end to the end of the WQE. 1106 */ 1107 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1108 struct mlx5_ib_srq *srq, 1109 void **wqe, void **wqe_end, 1110 int wqe_length) 1111 { 1112 int wqe_size = 1 << srq->msrq.wqe_shift; 1113 1114 if (wqe_size > wqe_length) { 1115 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1116 return -EFAULT; 1117 } 1118 1119 *wqe_end = *wqe + wqe_size; 1120 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1121 1122 return 0; 1123 } 1124 1125 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1126 struct mlx5_ib_qp *qp, 1127 void *wqe, void **wqe_end, 1128 int wqe_length) 1129 { 1130 struct mlx5_ib_wq *wq = &qp->rq; 1131 int wqe_size = 1 << wq->wqe_shift; 1132 1133 if (qp->wq_sig) { 1134 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1135 return -EFAULT; 1136 } 1137 1138 if (wqe_size > wqe_length) { 1139 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1140 return -EFAULT; 1141 } 1142 1143 switch (qp->ibqp.qp_type) { 1144 case IB_QPT_RC: 1145 if (!(dev->odp_caps.per_transport_caps.rc_odp_caps & 1146 IB_ODP_SUPPORT_RECV)) 1147 goto invalid_transport_or_opcode; 1148 break; 1149 default: 1150 invalid_transport_or_opcode: 1151 mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport. transport: 0x%x\n", 1152 qp->ibqp.qp_type); 1153 return -EFAULT; 1154 } 1155 1156 *wqe_end = wqe + wqe_size; 1157 1158 return 0; 1159 } 1160 1161 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1162 u32 wq_num, int pf_type) 1163 { 1164 struct mlx5_core_rsc_common *common = NULL; 1165 struct mlx5_core_srq *srq; 1166 1167 switch (pf_type) { 1168 case MLX5_WQE_PF_TYPE_RMP: 1169 srq = mlx5_cmd_get_srq(dev, wq_num); 1170 if (srq) 1171 common = &srq->common; 1172 break; 1173 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1174 case MLX5_WQE_PF_TYPE_RESP: 1175 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1176 common = mlx5_core_res_hold(dev->mdev, wq_num, MLX5_RES_QP); 1177 break; 1178 default: 1179 break; 1180 } 1181 1182 return common; 1183 } 1184 1185 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1186 { 1187 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1188 1189 return to_mibqp(mqp); 1190 } 1191 1192 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1193 { 1194 struct mlx5_core_srq *msrq = 1195 container_of(res, struct mlx5_core_srq, common); 1196 1197 return to_mibsrq(msrq); 1198 } 1199 1200 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1201 struct mlx5_pagefault *pfault) 1202 { 1203 int ret; 1204 void *wqe, *wqe_end; 1205 u32 bytes_mapped, total_wqe_bytes; 1206 char *buffer = NULL; 1207 int resume_with_error = 1; 1208 u16 wqe_index = pfault->wqe.wqe_index; 1209 int requestor = pfault->type & MLX5_PFAULT_REQUESTOR; 1210 struct mlx5_core_rsc_common *res = NULL; 1211 struct mlx5_ib_qp *qp = NULL; 1212 struct mlx5_ib_srq *srq = NULL; 1213 size_t bytes_copied; 1214 1215 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1216 if (!res) { 1217 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1218 return; 1219 } 1220 1221 switch (res->res) { 1222 case MLX5_RES_QP: 1223 qp = res_to_qp(res); 1224 break; 1225 case MLX5_RES_SRQ: 1226 case MLX5_RES_XSRQ: 1227 srq = res_to_srq(res); 1228 break; 1229 default: 1230 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", pfault->type); 1231 goto resolve_page_fault; 1232 } 1233 1234 buffer = (char *)__get_free_page(GFP_KERNEL); 1235 if (!buffer) { 1236 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1237 goto resolve_page_fault; 1238 } 1239 1240 if (qp) { 1241 if (requestor) { 1242 ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, 1243 buffer, PAGE_SIZE, 1244 &bytes_copied); 1245 } else { 1246 ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, 1247 buffer, PAGE_SIZE, 1248 &bytes_copied); 1249 } 1250 } else { 1251 ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, 1252 buffer, PAGE_SIZE, 1253 &bytes_copied); 1254 } 1255 1256 if (ret) { 1257 mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n", 1258 ret, wqe_index, pfault->token); 1259 goto resolve_page_fault; 1260 } 1261 1262 wqe = buffer; 1263 if (requestor) 1264 ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp, 1265 &wqe, &wqe_end, 1266 bytes_copied); 1267 else if (qp) 1268 ret = mlx5_ib_mr_responder_pfault_handler_rq(dev, qp, 1269 wqe, &wqe_end, 1270 bytes_copied); 1271 else 1272 ret = mlx5_ib_mr_responder_pfault_handler_srq(dev, srq, 1273 &wqe, &wqe_end, 1274 bytes_copied); 1275 1276 if (ret < 0) 1277 goto resolve_page_fault; 1278 1279 if (wqe >= wqe_end) { 1280 mlx5_ib_err(dev, "ODP fault on invalid WQE.\n"); 1281 goto resolve_page_fault; 1282 } 1283 1284 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, 1285 &bytes_mapped, &total_wqe_bytes, 1286 !requestor); 1287 if (ret == -EAGAIN) { 1288 resume_with_error = 0; 1289 goto resolve_page_fault; 1290 } else if (ret < 0 || total_wqe_bytes > bytes_mapped) { 1291 goto resolve_page_fault; 1292 } 1293 1294 resume_with_error = 0; 1295 resolve_page_fault: 1296 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1297 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1298 pfault->wqe.wq_num, resume_with_error, 1299 pfault->type); 1300 mlx5_core_res_put(res); 1301 free_page((unsigned long)buffer); 1302 } 1303 1304 static int pages_in_range(u64 address, u32 length) 1305 { 1306 return (ALIGN(address + length, PAGE_SIZE) - 1307 (address & PAGE_MASK)) >> PAGE_SHIFT; 1308 } 1309 1310 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1311 struct mlx5_pagefault *pfault) 1312 { 1313 u64 address; 1314 u32 length; 1315 u32 prefetch_len = pfault->bytes_committed; 1316 int prefetch_activated = 0; 1317 u32 rkey = pfault->rdma.r_key; 1318 int ret; 1319 1320 /* The RDMA responder handler handles the page fault in two parts. 1321 * First it brings the necessary pages for the current packet 1322 * (and uses the pfault context), and then (after resuming the QP) 1323 * prefetches more pages. The second operation cannot use the pfault 1324 * context and therefore uses the dummy_pfault context allocated on 1325 * the stack */ 1326 pfault->rdma.rdma_va += pfault->bytes_committed; 1327 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1328 pfault->rdma.rdma_op_len); 1329 pfault->bytes_committed = 0; 1330 1331 address = pfault->rdma.rdma_va; 1332 length = pfault->rdma.rdma_op_len; 1333 1334 /* For some operations, the hardware cannot tell the exact message 1335 * length, and in those cases it reports zero. Use prefetch 1336 * logic. */ 1337 if (length == 0) { 1338 prefetch_activated = 1; 1339 length = pfault->rdma.packet_size; 1340 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1341 } 1342 1343 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1344 &pfault->bytes_committed, NULL, 1345 0); 1346 if (ret == -EAGAIN) { 1347 /* We're racing with an invalidation, don't prefetch */ 1348 prefetch_activated = 0; 1349 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1350 mlx5_ib_page_fault_resume(dev, pfault, 1); 1351 if (ret != -ENOENT) 1352 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1353 ret, pfault->token, pfault->type); 1354 return; 1355 } 1356 1357 mlx5_ib_page_fault_resume(dev, pfault, 0); 1358 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1359 pfault->token, pfault->type, 1360 prefetch_activated); 1361 1362 /* At this point, there might be a new pagefault already arriving in 1363 * the eq, switch to the dummy pagefault for the rest of the 1364 * processing. We're still OK with the objects being alive as the 1365 * work-queue is being fenced. */ 1366 1367 if (prefetch_activated) { 1368 u32 bytes_committed = 0; 1369 1370 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1371 prefetch_len, 1372 &bytes_committed, NULL, 1373 0); 1374 if (ret < 0 && ret != -EAGAIN) { 1375 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1376 ret, pfault->token, address, prefetch_len); 1377 } 1378 } 1379 } 1380 1381 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1382 { 1383 u8 event_subtype = pfault->event_subtype; 1384 1385 switch (event_subtype) { 1386 case MLX5_PFAULT_SUBTYPE_WQE: 1387 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1388 break; 1389 case MLX5_PFAULT_SUBTYPE_RDMA: 1390 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1391 break; 1392 default: 1393 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1394 event_subtype); 1395 mlx5_ib_page_fault_resume(dev, pfault, 1); 1396 } 1397 } 1398 1399 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1400 { 1401 struct mlx5_pagefault *pfault = container_of(work, 1402 struct mlx5_pagefault, 1403 work); 1404 struct mlx5_ib_pf_eq *eq = pfault->eq; 1405 1406 mlx5_ib_pfault(eq->dev, pfault); 1407 mempool_free(pfault, eq->pool); 1408 } 1409 1410 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1411 { 1412 struct mlx5_eqe_page_fault *pf_eqe; 1413 struct mlx5_pagefault *pfault; 1414 struct mlx5_eqe *eqe; 1415 int cc = 0; 1416 1417 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1418 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1419 if (!pfault) { 1420 schedule_work(&eq->work); 1421 break; 1422 } 1423 1424 pf_eqe = &eqe->data.page_fault; 1425 pfault->event_subtype = eqe->sub_type; 1426 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); 1427 1428 mlx5_ib_dbg(eq->dev, 1429 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", 1430 eqe->sub_type, pfault->bytes_committed); 1431 1432 switch (eqe->sub_type) { 1433 case MLX5_PFAULT_SUBTYPE_RDMA: 1434 /* RDMA based event */ 1435 pfault->type = 1436 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1437 pfault->token = 1438 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1439 MLX5_24BIT_MASK; 1440 pfault->rdma.r_key = 1441 be32_to_cpu(pf_eqe->rdma.r_key); 1442 pfault->rdma.packet_size = 1443 be16_to_cpu(pf_eqe->rdma.packet_length); 1444 pfault->rdma.rdma_op_len = 1445 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1446 pfault->rdma.rdma_va = 1447 be64_to_cpu(pf_eqe->rdma.rdma_va); 1448 mlx5_ib_dbg(eq->dev, 1449 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", 1450 pfault->type, pfault->token, 1451 pfault->rdma.r_key); 1452 mlx5_ib_dbg(eq->dev, 1453 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1454 pfault->rdma.rdma_op_len, 1455 pfault->rdma.rdma_va); 1456 break; 1457 1458 case MLX5_PFAULT_SUBTYPE_WQE: 1459 /* WQE based event */ 1460 pfault->type = 1461 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1462 pfault->token = 1463 be32_to_cpu(pf_eqe->wqe.token); 1464 pfault->wqe.wq_num = 1465 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1466 MLX5_24BIT_MASK; 1467 pfault->wqe.wqe_index = 1468 be16_to_cpu(pf_eqe->wqe.wqe_index); 1469 pfault->wqe.packet_size = 1470 be16_to_cpu(pf_eqe->wqe.packet_length); 1471 mlx5_ib_dbg(eq->dev, 1472 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1473 pfault->type, pfault->token, 1474 pfault->wqe.wq_num, 1475 pfault->wqe.wqe_index); 1476 break; 1477 1478 default: 1479 mlx5_ib_warn(eq->dev, 1480 "Unsupported page fault event sub-type: 0x%02hhx\n", 1481 eqe->sub_type); 1482 /* Unsupported page faults should still be 1483 * resolved by the page fault handler 1484 */ 1485 } 1486 1487 pfault->eq = eq; 1488 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1489 queue_work(eq->wq, &pfault->work); 1490 1491 cc = mlx5_eq_update_cc(eq->core, ++cc); 1492 } 1493 1494 mlx5_eq_update_ci(eq->core, cc, 1); 1495 } 1496 1497 static irqreturn_t mlx5_ib_eq_pf_int(int irq, void *eq_ptr) 1498 { 1499 struct mlx5_ib_pf_eq *eq = eq_ptr; 1500 unsigned long flags; 1501 1502 if (spin_trylock_irqsave(&eq->lock, flags)) { 1503 mlx5_ib_eq_pf_process(eq); 1504 spin_unlock_irqrestore(&eq->lock, flags); 1505 } else { 1506 schedule_work(&eq->work); 1507 } 1508 1509 return IRQ_HANDLED; 1510 } 1511 1512 /* mempool_refill() was proposed but unfortunately wasn't accepted 1513 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1514 * Cheap workaround. 1515 */ 1516 static void mempool_refill(mempool_t *pool) 1517 { 1518 while (pool->curr_nr < pool->min_nr) 1519 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1520 } 1521 1522 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1523 { 1524 struct mlx5_ib_pf_eq *eq = 1525 container_of(work, struct mlx5_ib_pf_eq, work); 1526 1527 mempool_refill(eq->pool); 1528 1529 spin_lock_irq(&eq->lock); 1530 mlx5_ib_eq_pf_process(eq); 1531 spin_unlock_irq(&eq->lock); 1532 } 1533 1534 enum { 1535 MLX5_IB_NUM_PF_EQE = 0x1000, 1536 MLX5_IB_NUM_PF_DRAIN = 64, 1537 }; 1538 1539 static int 1540 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1541 { 1542 struct mlx5_eq_param param = {}; 1543 int err; 1544 1545 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1546 spin_lock_init(&eq->lock); 1547 eq->dev = dev; 1548 1549 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1550 sizeof(struct mlx5_pagefault)); 1551 if (!eq->pool) 1552 return -ENOMEM; 1553 1554 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1555 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1556 MLX5_NUM_CMD_EQE); 1557 if (!eq->wq) { 1558 err = -ENOMEM; 1559 goto err_mempool; 1560 } 1561 1562 param = (struct mlx5_eq_param) { 1563 .index = MLX5_EQ_PFAULT_IDX, 1564 .mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT, 1565 .nent = MLX5_IB_NUM_PF_EQE, 1566 .context = eq, 1567 .handler = mlx5_ib_eq_pf_int 1568 }; 1569 eq->core = mlx5_eq_create_generic(dev->mdev, "mlx5_ib_page_fault_eq", ¶m); 1570 if (IS_ERR(eq->core)) { 1571 err = PTR_ERR(eq->core); 1572 goto err_wq; 1573 } 1574 1575 return 0; 1576 err_wq: 1577 destroy_workqueue(eq->wq); 1578 err_mempool: 1579 mempool_destroy(eq->pool); 1580 return err; 1581 } 1582 1583 static int 1584 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1585 { 1586 int err; 1587 1588 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1589 cancel_work_sync(&eq->work); 1590 destroy_workqueue(eq->wq); 1591 mempool_destroy(eq->pool); 1592 1593 return err; 1594 } 1595 1596 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1597 { 1598 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1599 return; 1600 1601 switch (ent->order - 2) { 1602 case MLX5_IMR_MTT_CACHE_ENTRY: 1603 ent->page = PAGE_SHIFT; 1604 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1605 sizeof(struct mlx5_mtt) / 1606 MLX5_IB_UMR_OCTOWORD; 1607 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1608 ent->limit = 0; 1609 break; 1610 1611 case MLX5_IMR_KSM_CACHE_ENTRY: 1612 ent->page = MLX5_KSM_PAGE_SHIFT; 1613 ent->xlt = mlx5_imr_ksm_entries * 1614 sizeof(struct mlx5_klm) / 1615 MLX5_IB_UMR_OCTOWORD; 1616 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1617 ent->limit = 0; 1618 break; 1619 } 1620 } 1621 1622 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1623 .advise_mr = mlx5_ib_advise_mr, 1624 }; 1625 1626 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1627 { 1628 int ret = 0; 1629 1630 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT) 1631 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1632 1633 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1634 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1635 if (ret) { 1636 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1637 return ret; 1638 } 1639 } 1640 1641 if (!MLX5_CAP_GEN(dev->mdev, pg)) 1642 return ret; 1643 1644 ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1645 1646 return ret; 1647 } 1648 1649 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1650 { 1651 if (!MLX5_CAP_GEN(dev->mdev, pg)) 1652 return; 1653 1654 mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1655 } 1656 1657 int mlx5_ib_odp_init(void) 1658 { 1659 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1660 MLX5_IMR_MTT_BITS); 1661 1662 return 0; 1663 } 1664 1665 struct prefetch_mr_work { 1666 struct work_struct work; 1667 struct ib_pd *pd; 1668 u32 pf_flags; 1669 u32 num_sge; 1670 struct ib_sge sg_list[0]; 1671 }; 1672 1673 static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, 1674 struct ib_sge *sg_list, u32 num_sge, 1675 u32 from) 1676 { 1677 u32 i; 1678 int srcu_key; 1679 1680 srcu_key = srcu_read_lock(&dev->mr_srcu); 1681 1682 for (i = from; i < num_sge; ++i) { 1683 struct mlx5_core_mkey *mmkey; 1684 struct mlx5_ib_mr *mr; 1685 1686 mmkey = __mlx5_mr_lookup(dev->mdev, 1687 mlx5_base_mkey(sg_list[i].lkey)); 1688 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1689 atomic_dec(&mr->num_pending_prefetch); 1690 } 1691 1692 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1693 } 1694 1695 static bool num_pending_prefetch_inc(struct ib_pd *pd, 1696 struct ib_sge *sg_list, u32 num_sge) 1697 { 1698 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1699 bool ret = true; 1700 u32 i; 1701 1702 for (i = 0; i < num_sge; ++i) { 1703 struct mlx5_core_mkey *mmkey; 1704 struct mlx5_ib_mr *mr; 1705 1706 mmkey = __mlx5_mr_lookup(dev->mdev, 1707 mlx5_base_mkey(sg_list[i].lkey)); 1708 if (!mmkey || mmkey->key != sg_list[i].lkey) { 1709 ret = false; 1710 break; 1711 } 1712 1713 if (mmkey->type != MLX5_MKEY_MR) { 1714 ret = false; 1715 break; 1716 } 1717 1718 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1719 1720 if (mr->ibmr.pd != pd) { 1721 ret = false; 1722 break; 1723 } 1724 1725 if (!mr->live) { 1726 ret = false; 1727 break; 1728 } 1729 1730 atomic_inc(&mr->num_pending_prefetch); 1731 } 1732 1733 if (!ret) 1734 num_pending_prefetch_dec(dev, sg_list, i, 0); 1735 1736 return ret; 1737 } 1738 1739 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags, 1740 struct ib_sge *sg_list, u32 num_sge) 1741 { 1742 u32 i; 1743 int ret = 0; 1744 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1745 1746 for (i = 0; i < num_sge; ++i) { 1747 struct ib_sge *sg = &sg_list[i]; 1748 int bytes_committed = 0; 1749 1750 ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr, 1751 sg->length, 1752 &bytes_committed, NULL, 1753 pf_flags); 1754 if (ret < 0) 1755 break; 1756 } 1757 1758 return ret < 0 ? ret : 0; 1759 } 1760 1761 static void mlx5_ib_prefetch_mr_work(struct work_struct *work) 1762 { 1763 struct prefetch_mr_work *w = 1764 container_of(work, struct prefetch_mr_work, work); 1765 1766 if (ib_device_try_get(w->pd->device)) { 1767 mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list, 1768 w->num_sge); 1769 ib_device_put(w->pd->device); 1770 } 1771 1772 num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list, 1773 w->num_sge, 0); 1774 kfree(w); 1775 } 1776 1777 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1778 enum ib_uverbs_advise_mr_advice advice, 1779 u32 flags, struct ib_sge *sg_list, u32 num_sge) 1780 { 1781 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1782 u32 pf_flags = MLX5_PF_FLAGS_PREFETCH; 1783 struct prefetch_mr_work *work; 1784 bool valid_req; 1785 int srcu_key; 1786 1787 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 1788 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 1789 1790 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 1791 return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list, 1792 num_sge); 1793 1794 work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL); 1795 if (!work) 1796 return -ENOMEM; 1797 1798 memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge)); 1799 1800 /* It is guaranteed that the pd when work is executed is the pd when 1801 * work was queued since pd can't be destroyed while it holds MRs and 1802 * destroying a MR leads to flushing the workquque 1803 */ 1804 work->pd = pd; 1805 work->pf_flags = pf_flags; 1806 work->num_sge = num_sge; 1807 1808 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 1809 1810 srcu_key = srcu_read_lock(&dev->mr_srcu); 1811 1812 valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge); 1813 if (valid_req) 1814 queue_work(system_unbound_wq, &work->work); 1815 else 1816 kfree(work); 1817 1818 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1819 1820 return valid_req ? 0 : -EINVAL; 1821 } 1822