1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem.h> 34 #include <rdma/ib_umem_odp.h> 35 #include <linux/kernel.h> 36 37 #include "mlx5_ib.h" 38 #include "cmd.h" 39 #include "qp.h" 40 41 #include <linux/mlx5/eq.h> 42 43 /* Contains the details of a pagefault. */ 44 struct mlx5_pagefault { 45 u32 bytes_committed; 46 u32 token; 47 u8 event_subtype; 48 u8 type; 49 union { 50 /* Initiator or send message responder pagefault details. */ 51 struct { 52 /* Received packet size, only valid for responders. */ 53 u32 packet_size; 54 /* 55 * Number of resource holding WQE, depends on type. 56 */ 57 u32 wq_num; 58 /* 59 * WQE index. Refers to either the send queue or 60 * receive queue, according to event_subtype. 61 */ 62 u16 wqe_index; 63 } wqe; 64 /* RDMA responder pagefault details */ 65 struct { 66 u32 r_key; 67 /* 68 * Received packet size, minimal size page fault 69 * resolution required for forward progress. 70 */ 71 u32 packet_size; 72 u32 rdma_op_len; 73 u64 rdma_va; 74 } rdma; 75 }; 76 77 struct mlx5_ib_pf_eq *eq; 78 struct work_struct work; 79 }; 80 81 #define MAX_PREFETCH_LEN (4*1024*1024U) 82 83 /* Timeout in ms to wait for an active mmu notifier to complete when handling 84 * a pagefault. */ 85 #define MMU_NOTIFIER_TIMEOUT 1000 86 87 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 88 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 89 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 90 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 91 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 92 93 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 94 95 static u64 mlx5_imr_ksm_entries; 96 97 static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, 98 struct mlx5_ib_mr *imr, int flags) 99 { 100 struct mlx5_klm *end = pklm + nentries; 101 102 if (flags & MLX5_IB_UPD_XLT_ZAP) { 103 for (; pklm != end; pklm++, idx++) { 104 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 105 pklm->key = cpu_to_be32(imr->dev->null_mkey); 106 pklm->va = 0; 107 } 108 return; 109 } 110 111 /* 112 * The locking here is pretty subtle. Ideally the implicit_children 113 * xarray would be protected by the umem_mutex, however that is not 114 * possible. Instead this uses a weaker update-then-lock pattern: 115 * 116 * srcu_read_lock() 117 * xa_store() 118 * mutex_lock(umem_mutex) 119 * mlx5_ib_update_xlt() 120 * mutex_unlock(umem_mutex) 121 * destroy lkey 122 * 123 * ie any change the xarray must be followed by the locked update_xlt 124 * before destroying. 125 * 126 * The umem_mutex provides the acquire/release semantic needed to make 127 * the xa_store() visible to a racing thread. While SRCU is not 128 * technically required, using it gives consistent use of the SRCU 129 * locking around the xarray. 130 */ 131 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); 132 lockdep_assert_held(&imr->dev->odp_srcu); 133 134 for (; pklm != end; pklm++, idx++) { 135 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); 136 137 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 138 if (mtt) { 139 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 140 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE); 141 } else { 142 pklm->key = cpu_to_be32(imr->dev->null_mkey); 143 pklm->va = 0; 144 } 145 } 146 } 147 148 static u64 umem_dma_to_mtt(dma_addr_t umem_dma) 149 { 150 u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; 151 152 if (umem_dma & ODP_READ_ALLOWED_BIT) 153 mtt_entry |= MLX5_IB_MTT_READ; 154 if (umem_dma & ODP_WRITE_ALLOWED_BIT) 155 mtt_entry |= MLX5_IB_MTT_WRITE; 156 157 return mtt_entry; 158 } 159 160 static void populate_mtt(__be64 *pas, size_t idx, size_t nentries, 161 struct mlx5_ib_mr *mr, int flags) 162 { 163 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 164 dma_addr_t pa; 165 size_t i; 166 167 if (flags & MLX5_IB_UPD_XLT_ZAP) 168 return; 169 170 for (i = 0; i < nentries; i++) { 171 pa = odp->dma_list[idx + i]; 172 pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); 173 } 174 } 175 176 void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 177 struct mlx5_ib_mr *mr, int flags) 178 { 179 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 180 populate_klm(xlt, idx, nentries, mr, flags); 181 } else { 182 populate_mtt(xlt, idx, nentries, mr, flags); 183 } 184 } 185 186 static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) 187 { 188 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 189 190 /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */ 191 mutex_lock(&odp->umem_mutex); 192 if (odp->npages) { 193 mlx5_mr_cache_invalidate(mr); 194 ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp), 195 ib_umem_end(odp)); 196 WARN_ON(odp->npages); 197 } 198 odp->private = NULL; 199 mutex_unlock(&odp->umem_mutex); 200 201 if (!mr->cache_ent) { 202 mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey); 203 WARN_ON(mr->descs); 204 } 205 } 206 207 /* 208 * This must be called after the mr has been removed from implicit_children 209 * and the SRCU synchronized. NOTE: The MR does not necessarily have to be 210 * empty here, parallel page faults could have raced with the free process and 211 * added pages to it. 212 */ 213 static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) 214 { 215 struct mlx5_ib_mr *imr = mr->parent; 216 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 217 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 218 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 219 int srcu_key; 220 221 /* implicit_child_mr's are not allowed to have deferred work */ 222 WARN_ON(atomic_read(&mr->num_deferred_work)); 223 224 if (need_imr_xlt) { 225 srcu_key = srcu_read_lock(&mr->dev->odp_srcu); 226 mutex_lock(&odp_imr->umem_mutex); 227 mlx5_ib_update_xlt(mr->parent, idx, 1, 0, 228 MLX5_IB_UPD_XLT_INDIRECT | 229 MLX5_IB_UPD_XLT_ATOMIC); 230 mutex_unlock(&odp_imr->umem_mutex); 231 srcu_read_unlock(&mr->dev->odp_srcu, srcu_key); 232 } 233 234 dma_fence_odp_mr(mr); 235 236 mr->parent = NULL; 237 mlx5_mr_cache_free(mr->dev, mr); 238 ib_umem_odp_release(odp); 239 if (atomic_dec_and_test(&imr->num_deferred_work)) 240 wake_up(&imr->q_deferred_work); 241 } 242 243 static void free_implicit_child_mr_work(struct work_struct *work) 244 { 245 struct mlx5_ib_mr *mr = 246 container_of(work, struct mlx5_ib_mr, odp_destroy.work); 247 248 free_implicit_child_mr(mr, true); 249 } 250 251 static void free_implicit_child_mr_rcu(struct rcu_head *head) 252 { 253 struct mlx5_ib_mr *mr = 254 container_of(head, struct mlx5_ib_mr, odp_destroy.rcu); 255 256 /* Freeing a MR is a sleeping operation, so bounce to a work queue */ 257 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); 258 queue_work(system_unbound_wq, &mr->odp_destroy.work); 259 } 260 261 static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) 262 { 263 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 264 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 265 struct mlx5_ib_mr *imr = mr->parent; 266 267 xa_lock(&imr->implicit_children); 268 /* 269 * This can race with mlx5_ib_free_implicit_mr(), the first one to 270 * reach the xa lock wins the race and destroys the MR. 271 */ 272 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_ATOMIC) != 273 mr) 274 goto out_unlock; 275 276 atomic_inc(&imr->num_deferred_work); 277 call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu, 278 free_implicit_child_mr_rcu); 279 280 out_unlock: 281 xa_unlock(&imr->implicit_children); 282 } 283 284 static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, 285 const struct mmu_notifier_range *range, 286 unsigned long cur_seq) 287 { 288 struct ib_umem_odp *umem_odp = 289 container_of(mni, struct ib_umem_odp, notifier); 290 struct mlx5_ib_mr *mr; 291 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 292 sizeof(struct mlx5_mtt)) - 1; 293 u64 idx = 0, blk_start_idx = 0; 294 u64 invalidations = 0; 295 unsigned long start; 296 unsigned long end; 297 int in_block = 0; 298 u64 addr; 299 300 if (!mmu_notifier_range_blockable(range)) 301 return false; 302 303 mutex_lock(&umem_odp->umem_mutex); 304 mmu_interval_set_seq(mni, cur_seq); 305 /* 306 * If npages is zero then umem_odp->private may not be setup yet. This 307 * does not complete until after the first page is mapped for DMA. 308 */ 309 if (!umem_odp->npages) 310 goto out; 311 mr = umem_odp->private; 312 313 start = max_t(u64, ib_umem_start(umem_odp), range->start); 314 end = min_t(u64, ib_umem_end(umem_odp), range->end); 315 316 /* 317 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 318 * while we are doing the invalidation, no page fault will attempt to 319 * overwrite the same MTTs. Concurent invalidations might race us, 320 * but they will write 0s as well, so no difference in the end result. 321 */ 322 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 323 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 324 /* 325 * Strive to write the MTTs in chunks, but avoid overwriting 326 * non-existing MTTs. The huristic here can be improved to 327 * estimate the cost of another UMR vs. the cost of bigger 328 * UMR. 329 */ 330 if (umem_odp->dma_list[idx] & 331 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 332 if (!in_block) { 333 blk_start_idx = idx; 334 in_block = 1; 335 } 336 337 /* Count page invalidations */ 338 invalidations += idx - blk_start_idx + 1; 339 } else { 340 u64 umr_offset = idx & umr_block_mask; 341 342 if (in_block && umr_offset == 0) { 343 mlx5_ib_update_xlt(mr, blk_start_idx, 344 idx - blk_start_idx, 0, 345 MLX5_IB_UPD_XLT_ZAP | 346 MLX5_IB_UPD_XLT_ATOMIC); 347 in_block = 0; 348 } 349 } 350 } 351 if (in_block) 352 mlx5_ib_update_xlt(mr, blk_start_idx, 353 idx - blk_start_idx + 1, 0, 354 MLX5_IB_UPD_XLT_ZAP | 355 MLX5_IB_UPD_XLT_ATOMIC); 356 357 mlx5_update_odp_stats(mr, invalidations, invalidations); 358 359 /* 360 * We are now sure that the device will not access the 361 * memory. We can safely unmap it, and mark it as dirty if 362 * needed. 363 */ 364 365 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 366 367 if (unlikely(!umem_odp->npages && mr->parent)) 368 destroy_unused_implicit_child_mr(mr); 369 out: 370 mutex_unlock(&umem_odp->umem_mutex); 371 return true; 372 } 373 374 const struct mmu_interval_notifier_ops mlx5_mn_ops = { 375 .invalidate = mlx5_ib_invalidate_range, 376 }; 377 378 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 379 { 380 struct ib_odp_caps *caps = &dev->odp_caps; 381 382 memset(caps, 0, sizeof(*caps)); 383 384 if (!MLX5_CAP_GEN(dev->mdev, pg) || 385 !mlx5_ib_can_use_umr(dev, true, 0)) 386 return; 387 388 caps->general_caps = IB_ODP_SUPPORT; 389 390 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 391 dev->odp_max_size = U64_MAX; 392 else 393 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 394 395 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 396 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 397 398 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive)) 399 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 400 401 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 402 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 403 404 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 405 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 406 407 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 408 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 409 410 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 411 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 412 413 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 414 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 415 416 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive)) 417 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 418 419 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send)) 420 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 421 422 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive)) 423 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 424 425 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write)) 426 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 427 428 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read)) 429 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 430 431 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic)) 432 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 433 434 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive)) 435 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 436 437 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 438 MLX5_CAP_GEN(dev->mdev, null_mkey) && 439 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 440 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled)) 441 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 442 } 443 444 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 445 struct mlx5_pagefault *pfault, 446 int error) 447 { 448 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 449 pfault->wqe.wq_num : pfault->token; 450 u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { }; 451 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = { }; 452 int err; 453 454 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 455 MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type); 456 MLX5_SET(page_fault_resume_in, in, token, pfault->token); 457 MLX5_SET(page_fault_resume_in, in, wq_number, wq_num); 458 MLX5_SET(page_fault_resume_in, in, error, !!error); 459 460 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 461 if (err) 462 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 463 wq_num, err); 464 } 465 466 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 467 unsigned long idx) 468 { 469 struct ib_umem_odp *odp; 470 struct mlx5_ib_mr *mr; 471 struct mlx5_ib_mr *ret; 472 int err; 473 474 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), 475 idx * MLX5_IMR_MTT_SIZE, 476 MLX5_IMR_MTT_SIZE, &mlx5_mn_ops); 477 if (IS_ERR(odp)) 478 return ERR_CAST(odp); 479 480 ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY); 481 if (IS_ERR(mr)) 482 goto out_umem; 483 484 mr->ibmr.pd = imr->ibmr.pd; 485 mr->access_flags = imr->access_flags; 486 mr->umem = &odp->umem; 487 mr->ibmr.lkey = mr->mmkey.key; 488 mr->ibmr.rkey = mr->mmkey.key; 489 mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE; 490 mr->parent = imr; 491 odp->private = mr; 492 493 err = mlx5_ib_update_xlt(mr, 0, 494 MLX5_IMR_MTT_ENTRIES, 495 PAGE_SHIFT, 496 MLX5_IB_UPD_XLT_ZAP | 497 MLX5_IB_UPD_XLT_ENABLE); 498 if (err) { 499 ret = ERR_PTR(err); 500 goto out_mr; 501 } 502 503 /* 504 * Once the store to either xarray completes any error unwind has to 505 * use synchronize_srcu(). Avoid this with xa_reserve() 506 */ 507 ret = xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, 508 GFP_KERNEL); 509 if (unlikely(ret)) { 510 if (xa_is_err(ret)) { 511 ret = ERR_PTR(xa_err(ret)); 512 goto out_mr; 513 } 514 /* 515 * Another thread beat us to creating the child mr, use 516 * theirs. 517 */ 518 goto out_mr; 519 } 520 521 mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr); 522 return mr; 523 524 out_mr: 525 mlx5_mr_cache_free(imr->dev, mr); 526 out_umem: 527 ib_umem_odp_release(odp); 528 return ret; 529 } 530 531 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 532 struct ib_udata *udata, 533 int access_flags) 534 { 535 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 536 struct ib_umem_odp *umem_odp; 537 struct mlx5_ib_mr *imr; 538 int err; 539 540 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags); 541 if (IS_ERR(umem_odp)) 542 return ERR_CAST(umem_odp); 543 544 imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY); 545 if (IS_ERR(imr)) { 546 err = PTR_ERR(imr); 547 goto out_umem; 548 } 549 550 imr->ibmr.pd = &pd->ibpd; 551 imr->access_flags = access_flags; 552 imr->mmkey.iova = 0; 553 imr->umem = &umem_odp->umem; 554 imr->ibmr.lkey = imr->mmkey.key; 555 imr->ibmr.rkey = imr->mmkey.key; 556 imr->umem = &umem_odp->umem; 557 imr->is_odp_implicit = true; 558 atomic_set(&imr->num_deferred_work, 0); 559 init_waitqueue_head(&imr->q_deferred_work); 560 xa_init(&imr->implicit_children); 561 562 err = mlx5_ib_update_xlt(imr, 0, 563 mlx5_imr_ksm_entries, 564 MLX5_KSM_PAGE_SHIFT, 565 MLX5_IB_UPD_XLT_INDIRECT | 566 MLX5_IB_UPD_XLT_ZAP | 567 MLX5_IB_UPD_XLT_ENABLE); 568 if (err) 569 goto out_mr; 570 571 err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key), 572 &imr->mmkey, GFP_KERNEL)); 573 if (err) 574 goto out_mr; 575 576 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); 577 return imr; 578 out_mr: 579 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 580 mlx5_mr_cache_free(dev, imr); 581 out_umem: 582 ib_umem_odp_release(umem_odp); 583 return ERR_PTR(err); 584 } 585 586 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 587 { 588 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 589 struct mlx5_ib_dev *dev = imr->dev; 590 struct list_head destroy_list; 591 struct mlx5_ib_mr *mtt; 592 struct mlx5_ib_mr *tmp; 593 unsigned long idx; 594 595 INIT_LIST_HEAD(&destroy_list); 596 597 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key)); 598 /* 599 * This stops the SRCU protected page fault path from touching either 600 * the imr or any children. The page fault path can only reach the 601 * children xarray via the imr. 602 */ 603 synchronize_srcu(&dev->odp_srcu); 604 605 xa_lock(&imr->implicit_children); 606 xa_for_each (&imr->implicit_children, idx, mtt) { 607 __xa_erase(&imr->implicit_children, idx); 608 list_add(&mtt->odp_destroy.elm, &destroy_list); 609 } 610 xa_unlock(&imr->implicit_children); 611 612 /* 613 * num_deferred_work can only be incremented inside the odp_srcu, or 614 * under xa_lock while the child is in the xarray. Thus at this point 615 * it is only decreasing, and all work holding it is now on the wq. 616 */ 617 wait_event(imr->q_deferred_work, !atomic_read(&imr->num_deferred_work)); 618 619 /* 620 * Fence the imr before we destroy the children. This allows us to 621 * skip updating the XLT of the imr during destroy of the child mkey 622 * the imr points to. 623 */ 624 mlx5_mr_cache_invalidate(imr); 625 626 list_for_each_entry_safe (mtt, tmp, &destroy_list, odp_destroy.elm) 627 free_implicit_child_mr(mtt, false); 628 629 mlx5_mr_cache_free(dev, imr); 630 ib_umem_odp_release(odp_imr); 631 } 632 633 /** 634 * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR 635 * @mr: to fence 636 * 637 * On return no parallel threads will be touching this MR and no DMA will be 638 * active. 639 */ 640 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) 641 { 642 /* Prevent new page faults and prefetch requests from succeeding */ 643 xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)); 644 645 /* Wait for all running page-fault handlers to finish. */ 646 synchronize_srcu(&mr->dev->odp_srcu); 647 648 wait_event(mr->q_deferred_work, !atomic_read(&mr->num_deferred_work)); 649 650 dma_fence_odp_mr(mr); 651 } 652 653 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 654 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, 655 u64 user_va, size_t bcnt, u32 *bytes_mapped, 656 u32 flags) 657 { 658 int page_shift, ret, np; 659 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 660 unsigned long current_seq; 661 u64 access_mask; 662 u64 start_idx; 663 664 page_shift = odp->page_shift; 665 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 666 access_mask = ODP_READ_ALLOWED_BIT; 667 668 if (odp->umem.writable && !downgrade) 669 access_mask |= ODP_WRITE_ALLOWED_BIT; 670 671 current_seq = mmu_interval_read_begin(&odp->notifier); 672 673 np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask, 674 current_seq); 675 if (np < 0) 676 return np; 677 678 mutex_lock(&odp->umem_mutex); 679 if (!mmu_interval_read_retry(&odp->notifier, current_seq)) { 680 /* 681 * No need to check whether the MTTs really belong to 682 * this MR, since ib_umem_odp_map_dma_pages already 683 * checks this. 684 */ 685 ret = mlx5_ib_update_xlt(mr, start_idx, np, 686 page_shift, MLX5_IB_UPD_XLT_ATOMIC); 687 } else { 688 ret = -EAGAIN; 689 } 690 mutex_unlock(&odp->umem_mutex); 691 692 if (ret < 0) { 693 if (ret != -EAGAIN) 694 mlx5_ib_err(mr->dev, 695 "Failed to update mkey page tables\n"); 696 goto out; 697 } 698 699 if (bytes_mapped) { 700 u32 new_mappings = (np << page_shift) - 701 (user_va - round_down(user_va, 1 << page_shift)); 702 703 *bytes_mapped += min_t(u32, new_mappings, bcnt); 704 } 705 706 return np << (page_shift - PAGE_SHIFT); 707 708 out: 709 return ret; 710 } 711 712 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, 713 struct ib_umem_odp *odp_imr, u64 user_va, 714 size_t bcnt, u32 *bytes_mapped, u32 flags) 715 { 716 unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT; 717 unsigned long upd_start_idx = end_idx + 1; 718 unsigned long upd_len = 0; 719 unsigned long npages = 0; 720 int err; 721 int ret; 722 723 if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE || 724 mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt)) 725 return -EFAULT; 726 727 /* Fault each child mr that intersects with our interval. */ 728 while (bcnt) { 729 unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT; 730 struct ib_umem_odp *umem_odp; 731 struct mlx5_ib_mr *mtt; 732 u64 len; 733 734 mtt = xa_load(&imr->implicit_children, idx); 735 if (unlikely(!mtt)) { 736 mtt = implicit_get_child_mr(imr, idx); 737 if (IS_ERR(mtt)) { 738 ret = PTR_ERR(mtt); 739 goto out; 740 } 741 upd_start_idx = min(upd_start_idx, idx); 742 upd_len = idx - upd_start_idx + 1; 743 } 744 745 umem_odp = to_ib_umem_odp(mtt->umem); 746 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - 747 user_va; 748 749 ret = pagefault_real_mr(mtt, umem_odp, user_va, len, 750 bytes_mapped, flags); 751 if (ret < 0) 752 goto out; 753 user_va += len; 754 bcnt -= len; 755 npages += ret; 756 } 757 758 ret = npages; 759 760 /* 761 * Any time the implicit_children are changed we must perform an 762 * update of the xlt before exiting to ensure the HW and the 763 * implicit_children remains synchronized. 764 */ 765 out: 766 if (likely(!upd_len)) 767 return ret; 768 769 /* 770 * Notice this is not strictly ordered right, the KSM is updated after 771 * the implicit_children is updated, so a parallel page fault could 772 * see a MR that is not yet visible in the KSM. This is similar to a 773 * parallel page fault seeing a MR that is being concurrently removed 774 * from the KSM. Both of these improbable situations are resolved 775 * safely by resuming the HW and then taking another page fault. The 776 * next pagefault handler will see the new information. 777 */ 778 mutex_lock(&odp_imr->umem_mutex); 779 err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0, 780 MLX5_IB_UPD_XLT_INDIRECT | 781 MLX5_IB_UPD_XLT_ATOMIC); 782 mutex_unlock(&odp_imr->umem_mutex); 783 if (err) { 784 mlx5_ib_err(imr->dev, "Failed to update PAS\n"); 785 return err; 786 } 787 return ret; 788 } 789 790 /* 791 * Returns: 792 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are 793 * not accessible, or the MR is no longer valid. 794 * -EAGAIN/-ENOMEM: The operation should be retried 795 * 796 * -EINVAL/others: General internal malfunction 797 * >0: Number of pages mapped 798 */ 799 static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, 800 u32 *bytes_mapped, u32 flags) 801 { 802 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 803 804 if (unlikely(io_virt < mr->mmkey.iova)) 805 return -EFAULT; 806 807 if (!odp->is_implicit_odp) { 808 u64 user_va; 809 810 if (check_add_overflow(io_virt - mr->mmkey.iova, 811 (u64)odp->umem.address, &user_va)) 812 return -EFAULT; 813 if (unlikely(user_va >= ib_umem_end(odp) || 814 ib_umem_end(odp) - user_va < bcnt)) 815 return -EFAULT; 816 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, 817 flags); 818 } 819 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, 820 flags); 821 } 822 823 struct pf_frame { 824 struct pf_frame *next; 825 u32 key; 826 u64 io_virt; 827 size_t bcnt; 828 int depth; 829 }; 830 831 static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key) 832 { 833 if (!mmkey) 834 return false; 835 if (mmkey->type == MLX5_MKEY_MW) 836 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 837 return mmkey->key == key; 838 } 839 840 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey) 841 { 842 struct mlx5_ib_mw *mw; 843 struct mlx5_ib_devx_mr *devx_mr; 844 845 if (mmkey->type == MLX5_MKEY_MW) { 846 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 847 return mw->ndescs; 848 } 849 850 devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr, 851 mmkey); 852 return devx_mr->ndescs; 853 } 854 855 /* 856 * Handle a single data segment in a page-fault WQE or RDMA region. 857 * 858 * Returns number of OS pages retrieved on success. The caller may continue to 859 * the next data segment. 860 * Can return the following error codes: 861 * -EAGAIN to designate a temporary error. The caller will abort handling the 862 * page fault and resolve it. 863 * -EFAULT when there's an error mapping the requested pages. The caller will 864 * abort the page fault handling. 865 */ 866 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 867 struct ib_pd *pd, u32 key, 868 u64 io_virt, size_t bcnt, 869 u32 *bytes_committed, 870 u32 *bytes_mapped) 871 { 872 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 873 struct pf_frame *head = NULL, *frame; 874 struct mlx5_core_mkey *mmkey; 875 struct mlx5_ib_mr *mr; 876 struct mlx5_klm *pklm; 877 u32 *out = NULL; 878 size_t offset; 879 int ndescs; 880 881 srcu_key = srcu_read_lock(&dev->odp_srcu); 882 883 io_virt += *bytes_committed; 884 bcnt -= *bytes_committed; 885 886 next_mr: 887 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); 888 if (!mmkey) { 889 mlx5_ib_dbg( 890 dev, 891 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 892 key); 893 if (bytes_mapped) 894 *bytes_mapped += bcnt; 895 /* 896 * The user could specify a SGL with multiple lkeys and only 897 * some of them are ODP. Treat the non-ODP ones as fully 898 * faulted. 899 */ 900 ret = 0; 901 goto srcu_unlock; 902 } 903 if (!mkey_is_eq(mmkey, key)) { 904 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 905 ret = -EFAULT; 906 goto srcu_unlock; 907 } 908 909 switch (mmkey->type) { 910 case MLX5_MKEY_MR: 911 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 912 913 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0); 914 if (ret < 0) 915 goto srcu_unlock; 916 917 /* 918 * When prefetching a page, page fault is generated 919 * in order to bring the page to the main memory. 920 * In the current flow, page faults are being counted. 921 */ 922 mlx5_update_odp_stats(mr, faults, ret); 923 924 npages += ret; 925 ret = 0; 926 break; 927 928 case MLX5_MKEY_MW: 929 case MLX5_MKEY_INDIRECT_DEVX: 930 ndescs = get_indirect_num_descs(mmkey); 931 932 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 933 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 934 ret = -EFAULT; 935 goto srcu_unlock; 936 } 937 938 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 939 sizeof(*pklm) * (ndescs - 2); 940 941 if (outlen > cur_outlen) { 942 kfree(out); 943 out = kzalloc(outlen, GFP_KERNEL); 944 if (!out) { 945 ret = -ENOMEM; 946 goto srcu_unlock; 947 } 948 cur_outlen = outlen; 949 } 950 951 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 952 bsf0_klm0_pas_mtt0_1); 953 954 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); 955 if (ret) 956 goto srcu_unlock; 957 958 offset = io_virt - MLX5_GET64(query_mkey_out, out, 959 memory_key_mkey_entry.start_addr); 960 961 for (i = 0; bcnt && i < ndescs; i++, pklm++) { 962 if (offset >= be32_to_cpu(pklm->bcount)) { 963 offset -= be32_to_cpu(pklm->bcount); 964 continue; 965 } 966 967 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 968 if (!frame) { 969 ret = -ENOMEM; 970 goto srcu_unlock; 971 } 972 973 frame->key = be32_to_cpu(pklm->key); 974 frame->io_virt = be64_to_cpu(pklm->va) + offset; 975 frame->bcnt = min_t(size_t, bcnt, 976 be32_to_cpu(pklm->bcount) - offset); 977 frame->depth = depth + 1; 978 frame->next = head; 979 head = frame; 980 981 bcnt -= frame->bcnt; 982 offset = 0; 983 } 984 break; 985 986 default: 987 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 988 ret = -EFAULT; 989 goto srcu_unlock; 990 } 991 992 if (head) { 993 frame = head; 994 head = frame->next; 995 996 key = frame->key; 997 io_virt = frame->io_virt; 998 bcnt = frame->bcnt; 999 depth = frame->depth; 1000 kfree(frame); 1001 1002 goto next_mr; 1003 } 1004 1005 srcu_unlock: 1006 while (head) { 1007 frame = head; 1008 head = frame->next; 1009 kfree(frame); 1010 } 1011 kfree(out); 1012 1013 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1014 *bytes_committed = 0; 1015 return ret ? ret : npages; 1016 } 1017 1018 /** 1019 * Parse a series of data segments for page fault handling. 1020 * 1021 * @pfault contains page fault information. 1022 * @wqe points at the first data segment in the WQE. 1023 * @wqe_end points after the end of the WQE. 1024 * @bytes_mapped receives the number of bytes that the function was able to 1025 * map. This allows the caller to decide intelligently whether 1026 * enough memory was mapped to resolve the page fault 1027 * successfully (e.g. enough for the next MTU, or the entire 1028 * WQE). 1029 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 1030 * the committed bytes). 1031 * 1032 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 1033 * negative error code. 1034 */ 1035 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 1036 struct mlx5_pagefault *pfault, 1037 void *wqe, 1038 void *wqe_end, u32 *bytes_mapped, 1039 u32 *total_wqe_bytes, bool receive_queue) 1040 { 1041 int ret = 0, npages = 0; 1042 u64 io_virt; 1043 u32 key; 1044 u32 byte_count; 1045 size_t bcnt; 1046 int inline_segment; 1047 1048 if (bytes_mapped) 1049 *bytes_mapped = 0; 1050 if (total_wqe_bytes) 1051 *total_wqe_bytes = 0; 1052 1053 while (wqe < wqe_end) { 1054 struct mlx5_wqe_data_seg *dseg = wqe; 1055 1056 io_virt = be64_to_cpu(dseg->addr); 1057 key = be32_to_cpu(dseg->lkey); 1058 byte_count = be32_to_cpu(dseg->byte_count); 1059 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1060 bcnt = byte_count & ~MLX5_INLINE_SEG; 1061 1062 if (inline_segment) { 1063 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 1064 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 1065 16); 1066 } else { 1067 wqe += sizeof(*dseg); 1068 } 1069 1070 /* receive WQE end of sg list. */ 1071 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 1072 io_virt == 0) 1073 break; 1074 1075 if (!inline_segment && total_wqe_bytes) { 1076 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 1077 pfault->bytes_committed); 1078 } 1079 1080 /* A zero length data segment designates a length of 2GB. */ 1081 if (bcnt == 0) 1082 bcnt = 1U << 31; 1083 1084 if (inline_segment || bcnt <= pfault->bytes_committed) { 1085 pfault->bytes_committed -= 1086 min_t(size_t, bcnt, 1087 pfault->bytes_committed); 1088 continue; 1089 } 1090 1091 ret = pagefault_single_data_segment(dev, NULL, key, 1092 io_virt, bcnt, 1093 &pfault->bytes_committed, 1094 bytes_mapped); 1095 if (ret < 0) 1096 break; 1097 npages += ret; 1098 } 1099 1100 return ret < 0 ? ret : npages; 1101 } 1102 1103 /* 1104 * Parse initiator WQE. Advances the wqe pointer to point at the 1105 * scatter-gather list, and set wqe_end to the end of the WQE. 1106 */ 1107 static int mlx5_ib_mr_initiator_pfault_handler( 1108 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1109 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1110 { 1111 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1112 u16 wqe_index = pfault->wqe.wqe_index; 1113 struct mlx5_base_av *av; 1114 unsigned ds, opcode; 1115 u32 qpn = qp->trans_qp.base.mqp.qpn; 1116 1117 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1118 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1119 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1120 ds, wqe_length); 1121 return -EFAULT; 1122 } 1123 1124 if (ds == 0) { 1125 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1126 wqe_index, qpn); 1127 return -EFAULT; 1128 } 1129 1130 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1131 *wqe += sizeof(*ctrl); 1132 1133 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1134 MLX5_WQE_CTRL_OPCODE_MASK; 1135 1136 if (qp->ibqp.qp_type == IB_QPT_XRC_INI) 1137 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1138 1139 if (qp->ibqp.qp_type == IB_QPT_UD || 1140 qp->qp_sub_type == MLX5_IB_QPT_DCI) { 1141 av = *wqe; 1142 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1143 *wqe += sizeof(struct mlx5_av); 1144 else 1145 *wqe += sizeof(struct mlx5_base_av); 1146 } 1147 1148 switch (opcode) { 1149 case MLX5_OPCODE_RDMA_WRITE: 1150 case MLX5_OPCODE_RDMA_WRITE_IMM: 1151 case MLX5_OPCODE_RDMA_READ: 1152 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1153 break; 1154 case MLX5_OPCODE_ATOMIC_CS: 1155 case MLX5_OPCODE_ATOMIC_FA: 1156 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1157 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1158 break; 1159 } 1160 1161 return 0; 1162 } 1163 1164 /* 1165 * Parse responder WQE and set wqe_end to the end of the WQE. 1166 */ 1167 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1168 struct mlx5_ib_srq *srq, 1169 void **wqe, void **wqe_end, 1170 int wqe_length) 1171 { 1172 int wqe_size = 1 << srq->msrq.wqe_shift; 1173 1174 if (wqe_size > wqe_length) { 1175 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1176 return -EFAULT; 1177 } 1178 1179 *wqe_end = *wqe + wqe_size; 1180 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1181 1182 return 0; 1183 } 1184 1185 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1186 struct mlx5_ib_qp *qp, 1187 void *wqe, void **wqe_end, 1188 int wqe_length) 1189 { 1190 struct mlx5_ib_wq *wq = &qp->rq; 1191 int wqe_size = 1 << wq->wqe_shift; 1192 1193 if (qp->wq_sig) { 1194 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1195 return -EFAULT; 1196 } 1197 1198 if (wqe_size > wqe_length) { 1199 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1200 return -EFAULT; 1201 } 1202 1203 *wqe_end = wqe + wqe_size; 1204 1205 return 0; 1206 } 1207 1208 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1209 u32 wq_num, int pf_type) 1210 { 1211 struct mlx5_core_rsc_common *common = NULL; 1212 struct mlx5_core_srq *srq; 1213 1214 switch (pf_type) { 1215 case MLX5_WQE_PF_TYPE_RMP: 1216 srq = mlx5_cmd_get_srq(dev, wq_num); 1217 if (srq) 1218 common = &srq->common; 1219 break; 1220 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1221 case MLX5_WQE_PF_TYPE_RESP: 1222 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1223 common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP); 1224 break; 1225 default: 1226 break; 1227 } 1228 1229 return common; 1230 } 1231 1232 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1233 { 1234 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1235 1236 return to_mibqp(mqp); 1237 } 1238 1239 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1240 { 1241 struct mlx5_core_srq *msrq = 1242 container_of(res, struct mlx5_core_srq, common); 1243 1244 return to_mibsrq(msrq); 1245 } 1246 1247 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1248 struct mlx5_pagefault *pfault) 1249 { 1250 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1251 u16 wqe_index = pfault->wqe.wqe_index; 1252 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1253 u32 bytes_mapped, total_wqe_bytes; 1254 struct mlx5_core_rsc_common *res; 1255 int resume_with_error = 1; 1256 struct mlx5_ib_qp *qp; 1257 size_t bytes_copied; 1258 int ret = 0; 1259 1260 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1261 if (!res) { 1262 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1263 return; 1264 } 1265 1266 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1267 res->res != MLX5_RES_XSRQ) { 1268 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1269 pfault->type); 1270 goto resolve_page_fault; 1271 } 1272 1273 wqe_start = (void *)__get_free_page(GFP_KERNEL); 1274 if (!wqe_start) { 1275 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1276 goto resolve_page_fault; 1277 } 1278 1279 wqe = wqe_start; 1280 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1281 if (qp && sq) { 1282 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1283 &bytes_copied); 1284 if (ret) 1285 goto read_user; 1286 ret = mlx5_ib_mr_initiator_pfault_handler( 1287 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1288 } else if (qp && !sq) { 1289 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1290 &bytes_copied); 1291 if (ret) 1292 goto read_user; 1293 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1294 dev, qp, wqe, &wqe_end, bytes_copied); 1295 } else if (!qp) { 1296 struct mlx5_ib_srq *srq = res_to_srq(res); 1297 1298 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1299 &bytes_copied); 1300 if (ret) 1301 goto read_user; 1302 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1303 dev, srq, &wqe, &wqe_end, bytes_copied); 1304 } 1305 1306 if (ret < 0 || wqe >= wqe_end) 1307 goto resolve_page_fault; 1308 1309 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1310 &total_wqe_bytes, !sq); 1311 if (ret == -EAGAIN) 1312 goto out; 1313 1314 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1315 goto resolve_page_fault; 1316 1317 out: 1318 ret = 0; 1319 resume_with_error = 0; 1320 1321 read_user: 1322 if (ret) 1323 mlx5_ib_err( 1324 dev, 1325 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n", 1326 ret, wqe_index, pfault->token); 1327 1328 resolve_page_fault: 1329 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1330 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1331 pfault->wqe.wq_num, resume_with_error, 1332 pfault->type); 1333 mlx5_core_res_put(res); 1334 free_page((unsigned long)wqe_start); 1335 } 1336 1337 static int pages_in_range(u64 address, u32 length) 1338 { 1339 return (ALIGN(address + length, PAGE_SIZE) - 1340 (address & PAGE_MASK)) >> PAGE_SHIFT; 1341 } 1342 1343 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1344 struct mlx5_pagefault *pfault) 1345 { 1346 u64 address; 1347 u32 length; 1348 u32 prefetch_len = pfault->bytes_committed; 1349 int prefetch_activated = 0; 1350 u32 rkey = pfault->rdma.r_key; 1351 int ret; 1352 1353 /* The RDMA responder handler handles the page fault in two parts. 1354 * First it brings the necessary pages for the current packet 1355 * (and uses the pfault context), and then (after resuming the QP) 1356 * prefetches more pages. The second operation cannot use the pfault 1357 * context and therefore uses the dummy_pfault context allocated on 1358 * the stack */ 1359 pfault->rdma.rdma_va += pfault->bytes_committed; 1360 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1361 pfault->rdma.rdma_op_len); 1362 pfault->bytes_committed = 0; 1363 1364 address = pfault->rdma.rdma_va; 1365 length = pfault->rdma.rdma_op_len; 1366 1367 /* For some operations, the hardware cannot tell the exact message 1368 * length, and in those cases it reports zero. Use prefetch 1369 * logic. */ 1370 if (length == 0) { 1371 prefetch_activated = 1; 1372 length = pfault->rdma.packet_size; 1373 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1374 } 1375 1376 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1377 &pfault->bytes_committed, NULL); 1378 if (ret == -EAGAIN) { 1379 /* We're racing with an invalidation, don't prefetch */ 1380 prefetch_activated = 0; 1381 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1382 mlx5_ib_page_fault_resume(dev, pfault, 1); 1383 if (ret != -ENOENT) 1384 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1385 ret, pfault->token, pfault->type); 1386 return; 1387 } 1388 1389 mlx5_ib_page_fault_resume(dev, pfault, 0); 1390 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1391 pfault->token, pfault->type, 1392 prefetch_activated); 1393 1394 /* At this point, there might be a new pagefault already arriving in 1395 * the eq, switch to the dummy pagefault for the rest of the 1396 * processing. We're still OK with the objects being alive as the 1397 * work-queue is being fenced. */ 1398 1399 if (prefetch_activated) { 1400 u32 bytes_committed = 0; 1401 1402 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1403 prefetch_len, 1404 &bytes_committed, NULL); 1405 if (ret < 0 && ret != -EAGAIN) { 1406 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1407 ret, pfault->token, address, prefetch_len); 1408 } 1409 } 1410 } 1411 1412 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1413 { 1414 u8 event_subtype = pfault->event_subtype; 1415 1416 switch (event_subtype) { 1417 case MLX5_PFAULT_SUBTYPE_WQE: 1418 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1419 break; 1420 case MLX5_PFAULT_SUBTYPE_RDMA: 1421 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1422 break; 1423 default: 1424 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1425 event_subtype); 1426 mlx5_ib_page_fault_resume(dev, pfault, 1); 1427 } 1428 } 1429 1430 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1431 { 1432 struct mlx5_pagefault *pfault = container_of(work, 1433 struct mlx5_pagefault, 1434 work); 1435 struct mlx5_ib_pf_eq *eq = pfault->eq; 1436 1437 mlx5_ib_pfault(eq->dev, pfault); 1438 mempool_free(pfault, eq->pool); 1439 } 1440 1441 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1442 { 1443 struct mlx5_eqe_page_fault *pf_eqe; 1444 struct mlx5_pagefault *pfault; 1445 struct mlx5_eqe *eqe; 1446 int cc = 0; 1447 1448 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1449 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1450 if (!pfault) { 1451 schedule_work(&eq->work); 1452 break; 1453 } 1454 1455 pf_eqe = &eqe->data.page_fault; 1456 pfault->event_subtype = eqe->sub_type; 1457 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); 1458 1459 mlx5_ib_dbg(eq->dev, 1460 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", 1461 eqe->sub_type, pfault->bytes_committed); 1462 1463 switch (eqe->sub_type) { 1464 case MLX5_PFAULT_SUBTYPE_RDMA: 1465 /* RDMA based event */ 1466 pfault->type = 1467 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1468 pfault->token = 1469 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1470 MLX5_24BIT_MASK; 1471 pfault->rdma.r_key = 1472 be32_to_cpu(pf_eqe->rdma.r_key); 1473 pfault->rdma.packet_size = 1474 be16_to_cpu(pf_eqe->rdma.packet_length); 1475 pfault->rdma.rdma_op_len = 1476 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1477 pfault->rdma.rdma_va = 1478 be64_to_cpu(pf_eqe->rdma.rdma_va); 1479 mlx5_ib_dbg(eq->dev, 1480 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", 1481 pfault->type, pfault->token, 1482 pfault->rdma.r_key); 1483 mlx5_ib_dbg(eq->dev, 1484 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1485 pfault->rdma.rdma_op_len, 1486 pfault->rdma.rdma_va); 1487 break; 1488 1489 case MLX5_PFAULT_SUBTYPE_WQE: 1490 /* WQE based event */ 1491 pfault->type = 1492 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1493 pfault->token = 1494 be32_to_cpu(pf_eqe->wqe.token); 1495 pfault->wqe.wq_num = 1496 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1497 MLX5_24BIT_MASK; 1498 pfault->wqe.wqe_index = 1499 be16_to_cpu(pf_eqe->wqe.wqe_index); 1500 pfault->wqe.packet_size = 1501 be16_to_cpu(pf_eqe->wqe.packet_length); 1502 mlx5_ib_dbg(eq->dev, 1503 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1504 pfault->type, pfault->token, 1505 pfault->wqe.wq_num, 1506 pfault->wqe.wqe_index); 1507 break; 1508 1509 default: 1510 mlx5_ib_warn(eq->dev, 1511 "Unsupported page fault event sub-type: 0x%02hhx\n", 1512 eqe->sub_type); 1513 /* Unsupported page faults should still be 1514 * resolved by the page fault handler 1515 */ 1516 } 1517 1518 pfault->eq = eq; 1519 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1520 queue_work(eq->wq, &pfault->work); 1521 1522 cc = mlx5_eq_update_cc(eq->core, ++cc); 1523 } 1524 1525 mlx5_eq_update_ci(eq->core, cc, 1); 1526 } 1527 1528 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1529 void *data) 1530 { 1531 struct mlx5_ib_pf_eq *eq = 1532 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1533 unsigned long flags; 1534 1535 if (spin_trylock_irqsave(&eq->lock, flags)) { 1536 mlx5_ib_eq_pf_process(eq); 1537 spin_unlock_irqrestore(&eq->lock, flags); 1538 } else { 1539 schedule_work(&eq->work); 1540 } 1541 1542 return IRQ_HANDLED; 1543 } 1544 1545 /* mempool_refill() was proposed but unfortunately wasn't accepted 1546 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1547 * Cheap workaround. 1548 */ 1549 static void mempool_refill(mempool_t *pool) 1550 { 1551 while (pool->curr_nr < pool->min_nr) 1552 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1553 } 1554 1555 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1556 { 1557 struct mlx5_ib_pf_eq *eq = 1558 container_of(work, struct mlx5_ib_pf_eq, work); 1559 1560 mempool_refill(eq->pool); 1561 1562 spin_lock_irq(&eq->lock); 1563 mlx5_ib_eq_pf_process(eq); 1564 spin_unlock_irq(&eq->lock); 1565 } 1566 1567 enum { 1568 MLX5_IB_NUM_PF_EQE = 0x1000, 1569 MLX5_IB_NUM_PF_DRAIN = 64, 1570 }; 1571 1572 static int 1573 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1574 { 1575 struct mlx5_eq_param param = {}; 1576 int err; 1577 1578 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1579 spin_lock_init(&eq->lock); 1580 eq->dev = dev; 1581 1582 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1583 sizeof(struct mlx5_pagefault)); 1584 if (!eq->pool) 1585 return -ENOMEM; 1586 1587 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1588 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1589 MLX5_NUM_CMD_EQE); 1590 if (!eq->wq) { 1591 err = -ENOMEM; 1592 goto err_mempool; 1593 } 1594 1595 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1596 param = (struct mlx5_eq_param) { 1597 .irq_index = 0, 1598 .nent = MLX5_IB_NUM_PF_EQE, 1599 }; 1600 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1601 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1602 if (IS_ERR(eq->core)) { 1603 err = PTR_ERR(eq->core); 1604 goto err_wq; 1605 } 1606 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1607 if (err) { 1608 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1609 goto err_eq; 1610 } 1611 1612 return 0; 1613 err_eq: 1614 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1615 err_wq: 1616 destroy_workqueue(eq->wq); 1617 err_mempool: 1618 mempool_destroy(eq->pool); 1619 return err; 1620 } 1621 1622 static int 1623 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1624 { 1625 int err; 1626 1627 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1628 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1629 cancel_work_sync(&eq->work); 1630 destroy_workqueue(eq->wq); 1631 mempool_destroy(eq->pool); 1632 1633 return err; 1634 } 1635 1636 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1637 { 1638 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1639 return; 1640 1641 switch (ent->order - 2) { 1642 case MLX5_IMR_MTT_CACHE_ENTRY: 1643 ent->page = PAGE_SHIFT; 1644 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1645 sizeof(struct mlx5_mtt) / 1646 MLX5_IB_UMR_OCTOWORD; 1647 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1648 ent->limit = 0; 1649 break; 1650 1651 case MLX5_IMR_KSM_CACHE_ENTRY: 1652 ent->page = MLX5_KSM_PAGE_SHIFT; 1653 ent->xlt = mlx5_imr_ksm_entries * 1654 sizeof(struct mlx5_klm) / 1655 MLX5_IB_UMR_OCTOWORD; 1656 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1657 ent->limit = 0; 1658 break; 1659 } 1660 } 1661 1662 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1663 .advise_mr = mlx5_ib_advise_mr, 1664 }; 1665 1666 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1667 { 1668 int ret = 0; 1669 1670 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1671 return ret; 1672 1673 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1674 1675 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1676 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1677 if (ret) { 1678 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1679 return ret; 1680 } 1681 } 1682 1683 ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1684 1685 return ret; 1686 } 1687 1688 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1689 { 1690 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1691 return; 1692 1693 mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1694 } 1695 1696 int mlx5_ib_odp_init(void) 1697 { 1698 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1699 MLX5_IMR_MTT_BITS); 1700 1701 return 0; 1702 } 1703 1704 struct prefetch_mr_work { 1705 struct work_struct work; 1706 u32 pf_flags; 1707 u32 num_sge; 1708 struct { 1709 u64 io_virt; 1710 struct mlx5_ib_mr *mr; 1711 size_t length; 1712 } frags[]; 1713 }; 1714 1715 static void destroy_prefetch_work(struct prefetch_mr_work *work) 1716 { 1717 u32 i; 1718 1719 for (i = 0; i < work->num_sge; ++i) 1720 if (atomic_dec_and_test(&work->frags[i].mr->num_deferred_work)) 1721 wake_up(&work->frags[i].mr->q_deferred_work); 1722 kvfree(work); 1723 } 1724 1725 static struct mlx5_ib_mr * 1726 get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, 1727 u32 lkey) 1728 { 1729 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1730 struct mlx5_core_mkey *mmkey; 1731 struct ib_umem_odp *odp; 1732 struct mlx5_ib_mr *mr; 1733 1734 lockdep_assert_held(&dev->odp_srcu); 1735 1736 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); 1737 if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR) 1738 return NULL; 1739 1740 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1741 1742 if (mr->ibmr.pd != pd) 1743 return NULL; 1744 1745 odp = to_ib_umem_odp(mr->umem); 1746 1747 /* prefetch with write-access must be supported by the MR */ 1748 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1749 !odp->umem.writable) 1750 return NULL; 1751 1752 return mr; 1753 } 1754 1755 static void mlx5_ib_prefetch_mr_work(struct work_struct *w) 1756 { 1757 struct prefetch_mr_work *work = 1758 container_of(w, struct prefetch_mr_work, work); 1759 u32 bytes_mapped = 0; 1760 u32 i; 1761 1762 for (i = 0; i < work->num_sge; ++i) 1763 pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, 1764 work->frags[i].length, &bytes_mapped, 1765 work->pf_flags); 1766 1767 destroy_prefetch_work(work); 1768 } 1769 1770 static bool init_prefetch_work(struct ib_pd *pd, 1771 enum ib_uverbs_advise_mr_advice advice, 1772 u32 pf_flags, struct prefetch_mr_work *work, 1773 struct ib_sge *sg_list, u32 num_sge) 1774 { 1775 u32 i; 1776 1777 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 1778 work->pf_flags = pf_flags; 1779 1780 for (i = 0; i < num_sge; ++i) { 1781 work->frags[i].io_virt = sg_list[i].addr; 1782 work->frags[i].length = sg_list[i].length; 1783 work->frags[i].mr = 1784 get_prefetchable_mr(pd, advice, sg_list[i].lkey); 1785 if (!work->frags[i].mr) { 1786 work->num_sge = i - 1; 1787 if (i) 1788 destroy_prefetch_work(work); 1789 return false; 1790 } 1791 1792 /* Keep the MR pointer will valid outside the SRCU */ 1793 atomic_inc(&work->frags[i].mr->num_deferred_work); 1794 } 1795 work->num_sge = num_sge; 1796 return true; 1797 } 1798 1799 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, 1800 enum ib_uverbs_advise_mr_advice advice, 1801 u32 pf_flags, struct ib_sge *sg_list, 1802 u32 num_sge) 1803 { 1804 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1805 u32 bytes_mapped = 0; 1806 int srcu_key; 1807 int ret = 0; 1808 u32 i; 1809 1810 srcu_key = srcu_read_lock(&dev->odp_srcu); 1811 for (i = 0; i < num_sge; ++i) { 1812 struct mlx5_ib_mr *mr; 1813 1814 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 1815 if (!mr) { 1816 ret = -ENOENT; 1817 goto out; 1818 } 1819 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, 1820 &bytes_mapped, pf_flags); 1821 if (ret < 0) 1822 goto out; 1823 } 1824 ret = 0; 1825 1826 out: 1827 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1828 return ret; 1829 } 1830 1831 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1832 enum ib_uverbs_advise_mr_advice advice, 1833 u32 flags, struct ib_sge *sg_list, u32 num_sge) 1834 { 1835 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1836 u32 pf_flags = 0; 1837 struct prefetch_mr_work *work; 1838 int srcu_key; 1839 1840 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 1841 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 1842 1843 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 1844 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, 1845 num_sge); 1846 1847 work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL); 1848 if (!work) 1849 return -ENOMEM; 1850 1851 srcu_key = srcu_read_lock(&dev->odp_srcu); 1852 if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) { 1853 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1854 return -EINVAL; 1855 } 1856 queue_work(system_unbound_wq, &work->work); 1857 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1858 return 0; 1859 } 1860