1 /* 2 * Copyright (c) 2015 Oracle. All rights reserved. 3 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. 4 */ 5 6 /* Lightweight memory registration using Fast Registration Work 7 * Requests (FRWR). Also referred to sometimes as FRMR mode. 8 * 9 * FRWR features ordered asynchronous registration and deregistration 10 * of arbitrarily sized memory regions. This is the fastest and safest 11 * but most complex memory registration mode. 12 */ 13 14 /* Normal operation 15 * 16 * A Memory Region is prepared for RDMA READ or WRITE using a FAST_REG 17 * Work Request (frmr_op_map). When the RDMA operation is finished, this 18 * Memory Region is invalidated using a LOCAL_INV Work Request 19 * (frmr_op_unmap). 20 * 21 * Typically these Work Requests are not signaled, and neither are RDMA 22 * SEND Work Requests (with the exception of signaling occasionally to 23 * prevent provider work queue overflows). This greatly reduces HCA 24 * interrupt workload. 25 * 26 * As an optimization, frwr_op_unmap marks MRs INVALID before the 27 * LOCAL_INV WR is posted. If posting succeeds, the MR is placed on 28 * rb_mws immediately so that no work (like managing a linked list 29 * under a spinlock) is needed in the completion upcall. 30 * 31 * But this means that frwr_op_map() can occasionally encounter an MR 32 * that is INVALID but the LOCAL_INV WR has not completed. Work Queue 33 * ordering prevents a subsequent FAST_REG WR from executing against 34 * that MR while it is still being invalidated. 35 */ 36 37 /* Transport recovery 38 * 39 * ->op_map and the transport connect worker cannot run at the same 40 * time, but ->op_unmap can fire while the transport connect worker 41 * is running. Thus MR recovery is handled in ->op_map, to guarantee 42 * that recovered MRs are owned by a sending RPC, and not one where 43 * ->op_unmap could fire at the same time transport reconnect is 44 * being done. 45 * 46 * When the underlying transport disconnects, MRs are left in one of 47 * four states: 48 * 49 * INVALID: The MR was not in use before the QP entered ERROR state. 50 * 51 * VALID: The MR was registered before the QP entered ERROR state. 52 * 53 * FLUSHED_FR: The MR was being registered when the QP entered ERROR 54 * state, and the pending WR was flushed. 55 * 56 * FLUSHED_LI: The MR was being invalidated when the QP entered ERROR 57 * state, and the pending WR was flushed. 58 * 59 * When frwr_op_map encounters FLUSHED and VALID MRs, they are recovered 60 * with ib_dereg_mr and then are re-initialized. Because MR recovery 61 * allocates fresh resources, it is deferred to a workqueue, and the 62 * recovered MRs are placed back on the rb_mws list when recovery is 63 * complete. frwr_op_map allocates another MR for the current RPC while 64 * the broken MR is reset. 65 * 66 * To ensure that frwr_op_map doesn't encounter an MR that is marked 67 * INVALID but that is about to be flushed due to a previous transport 68 * disconnect, the transport connect worker attempts to drain all 69 * pending send queue WRs before the transport is reconnected. 70 */ 71 72 #include <linux/sunrpc/rpc_rdma.h> 73 74 #include "xprt_rdma.h" 75 76 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 77 # define RPCDBG_FACILITY RPCDBG_TRANS 78 #endif 79 80 bool 81 frwr_is_supported(struct rpcrdma_ia *ia) 82 { 83 struct ib_device_attr *attrs = &ia->ri_device->attrs; 84 85 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) 86 goto out_not_supported; 87 if (attrs->max_fast_reg_page_list_len == 0) 88 goto out_not_supported; 89 return true; 90 91 out_not_supported: 92 pr_info("rpcrdma: 'frwr' mode is not supported by device %s\n", 93 ia->ri_device->name); 94 return false; 95 } 96 97 static int 98 frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) 99 { 100 unsigned int depth = ia->ri_max_frmr_depth; 101 struct rpcrdma_frmr *f = &r->frmr; 102 int rc; 103 104 f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth); 105 if (IS_ERR(f->fr_mr)) 106 goto out_mr_err; 107 108 r->mw_sg = kcalloc(depth, sizeof(*r->mw_sg), GFP_KERNEL); 109 if (!r->mw_sg) 110 goto out_list_err; 111 112 sg_init_table(r->mw_sg, depth); 113 init_completion(&f->fr_linv_done); 114 return 0; 115 116 out_mr_err: 117 rc = PTR_ERR(f->fr_mr); 118 dprintk("RPC: %s: ib_alloc_mr status %i\n", 119 __func__, rc); 120 return rc; 121 122 out_list_err: 123 rc = -ENOMEM; 124 dprintk("RPC: %s: sg allocation failure\n", 125 __func__); 126 ib_dereg_mr(f->fr_mr); 127 return rc; 128 } 129 130 static void 131 frwr_op_release_mr(struct rpcrdma_mw *r) 132 { 133 int rc; 134 135 /* Ensure MW is not on any rl_registered list */ 136 if (!list_empty(&r->mw_list)) 137 list_del(&r->mw_list); 138 139 rc = ib_dereg_mr(r->frmr.fr_mr); 140 if (rc) 141 pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n", 142 r, rc); 143 kfree(r->mw_sg); 144 kfree(r); 145 } 146 147 static int 148 __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) 149 { 150 struct rpcrdma_frmr *f = &r->frmr; 151 int rc; 152 153 rc = ib_dereg_mr(f->fr_mr); 154 if (rc) { 155 pr_warn("rpcrdma: ib_dereg_mr status %d, frwr %p orphaned\n", 156 rc, r); 157 return rc; 158 } 159 160 f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, 161 ia->ri_max_frmr_depth); 162 if (IS_ERR(f->fr_mr)) { 163 pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n", 164 PTR_ERR(f->fr_mr), r); 165 return PTR_ERR(f->fr_mr); 166 } 167 168 dprintk("RPC: %s: recovered FRMR %p\n", __func__, f); 169 f->fr_state = FRMR_IS_INVALID; 170 return 0; 171 } 172 173 /* Reset of a single FRMR. Generate a fresh rkey by replacing the MR. 174 */ 175 static void 176 frwr_op_recover_mr(struct rpcrdma_mw *mw) 177 { 178 enum rpcrdma_frmr_state state = mw->frmr.fr_state; 179 struct rpcrdma_xprt *r_xprt = mw->mw_xprt; 180 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 181 int rc; 182 183 rc = __frwr_reset_mr(ia, mw); 184 if (state != FRMR_FLUSHED_LI) 185 ib_dma_unmap_sg(ia->ri_device, 186 mw->mw_sg, mw->mw_nents, mw->mw_dir); 187 if (rc) 188 goto out_release; 189 190 rpcrdma_put_mw(r_xprt, mw); 191 r_xprt->rx_stats.mrs_recovered++; 192 return; 193 194 out_release: 195 pr_err("rpcrdma: FRMR reset failed %d, %p release\n", rc, mw); 196 r_xprt->rx_stats.mrs_orphaned++; 197 198 spin_lock(&r_xprt->rx_buf.rb_mwlock); 199 list_del(&mw->mw_all); 200 spin_unlock(&r_xprt->rx_buf.rb_mwlock); 201 202 frwr_op_release_mr(mw); 203 } 204 205 static int 206 frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 207 struct rpcrdma_create_data_internal *cdata) 208 { 209 struct ib_device_attr *attrs = &ia->ri_device->attrs; 210 int depth, delta; 211 212 ia->ri_mrtype = IB_MR_TYPE_MEM_REG; 213 if (attrs->device_cap_flags & IB_DEVICE_SG_GAPS_REG) 214 ia->ri_mrtype = IB_MR_TYPE_SG_GAPS; 215 216 ia->ri_max_frmr_depth = 217 min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, 218 attrs->max_fast_reg_page_list_len); 219 dprintk("RPC: %s: device's max FR page list len = %u\n", 220 __func__, ia->ri_max_frmr_depth); 221 222 /* Add room for frmr register and invalidate WRs. 223 * 1. FRMR reg WR for head 224 * 2. FRMR invalidate WR for head 225 * 3. N FRMR reg WRs for pagelist 226 * 4. N FRMR invalidate WRs for pagelist 227 * 5. FRMR reg WR for tail 228 * 6. FRMR invalidate WR for tail 229 * 7. The RDMA_SEND WR 230 */ 231 depth = 7; 232 233 /* Calculate N if the device max FRMR depth is smaller than 234 * RPCRDMA_MAX_DATA_SEGS. 235 */ 236 if (ia->ri_max_frmr_depth < RPCRDMA_MAX_DATA_SEGS) { 237 delta = RPCRDMA_MAX_DATA_SEGS - ia->ri_max_frmr_depth; 238 do { 239 depth += 2; /* FRMR reg + invalidate */ 240 delta -= ia->ri_max_frmr_depth; 241 } while (delta > 0); 242 } 243 244 ep->rep_attr.cap.max_send_wr *= depth; 245 if (ep->rep_attr.cap.max_send_wr > attrs->max_qp_wr) { 246 cdata->max_requests = attrs->max_qp_wr / depth; 247 if (!cdata->max_requests) 248 return -EINVAL; 249 ep->rep_attr.cap.max_send_wr = cdata->max_requests * 250 depth; 251 } 252 253 ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS / 254 ia->ri_max_frmr_depth); 255 return 0; 256 } 257 258 /* FRWR mode conveys a list of pages per chunk segment. The 259 * maximum length of that list is the FRWR page list depth. 260 */ 261 static size_t 262 frwr_op_maxpages(struct rpcrdma_xprt *r_xprt) 263 { 264 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 265 266 return min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, 267 RPCRDMA_MAX_HDR_SEGS * ia->ri_max_frmr_depth); 268 } 269 270 static void 271 __frwr_sendcompletion_flush(struct ib_wc *wc, const char *wr) 272 { 273 if (wc->status != IB_WC_WR_FLUSH_ERR) 274 pr_err("rpcrdma: %s: %s (%u/0x%x)\n", 275 wr, ib_wc_status_msg(wc->status), 276 wc->status, wc->vendor_err); 277 } 278 279 /** 280 * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC 281 * @cq: completion queue (ignored) 282 * @wc: completed WR 283 * 284 */ 285 static void 286 frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) 287 { 288 struct rpcrdma_frmr *frmr; 289 struct ib_cqe *cqe; 290 291 /* WARNING: Only wr_cqe and status are reliable at this point */ 292 if (wc->status != IB_WC_SUCCESS) { 293 cqe = wc->wr_cqe; 294 frmr = container_of(cqe, struct rpcrdma_frmr, fr_cqe); 295 frmr->fr_state = FRMR_FLUSHED_FR; 296 __frwr_sendcompletion_flush(wc, "fastreg"); 297 } 298 } 299 300 /** 301 * frwr_wc_localinv - Invoked by RDMA provider for a flushed LocalInv WC 302 * @cq: completion queue (ignored) 303 * @wc: completed WR 304 * 305 */ 306 static void 307 frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) 308 { 309 struct rpcrdma_frmr *frmr; 310 struct ib_cqe *cqe; 311 312 /* WARNING: Only wr_cqe and status are reliable at this point */ 313 if (wc->status != IB_WC_SUCCESS) { 314 cqe = wc->wr_cqe; 315 frmr = container_of(cqe, struct rpcrdma_frmr, fr_cqe); 316 frmr->fr_state = FRMR_FLUSHED_LI; 317 __frwr_sendcompletion_flush(wc, "localinv"); 318 } 319 } 320 321 /** 322 * frwr_wc_localinv_wake - Invoked by RDMA provider for a signaled LocalInv WC 323 * @cq: completion queue (ignored) 324 * @wc: completed WR 325 * 326 * Awaken anyone waiting for an MR to finish being fenced. 327 */ 328 static void 329 frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) 330 { 331 struct rpcrdma_frmr *frmr; 332 struct ib_cqe *cqe; 333 334 /* WARNING: Only wr_cqe and status are reliable at this point */ 335 cqe = wc->wr_cqe; 336 frmr = container_of(cqe, struct rpcrdma_frmr, fr_cqe); 337 if (wc->status != IB_WC_SUCCESS) { 338 frmr->fr_state = FRMR_FLUSHED_LI; 339 __frwr_sendcompletion_flush(wc, "localinv"); 340 } 341 complete(&frmr->fr_linv_done); 342 } 343 344 /* Post a REG_MR Work Request to register a memory region 345 * for remote access via RDMA READ or RDMA WRITE. 346 */ 347 static struct rpcrdma_mr_seg * 348 frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 349 int nsegs, bool writing, struct rpcrdma_mw **out) 350 { 351 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 352 bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS; 353 struct rpcrdma_mw *mw; 354 struct rpcrdma_frmr *frmr; 355 struct ib_mr *mr; 356 struct ib_reg_wr *reg_wr; 357 struct ib_send_wr *bad_wr; 358 int rc, i, n; 359 u8 key; 360 361 mw = NULL; 362 do { 363 if (mw) 364 rpcrdma_defer_mr_recovery(mw); 365 mw = rpcrdma_get_mw(r_xprt); 366 if (!mw) 367 return ERR_PTR(-ENOBUFS); 368 } while (mw->frmr.fr_state != FRMR_IS_INVALID); 369 frmr = &mw->frmr; 370 frmr->fr_state = FRMR_IS_VALID; 371 mr = frmr->fr_mr; 372 reg_wr = &frmr->fr_regwr; 373 374 if (nsegs > ia->ri_max_frmr_depth) 375 nsegs = ia->ri_max_frmr_depth; 376 for (i = 0; i < nsegs;) { 377 if (seg->mr_page) 378 sg_set_page(&mw->mw_sg[i], 379 seg->mr_page, 380 seg->mr_len, 381 offset_in_page(seg->mr_offset)); 382 else 383 sg_set_buf(&mw->mw_sg[i], seg->mr_offset, 384 seg->mr_len); 385 386 ++seg; 387 ++i; 388 if (holes_ok) 389 continue; 390 if ((i < nsegs && offset_in_page(seg->mr_offset)) || 391 offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 392 break; 393 } 394 mw->mw_dir = rpcrdma_data_dir(writing); 395 396 mw->mw_nents = ib_dma_map_sg(ia->ri_device, mw->mw_sg, i, mw->mw_dir); 397 if (!mw->mw_nents) 398 goto out_dmamap_err; 399 400 n = ib_map_mr_sg(mr, mw->mw_sg, mw->mw_nents, NULL, PAGE_SIZE); 401 if (unlikely(n != mw->mw_nents)) 402 goto out_mapmr_err; 403 404 dprintk("RPC: %s: Using frmr %p to map %u segments (%llu bytes)\n", 405 __func__, frmr, mw->mw_nents, mr->length); 406 407 key = (u8)(mr->rkey & 0x000000FF); 408 ib_update_fast_reg_key(mr, ++key); 409 410 reg_wr->wr.next = NULL; 411 reg_wr->wr.opcode = IB_WR_REG_MR; 412 frmr->fr_cqe.done = frwr_wc_fastreg; 413 reg_wr->wr.wr_cqe = &frmr->fr_cqe; 414 reg_wr->wr.num_sge = 0; 415 reg_wr->wr.send_flags = 0; 416 reg_wr->mr = mr; 417 reg_wr->key = mr->rkey; 418 reg_wr->access = writing ? 419 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : 420 IB_ACCESS_REMOTE_READ; 421 422 rpcrdma_set_signaled(&r_xprt->rx_ep, ®_wr->wr); 423 rc = ib_post_send(ia->ri_id->qp, ®_wr->wr, &bad_wr); 424 if (rc) 425 goto out_senderr; 426 427 mw->mw_handle = mr->rkey; 428 mw->mw_length = mr->length; 429 mw->mw_offset = mr->iova; 430 431 *out = mw; 432 return seg; 433 434 out_dmamap_err: 435 pr_err("rpcrdma: failed to DMA map sg %p sg_nents %d\n", 436 mw->mw_sg, i); 437 frmr->fr_state = FRMR_IS_INVALID; 438 rpcrdma_put_mw(r_xprt, mw); 439 return ERR_PTR(-EIO); 440 441 out_mapmr_err: 442 pr_err("rpcrdma: failed to map mr %p (%d/%d)\n", 443 frmr->fr_mr, n, mw->mw_nents); 444 rpcrdma_defer_mr_recovery(mw); 445 return ERR_PTR(-EIO); 446 447 out_senderr: 448 pr_err("rpcrdma: FRMR registration ib_post_send returned %i\n", rc); 449 rpcrdma_defer_mr_recovery(mw); 450 return ERR_PTR(-ENOTCONN); 451 } 452 453 /* Invalidate all memory regions that were registered for "req". 454 * 455 * Sleeps until it is safe for the host CPU to access the 456 * previously mapped memory regions. 457 * 458 * Caller ensures that @mws is not empty before the call. This 459 * function empties the list. 460 */ 461 static void 462 frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 463 { 464 struct ib_send_wr *first, **prev, *last, *bad_wr; 465 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 466 struct rpcrdma_frmr *f; 467 struct rpcrdma_mw *mw; 468 int count, rc; 469 470 /* ORDER: Invalidate all of the MRs first 471 * 472 * Chain the LOCAL_INV Work Requests and post them with 473 * a single ib_post_send() call. 474 */ 475 f = NULL; 476 count = 0; 477 prev = &first; 478 list_for_each_entry(mw, mws, mw_list) { 479 mw->frmr.fr_state = FRMR_IS_INVALID; 480 481 if (mw->mw_flags & RPCRDMA_MW_F_RI) 482 continue; 483 484 f = &mw->frmr; 485 dprintk("RPC: %s: invalidating frmr %p\n", 486 __func__, f); 487 488 f->fr_cqe.done = frwr_wc_localinv; 489 last = &f->fr_invwr; 490 memset(last, 0, sizeof(*last)); 491 last->wr_cqe = &f->fr_cqe; 492 last->opcode = IB_WR_LOCAL_INV; 493 last->ex.invalidate_rkey = mw->mw_handle; 494 count++; 495 496 *prev = last; 497 prev = &last->next; 498 } 499 if (!f) 500 goto unmap; 501 502 /* Strong send queue ordering guarantees that when the 503 * last WR in the chain completes, all WRs in the chain 504 * are complete. 505 */ 506 last->send_flags = IB_SEND_SIGNALED; 507 f->fr_cqe.done = frwr_wc_localinv_wake; 508 reinit_completion(&f->fr_linv_done); 509 510 /* Initialize CQ count, since there is always a signaled 511 * WR being posted here. The new cqcount depends on how 512 * many SQEs are about to be consumed. 513 */ 514 rpcrdma_init_cqcount(&r_xprt->rx_ep, count); 515 516 /* Transport disconnect drains the receive CQ before it 517 * replaces the QP. The RPC reply handler won't call us 518 * unless ri_id->qp is a valid pointer. 519 */ 520 r_xprt->rx_stats.local_inv_needed++; 521 bad_wr = NULL; 522 rc = ib_post_send(ia->ri_id->qp, first, &bad_wr); 523 if (bad_wr != first) 524 wait_for_completion(&f->fr_linv_done); 525 if (rc) 526 goto reset_mrs; 527 528 /* ORDER: Now DMA unmap all of the MRs, and return 529 * them to the free MW list. 530 */ 531 unmap: 532 while (!list_empty(mws)) { 533 mw = rpcrdma_pop_mw(mws); 534 dprintk("RPC: %s: DMA unmapping frmr %p\n", 535 __func__, &mw->frmr); 536 ib_dma_unmap_sg(ia->ri_device, 537 mw->mw_sg, mw->mw_nents, mw->mw_dir); 538 rpcrdma_put_mw(r_xprt, mw); 539 } 540 return; 541 542 reset_mrs: 543 pr_err("rpcrdma: FRMR invalidate ib_post_send returned %i\n", rc); 544 545 /* Find and reset the MRs in the LOCAL_INV WRs that did not 546 * get posted. 547 */ 548 rpcrdma_init_cqcount(&r_xprt->rx_ep, -count); 549 while (bad_wr) { 550 f = container_of(bad_wr, struct rpcrdma_frmr, 551 fr_invwr); 552 mw = container_of(f, struct rpcrdma_mw, frmr); 553 554 __frwr_reset_mr(ia, mw); 555 556 bad_wr = bad_wr->next; 557 } 558 goto unmap; 559 } 560 561 /* Use a slow, safe mechanism to invalidate all memory regions 562 * that were registered for "req". 563 */ 564 static void 565 frwr_op_unmap_safe(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 566 bool sync) 567 { 568 struct rpcrdma_mw *mw; 569 570 while (!list_empty(&req->rl_registered)) { 571 mw = rpcrdma_pop_mw(&req->rl_registered); 572 if (sync) 573 frwr_op_recover_mr(mw); 574 else 575 rpcrdma_defer_mr_recovery(mw); 576 } 577 } 578 579 const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = { 580 .ro_map = frwr_op_map, 581 .ro_unmap_sync = frwr_op_unmap_sync, 582 .ro_unmap_safe = frwr_op_unmap_safe, 583 .ro_recover_mr = frwr_op_recover_mr, 584 .ro_open = frwr_op_open, 585 .ro_maxpages = frwr_op_maxpages, 586 .ro_init_mr = frwr_op_init_mr, 587 .ro_release_mr = frwr_op_release_mr, 588 .ro_displayname = "frwr", 589 .ro_send_w_inv_ok = RPCRDMA_CMP_F_SND_W_INV_OK, 590 }; 591