1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2f13193f5SChuck Lever /* 3ecf85b23SChuck Lever * Copyright (c) 2016-2018 Oracle. All rights reserved. 4f13193f5SChuck Lever * 5f13193f5SChuck Lever * Use the core R/W API to move RPC-over-RDMA Read and Write chunks. 6f13193f5SChuck Lever */ 7f13193f5SChuck Lever 898895edbSChuck Lever #include <rdma/rw.h> 998895edbSChuck Lever 10f60a0869SChuck Lever #include <linux/sunrpc/xdr.h> 11f13193f5SChuck Lever #include <linux/sunrpc/rpc_rdma.h> 12f13193f5SChuck Lever #include <linux/sunrpc/svc_rdma.h> 13f13193f5SChuck Lever 1498895edbSChuck Lever #include "xprt_rdma.h" 1598895edbSChuck Lever #include <trace/events/rpcrdma.h> 16f13193f5SChuck Lever 17026d958bSChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); 18026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); 19026d958bSChuck Lever 20f13193f5SChuck Lever /* Each R/W context contains state for one chain of RDMA Read or 21f13193f5SChuck Lever * Write Work Requests. 22f13193f5SChuck Lever * 23f13193f5SChuck Lever * Each WR chain handles a single contiguous server-side buffer, 24f13193f5SChuck Lever * because scatterlist entries after the first have to start on 25f13193f5SChuck Lever * page alignment. xdr_buf iovecs cannot guarantee alignment. 26f13193f5SChuck Lever * 27f13193f5SChuck Lever * Each WR chain handles only one R_key. Each RPC-over-RDMA segment 28f13193f5SChuck Lever * from a client may contain a unique R_key, so each WR chain moves 29f13193f5SChuck Lever * up to one segment at a time. 30f13193f5SChuck Lever * 31f13193f5SChuck Lever * The scatterlist makes this data structure over 4KB in size. To 32f13193f5SChuck Lever * make it less likely to fail, and to handle the allocation for 33f13193f5SChuck Lever * smaller I/O requests without disabling bottom-halves, these 34f13193f5SChuck Lever * contexts are created on demand, but cached and reused until the 35f13193f5SChuck Lever * controlling svcxprt_rdma is destroyed. 36f13193f5SChuck Lever */ 37f13193f5SChuck Lever struct svc_rdma_rw_ctxt { 38*07a92d00SChuck Lever struct llist_node rw_node; 39f13193f5SChuck Lever struct list_head rw_list; 40f13193f5SChuck Lever struct rdma_rw_ctx rw_ctx; 412abfbe7eSChuck Lever unsigned int rw_nents; 42f13193f5SChuck Lever struct sg_table rw_sg_table; 43c0fb23f8SGustavo A. R. Silva struct scatterlist rw_first_sgl[]; 44f13193f5SChuck Lever }; 45f13193f5SChuck Lever 46f13193f5SChuck Lever static inline struct svc_rdma_rw_ctxt * 47f13193f5SChuck Lever svc_rdma_next_ctxt(struct list_head *list) 48f13193f5SChuck Lever { 49f13193f5SChuck Lever return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt, 50f13193f5SChuck Lever rw_list); 51f13193f5SChuck Lever } 52f13193f5SChuck Lever 53f13193f5SChuck Lever static struct svc_rdma_rw_ctxt * 54f13193f5SChuck Lever svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) 55f13193f5SChuck Lever { 56f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 57*07a92d00SChuck Lever struct llist_node *node; 58f13193f5SChuck Lever 59f13193f5SChuck Lever spin_lock(&rdma->sc_rw_ctxt_lock); 60*07a92d00SChuck Lever node = llist_del_first(&rdma->sc_rw_ctxts); 61f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 62*07a92d00SChuck Lever if (node) { 63*07a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 64f13193f5SChuck Lever } else { 6514cfbd94SGustavo A. R. Silva ctxt = kmalloc(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), 66f13193f5SChuck Lever GFP_KERNEL); 67f13193f5SChuck Lever if (!ctxt) 68f4e53e1cSChuck Lever goto out_noctx; 69*07a92d00SChuck Lever 70f13193f5SChuck Lever INIT_LIST_HEAD(&ctxt->rw_list); 71f13193f5SChuck Lever } 72f13193f5SChuck Lever 73f13193f5SChuck Lever ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; 74f13193f5SChuck Lever if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, 754635873cSMing Lei ctxt->rw_sg_table.sgl, 76f4e53e1cSChuck Lever SG_CHUNK_SIZE)) 77f4e53e1cSChuck Lever goto out_free; 78f13193f5SChuck Lever return ctxt; 79f4e53e1cSChuck Lever 80f4e53e1cSChuck Lever out_free: 81f4e53e1cSChuck Lever kfree(ctxt); 82f4e53e1cSChuck Lever out_noctx: 83f4e53e1cSChuck Lever trace_svcrdma_no_rwctx_err(rdma, sges); 84f4e53e1cSChuck Lever return NULL; 85f13193f5SChuck Lever } 86f13193f5SChuck Lever 87*07a92d00SChuck Lever static void __svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 88*07a92d00SChuck Lever struct svc_rdma_rw_ctxt *ctxt, 89*07a92d00SChuck Lever struct llist_head *list) 90*07a92d00SChuck Lever { 91*07a92d00SChuck Lever sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE); 92*07a92d00SChuck Lever llist_add(&ctxt->rw_node, list); 93*07a92d00SChuck Lever } 94*07a92d00SChuck Lever 95f13193f5SChuck Lever static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 96f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 97f13193f5SChuck Lever { 98*07a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &rdma->sc_rw_ctxts); 99f13193f5SChuck Lever } 100f13193f5SChuck Lever 101f13193f5SChuck Lever /** 102f13193f5SChuck Lever * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts 103f13193f5SChuck Lever * @rdma: transport about to be destroyed 104f13193f5SChuck Lever * 105f13193f5SChuck Lever */ 106f13193f5SChuck Lever void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) 107f13193f5SChuck Lever { 108f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 109*07a92d00SChuck Lever struct llist_node *node; 110f13193f5SChuck Lever 111*07a92d00SChuck Lever while ((node = llist_del_first(&rdma->sc_rw_ctxts)) != NULL) { 112*07a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 113f13193f5SChuck Lever kfree(ctxt); 114f13193f5SChuck Lever } 115f13193f5SChuck Lever } 116f13193f5SChuck Lever 1172abfbe7eSChuck Lever /** 1182abfbe7eSChuck Lever * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O 1192abfbe7eSChuck Lever * @rdma: controlling transport instance 1202abfbe7eSChuck Lever * @ctxt: R/W context to prepare 1212abfbe7eSChuck Lever * @offset: RDMA offset 1222abfbe7eSChuck Lever * @handle: RDMA tag/handle 1232abfbe7eSChuck Lever * @direction: I/O direction 1242abfbe7eSChuck Lever * 1252abfbe7eSChuck Lever * Returns on success, the number of WQEs that will be needed 1262abfbe7eSChuck Lever * on the workqueue, or a negative errno. 1272abfbe7eSChuck Lever */ 1282abfbe7eSChuck Lever static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, 1292abfbe7eSChuck Lever struct svc_rdma_rw_ctxt *ctxt, 1302abfbe7eSChuck Lever u64 offset, u32 handle, 1312abfbe7eSChuck Lever enum dma_data_direction direction) 1322abfbe7eSChuck Lever { 1332abfbe7eSChuck Lever int ret; 1342abfbe7eSChuck Lever 1352abfbe7eSChuck Lever ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num, 1362abfbe7eSChuck Lever ctxt->rw_sg_table.sgl, ctxt->rw_nents, 1372abfbe7eSChuck Lever 0, offset, handle, direction); 1382abfbe7eSChuck Lever if (unlikely(ret < 0)) { 1392abfbe7eSChuck Lever svc_rdma_put_rw_ctxt(rdma, ctxt); 1402abfbe7eSChuck Lever trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret); 1412abfbe7eSChuck Lever } 1422abfbe7eSChuck Lever return ret; 1432abfbe7eSChuck Lever } 1442abfbe7eSChuck Lever 145f13193f5SChuck Lever /* A chunk context tracks all I/O for moving one Read or Write 1461cc5213bSRandy Dunlap * chunk. This is a set of rdma_rw's that handle data movement 147f13193f5SChuck Lever * for all segments of one chunk. 148f13193f5SChuck Lever * 149f13193f5SChuck Lever * These are small, acquired with a single allocator call, and 150f13193f5SChuck Lever * no more than one is needed per chunk. They are allocated on 151f13193f5SChuck Lever * demand, and not cached. 152f13193f5SChuck Lever */ 153f13193f5SChuck Lever struct svc_rdma_chunk_ctxt { 1546787f0beSChuck Lever struct rpc_rdma_cid cc_cid; 155f13193f5SChuck Lever struct ib_cqe cc_cqe; 156f13193f5SChuck Lever struct svcxprt_rdma *cc_rdma; 157f13193f5SChuck Lever struct list_head cc_rwctxts; 158f13193f5SChuck Lever int cc_sqecount; 1597d81ee87SChuck Lever enum ib_wc_status cc_status; 1607d81ee87SChuck Lever struct completion cc_done; 161f13193f5SChuck Lever }; 162f13193f5SChuck Lever 1636787f0beSChuck Lever static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma, 1646787f0beSChuck Lever struct rpc_rdma_cid *cid) 1656787f0beSChuck Lever { 1666787f0beSChuck Lever cid->ci_queue_id = rdma->sc_sq_cq->res.id; 1676787f0beSChuck Lever cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids); 1686787f0beSChuck Lever } 1696787f0beSChuck Lever 170f13193f5SChuck Lever static void svc_rdma_cc_init(struct svcxprt_rdma *rdma, 17135a30fc3SChuck Lever struct svc_rdma_chunk_ctxt *cc) 172f13193f5SChuck Lever { 1736787f0beSChuck Lever svc_rdma_cc_cid_init(rdma, &cc->cc_cid); 174f13193f5SChuck Lever cc->cc_rdma = rdma; 175f13193f5SChuck Lever 176f13193f5SChuck Lever INIT_LIST_HEAD(&cc->cc_rwctxts); 177f13193f5SChuck Lever cc->cc_sqecount = 0; 178f13193f5SChuck Lever } 179f13193f5SChuck Lever 180*07a92d00SChuck Lever /* 181*07a92d00SChuck Lever * The consumed rw_ctx's are cleaned and placed on a local llist so 182*07a92d00SChuck Lever * that only one atomic llist operation is needed to put them all 183*07a92d00SChuck Lever * back on the free list. 184*07a92d00SChuck Lever */ 18535a30fc3SChuck Lever static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc, 18635a30fc3SChuck Lever enum dma_data_direction dir) 187f13193f5SChuck Lever { 188f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 189*07a92d00SChuck Lever struct llist_node *first, *last; 190f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 191*07a92d00SChuck Lever LLIST_HEAD(free); 192f13193f5SChuck Lever 193*07a92d00SChuck Lever first = last = NULL; 194f13193f5SChuck Lever while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) { 195f13193f5SChuck Lever list_del(&ctxt->rw_list); 196f13193f5SChuck Lever 197f13193f5SChuck Lever rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp, 198f13193f5SChuck Lever rdma->sc_port_num, ctxt->rw_sg_table.sgl, 19935a30fc3SChuck Lever ctxt->rw_nents, dir); 200*07a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &free); 201*07a92d00SChuck Lever 202*07a92d00SChuck Lever ctxt->rw_node.next = first; 203*07a92d00SChuck Lever first = &ctxt->rw_node; 204*07a92d00SChuck Lever if (!last) 205*07a92d00SChuck Lever last = first; 206f13193f5SChuck Lever } 207*07a92d00SChuck Lever if (first) 208*07a92d00SChuck Lever llist_add_batch(first, last, &rdma->sc_rw_ctxts); 209f13193f5SChuck Lever } 210f13193f5SChuck Lever 211f13193f5SChuck Lever /* State for sending a Write or Reply chunk. 212f13193f5SChuck Lever * - Tracks progress of writing one chunk over all its segments 213f13193f5SChuck Lever * - Stores arguments for the SGL constructor functions 214f13193f5SChuck Lever */ 215f13193f5SChuck Lever struct svc_rdma_write_info { 2167a1cbfa1SChuck Lever const struct svc_rdma_chunk *wi_chunk; 2177a1cbfa1SChuck Lever 218f13193f5SChuck Lever /* write state of this chunk */ 219f13193f5SChuck Lever unsigned int wi_seg_off; 220f13193f5SChuck Lever unsigned int wi_seg_no; 221f13193f5SChuck Lever 222f13193f5SChuck Lever /* SGL constructor arguments */ 22351bad8ccSChuck Lever const struct xdr_buf *wi_xdr; 224f13193f5SChuck Lever unsigned char *wi_base; 225f13193f5SChuck Lever unsigned int wi_next_off; 226f13193f5SChuck Lever 227f13193f5SChuck Lever struct svc_rdma_chunk_ctxt wi_cc; 228f13193f5SChuck Lever }; 229f13193f5SChuck Lever 230f13193f5SChuck Lever static struct svc_rdma_write_info * 2317a1cbfa1SChuck Lever svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, 2327a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk) 233f13193f5SChuck Lever { 234f13193f5SChuck Lever struct svc_rdma_write_info *info; 235f13193f5SChuck Lever 236f13193f5SChuck Lever info = kmalloc(sizeof(*info), GFP_KERNEL); 237f13193f5SChuck Lever if (!info) 238f13193f5SChuck Lever return info; 239f13193f5SChuck Lever 2407a1cbfa1SChuck Lever info->wi_chunk = chunk; 241f13193f5SChuck Lever info->wi_seg_off = 0; 242f13193f5SChuck Lever info->wi_seg_no = 0; 24335a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->wi_cc); 244026d958bSChuck Lever info->wi_cc.cc_cqe.done = svc_rdma_write_done; 245f13193f5SChuck Lever return info; 246f13193f5SChuck Lever } 247f13193f5SChuck Lever 248f13193f5SChuck Lever static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) 249f13193f5SChuck Lever { 25035a30fc3SChuck Lever svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE); 251f13193f5SChuck Lever kfree(info); 252f13193f5SChuck Lever } 253f13193f5SChuck Lever 254f13193f5SChuck Lever /** 255f13193f5SChuck Lever * svc_rdma_write_done - Write chunk completion 256f13193f5SChuck Lever * @cq: controlling Completion Queue 257f13193f5SChuck Lever * @wc: Work Completion 258f13193f5SChuck Lever * 259f13193f5SChuck Lever * Pages under I/O are freed by a subsequent Send completion. 260f13193f5SChuck Lever */ 261f13193f5SChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 262f13193f5SChuck Lever { 263f13193f5SChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 264f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = 265f13193f5SChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 266f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 267f13193f5SChuck Lever struct svc_rdma_write_info *info = 268f13193f5SChuck Lever container_of(cc, struct svc_rdma_write_info, wi_cc); 269f13193f5SChuck Lever 2706787f0beSChuck Lever trace_svcrdma_wc_write(wc, &cc->cc_cid); 271bd2abef3SChuck Lever 2726c8c84f5SChuck Lever svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); 273f13193f5SChuck Lever 2748820bcaaSChuck Lever if (unlikely(wc->status != IB_WC_SUCCESS)) 275e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 276f13193f5SChuck Lever 277f13193f5SChuck Lever svc_rdma_write_info_free(info); 278f13193f5SChuck Lever } 279f13193f5SChuck Lever 280026d958bSChuck Lever /* State for pulling a Read chunk. 281026d958bSChuck Lever */ 282026d958bSChuck Lever struct svc_rdma_read_info { 283d96962e6SChuck Lever struct svc_rqst *ri_rqst; 284ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *ri_readctxt; 285026d958bSChuck Lever unsigned int ri_pageno; 286026d958bSChuck Lever unsigned int ri_pageoff; 287bafe9c27SChuck Lever unsigned int ri_totalbytes; 288026d958bSChuck Lever 289026d958bSChuck Lever struct svc_rdma_chunk_ctxt ri_cc; 290026d958bSChuck Lever }; 291026d958bSChuck Lever 292026d958bSChuck Lever static struct svc_rdma_read_info * 293026d958bSChuck Lever svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma) 294026d958bSChuck Lever { 295026d958bSChuck Lever struct svc_rdma_read_info *info; 296026d958bSChuck Lever 297026d958bSChuck Lever info = kmalloc(sizeof(*info), GFP_KERNEL); 298026d958bSChuck Lever if (!info) 299026d958bSChuck Lever return info; 300026d958bSChuck Lever 30135a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->ri_cc); 302026d958bSChuck Lever info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done; 303026d958bSChuck Lever return info; 304026d958bSChuck Lever } 305026d958bSChuck Lever 306026d958bSChuck Lever static void svc_rdma_read_info_free(struct svc_rdma_read_info *info) 307026d958bSChuck Lever { 30835a30fc3SChuck Lever svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE); 309026d958bSChuck Lever kfree(info); 310026d958bSChuck Lever } 311026d958bSChuck Lever 312026d958bSChuck Lever /** 313026d958bSChuck Lever * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx 314026d958bSChuck Lever * @cq: controlling Completion Queue 315026d958bSChuck Lever * @wc: Work Completion 316026d958bSChuck Lever * 317026d958bSChuck Lever */ 318026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) 319026d958bSChuck Lever { 320026d958bSChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 321026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = 322026d958bSChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 323026d958bSChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 324026d958bSChuck Lever 3256787f0beSChuck Lever trace_svcrdma_wc_read(wc, &cc->cc_cid); 326bd2abef3SChuck Lever 3276c8c84f5SChuck Lever svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); 3287d81ee87SChuck Lever cc->cc_status = wc->status; 3297d81ee87SChuck Lever complete(&cc->cc_done); 3307d81ee87SChuck Lever return; 331026d958bSChuck Lever } 332026d958bSChuck Lever 333f13193f5SChuck Lever /* This function sleeps when the transport's Send Queue is congested. 334f13193f5SChuck Lever * 335f13193f5SChuck Lever * Assumptions: 336f13193f5SChuck Lever * - If ib_post_send() succeeds, only one completion is expected, 337f13193f5SChuck Lever * even if one or more WRs are flushed. This is true when posting 338f13193f5SChuck Lever * an rdma_rw_ctx or when posting a single signaled WR. 339f13193f5SChuck Lever */ 340f13193f5SChuck Lever static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) 341f13193f5SChuck Lever { 342f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 343d34ac5cdSBart Van Assche struct ib_send_wr *first_wr; 344d34ac5cdSBart Van Assche const struct ib_send_wr *bad_wr; 345f13193f5SChuck Lever struct list_head *tmp; 346f13193f5SChuck Lever struct ib_cqe *cqe; 347f13193f5SChuck Lever int ret; 348f13193f5SChuck Lever 349107c1d0aSChuck Lever if (cc->cc_sqecount > rdma->sc_sq_depth) 350107c1d0aSChuck Lever return -EINVAL; 351107c1d0aSChuck Lever 352f13193f5SChuck Lever first_wr = NULL; 353f13193f5SChuck Lever cqe = &cc->cc_cqe; 354f13193f5SChuck Lever list_for_each(tmp, &cc->cc_rwctxts) { 355f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 356f13193f5SChuck Lever 357f13193f5SChuck Lever ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list); 358f13193f5SChuck Lever first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp, 359f13193f5SChuck Lever rdma->sc_port_num, cqe, first_wr); 360f13193f5SChuck Lever cqe = NULL; 361f13193f5SChuck Lever } 362f13193f5SChuck Lever 363f13193f5SChuck Lever do { 364f13193f5SChuck Lever if (atomic_sub_return(cc->cc_sqecount, 365f13193f5SChuck Lever &rdma->sc_sq_avail) > 0) { 3665d85a822SJason Gunthorpe ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); 367f13193f5SChuck Lever if (ret) 368f13193f5SChuck Lever break; 369f13193f5SChuck Lever return 0; 370f13193f5SChuck Lever } 371f13193f5SChuck Lever 37222df5a22SChuck Lever percpu_counter_inc(&svcrdma_stat_sq_starve); 373bd2abef3SChuck Lever trace_svcrdma_sq_full(rdma); 374f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 375f13193f5SChuck Lever wait_event(rdma->sc_send_wait, 376f13193f5SChuck Lever atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); 377bd2abef3SChuck Lever trace_svcrdma_sq_retry(rdma); 378f13193f5SChuck Lever } while (1); 379f13193f5SChuck Lever 380e28b4fc6SChuck Lever trace_svcrdma_sq_post_err(rdma, ret); 381e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 382f13193f5SChuck Lever 383f13193f5SChuck Lever /* If even one was posted, there will be a completion. */ 384f13193f5SChuck Lever if (bad_wr != first_wr) 385f13193f5SChuck Lever return 0; 386f13193f5SChuck Lever 387f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 388f13193f5SChuck Lever wake_up(&rdma->sc_send_wait); 389f13193f5SChuck Lever return -ENOTCONN; 390f13193f5SChuck Lever } 391f13193f5SChuck Lever 392f13193f5SChuck Lever /* Build and DMA-map an SGL that covers one kvec in an xdr_buf 393f13193f5SChuck Lever */ 394f13193f5SChuck Lever static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, 395f13193f5SChuck Lever unsigned int len, 396f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 397f13193f5SChuck Lever { 398f13193f5SChuck Lever struct scatterlist *sg = ctxt->rw_sg_table.sgl; 399f13193f5SChuck Lever 400f13193f5SChuck Lever sg_set_buf(&sg[0], info->wi_base, len); 401f13193f5SChuck Lever info->wi_base += len; 402f13193f5SChuck Lever 403f13193f5SChuck Lever ctxt->rw_nents = 1; 404f13193f5SChuck Lever } 405f13193f5SChuck Lever 406f13193f5SChuck Lever /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. 407f13193f5SChuck Lever */ 408f13193f5SChuck Lever static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, 409f13193f5SChuck Lever unsigned int remaining, 410f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 411f13193f5SChuck Lever { 412f13193f5SChuck Lever unsigned int sge_no, sge_bytes, page_off, page_no; 41351bad8ccSChuck Lever const struct xdr_buf *xdr = info->wi_xdr; 414f13193f5SChuck Lever struct scatterlist *sg; 415f13193f5SChuck Lever struct page **page; 416f13193f5SChuck Lever 41791b022ecSChuck Lever page_off = info->wi_next_off + xdr->page_base; 41891b022ecSChuck Lever page_no = page_off >> PAGE_SHIFT; 41991b022ecSChuck Lever page_off = offset_in_page(page_off); 420f13193f5SChuck Lever page = xdr->pages + page_no; 421f13193f5SChuck Lever info->wi_next_off += remaining; 422f13193f5SChuck Lever sg = ctxt->rw_sg_table.sgl; 423f13193f5SChuck Lever sge_no = 0; 424f13193f5SChuck Lever do { 425f13193f5SChuck Lever sge_bytes = min_t(unsigned int, remaining, 426f13193f5SChuck Lever PAGE_SIZE - page_off); 427f13193f5SChuck Lever sg_set_page(sg, *page, sge_bytes, page_off); 428f13193f5SChuck Lever 429f13193f5SChuck Lever remaining -= sge_bytes; 430f13193f5SChuck Lever sg = sg_next(sg); 431f13193f5SChuck Lever page_off = 0; 432f13193f5SChuck Lever sge_no++; 433f13193f5SChuck Lever page++; 434f13193f5SChuck Lever } while (remaining); 435f13193f5SChuck Lever 436f13193f5SChuck Lever ctxt->rw_nents = sge_no; 437f13193f5SChuck Lever } 438f13193f5SChuck Lever 439f13193f5SChuck Lever /* Construct RDMA Write WRs to send a portion of an xdr_buf containing 440f13193f5SChuck Lever * an RPC Reply. 441f13193f5SChuck Lever */ 442f13193f5SChuck Lever static int 443f13193f5SChuck Lever svc_rdma_build_writes(struct svc_rdma_write_info *info, 444f13193f5SChuck Lever void (*constructor)(struct svc_rdma_write_info *info, 445f13193f5SChuck Lever unsigned int len, 446f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt), 447f13193f5SChuck Lever unsigned int remaining) 448f13193f5SChuck Lever { 449f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; 450f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 4517a1cbfa1SChuck Lever const struct svc_rdma_segment *seg; 452f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 453f13193f5SChuck Lever int ret; 454f13193f5SChuck Lever 455f13193f5SChuck Lever do { 456f13193f5SChuck Lever unsigned int write_len; 457f60a0869SChuck Lever u64 offset; 458f13193f5SChuck Lever 4597a1cbfa1SChuck Lever seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; 4607a1cbfa1SChuck Lever if (!seg) 461f13193f5SChuck Lever goto out_overflow; 462f13193f5SChuck Lever 4637a1cbfa1SChuck Lever write_len = min(remaining, seg->rs_length - info->wi_seg_off); 464e5decb2eSChuck Lever if (!write_len) 465e5decb2eSChuck Lever goto out_overflow; 466f13193f5SChuck Lever ctxt = svc_rdma_get_rw_ctxt(rdma, 467f13193f5SChuck Lever (write_len >> PAGE_SHIFT) + 2); 468f13193f5SChuck Lever if (!ctxt) 469f4e53e1cSChuck Lever return -ENOMEM; 470f13193f5SChuck Lever 471f13193f5SChuck Lever constructor(info, write_len, ctxt); 4727a1cbfa1SChuck Lever offset = seg->rs_offset + info->wi_seg_off; 4737a1cbfa1SChuck Lever ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle, 4742abfbe7eSChuck Lever DMA_TO_DEVICE); 475f13193f5SChuck Lever if (ret < 0) 4762abfbe7eSChuck Lever return -EIO; 4771e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_write); 478f13193f5SChuck Lever 479f13193f5SChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 480f13193f5SChuck Lever cc->cc_sqecount += ret; 4817a1cbfa1SChuck Lever if (write_len == seg->rs_length - info->wi_seg_off) { 482f13193f5SChuck Lever info->wi_seg_no++; 483f13193f5SChuck Lever info->wi_seg_off = 0; 484f13193f5SChuck Lever } else { 485f13193f5SChuck Lever info->wi_seg_off += write_len; 486f13193f5SChuck Lever } 487f13193f5SChuck Lever remaining -= write_len; 488f13193f5SChuck Lever } while (remaining); 489f13193f5SChuck Lever 490f13193f5SChuck Lever return 0; 491f13193f5SChuck Lever 492f13193f5SChuck Lever out_overflow: 493dbc17acdSChuck Lever trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no, 4947a1cbfa1SChuck Lever info->wi_chunk->ch_segcount); 495f13193f5SChuck Lever return -E2BIG; 496f13193f5SChuck Lever } 497f13193f5SChuck Lever 498ab1394eeSChuck Lever /** 499ab1394eeSChuck Lever * svc_rdma_iov_write - Construct RDMA Writes from an iov 500ab1394eeSChuck Lever * @info: pointer to write arguments 501ab1394eeSChuck Lever * @iov: kvec to write 502ab1394eeSChuck Lever * 503ab1394eeSChuck Lever * Returns: 504d5029525SZheng Yongjun * On success, returns zero 505ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 506ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 507ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 508f13193f5SChuck Lever */ 509ab1394eeSChuck Lever static int svc_rdma_iov_write(struct svc_rdma_write_info *info, 510ab1394eeSChuck Lever const struct kvec *iov) 511f13193f5SChuck Lever { 512ab1394eeSChuck Lever info->wi_base = iov->iov_base; 513f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_vec_to_sg, 514ab1394eeSChuck Lever iov->iov_len); 515f13193f5SChuck Lever } 516f13193f5SChuck Lever 517ab1394eeSChuck Lever /** 518ab1394eeSChuck Lever * svc_rdma_pages_write - Construct RDMA Writes from pages 519ab1394eeSChuck Lever * @info: pointer to write arguments 520ab1394eeSChuck Lever * @xdr: xdr_buf with pages to write 521ab1394eeSChuck Lever * @offset: offset into the content of @xdr 522ab1394eeSChuck Lever * @length: number of bytes to write 523ab1394eeSChuck Lever * 524ab1394eeSChuck Lever * Returns: 525d5029525SZheng Yongjun * On success, returns zero 526ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 527ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 528ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 529f13193f5SChuck Lever */ 530ab1394eeSChuck Lever static int svc_rdma_pages_write(struct svc_rdma_write_info *info, 531ab1394eeSChuck Lever const struct xdr_buf *xdr, 53241205539SChuck Lever unsigned int offset, 53341205539SChuck Lever unsigned long length) 534f13193f5SChuck Lever { 535f13193f5SChuck Lever info->wi_xdr = xdr; 53641205539SChuck Lever info->wi_next_off = offset - xdr->head[0].iov_len; 537f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg, 53841205539SChuck Lever length); 539f13193f5SChuck Lever } 540f13193f5SChuck Lever 541f13193f5SChuck Lever /** 542f6ad7759SChuck Lever * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf 543f6ad7759SChuck Lever * @xdr: xdr_buf to write 54441bc163fSChuck Lever * @data: pointer to write arguments 545f6ad7759SChuck Lever * 546f6ad7759SChuck Lever * Returns: 547d5029525SZheng Yongjun * On success, returns zero 548f6ad7759SChuck Lever * %-E2BIG if the client-provided Write chunk is too small 549f6ad7759SChuck Lever * %-ENOMEM if a resource has been exhausted 550f6ad7759SChuck Lever * %-EIO if an rdma-rw error occurred 551f6ad7759SChuck Lever */ 55241bc163fSChuck Lever static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) 553f6ad7759SChuck Lever { 55441bc163fSChuck Lever struct svc_rdma_write_info *info = data; 555f6ad7759SChuck Lever int ret; 556f6ad7759SChuck Lever 557f6ad7759SChuck Lever if (xdr->head[0].iov_len) { 558f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->head[0]); 559f6ad7759SChuck Lever if (ret < 0) 560f6ad7759SChuck Lever return ret; 561f6ad7759SChuck Lever } 562f6ad7759SChuck Lever 563f6ad7759SChuck Lever if (xdr->page_len) { 564f6ad7759SChuck Lever ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len, 565f6ad7759SChuck Lever xdr->page_len); 566f6ad7759SChuck Lever if (ret < 0) 567f6ad7759SChuck Lever return ret; 568f6ad7759SChuck Lever } 569f6ad7759SChuck Lever 570f6ad7759SChuck Lever if (xdr->tail[0].iov_len) { 571f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->tail[0]); 572f6ad7759SChuck Lever if (ret < 0) 573f6ad7759SChuck Lever return ret; 574f6ad7759SChuck Lever } 575f6ad7759SChuck Lever 576f6ad7759SChuck Lever return xdr->len; 577f6ad7759SChuck Lever } 578f6ad7759SChuck Lever 579f6ad7759SChuck Lever /** 580f13193f5SChuck Lever * svc_rdma_send_write_chunk - Write all segments in a Write chunk 581f13193f5SChuck Lever * @rdma: controlling RDMA transport 5827a1cbfa1SChuck Lever * @chunk: Write chunk provided by the client 583f13193f5SChuck Lever * @xdr: xdr_buf containing the data payload 584f13193f5SChuck Lever * 585f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 586f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Write chunk, 587107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 588f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 589f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 590f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 591f13193f5SChuck Lever */ 5927a1cbfa1SChuck Lever int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, 5937a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk, 594f6ad7759SChuck Lever const struct xdr_buf *xdr) 595f13193f5SChuck Lever { 596f13193f5SChuck Lever struct svc_rdma_write_info *info; 597b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 598f13193f5SChuck Lever int ret; 599f13193f5SChuck Lever 6007a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 601f13193f5SChuck Lever if (!info) 602f13193f5SChuck Lever return -ENOMEM; 603b704be09SChuck Lever cc = &info->wi_cc; 604f13193f5SChuck Lever 605f6ad7759SChuck Lever ret = svc_rdma_xb_write(xdr, info); 606f6ad7759SChuck Lever if (ret != xdr->len) 607f13193f5SChuck Lever goto out_err; 608f13193f5SChuck Lever 609b704be09SChuck Lever trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); 610b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 611f13193f5SChuck Lever if (ret < 0) 612f13193f5SChuck Lever goto out_err; 613f6ad7759SChuck Lever return xdr->len; 614f13193f5SChuck Lever 615f13193f5SChuck Lever out_err: 616f13193f5SChuck Lever svc_rdma_write_info_free(info); 617f13193f5SChuck Lever return ret; 618f13193f5SChuck Lever } 619f13193f5SChuck Lever 620f13193f5SChuck Lever /** 621f13193f5SChuck Lever * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk 622f13193f5SChuck Lever * @rdma: controlling RDMA transport 6236fa5785eSChuck Lever * @rctxt: Write and Reply chunks from client 624f13193f5SChuck Lever * @xdr: xdr_buf containing an RPC Reply 625f13193f5SChuck Lever * 626f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 627f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Reply chunk, 628107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 629f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 630f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 631f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 632f13193f5SChuck Lever */ 6336fa5785eSChuck Lever int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, 6346fa5785eSChuck Lever const struct svc_rdma_recv_ctxt *rctxt, 63541bc163fSChuck Lever const struct xdr_buf *xdr) 636f13193f5SChuck Lever { 637f13193f5SChuck Lever struct svc_rdma_write_info *info; 638b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 6397a1cbfa1SChuck Lever struct svc_rdma_chunk *chunk; 64041bc163fSChuck Lever int ret; 641f13193f5SChuck Lever 6427a1cbfa1SChuck Lever if (pcl_is_empty(&rctxt->rc_reply_pcl)) 643ded380f1SChuck Lever return 0; 644ded380f1SChuck Lever 6457a1cbfa1SChuck Lever chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); 6467a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 647f13193f5SChuck Lever if (!info) 648f13193f5SChuck Lever return -ENOMEM; 649b704be09SChuck Lever cc = &info->wi_cc; 650f13193f5SChuck Lever 65141bc163fSChuck Lever ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, 65241bc163fSChuck Lever svc_rdma_xb_write, info); 653f13193f5SChuck Lever if (ret < 0) 654f13193f5SChuck Lever goto out_err; 655f13193f5SChuck Lever 656b704be09SChuck Lever trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); 657b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 658f13193f5SChuck Lever if (ret < 0) 659f13193f5SChuck Lever goto out_err; 66098895edbSChuck Lever 66141bc163fSChuck Lever return xdr->len; 662f13193f5SChuck Lever 663f13193f5SChuck Lever out_err: 664f13193f5SChuck Lever svc_rdma_write_info_free(info); 665f13193f5SChuck Lever return ret; 666f13193f5SChuck Lever } 667026d958bSChuck Lever 668d96962e6SChuck Lever /** 669d96962e6SChuck Lever * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment 670d96962e6SChuck Lever * @info: context for ongoing I/O 671d96962e6SChuck Lever * @segment: co-ordinates of remote memory to be read 672d96962e6SChuck Lever * 673d96962e6SChuck Lever * Returns: 674d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 675d96962e6SChuck Lever * %-EINVAL: there were not enough rq_pages to finish 676d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 677d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 678d96962e6SChuck Lever */ 679026d958bSChuck Lever static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, 680d96962e6SChuck Lever const struct svc_rdma_segment *segment) 681026d958bSChuck Lever { 682ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 683026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->ri_cc; 684d96962e6SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 685d96962e6SChuck Lever unsigned int sge_no, seg_len, len; 6867d81ee87SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 687026d958bSChuck Lever struct scatterlist *sg; 688026d958bSChuck Lever int ret; 689026d958bSChuck Lever 690d96962e6SChuck Lever len = segment->rs_length; 691026d958bSChuck Lever sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; 692026d958bSChuck Lever ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); 693026d958bSChuck Lever if (!ctxt) 694f4e53e1cSChuck Lever return -ENOMEM; 695026d958bSChuck Lever ctxt->rw_nents = sge_no; 696026d958bSChuck Lever 697026d958bSChuck Lever sg = ctxt->rw_sg_table.sgl; 698026d958bSChuck Lever for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) { 699026d958bSChuck Lever seg_len = min_t(unsigned int, len, 700026d958bSChuck Lever PAGE_SIZE - info->ri_pageoff); 701026d958bSChuck Lever 702026d958bSChuck Lever if (!info->ri_pageoff) 703ecf85b23SChuck Lever head->rc_page_count++; 704026d958bSChuck Lever 705026d958bSChuck Lever sg_set_page(sg, rqstp->rq_pages[info->ri_pageno], 706026d958bSChuck Lever seg_len, info->ri_pageoff); 707026d958bSChuck Lever sg = sg_next(sg); 708026d958bSChuck Lever 709026d958bSChuck Lever info->ri_pageoff += seg_len; 710026d958bSChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 711026d958bSChuck Lever info->ri_pageno++; 712026d958bSChuck Lever info->ri_pageoff = 0; 713026d958bSChuck Lever } 714026d958bSChuck Lever len -= seg_len; 715026d958bSChuck Lever 716026d958bSChuck Lever /* Safety check */ 717026d958bSChuck Lever if (len && 718026d958bSChuck Lever &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end) 719026d958bSChuck Lever goto out_overrun; 720026d958bSChuck Lever } 721026d958bSChuck Lever 722d96962e6SChuck Lever ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset, 723d96962e6SChuck Lever segment->rs_handle, DMA_FROM_DEVICE); 724026d958bSChuck Lever if (ret < 0) 7252abfbe7eSChuck Lever return -EIO; 7261e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_read); 727026d958bSChuck Lever 728026d958bSChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 729026d958bSChuck Lever cc->cc_sqecount += ret; 730026d958bSChuck Lever return 0; 731026d958bSChuck Lever 732026d958bSChuck Lever out_overrun: 7339d200638SChuck Lever trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno); 734026d958bSChuck Lever return -EINVAL; 735026d958bSChuck Lever } 736026d958bSChuck Lever 737d96962e6SChuck Lever /** 738d96962e6SChuck Lever * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk 739d96962e6SChuck Lever * @info: context for ongoing I/O 740d96962e6SChuck Lever * @chunk: Read chunk to pull 741d96962e6SChuck Lever * 742d96962e6SChuck Lever * Return values: 743d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 744d96962e6SChuck Lever * %-EINVAL: there were not enough resources to finish 745d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 746d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 7477075a867SChuck Lever */ 748d96962e6SChuck Lever static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info, 749d96962e6SChuck Lever const struct svc_rdma_chunk *chunk) 750026d958bSChuck Lever { 751d96962e6SChuck Lever const struct svc_rdma_segment *segment; 752026d958bSChuck Lever int ret; 753026d958bSChuck Lever 7547075a867SChuck Lever ret = -EINVAL; 755d96962e6SChuck Lever pcl_for_each_segment(segment, chunk) { 756d96962e6SChuck Lever ret = svc_rdma_build_read_segment(info, segment); 757026d958bSChuck Lever if (ret < 0) 758026d958bSChuck Lever break; 759d96962e6SChuck Lever info->ri_totalbytes += segment->rs_length; 760026d958bSChuck Lever } 761026d958bSChuck Lever return ret; 762026d958bSChuck Lever } 763026d958bSChuck Lever 764d96962e6SChuck Lever /** 765d7cc7397SChuck Lever * svc_rdma_copy_inline_range - Copy part of the inline content into pages 766d7cc7397SChuck Lever * @info: context for RDMA Reads 767d7cc7397SChuck Lever * @offset: offset into the Receive buffer of region to copy 768d7cc7397SChuck Lever * @remaining: length of region to copy 769d7cc7397SChuck Lever * 770d7cc7397SChuck Lever * Take a page at a time from rqstp->rq_pages and copy the inline 771d7cc7397SChuck Lever * content from the Receive buffer into that page. Update 772d7cc7397SChuck Lever * info->ri_pageno and info->ri_pageoff so that the next RDMA Read 773d7cc7397SChuck Lever * result will land contiguously with the copied content. 774d7cc7397SChuck Lever * 775d7cc7397SChuck Lever * Return values: 776d7cc7397SChuck Lever * %0: Inline content was successfully copied 777d7cc7397SChuck Lever * %-EINVAL: offset or length was incorrect 778d7cc7397SChuck Lever */ 779d7cc7397SChuck Lever static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info, 780d7cc7397SChuck Lever unsigned int offset, 781d7cc7397SChuck Lever unsigned int remaining) 782d7cc7397SChuck Lever { 783d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 784d7cc7397SChuck Lever unsigned char *dst, *src = head->rc_recv_buf; 785d7cc7397SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 786d7cc7397SChuck Lever unsigned int page_no, numpages; 787d7cc7397SChuck Lever 788d7cc7397SChuck Lever numpages = PAGE_ALIGN(info->ri_pageoff + remaining) >> PAGE_SHIFT; 789d7cc7397SChuck Lever for (page_no = 0; page_no < numpages; page_no++) { 790d7cc7397SChuck Lever unsigned int page_len; 791d7cc7397SChuck Lever 792d7cc7397SChuck Lever page_len = min_t(unsigned int, remaining, 793d7cc7397SChuck Lever PAGE_SIZE - info->ri_pageoff); 794d7cc7397SChuck Lever 795d7cc7397SChuck Lever if (!info->ri_pageoff) 796d7cc7397SChuck Lever head->rc_page_count++; 797d7cc7397SChuck Lever 7987d81ee87SChuck Lever dst = page_address(rqstp->rq_pages[info->ri_pageno]); 799d7cc7397SChuck Lever memcpy(dst + info->ri_pageno, src + offset, page_len); 800d7cc7397SChuck Lever 801d7cc7397SChuck Lever info->ri_totalbytes += page_len; 802d7cc7397SChuck Lever info->ri_pageoff += page_len; 803d7cc7397SChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 804d7cc7397SChuck Lever info->ri_pageno++; 805d7cc7397SChuck Lever info->ri_pageoff = 0; 806d7cc7397SChuck Lever } 807d7cc7397SChuck Lever remaining -= page_len; 808d7cc7397SChuck Lever offset += page_len; 809d7cc7397SChuck Lever } 810d7cc7397SChuck Lever 811d7cc7397SChuck Lever return -EINVAL; 812d7cc7397SChuck Lever } 813d7cc7397SChuck Lever 814d7cc7397SChuck Lever /** 815d7cc7397SChuck Lever * svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks 816d7cc7397SChuck Lever * @info: context for RDMA Reads 817d7cc7397SChuck Lever * 8187d81ee87SChuck Lever * The chunk data lands in rqstp->rq_arg as a series of contiguous pages, 819d7cc7397SChuck Lever * like an incoming TCP call. 820d7cc7397SChuck Lever * 821d7cc7397SChuck Lever * Return values: 822d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 823d7cc7397SChuck Lever * %-EINVAL: client provided too many chunks or segments, 824d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 825d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 826d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 827d7cc7397SChuck Lever */ 828d7cc7397SChuck Lever static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *info) 829d7cc7397SChuck Lever { 830d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 831d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 8327d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 833d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 834d7cc7397SChuck Lever unsigned int start, length; 835d7cc7397SChuck Lever int ret; 836d7cc7397SChuck Lever 837d7cc7397SChuck Lever start = 0; 838d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 839d7cc7397SChuck Lever length = chunk->ch_position; 840d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 841d7cc7397SChuck Lever if (ret < 0) 842d7cc7397SChuck Lever return ret; 843d7cc7397SChuck Lever 844d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 845d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 846d7cc7397SChuck Lever if (ret < 0) 847d7cc7397SChuck Lever return ret; 848d7cc7397SChuck Lever 849d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 850d7cc7397SChuck Lever if (!next) 851d7cc7397SChuck Lever break; 852d7cc7397SChuck Lever 853d7cc7397SChuck Lever start += length; 854d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 855d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 856d7cc7397SChuck Lever if (ret < 0) 857d7cc7397SChuck Lever return ret; 858d7cc7397SChuck Lever } 859d7cc7397SChuck Lever 860d7cc7397SChuck Lever start += length; 861d7cc7397SChuck Lever length = head->rc_byte_len - start; 862d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 863d7cc7397SChuck Lever if (ret < 0) 864d7cc7397SChuck Lever return ret; 865d7cc7397SChuck Lever 866d7cc7397SChuck Lever buf->len += info->ri_totalbytes; 867d7cc7397SChuck Lever buf->buflen += info->ri_totalbytes; 868d7cc7397SChuck Lever 8697d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 870d7cc7397SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 8717d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 872d7cc7397SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 873d7cc7397SChuck Lever return 0; 874d7cc7397SChuck Lever } 875d7cc7397SChuck Lever 876d7cc7397SChuck Lever /** 877d7cc7397SChuck Lever * svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks 878d96962e6SChuck Lever * @info: context for RDMA Reads 879d96962e6SChuck Lever * 8807d81ee87SChuck Lever * The chunk data lands in the page list of rqstp->rq_arg.pages. 881026d958bSChuck Lever * 8827d81ee87SChuck Lever * Currently NFSD does not look at the rqstp->rq_arg.tail[0] kvec. 883026d958bSChuck Lever * Therefore, XDR round-up of the Read chunk and trailing 884026d958bSChuck Lever * inline content must both be added at the end of the pagelist. 885d96962e6SChuck Lever * 886d96962e6SChuck Lever * Return values: 887d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 888d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 889d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 890d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 891d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 892026d958bSChuck Lever */ 893d7cc7397SChuck Lever static int svc_rdma_read_data_item(struct svc_rdma_read_info *info) 894026d958bSChuck Lever { 895ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 8967d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 897d96962e6SChuck Lever struct svc_rdma_chunk *chunk; 898bafe9c27SChuck Lever unsigned int length; 899026d958bSChuck Lever int ret; 900026d958bSChuck Lever 901d96962e6SChuck Lever chunk = pcl_first_chunk(&head->rc_read_pcl); 902d96962e6SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 903026d958bSChuck Lever if (ret < 0) 904026d958bSChuck Lever goto out; 905026d958bSChuck Lever 906193bcb7bSChuck Lever /* Split the Receive buffer between the head and tail 907193bcb7bSChuck Lever * buffers at Read chunk's position. XDR roundup of the 908193bcb7bSChuck Lever * chunk is not included in either the pagelist or in 909193bcb7bSChuck Lever * the tail. 910026d958bSChuck Lever */ 911d96962e6SChuck Lever buf->tail[0].iov_base = buf->head[0].iov_base + chunk->ch_position; 912d96962e6SChuck Lever buf->tail[0].iov_len = buf->head[0].iov_len - chunk->ch_position; 913d96962e6SChuck Lever buf->head[0].iov_len = chunk->ch_position; 914026d958bSChuck Lever 915175e0310SChuck Lever /* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2). 916193bcb7bSChuck Lever * 917175e0310SChuck Lever * If the client already rounded up the chunk length, the 918175e0310SChuck Lever * length does not change. Otherwise, the length of the page 919175e0310SChuck Lever * list is increased to include XDR round-up. 920175e0310SChuck Lever * 921175e0310SChuck Lever * Currently these chunks always start at page offset 0, 922175e0310SChuck Lever * thus the rounded-up length never crosses a page boundary. 923026d958bSChuck Lever */ 9247d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[0]; 9257d81ee87SChuck Lever length = xdr_align_size(chunk->ch_length); 926bafe9c27SChuck Lever buf->page_len = length; 927bafe9c27SChuck Lever buf->len += length; 928bafe9c27SChuck Lever buf->buflen += length; 929026d958bSChuck Lever 930026d958bSChuck Lever out: 931026d958bSChuck Lever return ret; 932026d958bSChuck Lever } 933026d958bSChuck Lever 934d96962e6SChuck Lever /** 935d7cc7397SChuck Lever * svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk 936d7cc7397SChuck Lever * @info: context for RDMA Reads 937d7cc7397SChuck Lever * @chunk: parsed Call chunk to pull 938d7cc7397SChuck Lever * @offset: offset of region to pull 939d7cc7397SChuck Lever * @length: length of region to pull 940d7cc7397SChuck Lever * 941d7cc7397SChuck Lever * Return values: 942d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 943d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 944d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 945d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 946d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 947d7cc7397SChuck Lever */ 948d7cc7397SChuck Lever static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info, 949d7cc7397SChuck Lever const struct svc_rdma_chunk *chunk, 950d7cc7397SChuck Lever unsigned int offset, unsigned int length) 951d7cc7397SChuck Lever { 952d7cc7397SChuck Lever const struct svc_rdma_segment *segment; 953d7cc7397SChuck Lever int ret; 954d7cc7397SChuck Lever 955d7cc7397SChuck Lever ret = -EINVAL; 956d7cc7397SChuck Lever pcl_for_each_segment(segment, chunk) { 957d7cc7397SChuck Lever struct svc_rdma_segment dummy; 958d7cc7397SChuck Lever 959d7cc7397SChuck Lever if (offset > segment->rs_length) { 960d7cc7397SChuck Lever offset -= segment->rs_length; 961d7cc7397SChuck Lever continue; 962d7cc7397SChuck Lever } 963d7cc7397SChuck Lever 964d7cc7397SChuck Lever dummy.rs_handle = segment->rs_handle; 965d7cc7397SChuck Lever dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; 966d7cc7397SChuck Lever dummy.rs_offset = segment->rs_offset + offset; 967d7cc7397SChuck Lever 968d7cc7397SChuck Lever ret = svc_rdma_build_read_segment(info, &dummy); 969d7cc7397SChuck Lever if (ret < 0) 970d7cc7397SChuck Lever break; 971d7cc7397SChuck Lever 972d7cc7397SChuck Lever info->ri_totalbytes += dummy.rs_length; 973d7cc7397SChuck Lever length -= dummy.rs_length; 974d7cc7397SChuck Lever offset = 0; 975d7cc7397SChuck Lever } 976d7cc7397SChuck Lever return ret; 977d7cc7397SChuck Lever } 978d7cc7397SChuck Lever 979d7cc7397SChuck Lever /** 980d7cc7397SChuck Lever * svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message 981d7cc7397SChuck Lever * @info: context for RDMA Reads 982d7cc7397SChuck Lever * 983d7cc7397SChuck Lever * Return values: 984d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 985d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 986d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 987d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 988d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 989d7cc7397SChuck Lever */ 990d7cc7397SChuck Lever static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info) 991d7cc7397SChuck Lever { 992d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 993d7cc7397SChuck Lever const struct svc_rdma_chunk *call_chunk = 994d7cc7397SChuck Lever pcl_first_chunk(&head->rc_call_pcl); 995d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 996d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 997d7cc7397SChuck Lever unsigned int start, length; 998d7cc7397SChuck Lever int ret; 999d7cc7397SChuck Lever 1000d7cc7397SChuck Lever if (pcl_is_empty(pcl)) 1001d7cc7397SChuck Lever return svc_rdma_build_read_chunk(info, call_chunk); 1002d7cc7397SChuck Lever 1003d7cc7397SChuck Lever start = 0; 1004d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 1005d7cc7397SChuck Lever length = chunk->ch_position; 1006d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, start, length); 1007d7cc7397SChuck Lever if (ret < 0) 1008d7cc7397SChuck Lever return ret; 1009d7cc7397SChuck Lever 1010d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 1011d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 1012d7cc7397SChuck Lever if (ret < 0) 1013d7cc7397SChuck Lever return ret; 1014d7cc7397SChuck Lever 1015d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 1016d7cc7397SChuck Lever if (!next) 1017d7cc7397SChuck Lever break; 1018d7cc7397SChuck Lever 1019d7cc7397SChuck Lever start += length; 1020d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 1021d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, 1022d7cc7397SChuck Lever start, length); 1023d7cc7397SChuck Lever if (ret < 0) 1024d7cc7397SChuck Lever return ret; 1025d7cc7397SChuck Lever } 1026d7cc7397SChuck Lever 1027d7cc7397SChuck Lever start += length; 1028d7cc7397SChuck Lever length = call_chunk->ch_length - start; 1029d7cc7397SChuck Lever return svc_rdma_read_chunk_range(info, call_chunk, start, length); 1030d7cc7397SChuck Lever } 1031d7cc7397SChuck Lever 1032d7cc7397SChuck Lever /** 1033d96962e6SChuck Lever * svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message 1034d96962e6SChuck Lever * @info: context for RDMA Reads 1035d96962e6SChuck Lever * 1036d96962e6SChuck Lever * The start of the data lands in the first page just after the 10377d81ee87SChuck Lever * Transport header, and the rest lands in rqstp->rq_arg.pages. 1038026d958bSChuck Lever * 1039026d958bSChuck Lever * Assumptions: 1040d96962e6SChuck Lever * - A PZRC is never sent in an RDMA_MSG message, though it's 1041d96962e6SChuck Lever * allowed by spec. 1042d96962e6SChuck Lever * 1043d96962e6SChuck Lever * Return values: 1044d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 1045d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1046d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1047d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1048d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1049026d958bSChuck Lever */ 1050d7cc7397SChuck Lever static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info) 1051026d958bSChuck Lever { 10527d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 1053026d958bSChuck Lever int ret; 1054026d958bSChuck Lever 1055d7cc7397SChuck Lever ret = svc_rdma_read_call_chunk(info); 1056026d958bSChuck Lever if (ret < 0) 1057026d958bSChuck Lever goto out; 1058026d958bSChuck Lever 1059bafe9c27SChuck Lever buf->len += info->ri_totalbytes; 1060bafe9c27SChuck Lever buf->buflen += info->ri_totalbytes; 1061026d958bSChuck Lever 10627d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 1063bafe9c27SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 10647d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 1065bafe9c27SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 1066026d958bSChuck Lever 1067026d958bSChuck Lever out: 1068026d958bSChuck Lever return ret; 1069026d958bSChuck Lever } 1070026d958bSChuck Lever 1071026d958bSChuck Lever /** 1072d96962e6SChuck Lever * svc_rdma_process_read_list - Pull list of Read chunks from the client 1073026d958bSChuck Lever * @rdma: controlling RDMA transport 1074026d958bSChuck Lever * @rqstp: set of pages to use as Read sink buffers 1075026d958bSChuck Lever * @head: pages under I/O collect here 1076026d958bSChuck Lever * 1077d96962e6SChuck Lever * The RPC/RDMA protocol assumes that the upper layer's XDR decoders 1078d96962e6SChuck Lever * pull each Read chunk as they decode an incoming RPC message. 1079026d958bSChuck Lever * 1080d96962e6SChuck Lever * On Linux, however, the server needs to have a fully-constructed RPC 1081d96962e6SChuck Lever * message in rqstp->rq_arg when there is a positive return code from 1082d96962e6SChuck Lever * ->xpo_recvfrom. So the Read list is safety-checked immediately when 1083d96962e6SChuck Lever * it is received, then here the whole Read list is pulled all at once. 1084d96962e6SChuck Lever * The ingress RPC message is fully reconstructed once all associated 1085d96962e6SChuck Lever * RDMA Reads have completed. 1086d96962e6SChuck Lever * 1087d96962e6SChuck Lever * Return values: 1088d96962e6SChuck Lever * %1: all needed RDMA Reads were posted successfully, 1089d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1090d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1091d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1092d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1093026d958bSChuck Lever */ 1094d96962e6SChuck Lever int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, 1095d96962e6SChuck Lever struct svc_rqst *rqstp, 1096d96962e6SChuck Lever struct svc_rdma_recv_ctxt *head) 1097026d958bSChuck Lever { 1098026d958bSChuck Lever struct svc_rdma_read_info *info; 1099b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 1100026d958bSChuck Lever int ret; 1101026d958bSChuck Lever 1102026d958bSChuck Lever info = svc_rdma_read_info_alloc(rdma); 1103026d958bSChuck Lever if (!info) 1104026d958bSChuck Lever return -ENOMEM; 1105b704be09SChuck Lever cc = &info->ri_cc; 1106d96962e6SChuck Lever info->ri_rqst = rqstp; 1107026d958bSChuck Lever info->ri_readctxt = head; 11083316f063SChuck Lever info->ri_pageno = 0; 11093316f063SChuck Lever info->ri_pageoff = 0; 1110bafe9c27SChuck Lever info->ri_totalbytes = 0; 1111026d958bSChuck Lever 1112d7cc7397SChuck Lever if (pcl_is_empty(&head->rc_call_pcl)) { 1113d7cc7397SChuck Lever if (head->rc_read_pcl.cl_count == 1) 1114d7cc7397SChuck Lever ret = svc_rdma_read_data_item(info); 1115026d958bSChuck Lever else 1116d7cc7397SChuck Lever ret = svc_rdma_read_multiple_chunks(info); 1117d7cc7397SChuck Lever } else 1118d96962e6SChuck Lever ret = svc_rdma_read_special(info); 1119026d958bSChuck Lever if (ret < 0) 112007d0ff3bSChuck Lever goto out_err; 1121026d958bSChuck Lever 1122b704be09SChuck Lever trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount); 11237d81ee87SChuck Lever init_completion(&cc->cc_done); 1124b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 1125026d958bSChuck Lever if (ret < 0) 112607d0ff3bSChuck Lever goto out_err; 11277d81ee87SChuck Lever 11287d81ee87SChuck Lever ret = 1; 11297d81ee87SChuck Lever wait_for_completion(&cc->cc_done); 11307d81ee87SChuck Lever if (cc->cc_status != IB_WC_SUCCESS) 11317d81ee87SChuck Lever ret = -EIO; 11327d81ee87SChuck Lever 11337d81ee87SChuck Lever /* rq_respages starts after the last arg page */ 11347d81ee87SChuck Lever rqstp->rq_respages = &rqstp->rq_pages[head->rc_page_count]; 11357d81ee87SChuck Lever rqstp->rq_next_page = rqstp->rq_respages + 1; 11367d81ee87SChuck Lever 11377d81ee87SChuck Lever /* Ensure svc_rdma_recv_ctxt_put() does not try to release pages */ 11387d81ee87SChuck Lever head->rc_page_count = 0; 113907d0ff3bSChuck Lever 114007d0ff3bSChuck Lever out_err: 1141026d958bSChuck Lever svc_rdma_read_info_free(info); 1142026d958bSChuck Lever return ret; 1143026d958bSChuck Lever } 1144