1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2f13193f5SChuck Lever /* 3ecf85b23SChuck Lever * Copyright (c) 2016-2018 Oracle. All rights reserved. 4f13193f5SChuck Lever * 5f13193f5SChuck Lever * Use the core R/W API to move RPC-over-RDMA Read and Write chunks. 6f13193f5SChuck Lever */ 7f13193f5SChuck Lever 898895edbSChuck Lever #include <rdma/rw.h> 998895edbSChuck Lever 10f60a0869SChuck Lever #include <linux/sunrpc/xdr.h> 11f13193f5SChuck Lever #include <linux/sunrpc/rpc_rdma.h> 12f13193f5SChuck Lever #include <linux/sunrpc/svc_rdma.h> 13f13193f5SChuck Lever 1498895edbSChuck Lever #include "xprt_rdma.h" 1598895edbSChuck Lever #include <trace/events/rpcrdma.h> 16f13193f5SChuck Lever 17026d958bSChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); 18026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); 19026d958bSChuck Lever 20f13193f5SChuck Lever /* Each R/W context contains state for one chain of RDMA Read or 21f13193f5SChuck Lever * Write Work Requests. 22f13193f5SChuck Lever * 23f13193f5SChuck Lever * Each WR chain handles a single contiguous server-side buffer, 24f13193f5SChuck Lever * because scatterlist entries after the first have to start on 25f13193f5SChuck Lever * page alignment. xdr_buf iovecs cannot guarantee alignment. 26f13193f5SChuck Lever * 27f13193f5SChuck Lever * Each WR chain handles only one R_key. Each RPC-over-RDMA segment 28f13193f5SChuck Lever * from a client may contain a unique R_key, so each WR chain moves 29f13193f5SChuck Lever * up to one segment at a time. 30f13193f5SChuck Lever * 31f13193f5SChuck Lever * The scatterlist makes this data structure over 4KB in size. To 32f13193f5SChuck Lever * make it less likely to fail, and to handle the allocation for 33f13193f5SChuck Lever * smaller I/O requests without disabling bottom-halves, these 34f13193f5SChuck Lever * contexts are created on demand, but cached and reused until the 35f13193f5SChuck Lever * controlling svcxprt_rdma is destroyed. 36f13193f5SChuck Lever */ 37f13193f5SChuck Lever struct svc_rdma_rw_ctxt { 3807a92d00SChuck Lever struct llist_node rw_node; 39f13193f5SChuck Lever struct list_head rw_list; 40f13193f5SChuck Lever struct rdma_rw_ctx rw_ctx; 412abfbe7eSChuck Lever unsigned int rw_nents; 42f13193f5SChuck Lever struct sg_table rw_sg_table; 43c0fb23f8SGustavo A. R. Silva struct scatterlist rw_first_sgl[]; 44f13193f5SChuck Lever }; 45f13193f5SChuck Lever 46f13193f5SChuck Lever static inline struct svc_rdma_rw_ctxt * 47f13193f5SChuck Lever svc_rdma_next_ctxt(struct list_head *list) 48f13193f5SChuck Lever { 49f13193f5SChuck Lever return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt, 50f13193f5SChuck Lever rw_list); 51f13193f5SChuck Lever } 52f13193f5SChuck Lever 53f13193f5SChuck Lever static struct svc_rdma_rw_ctxt * 54f13193f5SChuck Lever svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) 55f13193f5SChuck Lever { 56f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 5707a92d00SChuck Lever struct llist_node *node; 58f13193f5SChuck Lever 59f13193f5SChuck Lever spin_lock(&rdma->sc_rw_ctxt_lock); 6007a92d00SChuck Lever node = llist_del_first(&rdma->sc_rw_ctxts); 61f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 6207a92d00SChuck Lever if (node) { 6307a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 64f13193f5SChuck Lever } else { 65ac3c32bbSChuck Lever ctxt = kmalloc_node(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), 66ac3c32bbSChuck Lever GFP_KERNEL, ibdev_to_node(rdma->sc_cm_id->device)); 67f13193f5SChuck Lever if (!ctxt) 68f4e53e1cSChuck Lever goto out_noctx; 6907a92d00SChuck Lever 70f13193f5SChuck Lever INIT_LIST_HEAD(&ctxt->rw_list); 71f13193f5SChuck Lever } 72f13193f5SChuck Lever 73f13193f5SChuck Lever ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; 74f13193f5SChuck Lever if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, 754635873cSMing Lei ctxt->rw_sg_table.sgl, 76f4e53e1cSChuck Lever SG_CHUNK_SIZE)) 77f4e53e1cSChuck Lever goto out_free; 78f13193f5SChuck Lever return ctxt; 79f4e53e1cSChuck Lever 80f4e53e1cSChuck Lever out_free: 81f4e53e1cSChuck Lever kfree(ctxt); 82f4e53e1cSChuck Lever out_noctx: 83f4e53e1cSChuck Lever trace_svcrdma_no_rwctx_err(rdma, sges); 84f4e53e1cSChuck Lever return NULL; 85f13193f5SChuck Lever } 86f13193f5SChuck Lever 8707a92d00SChuck Lever static void __svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 8807a92d00SChuck Lever struct svc_rdma_rw_ctxt *ctxt, 8907a92d00SChuck Lever struct llist_head *list) 9007a92d00SChuck Lever { 9107a92d00SChuck Lever sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE); 9207a92d00SChuck Lever llist_add(&ctxt->rw_node, list); 9307a92d00SChuck Lever } 9407a92d00SChuck Lever 95f13193f5SChuck Lever static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 96f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 97f13193f5SChuck Lever { 9807a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &rdma->sc_rw_ctxts); 99f13193f5SChuck Lever } 100f13193f5SChuck Lever 101f13193f5SChuck Lever /** 102f13193f5SChuck Lever * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts 103f13193f5SChuck Lever * @rdma: transport about to be destroyed 104f13193f5SChuck Lever * 105f13193f5SChuck Lever */ 106f13193f5SChuck Lever void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) 107f13193f5SChuck Lever { 108f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 10907a92d00SChuck Lever struct llist_node *node; 110f13193f5SChuck Lever 11107a92d00SChuck Lever while ((node = llist_del_first(&rdma->sc_rw_ctxts)) != NULL) { 11207a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 113f13193f5SChuck Lever kfree(ctxt); 114f13193f5SChuck Lever } 115f13193f5SChuck Lever } 116f13193f5SChuck Lever 1172abfbe7eSChuck Lever /** 1182abfbe7eSChuck Lever * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O 1192abfbe7eSChuck Lever * @rdma: controlling transport instance 1202abfbe7eSChuck Lever * @ctxt: R/W context to prepare 1212abfbe7eSChuck Lever * @offset: RDMA offset 1222abfbe7eSChuck Lever * @handle: RDMA tag/handle 1232abfbe7eSChuck Lever * @direction: I/O direction 1242abfbe7eSChuck Lever * 1252abfbe7eSChuck Lever * Returns on success, the number of WQEs that will be needed 1262abfbe7eSChuck Lever * on the workqueue, or a negative errno. 1272abfbe7eSChuck Lever */ 1282abfbe7eSChuck Lever static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, 1292abfbe7eSChuck Lever struct svc_rdma_rw_ctxt *ctxt, 1302abfbe7eSChuck Lever u64 offset, u32 handle, 1312abfbe7eSChuck Lever enum dma_data_direction direction) 1322abfbe7eSChuck Lever { 1332abfbe7eSChuck Lever int ret; 1342abfbe7eSChuck Lever 1352abfbe7eSChuck Lever ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num, 1362abfbe7eSChuck Lever ctxt->rw_sg_table.sgl, ctxt->rw_nents, 1372abfbe7eSChuck Lever 0, offset, handle, direction); 1382abfbe7eSChuck Lever if (unlikely(ret < 0)) { 1392abfbe7eSChuck Lever svc_rdma_put_rw_ctxt(rdma, ctxt); 1402abfbe7eSChuck Lever trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret); 1412abfbe7eSChuck Lever } 1422abfbe7eSChuck Lever return ret; 1432abfbe7eSChuck Lever } 1442abfbe7eSChuck Lever 145f13193f5SChuck Lever /* A chunk context tracks all I/O for moving one Read or Write 1461cc5213bSRandy Dunlap * chunk. This is a set of rdma_rw's that handle data movement 147f13193f5SChuck Lever * for all segments of one chunk. 148f13193f5SChuck Lever * 149f13193f5SChuck Lever * These are small, acquired with a single allocator call, and 150f13193f5SChuck Lever * no more than one is needed per chunk. They are allocated on 151f13193f5SChuck Lever * demand, and not cached. 152f13193f5SChuck Lever */ 153f13193f5SChuck Lever struct svc_rdma_chunk_ctxt { 1546787f0beSChuck Lever struct rpc_rdma_cid cc_cid; 155f13193f5SChuck Lever struct ib_cqe cc_cqe; 156f13193f5SChuck Lever struct svcxprt_rdma *cc_rdma; 157f13193f5SChuck Lever struct list_head cc_rwctxts; 15845f13584SChuck Lever ktime_t cc_posttime; 159f13193f5SChuck Lever int cc_sqecount; 1607d81ee87SChuck Lever enum ib_wc_status cc_status; 1617d81ee87SChuck Lever struct completion cc_done; 162f13193f5SChuck Lever }; 163f13193f5SChuck Lever 1646787f0beSChuck Lever static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma, 1656787f0beSChuck Lever struct rpc_rdma_cid *cid) 1666787f0beSChuck Lever { 1676787f0beSChuck Lever cid->ci_queue_id = rdma->sc_sq_cq->res.id; 1686787f0beSChuck Lever cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids); 1696787f0beSChuck Lever } 1706787f0beSChuck Lever 171f13193f5SChuck Lever static void svc_rdma_cc_init(struct svcxprt_rdma *rdma, 17235a30fc3SChuck Lever struct svc_rdma_chunk_ctxt *cc) 173f13193f5SChuck Lever { 1746787f0beSChuck Lever svc_rdma_cc_cid_init(rdma, &cc->cc_cid); 175f13193f5SChuck Lever cc->cc_rdma = rdma; 176f13193f5SChuck Lever 177f13193f5SChuck Lever INIT_LIST_HEAD(&cc->cc_rwctxts); 178f13193f5SChuck Lever cc->cc_sqecount = 0; 179f13193f5SChuck Lever } 180f13193f5SChuck Lever 18107a92d00SChuck Lever /* 18207a92d00SChuck Lever * The consumed rw_ctx's are cleaned and placed on a local llist so 18307a92d00SChuck Lever * that only one atomic llist operation is needed to put them all 18407a92d00SChuck Lever * back on the free list. 18507a92d00SChuck Lever */ 18635a30fc3SChuck Lever static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc, 18735a30fc3SChuck Lever enum dma_data_direction dir) 188f13193f5SChuck Lever { 189f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 19007a92d00SChuck Lever struct llist_node *first, *last; 191f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 19207a92d00SChuck Lever LLIST_HEAD(free); 193f13193f5SChuck Lever 19407a92d00SChuck Lever first = last = NULL; 195f13193f5SChuck Lever while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) { 196f13193f5SChuck Lever list_del(&ctxt->rw_list); 197f13193f5SChuck Lever 198f13193f5SChuck Lever rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp, 199f13193f5SChuck Lever rdma->sc_port_num, ctxt->rw_sg_table.sgl, 20035a30fc3SChuck Lever ctxt->rw_nents, dir); 20107a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &free); 20207a92d00SChuck Lever 20307a92d00SChuck Lever ctxt->rw_node.next = first; 20407a92d00SChuck Lever first = &ctxt->rw_node; 20507a92d00SChuck Lever if (!last) 20607a92d00SChuck Lever last = first; 207f13193f5SChuck Lever } 20807a92d00SChuck Lever if (first) 20907a92d00SChuck Lever llist_add_batch(first, last, &rdma->sc_rw_ctxts); 210f13193f5SChuck Lever } 211f13193f5SChuck Lever 212f13193f5SChuck Lever /* State for sending a Write or Reply chunk. 213f13193f5SChuck Lever * - Tracks progress of writing one chunk over all its segments 214f13193f5SChuck Lever * - Stores arguments for the SGL constructor functions 215f13193f5SChuck Lever */ 216f13193f5SChuck Lever struct svc_rdma_write_info { 2177a1cbfa1SChuck Lever const struct svc_rdma_chunk *wi_chunk; 2187a1cbfa1SChuck Lever 219f13193f5SChuck Lever /* write state of this chunk */ 220f13193f5SChuck Lever unsigned int wi_seg_off; 221f13193f5SChuck Lever unsigned int wi_seg_no; 222f13193f5SChuck Lever 223f13193f5SChuck Lever /* SGL constructor arguments */ 22451bad8ccSChuck Lever const struct xdr_buf *wi_xdr; 225f13193f5SChuck Lever unsigned char *wi_base; 226f13193f5SChuck Lever unsigned int wi_next_off; 227f13193f5SChuck Lever 228f13193f5SChuck Lever struct svc_rdma_chunk_ctxt wi_cc; 229f13193f5SChuck Lever }; 230f13193f5SChuck Lever 231f13193f5SChuck Lever static struct svc_rdma_write_info * 2327a1cbfa1SChuck Lever svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, 2337a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk) 234f13193f5SChuck Lever { 235f13193f5SChuck Lever struct svc_rdma_write_info *info; 236f13193f5SChuck Lever 237ac3c32bbSChuck Lever info = kmalloc_node(sizeof(*info), GFP_KERNEL, 238ac3c32bbSChuck Lever ibdev_to_node(rdma->sc_cm_id->device)); 239f13193f5SChuck Lever if (!info) 240f13193f5SChuck Lever return info; 241f13193f5SChuck Lever 2427a1cbfa1SChuck Lever info->wi_chunk = chunk; 243f13193f5SChuck Lever info->wi_seg_off = 0; 244f13193f5SChuck Lever info->wi_seg_no = 0; 24535a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->wi_cc); 246026d958bSChuck Lever info->wi_cc.cc_cqe.done = svc_rdma_write_done; 247f13193f5SChuck Lever return info; 248f13193f5SChuck Lever } 249f13193f5SChuck Lever 250f13193f5SChuck Lever static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) 251f13193f5SChuck Lever { 25235a30fc3SChuck Lever svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE); 253f13193f5SChuck Lever kfree(info); 254f13193f5SChuck Lever } 255f13193f5SChuck Lever 256f13193f5SChuck Lever /** 257f13193f5SChuck Lever * svc_rdma_write_done - Write chunk completion 258f13193f5SChuck Lever * @cq: controlling Completion Queue 259f13193f5SChuck Lever * @wc: Work Completion 260f13193f5SChuck Lever * 261f13193f5SChuck Lever * Pages under I/O are freed by a subsequent Send completion. 262f13193f5SChuck Lever */ 263f13193f5SChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 264f13193f5SChuck Lever { 265f13193f5SChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 266f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = 267f13193f5SChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 268f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 269f13193f5SChuck Lever struct svc_rdma_write_info *info = 270f13193f5SChuck Lever container_of(cc, struct svc_rdma_write_info, wi_cc); 271f13193f5SChuck Lever 27245f13584SChuck Lever switch (wc->status) { 27345f13584SChuck Lever case IB_WC_SUCCESS: 2746787f0beSChuck Lever trace_svcrdma_wc_write(wc, &cc->cc_cid); 27545f13584SChuck Lever break; 27645f13584SChuck Lever case IB_WC_WR_FLUSH_ERR: 27745f13584SChuck Lever trace_svcrdma_wc_write_flush(wc, &cc->cc_cid); 27845f13584SChuck Lever break; 27945f13584SChuck Lever default: 28045f13584SChuck Lever trace_svcrdma_wc_write_err(wc, &cc->cc_cid); 28145f13584SChuck Lever } 282bd2abef3SChuck Lever 2836c8c84f5SChuck Lever svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); 284f13193f5SChuck Lever 2858820bcaaSChuck Lever if (unlikely(wc->status != IB_WC_SUCCESS)) 286e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 287f13193f5SChuck Lever 288f13193f5SChuck Lever svc_rdma_write_info_free(info); 289f13193f5SChuck Lever } 290f13193f5SChuck Lever 291026d958bSChuck Lever /* State for pulling a Read chunk. 292026d958bSChuck Lever */ 293026d958bSChuck Lever struct svc_rdma_read_info { 294d96962e6SChuck Lever struct svc_rqst *ri_rqst; 295ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *ri_readctxt; 296026d958bSChuck Lever unsigned int ri_pageno; 297026d958bSChuck Lever unsigned int ri_pageoff; 298bafe9c27SChuck Lever unsigned int ri_totalbytes; 299026d958bSChuck Lever 300026d958bSChuck Lever struct svc_rdma_chunk_ctxt ri_cc; 301026d958bSChuck Lever }; 302026d958bSChuck Lever 303026d958bSChuck Lever static struct svc_rdma_read_info * 304026d958bSChuck Lever svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma) 305026d958bSChuck Lever { 306026d958bSChuck Lever struct svc_rdma_read_info *info; 307026d958bSChuck Lever 308ac3c32bbSChuck Lever info = kmalloc_node(sizeof(*info), GFP_KERNEL, 309ac3c32bbSChuck Lever ibdev_to_node(rdma->sc_cm_id->device)); 310026d958bSChuck Lever if (!info) 311026d958bSChuck Lever return info; 312026d958bSChuck Lever 31335a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->ri_cc); 314026d958bSChuck Lever info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done; 315026d958bSChuck Lever return info; 316026d958bSChuck Lever } 317026d958bSChuck Lever 318026d958bSChuck Lever static void svc_rdma_read_info_free(struct svc_rdma_read_info *info) 319026d958bSChuck Lever { 32035a30fc3SChuck Lever svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE); 321026d958bSChuck Lever kfree(info); 322026d958bSChuck Lever } 323026d958bSChuck Lever 324026d958bSChuck Lever /** 325026d958bSChuck Lever * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx 326026d958bSChuck Lever * @cq: controlling Completion Queue 327026d958bSChuck Lever * @wc: Work Completion 328026d958bSChuck Lever * 329026d958bSChuck Lever */ 330026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) 331026d958bSChuck Lever { 332026d958bSChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 333026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = 334026d958bSChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 33545f13584SChuck Lever struct svc_rdma_read_info *info; 336026d958bSChuck Lever 33745f13584SChuck Lever switch (wc->status) { 33845f13584SChuck Lever case IB_WC_SUCCESS: 33945f13584SChuck Lever info = container_of(cc, struct svc_rdma_read_info, ri_cc); 34045f13584SChuck Lever trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_totalbytes, 34145f13584SChuck Lever cc->cc_posttime); 34245f13584SChuck Lever break; 34345f13584SChuck Lever case IB_WC_WR_FLUSH_ERR: 34445f13584SChuck Lever trace_svcrdma_wc_read_flush(wc, &cc->cc_cid); 34545f13584SChuck Lever break; 34645f13584SChuck Lever default: 34745f13584SChuck Lever trace_svcrdma_wc_read_err(wc, &cc->cc_cid); 34845f13584SChuck Lever } 349bd2abef3SChuck Lever 35045f13584SChuck Lever svc_rdma_wake_send_waiters(cc->cc_rdma, cc->cc_sqecount); 3517d81ee87SChuck Lever cc->cc_status = wc->status; 3527d81ee87SChuck Lever complete(&cc->cc_done); 3537d81ee87SChuck Lever return; 354026d958bSChuck Lever } 355026d958bSChuck Lever 356*91f8ce28SChuck Lever /* 357f13193f5SChuck Lever * Assumptions: 358f13193f5SChuck Lever * - If ib_post_send() succeeds, only one completion is expected, 359f13193f5SChuck Lever * even if one or more WRs are flushed. This is true when posting 360f13193f5SChuck Lever * an rdma_rw_ctx or when posting a single signaled WR. 361f13193f5SChuck Lever */ 362f13193f5SChuck Lever static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) 363f13193f5SChuck Lever { 364f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 365d34ac5cdSBart Van Assche struct ib_send_wr *first_wr; 366d34ac5cdSBart Van Assche const struct ib_send_wr *bad_wr; 367f13193f5SChuck Lever struct list_head *tmp; 368f13193f5SChuck Lever struct ib_cqe *cqe; 369f13193f5SChuck Lever int ret; 370f13193f5SChuck Lever 371*91f8ce28SChuck Lever might_sleep(); 372*91f8ce28SChuck Lever 373107c1d0aSChuck Lever if (cc->cc_sqecount > rdma->sc_sq_depth) 374107c1d0aSChuck Lever return -EINVAL; 375107c1d0aSChuck Lever 376f13193f5SChuck Lever first_wr = NULL; 377f13193f5SChuck Lever cqe = &cc->cc_cqe; 378f13193f5SChuck Lever list_for_each(tmp, &cc->cc_rwctxts) { 379f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 380f13193f5SChuck Lever 381f13193f5SChuck Lever ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list); 382f13193f5SChuck Lever first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp, 383f13193f5SChuck Lever rdma->sc_port_num, cqe, first_wr); 384f13193f5SChuck Lever cqe = NULL; 385f13193f5SChuck Lever } 386f13193f5SChuck Lever 387f13193f5SChuck Lever do { 388f13193f5SChuck Lever if (atomic_sub_return(cc->cc_sqecount, 389f13193f5SChuck Lever &rdma->sc_sq_avail) > 0) { 39045f13584SChuck Lever cc->cc_posttime = ktime_get(); 3915d85a822SJason Gunthorpe ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); 392f13193f5SChuck Lever if (ret) 393f13193f5SChuck Lever break; 394f13193f5SChuck Lever return 0; 395f13193f5SChuck Lever } 396f13193f5SChuck Lever 39722df5a22SChuck Lever percpu_counter_inc(&svcrdma_stat_sq_starve); 398bd2abef3SChuck Lever trace_svcrdma_sq_full(rdma); 399f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 400f13193f5SChuck Lever wait_event(rdma->sc_send_wait, 401f13193f5SChuck Lever atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); 402bd2abef3SChuck Lever trace_svcrdma_sq_retry(rdma); 403f13193f5SChuck Lever } while (1); 404f13193f5SChuck Lever 405e28b4fc6SChuck Lever trace_svcrdma_sq_post_err(rdma, ret); 406e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 407f13193f5SChuck Lever 408f13193f5SChuck Lever /* If even one was posted, there will be a completion. */ 409f13193f5SChuck Lever if (bad_wr != first_wr) 410f13193f5SChuck Lever return 0; 411f13193f5SChuck Lever 412f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 413f13193f5SChuck Lever wake_up(&rdma->sc_send_wait); 414f13193f5SChuck Lever return -ENOTCONN; 415f13193f5SChuck Lever } 416f13193f5SChuck Lever 417f13193f5SChuck Lever /* Build and DMA-map an SGL that covers one kvec in an xdr_buf 418f13193f5SChuck Lever */ 419f13193f5SChuck Lever static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, 420f13193f5SChuck Lever unsigned int len, 421f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 422f13193f5SChuck Lever { 423f13193f5SChuck Lever struct scatterlist *sg = ctxt->rw_sg_table.sgl; 424f13193f5SChuck Lever 425f13193f5SChuck Lever sg_set_buf(&sg[0], info->wi_base, len); 426f13193f5SChuck Lever info->wi_base += len; 427f13193f5SChuck Lever 428f13193f5SChuck Lever ctxt->rw_nents = 1; 429f13193f5SChuck Lever } 430f13193f5SChuck Lever 431f13193f5SChuck Lever /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. 432f13193f5SChuck Lever */ 433f13193f5SChuck Lever static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, 434f13193f5SChuck Lever unsigned int remaining, 435f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 436f13193f5SChuck Lever { 437f13193f5SChuck Lever unsigned int sge_no, sge_bytes, page_off, page_no; 43851bad8ccSChuck Lever const struct xdr_buf *xdr = info->wi_xdr; 439f13193f5SChuck Lever struct scatterlist *sg; 440f13193f5SChuck Lever struct page **page; 441f13193f5SChuck Lever 44291b022ecSChuck Lever page_off = info->wi_next_off + xdr->page_base; 44391b022ecSChuck Lever page_no = page_off >> PAGE_SHIFT; 44491b022ecSChuck Lever page_off = offset_in_page(page_off); 445f13193f5SChuck Lever page = xdr->pages + page_no; 446f13193f5SChuck Lever info->wi_next_off += remaining; 447f13193f5SChuck Lever sg = ctxt->rw_sg_table.sgl; 448f13193f5SChuck Lever sge_no = 0; 449f13193f5SChuck Lever do { 450f13193f5SChuck Lever sge_bytes = min_t(unsigned int, remaining, 451f13193f5SChuck Lever PAGE_SIZE - page_off); 452f13193f5SChuck Lever sg_set_page(sg, *page, sge_bytes, page_off); 453f13193f5SChuck Lever 454f13193f5SChuck Lever remaining -= sge_bytes; 455f13193f5SChuck Lever sg = sg_next(sg); 456f13193f5SChuck Lever page_off = 0; 457f13193f5SChuck Lever sge_no++; 458f13193f5SChuck Lever page++; 459f13193f5SChuck Lever } while (remaining); 460f13193f5SChuck Lever 461f13193f5SChuck Lever ctxt->rw_nents = sge_no; 462f13193f5SChuck Lever } 463f13193f5SChuck Lever 464f13193f5SChuck Lever /* Construct RDMA Write WRs to send a portion of an xdr_buf containing 465f13193f5SChuck Lever * an RPC Reply. 466f13193f5SChuck Lever */ 467f13193f5SChuck Lever static int 468f13193f5SChuck Lever svc_rdma_build_writes(struct svc_rdma_write_info *info, 469f13193f5SChuck Lever void (*constructor)(struct svc_rdma_write_info *info, 470f13193f5SChuck Lever unsigned int len, 471f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt), 472f13193f5SChuck Lever unsigned int remaining) 473f13193f5SChuck Lever { 474f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; 475f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 4767a1cbfa1SChuck Lever const struct svc_rdma_segment *seg; 477f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 478f13193f5SChuck Lever int ret; 479f13193f5SChuck Lever 480f13193f5SChuck Lever do { 481f13193f5SChuck Lever unsigned int write_len; 482f60a0869SChuck Lever u64 offset; 483f13193f5SChuck Lever 484f012e95bSChuck Lever if (info->wi_seg_no >= info->wi_chunk->ch_segcount) 485f13193f5SChuck Lever goto out_overflow; 486f13193f5SChuck Lever 487f012e95bSChuck Lever seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; 4887a1cbfa1SChuck Lever write_len = min(remaining, seg->rs_length - info->wi_seg_off); 489e5decb2eSChuck Lever if (!write_len) 490e5decb2eSChuck Lever goto out_overflow; 491f13193f5SChuck Lever ctxt = svc_rdma_get_rw_ctxt(rdma, 492f13193f5SChuck Lever (write_len >> PAGE_SHIFT) + 2); 493f13193f5SChuck Lever if (!ctxt) 494f4e53e1cSChuck Lever return -ENOMEM; 495f13193f5SChuck Lever 496f13193f5SChuck Lever constructor(info, write_len, ctxt); 4977a1cbfa1SChuck Lever offset = seg->rs_offset + info->wi_seg_off; 4987a1cbfa1SChuck Lever ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle, 4992abfbe7eSChuck Lever DMA_TO_DEVICE); 500f13193f5SChuck Lever if (ret < 0) 5012abfbe7eSChuck Lever return -EIO; 5021e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_write); 503f13193f5SChuck Lever 504f13193f5SChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 505f13193f5SChuck Lever cc->cc_sqecount += ret; 5067a1cbfa1SChuck Lever if (write_len == seg->rs_length - info->wi_seg_off) { 507f13193f5SChuck Lever info->wi_seg_no++; 508f13193f5SChuck Lever info->wi_seg_off = 0; 509f13193f5SChuck Lever } else { 510f13193f5SChuck Lever info->wi_seg_off += write_len; 511f13193f5SChuck Lever } 512f13193f5SChuck Lever remaining -= write_len; 513f13193f5SChuck Lever } while (remaining); 514f13193f5SChuck Lever 515f13193f5SChuck Lever return 0; 516f13193f5SChuck Lever 517f13193f5SChuck Lever out_overflow: 518dbc17acdSChuck Lever trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no, 5197a1cbfa1SChuck Lever info->wi_chunk->ch_segcount); 520f13193f5SChuck Lever return -E2BIG; 521f13193f5SChuck Lever } 522f13193f5SChuck Lever 523ab1394eeSChuck Lever /** 524ab1394eeSChuck Lever * svc_rdma_iov_write - Construct RDMA Writes from an iov 525ab1394eeSChuck Lever * @info: pointer to write arguments 526ab1394eeSChuck Lever * @iov: kvec to write 527ab1394eeSChuck Lever * 528ab1394eeSChuck Lever * Returns: 529d5029525SZheng Yongjun * On success, returns zero 530ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 531ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 532ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 533f13193f5SChuck Lever */ 534ab1394eeSChuck Lever static int svc_rdma_iov_write(struct svc_rdma_write_info *info, 535ab1394eeSChuck Lever const struct kvec *iov) 536f13193f5SChuck Lever { 537ab1394eeSChuck Lever info->wi_base = iov->iov_base; 538f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_vec_to_sg, 539ab1394eeSChuck Lever iov->iov_len); 540f13193f5SChuck Lever } 541f13193f5SChuck Lever 542ab1394eeSChuck Lever /** 543ab1394eeSChuck Lever * svc_rdma_pages_write - Construct RDMA Writes from pages 544ab1394eeSChuck Lever * @info: pointer to write arguments 545ab1394eeSChuck Lever * @xdr: xdr_buf with pages to write 546ab1394eeSChuck Lever * @offset: offset into the content of @xdr 547ab1394eeSChuck Lever * @length: number of bytes to write 548ab1394eeSChuck Lever * 549ab1394eeSChuck Lever * Returns: 550d5029525SZheng Yongjun * On success, returns zero 551ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 552ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 553ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 554f13193f5SChuck Lever */ 555ab1394eeSChuck Lever static int svc_rdma_pages_write(struct svc_rdma_write_info *info, 556ab1394eeSChuck Lever const struct xdr_buf *xdr, 55741205539SChuck Lever unsigned int offset, 55841205539SChuck Lever unsigned long length) 559f13193f5SChuck Lever { 560f13193f5SChuck Lever info->wi_xdr = xdr; 56141205539SChuck Lever info->wi_next_off = offset - xdr->head[0].iov_len; 562f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg, 56341205539SChuck Lever length); 564f13193f5SChuck Lever } 565f13193f5SChuck Lever 566f13193f5SChuck Lever /** 567f6ad7759SChuck Lever * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf 568f6ad7759SChuck Lever * @xdr: xdr_buf to write 56941bc163fSChuck Lever * @data: pointer to write arguments 570f6ad7759SChuck Lever * 571f6ad7759SChuck Lever * Returns: 572d5029525SZheng Yongjun * On success, returns zero 573f6ad7759SChuck Lever * %-E2BIG if the client-provided Write chunk is too small 574f6ad7759SChuck Lever * %-ENOMEM if a resource has been exhausted 575f6ad7759SChuck Lever * %-EIO if an rdma-rw error occurred 576f6ad7759SChuck Lever */ 57741bc163fSChuck Lever static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) 578f6ad7759SChuck Lever { 57941bc163fSChuck Lever struct svc_rdma_write_info *info = data; 580f6ad7759SChuck Lever int ret; 581f6ad7759SChuck Lever 582f6ad7759SChuck Lever if (xdr->head[0].iov_len) { 583f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->head[0]); 584f6ad7759SChuck Lever if (ret < 0) 585f6ad7759SChuck Lever return ret; 586f6ad7759SChuck Lever } 587f6ad7759SChuck Lever 588f6ad7759SChuck Lever if (xdr->page_len) { 589f6ad7759SChuck Lever ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len, 590f6ad7759SChuck Lever xdr->page_len); 591f6ad7759SChuck Lever if (ret < 0) 592f6ad7759SChuck Lever return ret; 593f6ad7759SChuck Lever } 594f6ad7759SChuck Lever 595f6ad7759SChuck Lever if (xdr->tail[0].iov_len) { 596f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->tail[0]); 597f6ad7759SChuck Lever if (ret < 0) 598f6ad7759SChuck Lever return ret; 599f6ad7759SChuck Lever } 600f6ad7759SChuck Lever 601f6ad7759SChuck Lever return xdr->len; 602f6ad7759SChuck Lever } 603f6ad7759SChuck Lever 604f6ad7759SChuck Lever /** 605f13193f5SChuck Lever * svc_rdma_send_write_chunk - Write all segments in a Write chunk 606f13193f5SChuck Lever * @rdma: controlling RDMA transport 6077a1cbfa1SChuck Lever * @chunk: Write chunk provided by the client 608f13193f5SChuck Lever * @xdr: xdr_buf containing the data payload 609f13193f5SChuck Lever * 610f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 611f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Write chunk, 612107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 613f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 614f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 615f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 616f13193f5SChuck Lever */ 6177a1cbfa1SChuck Lever int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, 6187a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk, 619f6ad7759SChuck Lever const struct xdr_buf *xdr) 620f13193f5SChuck Lever { 621f13193f5SChuck Lever struct svc_rdma_write_info *info; 622b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 623f13193f5SChuck Lever int ret; 624f13193f5SChuck Lever 6257a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 626f13193f5SChuck Lever if (!info) 627f13193f5SChuck Lever return -ENOMEM; 628b704be09SChuck Lever cc = &info->wi_cc; 629f13193f5SChuck Lever 630f6ad7759SChuck Lever ret = svc_rdma_xb_write(xdr, info); 631f6ad7759SChuck Lever if (ret != xdr->len) 632f13193f5SChuck Lever goto out_err; 633f13193f5SChuck Lever 634b704be09SChuck Lever trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); 635b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 636f13193f5SChuck Lever if (ret < 0) 637f13193f5SChuck Lever goto out_err; 638f6ad7759SChuck Lever return xdr->len; 639f13193f5SChuck Lever 640f13193f5SChuck Lever out_err: 641f13193f5SChuck Lever svc_rdma_write_info_free(info); 642f13193f5SChuck Lever return ret; 643f13193f5SChuck Lever } 644f13193f5SChuck Lever 645f13193f5SChuck Lever /** 646f13193f5SChuck Lever * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk 647f13193f5SChuck Lever * @rdma: controlling RDMA transport 6486fa5785eSChuck Lever * @rctxt: Write and Reply chunks from client 649f13193f5SChuck Lever * @xdr: xdr_buf containing an RPC Reply 650f13193f5SChuck Lever * 651f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 652f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Reply chunk, 653107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 654f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 655f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 656f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 657f13193f5SChuck Lever */ 6586fa5785eSChuck Lever int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, 6596fa5785eSChuck Lever const struct svc_rdma_recv_ctxt *rctxt, 66041bc163fSChuck Lever const struct xdr_buf *xdr) 661f13193f5SChuck Lever { 662f13193f5SChuck Lever struct svc_rdma_write_info *info; 663b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 6647a1cbfa1SChuck Lever struct svc_rdma_chunk *chunk; 66541bc163fSChuck Lever int ret; 666f13193f5SChuck Lever 6677a1cbfa1SChuck Lever if (pcl_is_empty(&rctxt->rc_reply_pcl)) 668ded380f1SChuck Lever return 0; 669ded380f1SChuck Lever 6707a1cbfa1SChuck Lever chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); 6717a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 672f13193f5SChuck Lever if (!info) 673f13193f5SChuck Lever return -ENOMEM; 674b704be09SChuck Lever cc = &info->wi_cc; 675f13193f5SChuck Lever 67641bc163fSChuck Lever ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, 67741bc163fSChuck Lever svc_rdma_xb_write, info); 678f13193f5SChuck Lever if (ret < 0) 679f13193f5SChuck Lever goto out_err; 680f13193f5SChuck Lever 681b704be09SChuck Lever trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); 682b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 683f13193f5SChuck Lever if (ret < 0) 684f13193f5SChuck Lever goto out_err; 68598895edbSChuck Lever 68641bc163fSChuck Lever return xdr->len; 687f13193f5SChuck Lever 688f13193f5SChuck Lever out_err: 689f13193f5SChuck Lever svc_rdma_write_info_free(info); 690f13193f5SChuck Lever return ret; 691f13193f5SChuck Lever } 692026d958bSChuck Lever 693d96962e6SChuck Lever /** 694d96962e6SChuck Lever * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment 695d96962e6SChuck Lever * @info: context for ongoing I/O 696d96962e6SChuck Lever * @segment: co-ordinates of remote memory to be read 697d96962e6SChuck Lever * 698d96962e6SChuck Lever * Returns: 699d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 700d96962e6SChuck Lever * %-EINVAL: there were not enough rq_pages to finish 701d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 702d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 703d96962e6SChuck Lever */ 704026d958bSChuck Lever static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, 705d96962e6SChuck Lever const struct svc_rdma_segment *segment) 706026d958bSChuck Lever { 707ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 708026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->ri_cc; 709d96962e6SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 710d96962e6SChuck Lever unsigned int sge_no, seg_len, len; 7117d81ee87SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 712026d958bSChuck Lever struct scatterlist *sg; 713026d958bSChuck Lever int ret; 714026d958bSChuck Lever 715d96962e6SChuck Lever len = segment->rs_length; 716026d958bSChuck Lever sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; 717026d958bSChuck Lever ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); 718026d958bSChuck Lever if (!ctxt) 719f4e53e1cSChuck Lever return -ENOMEM; 720026d958bSChuck Lever ctxt->rw_nents = sge_no; 721026d958bSChuck Lever 722026d958bSChuck Lever sg = ctxt->rw_sg_table.sgl; 723026d958bSChuck Lever for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) { 724026d958bSChuck Lever seg_len = min_t(unsigned int, len, 725026d958bSChuck Lever PAGE_SIZE - info->ri_pageoff); 726026d958bSChuck Lever 727026d958bSChuck Lever if (!info->ri_pageoff) 728ecf85b23SChuck Lever head->rc_page_count++; 729026d958bSChuck Lever 730026d958bSChuck Lever sg_set_page(sg, rqstp->rq_pages[info->ri_pageno], 731026d958bSChuck Lever seg_len, info->ri_pageoff); 732026d958bSChuck Lever sg = sg_next(sg); 733026d958bSChuck Lever 734026d958bSChuck Lever info->ri_pageoff += seg_len; 735026d958bSChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 736026d958bSChuck Lever info->ri_pageno++; 737026d958bSChuck Lever info->ri_pageoff = 0; 738026d958bSChuck Lever } 739026d958bSChuck Lever len -= seg_len; 740026d958bSChuck Lever 741026d958bSChuck Lever /* Safety check */ 742026d958bSChuck Lever if (len && 743026d958bSChuck Lever &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end) 744026d958bSChuck Lever goto out_overrun; 745026d958bSChuck Lever } 746026d958bSChuck Lever 747d96962e6SChuck Lever ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset, 748d96962e6SChuck Lever segment->rs_handle, DMA_FROM_DEVICE); 749026d958bSChuck Lever if (ret < 0) 7502abfbe7eSChuck Lever return -EIO; 7511e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_read); 752026d958bSChuck Lever 753026d958bSChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 754026d958bSChuck Lever cc->cc_sqecount += ret; 755026d958bSChuck Lever return 0; 756026d958bSChuck Lever 757026d958bSChuck Lever out_overrun: 7589d200638SChuck Lever trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno); 759026d958bSChuck Lever return -EINVAL; 760026d958bSChuck Lever } 761026d958bSChuck Lever 762d96962e6SChuck Lever /** 763d96962e6SChuck Lever * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk 764d96962e6SChuck Lever * @info: context for ongoing I/O 765d96962e6SChuck Lever * @chunk: Read chunk to pull 766d96962e6SChuck Lever * 767d96962e6SChuck Lever * Return values: 768d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 769d96962e6SChuck Lever * %-EINVAL: there were not enough resources to finish 770d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 771d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 7727075a867SChuck Lever */ 773d96962e6SChuck Lever static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info, 774d96962e6SChuck Lever const struct svc_rdma_chunk *chunk) 775026d958bSChuck Lever { 776d96962e6SChuck Lever const struct svc_rdma_segment *segment; 777026d958bSChuck Lever int ret; 778026d958bSChuck Lever 7797075a867SChuck Lever ret = -EINVAL; 780d96962e6SChuck Lever pcl_for_each_segment(segment, chunk) { 781d96962e6SChuck Lever ret = svc_rdma_build_read_segment(info, segment); 782026d958bSChuck Lever if (ret < 0) 783026d958bSChuck Lever break; 784d96962e6SChuck Lever info->ri_totalbytes += segment->rs_length; 785026d958bSChuck Lever } 786026d958bSChuck Lever return ret; 787026d958bSChuck Lever } 788026d958bSChuck Lever 789d96962e6SChuck Lever /** 790d7cc7397SChuck Lever * svc_rdma_copy_inline_range - Copy part of the inline content into pages 791d7cc7397SChuck Lever * @info: context for RDMA Reads 792d7cc7397SChuck Lever * @offset: offset into the Receive buffer of region to copy 793d7cc7397SChuck Lever * @remaining: length of region to copy 794d7cc7397SChuck Lever * 795d7cc7397SChuck Lever * Take a page at a time from rqstp->rq_pages and copy the inline 796d7cc7397SChuck Lever * content from the Receive buffer into that page. Update 797d7cc7397SChuck Lever * info->ri_pageno and info->ri_pageoff so that the next RDMA Read 798d7cc7397SChuck Lever * result will land contiguously with the copied content. 799d7cc7397SChuck Lever * 800d7cc7397SChuck Lever * Return values: 801d7cc7397SChuck Lever * %0: Inline content was successfully copied 802d7cc7397SChuck Lever * %-EINVAL: offset or length was incorrect 803d7cc7397SChuck Lever */ 804d7cc7397SChuck Lever static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info, 805d7cc7397SChuck Lever unsigned int offset, 806d7cc7397SChuck Lever unsigned int remaining) 807d7cc7397SChuck Lever { 808d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 809d7cc7397SChuck Lever unsigned char *dst, *src = head->rc_recv_buf; 810d7cc7397SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 811d7cc7397SChuck Lever unsigned int page_no, numpages; 812d7cc7397SChuck Lever 813d7cc7397SChuck Lever numpages = PAGE_ALIGN(info->ri_pageoff + remaining) >> PAGE_SHIFT; 814d7cc7397SChuck Lever for (page_no = 0; page_no < numpages; page_no++) { 815d7cc7397SChuck Lever unsigned int page_len; 816d7cc7397SChuck Lever 817d7cc7397SChuck Lever page_len = min_t(unsigned int, remaining, 818d7cc7397SChuck Lever PAGE_SIZE - info->ri_pageoff); 819d7cc7397SChuck Lever 820d7cc7397SChuck Lever if (!info->ri_pageoff) 821d7cc7397SChuck Lever head->rc_page_count++; 822d7cc7397SChuck Lever 8237d81ee87SChuck Lever dst = page_address(rqstp->rq_pages[info->ri_pageno]); 824d7cc7397SChuck Lever memcpy(dst + info->ri_pageno, src + offset, page_len); 825d7cc7397SChuck Lever 826d7cc7397SChuck Lever info->ri_totalbytes += page_len; 827d7cc7397SChuck Lever info->ri_pageoff += page_len; 828d7cc7397SChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 829d7cc7397SChuck Lever info->ri_pageno++; 830d7cc7397SChuck Lever info->ri_pageoff = 0; 831d7cc7397SChuck Lever } 832d7cc7397SChuck Lever remaining -= page_len; 833d7cc7397SChuck Lever offset += page_len; 834d7cc7397SChuck Lever } 835d7cc7397SChuck Lever 836d7cc7397SChuck Lever return -EINVAL; 837d7cc7397SChuck Lever } 838d7cc7397SChuck Lever 839d7cc7397SChuck Lever /** 840d7cc7397SChuck Lever * svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks 841d7cc7397SChuck Lever * @info: context for RDMA Reads 842d7cc7397SChuck Lever * 8437d81ee87SChuck Lever * The chunk data lands in rqstp->rq_arg as a series of contiguous pages, 844d7cc7397SChuck Lever * like an incoming TCP call. 845d7cc7397SChuck Lever * 846d7cc7397SChuck Lever * Return values: 847d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 848d7cc7397SChuck Lever * %-EINVAL: client provided too many chunks or segments, 849d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 850d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 851d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 852d7cc7397SChuck Lever */ 853d7cc7397SChuck Lever static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *info) 854d7cc7397SChuck Lever { 855d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 856d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 8577d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 858d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 859d7cc7397SChuck Lever unsigned int start, length; 860d7cc7397SChuck Lever int ret; 861d7cc7397SChuck Lever 862d7cc7397SChuck Lever start = 0; 863d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 864d7cc7397SChuck Lever length = chunk->ch_position; 865d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 866d7cc7397SChuck Lever if (ret < 0) 867d7cc7397SChuck Lever return ret; 868d7cc7397SChuck Lever 869d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 870d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 871d7cc7397SChuck Lever if (ret < 0) 872d7cc7397SChuck Lever return ret; 873d7cc7397SChuck Lever 874d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 875d7cc7397SChuck Lever if (!next) 876d7cc7397SChuck Lever break; 877d7cc7397SChuck Lever 878d7cc7397SChuck Lever start += length; 879d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 880d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 881d7cc7397SChuck Lever if (ret < 0) 882d7cc7397SChuck Lever return ret; 883d7cc7397SChuck Lever } 884d7cc7397SChuck Lever 885d7cc7397SChuck Lever start += length; 886d7cc7397SChuck Lever length = head->rc_byte_len - start; 887d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 888d7cc7397SChuck Lever if (ret < 0) 889d7cc7397SChuck Lever return ret; 890d7cc7397SChuck Lever 891d7cc7397SChuck Lever buf->len += info->ri_totalbytes; 892d7cc7397SChuck Lever buf->buflen += info->ri_totalbytes; 893d7cc7397SChuck Lever 8947d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 895d7cc7397SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 8967d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 897d7cc7397SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 898d7cc7397SChuck Lever return 0; 899d7cc7397SChuck Lever } 900d7cc7397SChuck Lever 901d7cc7397SChuck Lever /** 902d7cc7397SChuck Lever * svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks 903d96962e6SChuck Lever * @info: context for RDMA Reads 904d96962e6SChuck Lever * 9057d81ee87SChuck Lever * The chunk data lands in the page list of rqstp->rq_arg.pages. 906026d958bSChuck Lever * 9077d81ee87SChuck Lever * Currently NFSD does not look at the rqstp->rq_arg.tail[0] kvec. 908026d958bSChuck Lever * Therefore, XDR round-up of the Read chunk and trailing 909026d958bSChuck Lever * inline content must both be added at the end of the pagelist. 910d96962e6SChuck Lever * 911d96962e6SChuck Lever * Return values: 912d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 913d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 914d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 915d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 916d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 917026d958bSChuck Lever */ 918d7cc7397SChuck Lever static int svc_rdma_read_data_item(struct svc_rdma_read_info *info) 919026d958bSChuck Lever { 920ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 9217d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 922d96962e6SChuck Lever struct svc_rdma_chunk *chunk; 923bafe9c27SChuck Lever unsigned int length; 924026d958bSChuck Lever int ret; 925026d958bSChuck Lever 926d96962e6SChuck Lever chunk = pcl_first_chunk(&head->rc_read_pcl); 927d96962e6SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 928026d958bSChuck Lever if (ret < 0) 929026d958bSChuck Lever goto out; 930026d958bSChuck Lever 931193bcb7bSChuck Lever /* Split the Receive buffer between the head and tail 932193bcb7bSChuck Lever * buffers at Read chunk's position. XDR roundup of the 933193bcb7bSChuck Lever * chunk is not included in either the pagelist or in 934193bcb7bSChuck Lever * the tail. 935026d958bSChuck Lever */ 936d96962e6SChuck Lever buf->tail[0].iov_base = buf->head[0].iov_base + chunk->ch_position; 937d96962e6SChuck Lever buf->tail[0].iov_len = buf->head[0].iov_len - chunk->ch_position; 938d96962e6SChuck Lever buf->head[0].iov_len = chunk->ch_position; 939026d958bSChuck Lever 940175e0310SChuck Lever /* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2). 941193bcb7bSChuck Lever * 942175e0310SChuck Lever * If the client already rounded up the chunk length, the 943175e0310SChuck Lever * length does not change. Otherwise, the length of the page 944175e0310SChuck Lever * list is increased to include XDR round-up. 945175e0310SChuck Lever * 946175e0310SChuck Lever * Currently these chunks always start at page offset 0, 947175e0310SChuck Lever * thus the rounded-up length never crosses a page boundary. 948026d958bSChuck Lever */ 9497d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[0]; 9507d81ee87SChuck Lever length = xdr_align_size(chunk->ch_length); 951bafe9c27SChuck Lever buf->page_len = length; 952bafe9c27SChuck Lever buf->len += length; 953bafe9c27SChuck Lever buf->buflen += length; 954026d958bSChuck Lever 955026d958bSChuck Lever out: 956026d958bSChuck Lever return ret; 957026d958bSChuck Lever } 958026d958bSChuck Lever 959d96962e6SChuck Lever /** 960d7cc7397SChuck Lever * svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk 961d7cc7397SChuck Lever * @info: context for RDMA Reads 962d7cc7397SChuck Lever * @chunk: parsed Call chunk to pull 963d7cc7397SChuck Lever * @offset: offset of region to pull 964d7cc7397SChuck Lever * @length: length of region to pull 965d7cc7397SChuck Lever * 966d7cc7397SChuck Lever * Return values: 967d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 968d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 969d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 970d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 971d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 972d7cc7397SChuck Lever */ 973d7cc7397SChuck Lever static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info, 974d7cc7397SChuck Lever const struct svc_rdma_chunk *chunk, 975d7cc7397SChuck Lever unsigned int offset, unsigned int length) 976d7cc7397SChuck Lever { 977d7cc7397SChuck Lever const struct svc_rdma_segment *segment; 978d7cc7397SChuck Lever int ret; 979d7cc7397SChuck Lever 980d7cc7397SChuck Lever ret = -EINVAL; 981d7cc7397SChuck Lever pcl_for_each_segment(segment, chunk) { 982d7cc7397SChuck Lever struct svc_rdma_segment dummy; 983d7cc7397SChuck Lever 984d7cc7397SChuck Lever if (offset > segment->rs_length) { 985d7cc7397SChuck Lever offset -= segment->rs_length; 986d7cc7397SChuck Lever continue; 987d7cc7397SChuck Lever } 988d7cc7397SChuck Lever 989d7cc7397SChuck Lever dummy.rs_handle = segment->rs_handle; 990d7cc7397SChuck Lever dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; 991d7cc7397SChuck Lever dummy.rs_offset = segment->rs_offset + offset; 992d7cc7397SChuck Lever 993d7cc7397SChuck Lever ret = svc_rdma_build_read_segment(info, &dummy); 994d7cc7397SChuck Lever if (ret < 0) 995d7cc7397SChuck Lever break; 996d7cc7397SChuck Lever 997d7cc7397SChuck Lever info->ri_totalbytes += dummy.rs_length; 998d7cc7397SChuck Lever length -= dummy.rs_length; 999d7cc7397SChuck Lever offset = 0; 1000d7cc7397SChuck Lever } 1001d7cc7397SChuck Lever return ret; 1002d7cc7397SChuck Lever } 1003d7cc7397SChuck Lever 1004d7cc7397SChuck Lever /** 1005d7cc7397SChuck Lever * svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message 1006d7cc7397SChuck Lever * @info: context for RDMA Reads 1007d7cc7397SChuck Lever * 1008d7cc7397SChuck Lever * Return values: 1009d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 1010d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 1011d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1012d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1013d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1014d7cc7397SChuck Lever */ 1015d7cc7397SChuck Lever static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info) 1016d7cc7397SChuck Lever { 1017d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 1018d7cc7397SChuck Lever const struct svc_rdma_chunk *call_chunk = 1019d7cc7397SChuck Lever pcl_first_chunk(&head->rc_call_pcl); 1020d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 1021d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 1022d7cc7397SChuck Lever unsigned int start, length; 1023d7cc7397SChuck Lever int ret; 1024d7cc7397SChuck Lever 1025d7cc7397SChuck Lever if (pcl_is_empty(pcl)) 1026d7cc7397SChuck Lever return svc_rdma_build_read_chunk(info, call_chunk); 1027d7cc7397SChuck Lever 1028d7cc7397SChuck Lever start = 0; 1029d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 1030d7cc7397SChuck Lever length = chunk->ch_position; 1031d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, start, length); 1032d7cc7397SChuck Lever if (ret < 0) 1033d7cc7397SChuck Lever return ret; 1034d7cc7397SChuck Lever 1035d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 1036d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 1037d7cc7397SChuck Lever if (ret < 0) 1038d7cc7397SChuck Lever return ret; 1039d7cc7397SChuck Lever 1040d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 1041d7cc7397SChuck Lever if (!next) 1042d7cc7397SChuck Lever break; 1043d7cc7397SChuck Lever 1044d7cc7397SChuck Lever start += length; 1045d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 1046d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, 1047d7cc7397SChuck Lever start, length); 1048d7cc7397SChuck Lever if (ret < 0) 1049d7cc7397SChuck Lever return ret; 1050d7cc7397SChuck Lever } 1051d7cc7397SChuck Lever 1052d7cc7397SChuck Lever start += length; 1053d7cc7397SChuck Lever length = call_chunk->ch_length - start; 1054d7cc7397SChuck Lever return svc_rdma_read_chunk_range(info, call_chunk, start, length); 1055d7cc7397SChuck Lever } 1056d7cc7397SChuck Lever 1057d7cc7397SChuck Lever /** 1058d96962e6SChuck Lever * svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message 1059d96962e6SChuck Lever * @info: context for RDMA Reads 1060d96962e6SChuck Lever * 1061d96962e6SChuck Lever * The start of the data lands in the first page just after the 10627d81ee87SChuck Lever * Transport header, and the rest lands in rqstp->rq_arg.pages. 1063026d958bSChuck Lever * 1064026d958bSChuck Lever * Assumptions: 1065d96962e6SChuck Lever * - A PZRC is never sent in an RDMA_MSG message, though it's 1066d96962e6SChuck Lever * allowed by spec. 1067d96962e6SChuck Lever * 1068d96962e6SChuck Lever * Return values: 1069d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 1070d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1071d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1072d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1073d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1074026d958bSChuck Lever */ 1075d7cc7397SChuck Lever static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info) 1076026d958bSChuck Lever { 10777d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 1078026d958bSChuck Lever int ret; 1079026d958bSChuck Lever 1080d7cc7397SChuck Lever ret = svc_rdma_read_call_chunk(info); 1081026d958bSChuck Lever if (ret < 0) 1082026d958bSChuck Lever goto out; 1083026d958bSChuck Lever 1084bafe9c27SChuck Lever buf->len += info->ri_totalbytes; 1085bafe9c27SChuck Lever buf->buflen += info->ri_totalbytes; 1086026d958bSChuck Lever 10877d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 1088bafe9c27SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 10897d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 1090bafe9c27SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 1091026d958bSChuck Lever 1092026d958bSChuck Lever out: 1093026d958bSChuck Lever return ret; 1094026d958bSChuck Lever } 1095026d958bSChuck Lever 1096026d958bSChuck Lever /** 1097d96962e6SChuck Lever * svc_rdma_process_read_list - Pull list of Read chunks from the client 1098026d958bSChuck Lever * @rdma: controlling RDMA transport 1099026d958bSChuck Lever * @rqstp: set of pages to use as Read sink buffers 1100026d958bSChuck Lever * @head: pages under I/O collect here 1101026d958bSChuck Lever * 1102d96962e6SChuck Lever * The RPC/RDMA protocol assumes that the upper layer's XDR decoders 1103d96962e6SChuck Lever * pull each Read chunk as they decode an incoming RPC message. 1104026d958bSChuck Lever * 1105d96962e6SChuck Lever * On Linux, however, the server needs to have a fully-constructed RPC 1106d96962e6SChuck Lever * message in rqstp->rq_arg when there is a positive return code from 1107d96962e6SChuck Lever * ->xpo_recvfrom. So the Read list is safety-checked immediately when 1108d96962e6SChuck Lever * it is received, then here the whole Read list is pulled all at once. 1109d96962e6SChuck Lever * The ingress RPC message is fully reconstructed once all associated 1110d96962e6SChuck Lever * RDMA Reads have completed. 1111d96962e6SChuck Lever * 1112d96962e6SChuck Lever * Return values: 1113d96962e6SChuck Lever * %1: all needed RDMA Reads were posted successfully, 1114d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1115d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1116d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1117d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1118026d958bSChuck Lever */ 1119d96962e6SChuck Lever int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, 1120d96962e6SChuck Lever struct svc_rqst *rqstp, 1121d96962e6SChuck Lever struct svc_rdma_recv_ctxt *head) 1122026d958bSChuck Lever { 1123026d958bSChuck Lever struct svc_rdma_read_info *info; 1124b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 1125026d958bSChuck Lever int ret; 1126026d958bSChuck Lever 1127026d958bSChuck Lever info = svc_rdma_read_info_alloc(rdma); 1128026d958bSChuck Lever if (!info) 1129026d958bSChuck Lever return -ENOMEM; 1130b704be09SChuck Lever cc = &info->ri_cc; 1131d96962e6SChuck Lever info->ri_rqst = rqstp; 1132026d958bSChuck Lever info->ri_readctxt = head; 11333316f063SChuck Lever info->ri_pageno = 0; 11343316f063SChuck Lever info->ri_pageoff = 0; 1135bafe9c27SChuck Lever info->ri_totalbytes = 0; 1136026d958bSChuck Lever 1137d7cc7397SChuck Lever if (pcl_is_empty(&head->rc_call_pcl)) { 1138d7cc7397SChuck Lever if (head->rc_read_pcl.cl_count == 1) 1139d7cc7397SChuck Lever ret = svc_rdma_read_data_item(info); 1140026d958bSChuck Lever else 1141d7cc7397SChuck Lever ret = svc_rdma_read_multiple_chunks(info); 1142d7cc7397SChuck Lever } else 1143d96962e6SChuck Lever ret = svc_rdma_read_special(info); 1144026d958bSChuck Lever if (ret < 0) 114507d0ff3bSChuck Lever goto out_err; 1146026d958bSChuck Lever 1147b704be09SChuck Lever trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount); 11487d81ee87SChuck Lever init_completion(&cc->cc_done); 1149b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 1150026d958bSChuck Lever if (ret < 0) 115107d0ff3bSChuck Lever goto out_err; 11527d81ee87SChuck Lever 11537d81ee87SChuck Lever ret = 1; 11547d81ee87SChuck Lever wait_for_completion(&cc->cc_done); 11557d81ee87SChuck Lever if (cc->cc_status != IB_WC_SUCCESS) 11567d81ee87SChuck Lever ret = -EIO; 11577d81ee87SChuck Lever 11587d81ee87SChuck Lever /* rq_respages starts after the last arg page */ 11597d81ee87SChuck Lever rqstp->rq_respages = &rqstp->rq_pages[head->rc_page_count]; 11607d81ee87SChuck Lever rqstp->rq_next_page = rqstp->rq_respages + 1; 11617d81ee87SChuck Lever 11627d81ee87SChuck Lever /* Ensure svc_rdma_recv_ctxt_put() does not try to release pages */ 11637d81ee87SChuck Lever head->rc_page_count = 0; 116407d0ff3bSChuck Lever 116507d0ff3bSChuck Lever out_err: 1166026d958bSChuck Lever svc_rdma_read_info_free(info); 1167026d958bSChuck Lever return ret; 1168026d958bSChuck Lever } 1169