1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2f13193f5SChuck Lever /* 3ecf85b23SChuck Lever * Copyright (c) 2016-2018 Oracle. All rights reserved. 4f13193f5SChuck Lever * 5f13193f5SChuck Lever * Use the core R/W API to move RPC-over-RDMA Read and Write chunks. 6f13193f5SChuck Lever */ 7f13193f5SChuck Lever 898895edbSChuck Lever #include <rdma/rw.h> 998895edbSChuck Lever 10f60a0869SChuck Lever #include <linux/sunrpc/xdr.h> 11f13193f5SChuck Lever #include <linux/sunrpc/rpc_rdma.h> 12f13193f5SChuck Lever #include <linux/sunrpc/svc_rdma.h> 13f13193f5SChuck Lever 1498895edbSChuck Lever #include "xprt_rdma.h" 1598895edbSChuck Lever #include <trace/events/rpcrdma.h> 16f13193f5SChuck Lever 17026d958bSChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); 18026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); 19026d958bSChuck Lever 20f13193f5SChuck Lever /* Each R/W context contains state for one chain of RDMA Read or 21f13193f5SChuck Lever * Write Work Requests. 22f13193f5SChuck Lever * 23f13193f5SChuck Lever * Each WR chain handles a single contiguous server-side buffer, 24f13193f5SChuck Lever * because scatterlist entries after the first have to start on 25f13193f5SChuck Lever * page alignment. xdr_buf iovecs cannot guarantee alignment. 26f13193f5SChuck Lever * 27f13193f5SChuck Lever * Each WR chain handles only one R_key. Each RPC-over-RDMA segment 28f13193f5SChuck Lever * from a client may contain a unique R_key, so each WR chain moves 29f13193f5SChuck Lever * up to one segment at a time. 30f13193f5SChuck Lever * 31f13193f5SChuck Lever * The scatterlist makes this data structure over 4KB in size. To 32f13193f5SChuck Lever * make it less likely to fail, and to handle the allocation for 33f13193f5SChuck Lever * smaller I/O requests without disabling bottom-halves, these 34f13193f5SChuck Lever * contexts are created on demand, but cached and reused until the 35f13193f5SChuck Lever * controlling svcxprt_rdma is destroyed. 36f13193f5SChuck Lever */ 37f13193f5SChuck Lever struct svc_rdma_rw_ctxt { 3807a92d00SChuck Lever struct llist_node rw_node; 39f13193f5SChuck Lever struct list_head rw_list; 40f13193f5SChuck Lever struct rdma_rw_ctx rw_ctx; 412abfbe7eSChuck Lever unsigned int rw_nents; 42f13193f5SChuck Lever struct sg_table rw_sg_table; 43c0fb23f8SGustavo A. R. Silva struct scatterlist rw_first_sgl[]; 44f13193f5SChuck Lever }; 45f13193f5SChuck Lever 46f13193f5SChuck Lever static inline struct svc_rdma_rw_ctxt * 47f13193f5SChuck Lever svc_rdma_next_ctxt(struct list_head *list) 48f13193f5SChuck Lever { 49f13193f5SChuck Lever return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt, 50f13193f5SChuck Lever rw_list); 51f13193f5SChuck Lever } 52f13193f5SChuck Lever 53f13193f5SChuck Lever static struct svc_rdma_rw_ctxt * 54f13193f5SChuck Lever svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) 55f13193f5SChuck Lever { 56f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 5707a92d00SChuck Lever struct llist_node *node; 58f13193f5SChuck Lever 59f13193f5SChuck Lever spin_lock(&rdma->sc_rw_ctxt_lock); 6007a92d00SChuck Lever node = llist_del_first(&rdma->sc_rw_ctxts); 61f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 6207a92d00SChuck Lever if (node) { 6307a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 64f13193f5SChuck Lever } else { 65*ac3c32bbSChuck Lever ctxt = kmalloc_node(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), 66*ac3c32bbSChuck Lever GFP_KERNEL, ibdev_to_node(rdma->sc_cm_id->device)); 67f13193f5SChuck Lever if (!ctxt) 68f4e53e1cSChuck Lever goto out_noctx; 6907a92d00SChuck Lever 70f13193f5SChuck Lever INIT_LIST_HEAD(&ctxt->rw_list); 71f13193f5SChuck Lever } 72f13193f5SChuck Lever 73f13193f5SChuck Lever ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; 74f13193f5SChuck Lever if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, 754635873cSMing Lei ctxt->rw_sg_table.sgl, 76f4e53e1cSChuck Lever SG_CHUNK_SIZE)) 77f4e53e1cSChuck Lever goto out_free; 78f13193f5SChuck Lever return ctxt; 79f4e53e1cSChuck Lever 80f4e53e1cSChuck Lever out_free: 81f4e53e1cSChuck Lever kfree(ctxt); 82f4e53e1cSChuck Lever out_noctx: 83f4e53e1cSChuck Lever trace_svcrdma_no_rwctx_err(rdma, sges); 84f4e53e1cSChuck Lever return NULL; 85f13193f5SChuck Lever } 86f13193f5SChuck Lever 8707a92d00SChuck Lever static void __svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 8807a92d00SChuck Lever struct svc_rdma_rw_ctxt *ctxt, 8907a92d00SChuck Lever struct llist_head *list) 9007a92d00SChuck Lever { 9107a92d00SChuck Lever sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE); 9207a92d00SChuck Lever llist_add(&ctxt->rw_node, list); 9307a92d00SChuck Lever } 9407a92d00SChuck Lever 95f13193f5SChuck Lever static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 96f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 97f13193f5SChuck Lever { 9807a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &rdma->sc_rw_ctxts); 99f13193f5SChuck Lever } 100f13193f5SChuck Lever 101f13193f5SChuck Lever /** 102f13193f5SChuck Lever * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts 103f13193f5SChuck Lever * @rdma: transport about to be destroyed 104f13193f5SChuck Lever * 105f13193f5SChuck Lever */ 106f13193f5SChuck Lever void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) 107f13193f5SChuck Lever { 108f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 10907a92d00SChuck Lever struct llist_node *node; 110f13193f5SChuck Lever 11107a92d00SChuck Lever while ((node = llist_del_first(&rdma->sc_rw_ctxts)) != NULL) { 11207a92d00SChuck Lever ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); 113f13193f5SChuck Lever kfree(ctxt); 114f13193f5SChuck Lever } 115f13193f5SChuck Lever } 116f13193f5SChuck Lever 1172abfbe7eSChuck Lever /** 1182abfbe7eSChuck Lever * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O 1192abfbe7eSChuck Lever * @rdma: controlling transport instance 1202abfbe7eSChuck Lever * @ctxt: R/W context to prepare 1212abfbe7eSChuck Lever * @offset: RDMA offset 1222abfbe7eSChuck Lever * @handle: RDMA tag/handle 1232abfbe7eSChuck Lever * @direction: I/O direction 1242abfbe7eSChuck Lever * 1252abfbe7eSChuck Lever * Returns on success, the number of WQEs that will be needed 1262abfbe7eSChuck Lever * on the workqueue, or a negative errno. 1272abfbe7eSChuck Lever */ 1282abfbe7eSChuck Lever static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, 1292abfbe7eSChuck Lever struct svc_rdma_rw_ctxt *ctxt, 1302abfbe7eSChuck Lever u64 offset, u32 handle, 1312abfbe7eSChuck Lever enum dma_data_direction direction) 1322abfbe7eSChuck Lever { 1332abfbe7eSChuck Lever int ret; 1342abfbe7eSChuck Lever 1352abfbe7eSChuck Lever ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num, 1362abfbe7eSChuck Lever ctxt->rw_sg_table.sgl, ctxt->rw_nents, 1372abfbe7eSChuck Lever 0, offset, handle, direction); 1382abfbe7eSChuck Lever if (unlikely(ret < 0)) { 1392abfbe7eSChuck Lever svc_rdma_put_rw_ctxt(rdma, ctxt); 1402abfbe7eSChuck Lever trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret); 1412abfbe7eSChuck Lever } 1422abfbe7eSChuck Lever return ret; 1432abfbe7eSChuck Lever } 1442abfbe7eSChuck Lever 145f13193f5SChuck Lever /* A chunk context tracks all I/O for moving one Read or Write 1461cc5213bSRandy Dunlap * chunk. This is a set of rdma_rw's that handle data movement 147f13193f5SChuck Lever * for all segments of one chunk. 148f13193f5SChuck Lever * 149f13193f5SChuck Lever * These are small, acquired with a single allocator call, and 150f13193f5SChuck Lever * no more than one is needed per chunk. They are allocated on 151f13193f5SChuck Lever * demand, and not cached. 152f13193f5SChuck Lever */ 153f13193f5SChuck Lever struct svc_rdma_chunk_ctxt { 1546787f0beSChuck Lever struct rpc_rdma_cid cc_cid; 155f13193f5SChuck Lever struct ib_cqe cc_cqe; 156f13193f5SChuck Lever struct svcxprt_rdma *cc_rdma; 157f13193f5SChuck Lever struct list_head cc_rwctxts; 15845f13584SChuck Lever ktime_t cc_posttime; 159f13193f5SChuck Lever int cc_sqecount; 1607d81ee87SChuck Lever enum ib_wc_status cc_status; 1617d81ee87SChuck Lever struct completion cc_done; 162f13193f5SChuck Lever }; 163f13193f5SChuck Lever 1646787f0beSChuck Lever static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma, 1656787f0beSChuck Lever struct rpc_rdma_cid *cid) 1666787f0beSChuck Lever { 1676787f0beSChuck Lever cid->ci_queue_id = rdma->sc_sq_cq->res.id; 1686787f0beSChuck Lever cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids); 1696787f0beSChuck Lever } 1706787f0beSChuck Lever 171f13193f5SChuck Lever static void svc_rdma_cc_init(struct svcxprt_rdma *rdma, 17235a30fc3SChuck Lever struct svc_rdma_chunk_ctxt *cc) 173f13193f5SChuck Lever { 1746787f0beSChuck Lever svc_rdma_cc_cid_init(rdma, &cc->cc_cid); 175f13193f5SChuck Lever cc->cc_rdma = rdma; 176f13193f5SChuck Lever 177f13193f5SChuck Lever INIT_LIST_HEAD(&cc->cc_rwctxts); 178f13193f5SChuck Lever cc->cc_sqecount = 0; 179f13193f5SChuck Lever } 180f13193f5SChuck Lever 18107a92d00SChuck Lever /* 18207a92d00SChuck Lever * The consumed rw_ctx's are cleaned and placed on a local llist so 18307a92d00SChuck Lever * that only one atomic llist operation is needed to put them all 18407a92d00SChuck Lever * back on the free list. 18507a92d00SChuck Lever */ 18635a30fc3SChuck Lever static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc, 18735a30fc3SChuck Lever enum dma_data_direction dir) 188f13193f5SChuck Lever { 189f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 19007a92d00SChuck Lever struct llist_node *first, *last; 191f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 19207a92d00SChuck Lever LLIST_HEAD(free); 193f13193f5SChuck Lever 19407a92d00SChuck Lever first = last = NULL; 195f13193f5SChuck Lever while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) { 196f13193f5SChuck Lever list_del(&ctxt->rw_list); 197f13193f5SChuck Lever 198f13193f5SChuck Lever rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp, 199f13193f5SChuck Lever rdma->sc_port_num, ctxt->rw_sg_table.sgl, 20035a30fc3SChuck Lever ctxt->rw_nents, dir); 20107a92d00SChuck Lever __svc_rdma_put_rw_ctxt(rdma, ctxt, &free); 20207a92d00SChuck Lever 20307a92d00SChuck Lever ctxt->rw_node.next = first; 20407a92d00SChuck Lever first = &ctxt->rw_node; 20507a92d00SChuck Lever if (!last) 20607a92d00SChuck Lever last = first; 207f13193f5SChuck Lever } 20807a92d00SChuck Lever if (first) 20907a92d00SChuck Lever llist_add_batch(first, last, &rdma->sc_rw_ctxts); 210f13193f5SChuck Lever } 211f13193f5SChuck Lever 212f13193f5SChuck Lever /* State for sending a Write or Reply chunk. 213f13193f5SChuck Lever * - Tracks progress of writing one chunk over all its segments 214f13193f5SChuck Lever * - Stores arguments for the SGL constructor functions 215f13193f5SChuck Lever */ 216f13193f5SChuck Lever struct svc_rdma_write_info { 2177a1cbfa1SChuck Lever const struct svc_rdma_chunk *wi_chunk; 2187a1cbfa1SChuck Lever 219f13193f5SChuck Lever /* write state of this chunk */ 220f13193f5SChuck Lever unsigned int wi_seg_off; 221f13193f5SChuck Lever unsigned int wi_seg_no; 222f13193f5SChuck Lever 223f13193f5SChuck Lever /* SGL constructor arguments */ 22451bad8ccSChuck Lever const struct xdr_buf *wi_xdr; 225f13193f5SChuck Lever unsigned char *wi_base; 226f13193f5SChuck Lever unsigned int wi_next_off; 227f13193f5SChuck Lever 228f13193f5SChuck Lever struct svc_rdma_chunk_ctxt wi_cc; 229f13193f5SChuck Lever }; 230f13193f5SChuck Lever 231f13193f5SChuck Lever static struct svc_rdma_write_info * 2327a1cbfa1SChuck Lever svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, 2337a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk) 234f13193f5SChuck Lever { 235f13193f5SChuck Lever struct svc_rdma_write_info *info; 236f13193f5SChuck Lever 237*ac3c32bbSChuck Lever info = kmalloc_node(sizeof(*info), GFP_KERNEL, 238*ac3c32bbSChuck Lever ibdev_to_node(rdma->sc_cm_id->device)); 239f13193f5SChuck Lever if (!info) 240f13193f5SChuck Lever return info; 241f13193f5SChuck Lever 2427a1cbfa1SChuck Lever info->wi_chunk = chunk; 243f13193f5SChuck Lever info->wi_seg_off = 0; 244f13193f5SChuck Lever info->wi_seg_no = 0; 24535a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->wi_cc); 246026d958bSChuck Lever info->wi_cc.cc_cqe.done = svc_rdma_write_done; 247f13193f5SChuck Lever return info; 248f13193f5SChuck Lever } 249f13193f5SChuck Lever 250f13193f5SChuck Lever static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) 251f13193f5SChuck Lever { 25235a30fc3SChuck Lever svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE); 253f13193f5SChuck Lever kfree(info); 254f13193f5SChuck Lever } 255f13193f5SChuck Lever 256f13193f5SChuck Lever /** 257f13193f5SChuck Lever * svc_rdma_write_done - Write chunk completion 258f13193f5SChuck Lever * @cq: controlling Completion Queue 259f13193f5SChuck Lever * @wc: Work Completion 260f13193f5SChuck Lever * 261f13193f5SChuck Lever * Pages under I/O are freed by a subsequent Send completion. 262f13193f5SChuck Lever */ 263f13193f5SChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 264f13193f5SChuck Lever { 265f13193f5SChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 266f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = 267f13193f5SChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 268f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 269f13193f5SChuck Lever struct svc_rdma_write_info *info = 270f13193f5SChuck Lever container_of(cc, struct svc_rdma_write_info, wi_cc); 271f13193f5SChuck Lever 27245f13584SChuck Lever switch (wc->status) { 27345f13584SChuck Lever case IB_WC_SUCCESS: 2746787f0beSChuck Lever trace_svcrdma_wc_write(wc, &cc->cc_cid); 27545f13584SChuck Lever break; 27645f13584SChuck Lever case IB_WC_WR_FLUSH_ERR: 27745f13584SChuck Lever trace_svcrdma_wc_write_flush(wc, &cc->cc_cid); 27845f13584SChuck Lever break; 27945f13584SChuck Lever default: 28045f13584SChuck Lever trace_svcrdma_wc_write_err(wc, &cc->cc_cid); 28145f13584SChuck Lever } 282bd2abef3SChuck Lever 2836c8c84f5SChuck Lever svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); 284f13193f5SChuck Lever 2858820bcaaSChuck Lever if (unlikely(wc->status != IB_WC_SUCCESS)) 286e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 287f13193f5SChuck Lever 288f13193f5SChuck Lever svc_rdma_write_info_free(info); 289f13193f5SChuck Lever } 290f13193f5SChuck Lever 291026d958bSChuck Lever /* State for pulling a Read chunk. 292026d958bSChuck Lever */ 293026d958bSChuck Lever struct svc_rdma_read_info { 294d96962e6SChuck Lever struct svc_rqst *ri_rqst; 295ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *ri_readctxt; 296026d958bSChuck Lever unsigned int ri_pageno; 297026d958bSChuck Lever unsigned int ri_pageoff; 298bafe9c27SChuck Lever unsigned int ri_totalbytes; 299026d958bSChuck Lever 300026d958bSChuck Lever struct svc_rdma_chunk_ctxt ri_cc; 301026d958bSChuck Lever }; 302026d958bSChuck Lever 303026d958bSChuck Lever static struct svc_rdma_read_info * 304026d958bSChuck Lever svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma) 305026d958bSChuck Lever { 306026d958bSChuck Lever struct svc_rdma_read_info *info; 307026d958bSChuck Lever 308*ac3c32bbSChuck Lever info = kmalloc_node(sizeof(*info), GFP_KERNEL, 309*ac3c32bbSChuck Lever ibdev_to_node(rdma->sc_cm_id->device)); 310026d958bSChuck Lever if (!info) 311026d958bSChuck Lever return info; 312026d958bSChuck Lever 31335a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->ri_cc); 314026d958bSChuck Lever info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done; 315026d958bSChuck Lever return info; 316026d958bSChuck Lever } 317026d958bSChuck Lever 318026d958bSChuck Lever static void svc_rdma_read_info_free(struct svc_rdma_read_info *info) 319026d958bSChuck Lever { 32035a30fc3SChuck Lever svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE); 321026d958bSChuck Lever kfree(info); 322026d958bSChuck Lever } 323026d958bSChuck Lever 324026d958bSChuck Lever /** 325026d958bSChuck Lever * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx 326026d958bSChuck Lever * @cq: controlling Completion Queue 327026d958bSChuck Lever * @wc: Work Completion 328026d958bSChuck Lever * 329026d958bSChuck Lever */ 330026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) 331026d958bSChuck Lever { 332026d958bSChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 333026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = 334026d958bSChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 33545f13584SChuck Lever struct svc_rdma_read_info *info; 336026d958bSChuck Lever 33745f13584SChuck Lever switch (wc->status) { 33845f13584SChuck Lever case IB_WC_SUCCESS: 33945f13584SChuck Lever info = container_of(cc, struct svc_rdma_read_info, ri_cc); 34045f13584SChuck Lever trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_totalbytes, 34145f13584SChuck Lever cc->cc_posttime); 34245f13584SChuck Lever break; 34345f13584SChuck Lever case IB_WC_WR_FLUSH_ERR: 34445f13584SChuck Lever trace_svcrdma_wc_read_flush(wc, &cc->cc_cid); 34545f13584SChuck Lever break; 34645f13584SChuck Lever default: 34745f13584SChuck Lever trace_svcrdma_wc_read_err(wc, &cc->cc_cid); 34845f13584SChuck Lever } 349bd2abef3SChuck Lever 35045f13584SChuck Lever svc_rdma_wake_send_waiters(cc->cc_rdma, cc->cc_sqecount); 3517d81ee87SChuck Lever cc->cc_status = wc->status; 3527d81ee87SChuck Lever complete(&cc->cc_done); 3537d81ee87SChuck Lever return; 354026d958bSChuck Lever } 355026d958bSChuck Lever 356f13193f5SChuck Lever /* This function sleeps when the transport's Send Queue is congested. 357f13193f5SChuck Lever * 358f13193f5SChuck Lever * Assumptions: 359f13193f5SChuck Lever * - If ib_post_send() succeeds, only one completion is expected, 360f13193f5SChuck Lever * even if one or more WRs are flushed. This is true when posting 361f13193f5SChuck Lever * an rdma_rw_ctx or when posting a single signaled WR. 362f13193f5SChuck Lever */ 363f13193f5SChuck Lever static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) 364f13193f5SChuck Lever { 365f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 366d34ac5cdSBart Van Assche struct ib_send_wr *first_wr; 367d34ac5cdSBart Van Assche const struct ib_send_wr *bad_wr; 368f13193f5SChuck Lever struct list_head *tmp; 369f13193f5SChuck Lever struct ib_cqe *cqe; 370f13193f5SChuck Lever int ret; 371f13193f5SChuck Lever 372107c1d0aSChuck Lever if (cc->cc_sqecount > rdma->sc_sq_depth) 373107c1d0aSChuck Lever return -EINVAL; 374107c1d0aSChuck Lever 375f13193f5SChuck Lever first_wr = NULL; 376f13193f5SChuck Lever cqe = &cc->cc_cqe; 377f13193f5SChuck Lever list_for_each(tmp, &cc->cc_rwctxts) { 378f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 379f13193f5SChuck Lever 380f13193f5SChuck Lever ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list); 381f13193f5SChuck Lever first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp, 382f13193f5SChuck Lever rdma->sc_port_num, cqe, first_wr); 383f13193f5SChuck Lever cqe = NULL; 384f13193f5SChuck Lever } 385f13193f5SChuck Lever 386f13193f5SChuck Lever do { 387f13193f5SChuck Lever if (atomic_sub_return(cc->cc_sqecount, 388f13193f5SChuck Lever &rdma->sc_sq_avail) > 0) { 38945f13584SChuck Lever cc->cc_posttime = ktime_get(); 3905d85a822SJason Gunthorpe ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); 391f13193f5SChuck Lever if (ret) 392f13193f5SChuck Lever break; 393f13193f5SChuck Lever return 0; 394f13193f5SChuck Lever } 395f13193f5SChuck Lever 39622df5a22SChuck Lever percpu_counter_inc(&svcrdma_stat_sq_starve); 397bd2abef3SChuck Lever trace_svcrdma_sq_full(rdma); 398f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 399f13193f5SChuck Lever wait_event(rdma->sc_send_wait, 400f13193f5SChuck Lever atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); 401bd2abef3SChuck Lever trace_svcrdma_sq_retry(rdma); 402f13193f5SChuck Lever } while (1); 403f13193f5SChuck Lever 404e28b4fc6SChuck Lever trace_svcrdma_sq_post_err(rdma, ret); 405e844d307SChuck Lever svc_xprt_deferred_close(&rdma->sc_xprt); 406f13193f5SChuck Lever 407f13193f5SChuck Lever /* If even one was posted, there will be a completion. */ 408f13193f5SChuck Lever if (bad_wr != first_wr) 409f13193f5SChuck Lever return 0; 410f13193f5SChuck Lever 411f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 412f13193f5SChuck Lever wake_up(&rdma->sc_send_wait); 413f13193f5SChuck Lever return -ENOTCONN; 414f13193f5SChuck Lever } 415f13193f5SChuck Lever 416f13193f5SChuck Lever /* Build and DMA-map an SGL that covers one kvec in an xdr_buf 417f13193f5SChuck Lever */ 418f13193f5SChuck Lever static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, 419f13193f5SChuck Lever unsigned int len, 420f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 421f13193f5SChuck Lever { 422f13193f5SChuck Lever struct scatterlist *sg = ctxt->rw_sg_table.sgl; 423f13193f5SChuck Lever 424f13193f5SChuck Lever sg_set_buf(&sg[0], info->wi_base, len); 425f13193f5SChuck Lever info->wi_base += len; 426f13193f5SChuck Lever 427f13193f5SChuck Lever ctxt->rw_nents = 1; 428f13193f5SChuck Lever } 429f13193f5SChuck Lever 430f13193f5SChuck Lever /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. 431f13193f5SChuck Lever */ 432f13193f5SChuck Lever static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, 433f13193f5SChuck Lever unsigned int remaining, 434f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 435f13193f5SChuck Lever { 436f13193f5SChuck Lever unsigned int sge_no, sge_bytes, page_off, page_no; 43751bad8ccSChuck Lever const struct xdr_buf *xdr = info->wi_xdr; 438f13193f5SChuck Lever struct scatterlist *sg; 439f13193f5SChuck Lever struct page **page; 440f13193f5SChuck Lever 44191b022ecSChuck Lever page_off = info->wi_next_off + xdr->page_base; 44291b022ecSChuck Lever page_no = page_off >> PAGE_SHIFT; 44391b022ecSChuck Lever page_off = offset_in_page(page_off); 444f13193f5SChuck Lever page = xdr->pages + page_no; 445f13193f5SChuck Lever info->wi_next_off += remaining; 446f13193f5SChuck Lever sg = ctxt->rw_sg_table.sgl; 447f13193f5SChuck Lever sge_no = 0; 448f13193f5SChuck Lever do { 449f13193f5SChuck Lever sge_bytes = min_t(unsigned int, remaining, 450f13193f5SChuck Lever PAGE_SIZE - page_off); 451f13193f5SChuck Lever sg_set_page(sg, *page, sge_bytes, page_off); 452f13193f5SChuck Lever 453f13193f5SChuck Lever remaining -= sge_bytes; 454f13193f5SChuck Lever sg = sg_next(sg); 455f13193f5SChuck Lever page_off = 0; 456f13193f5SChuck Lever sge_no++; 457f13193f5SChuck Lever page++; 458f13193f5SChuck Lever } while (remaining); 459f13193f5SChuck Lever 460f13193f5SChuck Lever ctxt->rw_nents = sge_no; 461f13193f5SChuck Lever } 462f13193f5SChuck Lever 463f13193f5SChuck Lever /* Construct RDMA Write WRs to send a portion of an xdr_buf containing 464f13193f5SChuck Lever * an RPC Reply. 465f13193f5SChuck Lever */ 466f13193f5SChuck Lever static int 467f13193f5SChuck Lever svc_rdma_build_writes(struct svc_rdma_write_info *info, 468f13193f5SChuck Lever void (*constructor)(struct svc_rdma_write_info *info, 469f13193f5SChuck Lever unsigned int len, 470f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt), 471f13193f5SChuck Lever unsigned int remaining) 472f13193f5SChuck Lever { 473f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; 474f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 4757a1cbfa1SChuck Lever const struct svc_rdma_segment *seg; 476f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 477f13193f5SChuck Lever int ret; 478f13193f5SChuck Lever 479f13193f5SChuck Lever do { 480f13193f5SChuck Lever unsigned int write_len; 481f60a0869SChuck Lever u64 offset; 482f13193f5SChuck Lever 483f012e95bSChuck Lever if (info->wi_seg_no >= info->wi_chunk->ch_segcount) 484f13193f5SChuck Lever goto out_overflow; 485f13193f5SChuck Lever 486f012e95bSChuck Lever seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; 4877a1cbfa1SChuck Lever write_len = min(remaining, seg->rs_length - info->wi_seg_off); 488e5decb2eSChuck Lever if (!write_len) 489e5decb2eSChuck Lever goto out_overflow; 490f13193f5SChuck Lever ctxt = svc_rdma_get_rw_ctxt(rdma, 491f13193f5SChuck Lever (write_len >> PAGE_SHIFT) + 2); 492f13193f5SChuck Lever if (!ctxt) 493f4e53e1cSChuck Lever return -ENOMEM; 494f13193f5SChuck Lever 495f13193f5SChuck Lever constructor(info, write_len, ctxt); 4967a1cbfa1SChuck Lever offset = seg->rs_offset + info->wi_seg_off; 4977a1cbfa1SChuck Lever ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle, 4982abfbe7eSChuck Lever DMA_TO_DEVICE); 499f13193f5SChuck Lever if (ret < 0) 5002abfbe7eSChuck Lever return -EIO; 5011e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_write); 502f13193f5SChuck Lever 503f13193f5SChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 504f13193f5SChuck Lever cc->cc_sqecount += ret; 5057a1cbfa1SChuck Lever if (write_len == seg->rs_length - info->wi_seg_off) { 506f13193f5SChuck Lever info->wi_seg_no++; 507f13193f5SChuck Lever info->wi_seg_off = 0; 508f13193f5SChuck Lever } else { 509f13193f5SChuck Lever info->wi_seg_off += write_len; 510f13193f5SChuck Lever } 511f13193f5SChuck Lever remaining -= write_len; 512f13193f5SChuck Lever } while (remaining); 513f13193f5SChuck Lever 514f13193f5SChuck Lever return 0; 515f13193f5SChuck Lever 516f13193f5SChuck Lever out_overflow: 517dbc17acdSChuck Lever trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no, 5187a1cbfa1SChuck Lever info->wi_chunk->ch_segcount); 519f13193f5SChuck Lever return -E2BIG; 520f13193f5SChuck Lever } 521f13193f5SChuck Lever 522ab1394eeSChuck Lever /** 523ab1394eeSChuck Lever * svc_rdma_iov_write - Construct RDMA Writes from an iov 524ab1394eeSChuck Lever * @info: pointer to write arguments 525ab1394eeSChuck Lever * @iov: kvec to write 526ab1394eeSChuck Lever * 527ab1394eeSChuck Lever * Returns: 528d5029525SZheng Yongjun * On success, returns zero 529ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 530ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 531ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 532f13193f5SChuck Lever */ 533ab1394eeSChuck Lever static int svc_rdma_iov_write(struct svc_rdma_write_info *info, 534ab1394eeSChuck Lever const struct kvec *iov) 535f13193f5SChuck Lever { 536ab1394eeSChuck Lever info->wi_base = iov->iov_base; 537f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_vec_to_sg, 538ab1394eeSChuck Lever iov->iov_len); 539f13193f5SChuck Lever } 540f13193f5SChuck Lever 541ab1394eeSChuck Lever /** 542ab1394eeSChuck Lever * svc_rdma_pages_write - Construct RDMA Writes from pages 543ab1394eeSChuck Lever * @info: pointer to write arguments 544ab1394eeSChuck Lever * @xdr: xdr_buf with pages to write 545ab1394eeSChuck Lever * @offset: offset into the content of @xdr 546ab1394eeSChuck Lever * @length: number of bytes to write 547ab1394eeSChuck Lever * 548ab1394eeSChuck Lever * Returns: 549d5029525SZheng Yongjun * On success, returns zero 550ab1394eeSChuck Lever * %-E2BIG if the client-provided Write chunk is too small 551ab1394eeSChuck Lever * %-ENOMEM if a resource has been exhausted 552ab1394eeSChuck Lever * %-EIO if an rdma-rw error occurred 553f13193f5SChuck Lever */ 554ab1394eeSChuck Lever static int svc_rdma_pages_write(struct svc_rdma_write_info *info, 555ab1394eeSChuck Lever const struct xdr_buf *xdr, 55641205539SChuck Lever unsigned int offset, 55741205539SChuck Lever unsigned long length) 558f13193f5SChuck Lever { 559f13193f5SChuck Lever info->wi_xdr = xdr; 56041205539SChuck Lever info->wi_next_off = offset - xdr->head[0].iov_len; 561f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg, 56241205539SChuck Lever length); 563f13193f5SChuck Lever } 564f13193f5SChuck Lever 565f13193f5SChuck Lever /** 566f6ad7759SChuck Lever * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf 567f6ad7759SChuck Lever * @xdr: xdr_buf to write 56841bc163fSChuck Lever * @data: pointer to write arguments 569f6ad7759SChuck Lever * 570f6ad7759SChuck Lever * Returns: 571d5029525SZheng Yongjun * On success, returns zero 572f6ad7759SChuck Lever * %-E2BIG if the client-provided Write chunk is too small 573f6ad7759SChuck Lever * %-ENOMEM if a resource has been exhausted 574f6ad7759SChuck Lever * %-EIO if an rdma-rw error occurred 575f6ad7759SChuck Lever */ 57641bc163fSChuck Lever static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) 577f6ad7759SChuck Lever { 57841bc163fSChuck Lever struct svc_rdma_write_info *info = data; 579f6ad7759SChuck Lever int ret; 580f6ad7759SChuck Lever 581f6ad7759SChuck Lever if (xdr->head[0].iov_len) { 582f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->head[0]); 583f6ad7759SChuck Lever if (ret < 0) 584f6ad7759SChuck Lever return ret; 585f6ad7759SChuck Lever } 586f6ad7759SChuck Lever 587f6ad7759SChuck Lever if (xdr->page_len) { 588f6ad7759SChuck Lever ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len, 589f6ad7759SChuck Lever xdr->page_len); 590f6ad7759SChuck Lever if (ret < 0) 591f6ad7759SChuck Lever return ret; 592f6ad7759SChuck Lever } 593f6ad7759SChuck Lever 594f6ad7759SChuck Lever if (xdr->tail[0].iov_len) { 595f6ad7759SChuck Lever ret = svc_rdma_iov_write(info, &xdr->tail[0]); 596f6ad7759SChuck Lever if (ret < 0) 597f6ad7759SChuck Lever return ret; 598f6ad7759SChuck Lever } 599f6ad7759SChuck Lever 600f6ad7759SChuck Lever return xdr->len; 601f6ad7759SChuck Lever } 602f6ad7759SChuck Lever 603f6ad7759SChuck Lever /** 604f13193f5SChuck Lever * svc_rdma_send_write_chunk - Write all segments in a Write chunk 605f13193f5SChuck Lever * @rdma: controlling RDMA transport 6067a1cbfa1SChuck Lever * @chunk: Write chunk provided by the client 607f13193f5SChuck Lever * @xdr: xdr_buf containing the data payload 608f13193f5SChuck Lever * 609f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 610f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Write chunk, 611107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 612f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 613f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 614f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 615f13193f5SChuck Lever */ 6167a1cbfa1SChuck Lever int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, 6177a1cbfa1SChuck Lever const struct svc_rdma_chunk *chunk, 618f6ad7759SChuck Lever const struct xdr_buf *xdr) 619f13193f5SChuck Lever { 620f13193f5SChuck Lever struct svc_rdma_write_info *info; 621b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 622f13193f5SChuck Lever int ret; 623f13193f5SChuck Lever 6247a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 625f13193f5SChuck Lever if (!info) 626f13193f5SChuck Lever return -ENOMEM; 627b704be09SChuck Lever cc = &info->wi_cc; 628f13193f5SChuck Lever 629f6ad7759SChuck Lever ret = svc_rdma_xb_write(xdr, info); 630f6ad7759SChuck Lever if (ret != xdr->len) 631f13193f5SChuck Lever goto out_err; 632f13193f5SChuck Lever 633b704be09SChuck Lever trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); 634b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 635f13193f5SChuck Lever if (ret < 0) 636f13193f5SChuck Lever goto out_err; 637f6ad7759SChuck Lever return xdr->len; 638f13193f5SChuck Lever 639f13193f5SChuck Lever out_err: 640f13193f5SChuck Lever svc_rdma_write_info_free(info); 641f13193f5SChuck Lever return ret; 642f13193f5SChuck Lever } 643f13193f5SChuck Lever 644f13193f5SChuck Lever /** 645f13193f5SChuck Lever * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk 646f13193f5SChuck Lever * @rdma: controlling RDMA transport 6476fa5785eSChuck Lever * @rctxt: Write and Reply chunks from client 648f13193f5SChuck Lever * @xdr: xdr_buf containing an RPC Reply 649f13193f5SChuck Lever * 650f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 651f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Reply chunk, 652107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 653f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 654f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 655f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 656f13193f5SChuck Lever */ 6576fa5785eSChuck Lever int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, 6586fa5785eSChuck Lever const struct svc_rdma_recv_ctxt *rctxt, 65941bc163fSChuck Lever const struct xdr_buf *xdr) 660f13193f5SChuck Lever { 661f13193f5SChuck Lever struct svc_rdma_write_info *info; 662b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 6637a1cbfa1SChuck Lever struct svc_rdma_chunk *chunk; 66441bc163fSChuck Lever int ret; 665f13193f5SChuck Lever 6667a1cbfa1SChuck Lever if (pcl_is_empty(&rctxt->rc_reply_pcl)) 667ded380f1SChuck Lever return 0; 668ded380f1SChuck Lever 6697a1cbfa1SChuck Lever chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); 6707a1cbfa1SChuck Lever info = svc_rdma_write_info_alloc(rdma, chunk); 671f13193f5SChuck Lever if (!info) 672f13193f5SChuck Lever return -ENOMEM; 673b704be09SChuck Lever cc = &info->wi_cc; 674f13193f5SChuck Lever 67541bc163fSChuck Lever ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, 67641bc163fSChuck Lever svc_rdma_xb_write, info); 677f13193f5SChuck Lever if (ret < 0) 678f13193f5SChuck Lever goto out_err; 679f13193f5SChuck Lever 680b704be09SChuck Lever trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); 681b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 682f13193f5SChuck Lever if (ret < 0) 683f13193f5SChuck Lever goto out_err; 68498895edbSChuck Lever 68541bc163fSChuck Lever return xdr->len; 686f13193f5SChuck Lever 687f13193f5SChuck Lever out_err: 688f13193f5SChuck Lever svc_rdma_write_info_free(info); 689f13193f5SChuck Lever return ret; 690f13193f5SChuck Lever } 691026d958bSChuck Lever 692d96962e6SChuck Lever /** 693d96962e6SChuck Lever * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment 694d96962e6SChuck Lever * @info: context for ongoing I/O 695d96962e6SChuck Lever * @segment: co-ordinates of remote memory to be read 696d96962e6SChuck Lever * 697d96962e6SChuck Lever * Returns: 698d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 699d96962e6SChuck Lever * %-EINVAL: there were not enough rq_pages to finish 700d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 701d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 702d96962e6SChuck Lever */ 703026d958bSChuck Lever static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, 704d96962e6SChuck Lever const struct svc_rdma_segment *segment) 705026d958bSChuck Lever { 706ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 707026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->ri_cc; 708d96962e6SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 709d96962e6SChuck Lever unsigned int sge_no, seg_len, len; 7107d81ee87SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 711026d958bSChuck Lever struct scatterlist *sg; 712026d958bSChuck Lever int ret; 713026d958bSChuck Lever 714d96962e6SChuck Lever len = segment->rs_length; 715026d958bSChuck Lever sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; 716026d958bSChuck Lever ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); 717026d958bSChuck Lever if (!ctxt) 718f4e53e1cSChuck Lever return -ENOMEM; 719026d958bSChuck Lever ctxt->rw_nents = sge_no; 720026d958bSChuck Lever 721026d958bSChuck Lever sg = ctxt->rw_sg_table.sgl; 722026d958bSChuck Lever for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) { 723026d958bSChuck Lever seg_len = min_t(unsigned int, len, 724026d958bSChuck Lever PAGE_SIZE - info->ri_pageoff); 725026d958bSChuck Lever 726026d958bSChuck Lever if (!info->ri_pageoff) 727ecf85b23SChuck Lever head->rc_page_count++; 728026d958bSChuck Lever 729026d958bSChuck Lever sg_set_page(sg, rqstp->rq_pages[info->ri_pageno], 730026d958bSChuck Lever seg_len, info->ri_pageoff); 731026d958bSChuck Lever sg = sg_next(sg); 732026d958bSChuck Lever 733026d958bSChuck Lever info->ri_pageoff += seg_len; 734026d958bSChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 735026d958bSChuck Lever info->ri_pageno++; 736026d958bSChuck Lever info->ri_pageoff = 0; 737026d958bSChuck Lever } 738026d958bSChuck Lever len -= seg_len; 739026d958bSChuck Lever 740026d958bSChuck Lever /* Safety check */ 741026d958bSChuck Lever if (len && 742026d958bSChuck Lever &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end) 743026d958bSChuck Lever goto out_overrun; 744026d958bSChuck Lever } 745026d958bSChuck Lever 746d96962e6SChuck Lever ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset, 747d96962e6SChuck Lever segment->rs_handle, DMA_FROM_DEVICE); 748026d958bSChuck Lever if (ret < 0) 7492abfbe7eSChuck Lever return -EIO; 7501e7e5573SChuck Lever percpu_counter_inc(&svcrdma_stat_read); 751026d958bSChuck Lever 752026d958bSChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 753026d958bSChuck Lever cc->cc_sqecount += ret; 754026d958bSChuck Lever return 0; 755026d958bSChuck Lever 756026d958bSChuck Lever out_overrun: 7579d200638SChuck Lever trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno); 758026d958bSChuck Lever return -EINVAL; 759026d958bSChuck Lever } 760026d958bSChuck Lever 761d96962e6SChuck Lever /** 762d96962e6SChuck Lever * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk 763d96962e6SChuck Lever * @info: context for ongoing I/O 764d96962e6SChuck Lever * @chunk: Read chunk to pull 765d96962e6SChuck Lever * 766d96962e6SChuck Lever * Return values: 767d96962e6SChuck Lever * %0: the Read WR chain was constructed successfully 768d96962e6SChuck Lever * %-EINVAL: there were not enough resources to finish 769d96962e6SChuck Lever * %-ENOMEM: allocating a local resources failed 770d96962e6SChuck Lever * %-EIO: a DMA mapping error occurred 7717075a867SChuck Lever */ 772d96962e6SChuck Lever static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info, 773d96962e6SChuck Lever const struct svc_rdma_chunk *chunk) 774026d958bSChuck Lever { 775d96962e6SChuck Lever const struct svc_rdma_segment *segment; 776026d958bSChuck Lever int ret; 777026d958bSChuck Lever 7787075a867SChuck Lever ret = -EINVAL; 779d96962e6SChuck Lever pcl_for_each_segment(segment, chunk) { 780d96962e6SChuck Lever ret = svc_rdma_build_read_segment(info, segment); 781026d958bSChuck Lever if (ret < 0) 782026d958bSChuck Lever break; 783d96962e6SChuck Lever info->ri_totalbytes += segment->rs_length; 784026d958bSChuck Lever } 785026d958bSChuck Lever return ret; 786026d958bSChuck Lever } 787026d958bSChuck Lever 788d96962e6SChuck Lever /** 789d7cc7397SChuck Lever * svc_rdma_copy_inline_range - Copy part of the inline content into pages 790d7cc7397SChuck Lever * @info: context for RDMA Reads 791d7cc7397SChuck Lever * @offset: offset into the Receive buffer of region to copy 792d7cc7397SChuck Lever * @remaining: length of region to copy 793d7cc7397SChuck Lever * 794d7cc7397SChuck Lever * Take a page at a time from rqstp->rq_pages and copy the inline 795d7cc7397SChuck Lever * content from the Receive buffer into that page. Update 796d7cc7397SChuck Lever * info->ri_pageno and info->ri_pageoff so that the next RDMA Read 797d7cc7397SChuck Lever * result will land contiguously with the copied content. 798d7cc7397SChuck Lever * 799d7cc7397SChuck Lever * Return values: 800d7cc7397SChuck Lever * %0: Inline content was successfully copied 801d7cc7397SChuck Lever * %-EINVAL: offset or length was incorrect 802d7cc7397SChuck Lever */ 803d7cc7397SChuck Lever static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info, 804d7cc7397SChuck Lever unsigned int offset, 805d7cc7397SChuck Lever unsigned int remaining) 806d7cc7397SChuck Lever { 807d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 808d7cc7397SChuck Lever unsigned char *dst, *src = head->rc_recv_buf; 809d7cc7397SChuck Lever struct svc_rqst *rqstp = info->ri_rqst; 810d7cc7397SChuck Lever unsigned int page_no, numpages; 811d7cc7397SChuck Lever 812d7cc7397SChuck Lever numpages = PAGE_ALIGN(info->ri_pageoff + remaining) >> PAGE_SHIFT; 813d7cc7397SChuck Lever for (page_no = 0; page_no < numpages; page_no++) { 814d7cc7397SChuck Lever unsigned int page_len; 815d7cc7397SChuck Lever 816d7cc7397SChuck Lever page_len = min_t(unsigned int, remaining, 817d7cc7397SChuck Lever PAGE_SIZE - info->ri_pageoff); 818d7cc7397SChuck Lever 819d7cc7397SChuck Lever if (!info->ri_pageoff) 820d7cc7397SChuck Lever head->rc_page_count++; 821d7cc7397SChuck Lever 8227d81ee87SChuck Lever dst = page_address(rqstp->rq_pages[info->ri_pageno]); 823d7cc7397SChuck Lever memcpy(dst + info->ri_pageno, src + offset, page_len); 824d7cc7397SChuck Lever 825d7cc7397SChuck Lever info->ri_totalbytes += page_len; 826d7cc7397SChuck Lever info->ri_pageoff += page_len; 827d7cc7397SChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 828d7cc7397SChuck Lever info->ri_pageno++; 829d7cc7397SChuck Lever info->ri_pageoff = 0; 830d7cc7397SChuck Lever } 831d7cc7397SChuck Lever remaining -= page_len; 832d7cc7397SChuck Lever offset += page_len; 833d7cc7397SChuck Lever } 834d7cc7397SChuck Lever 835d7cc7397SChuck Lever return -EINVAL; 836d7cc7397SChuck Lever } 837d7cc7397SChuck Lever 838d7cc7397SChuck Lever /** 839d7cc7397SChuck Lever * svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks 840d7cc7397SChuck Lever * @info: context for RDMA Reads 841d7cc7397SChuck Lever * 8427d81ee87SChuck Lever * The chunk data lands in rqstp->rq_arg as a series of contiguous pages, 843d7cc7397SChuck Lever * like an incoming TCP call. 844d7cc7397SChuck Lever * 845d7cc7397SChuck Lever * Return values: 846d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 847d7cc7397SChuck Lever * %-EINVAL: client provided too many chunks or segments, 848d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 849d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 850d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 851d7cc7397SChuck Lever */ 852d7cc7397SChuck Lever static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *info) 853d7cc7397SChuck Lever { 854d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 855d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 8567d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 857d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 858d7cc7397SChuck Lever unsigned int start, length; 859d7cc7397SChuck Lever int ret; 860d7cc7397SChuck Lever 861d7cc7397SChuck Lever start = 0; 862d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 863d7cc7397SChuck Lever length = chunk->ch_position; 864d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 865d7cc7397SChuck Lever if (ret < 0) 866d7cc7397SChuck Lever return ret; 867d7cc7397SChuck Lever 868d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 869d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 870d7cc7397SChuck Lever if (ret < 0) 871d7cc7397SChuck Lever return ret; 872d7cc7397SChuck Lever 873d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 874d7cc7397SChuck Lever if (!next) 875d7cc7397SChuck Lever break; 876d7cc7397SChuck Lever 877d7cc7397SChuck Lever start += length; 878d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 879d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 880d7cc7397SChuck Lever if (ret < 0) 881d7cc7397SChuck Lever return ret; 882d7cc7397SChuck Lever } 883d7cc7397SChuck Lever 884d7cc7397SChuck Lever start += length; 885d7cc7397SChuck Lever length = head->rc_byte_len - start; 886d7cc7397SChuck Lever ret = svc_rdma_copy_inline_range(info, start, length); 887d7cc7397SChuck Lever if (ret < 0) 888d7cc7397SChuck Lever return ret; 889d7cc7397SChuck Lever 890d7cc7397SChuck Lever buf->len += info->ri_totalbytes; 891d7cc7397SChuck Lever buf->buflen += info->ri_totalbytes; 892d7cc7397SChuck Lever 8937d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 894d7cc7397SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 8957d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 896d7cc7397SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 897d7cc7397SChuck Lever return 0; 898d7cc7397SChuck Lever } 899d7cc7397SChuck Lever 900d7cc7397SChuck Lever /** 901d7cc7397SChuck Lever * svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks 902d96962e6SChuck Lever * @info: context for RDMA Reads 903d96962e6SChuck Lever * 9047d81ee87SChuck Lever * The chunk data lands in the page list of rqstp->rq_arg.pages. 905026d958bSChuck Lever * 9067d81ee87SChuck Lever * Currently NFSD does not look at the rqstp->rq_arg.tail[0] kvec. 907026d958bSChuck Lever * Therefore, XDR round-up of the Read chunk and trailing 908026d958bSChuck Lever * inline content must both be added at the end of the pagelist. 909d96962e6SChuck Lever * 910d96962e6SChuck Lever * Return values: 911d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 912d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 913d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 914d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 915d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 916026d958bSChuck Lever */ 917d7cc7397SChuck Lever static int svc_rdma_read_data_item(struct svc_rdma_read_info *info) 918026d958bSChuck Lever { 919ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 9207d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 921d96962e6SChuck Lever struct svc_rdma_chunk *chunk; 922bafe9c27SChuck Lever unsigned int length; 923026d958bSChuck Lever int ret; 924026d958bSChuck Lever 925d96962e6SChuck Lever chunk = pcl_first_chunk(&head->rc_read_pcl); 926d96962e6SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 927026d958bSChuck Lever if (ret < 0) 928026d958bSChuck Lever goto out; 929026d958bSChuck Lever 930193bcb7bSChuck Lever /* Split the Receive buffer between the head and tail 931193bcb7bSChuck Lever * buffers at Read chunk's position. XDR roundup of the 932193bcb7bSChuck Lever * chunk is not included in either the pagelist or in 933193bcb7bSChuck Lever * the tail. 934026d958bSChuck Lever */ 935d96962e6SChuck Lever buf->tail[0].iov_base = buf->head[0].iov_base + chunk->ch_position; 936d96962e6SChuck Lever buf->tail[0].iov_len = buf->head[0].iov_len - chunk->ch_position; 937d96962e6SChuck Lever buf->head[0].iov_len = chunk->ch_position; 938026d958bSChuck Lever 939175e0310SChuck Lever /* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2). 940193bcb7bSChuck Lever * 941175e0310SChuck Lever * If the client already rounded up the chunk length, the 942175e0310SChuck Lever * length does not change. Otherwise, the length of the page 943175e0310SChuck Lever * list is increased to include XDR round-up. 944175e0310SChuck Lever * 945175e0310SChuck Lever * Currently these chunks always start at page offset 0, 946175e0310SChuck Lever * thus the rounded-up length never crosses a page boundary. 947026d958bSChuck Lever */ 9487d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[0]; 9497d81ee87SChuck Lever length = xdr_align_size(chunk->ch_length); 950bafe9c27SChuck Lever buf->page_len = length; 951bafe9c27SChuck Lever buf->len += length; 952bafe9c27SChuck Lever buf->buflen += length; 953026d958bSChuck Lever 954026d958bSChuck Lever out: 955026d958bSChuck Lever return ret; 956026d958bSChuck Lever } 957026d958bSChuck Lever 958d96962e6SChuck Lever /** 959d7cc7397SChuck Lever * svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk 960d7cc7397SChuck Lever * @info: context for RDMA Reads 961d7cc7397SChuck Lever * @chunk: parsed Call chunk to pull 962d7cc7397SChuck Lever * @offset: offset of region to pull 963d7cc7397SChuck Lever * @length: length of region to pull 964d7cc7397SChuck Lever * 965d7cc7397SChuck Lever * Return values: 966d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 967d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 968d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 969d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 970d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 971d7cc7397SChuck Lever */ 972d7cc7397SChuck Lever static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info, 973d7cc7397SChuck Lever const struct svc_rdma_chunk *chunk, 974d7cc7397SChuck Lever unsigned int offset, unsigned int length) 975d7cc7397SChuck Lever { 976d7cc7397SChuck Lever const struct svc_rdma_segment *segment; 977d7cc7397SChuck Lever int ret; 978d7cc7397SChuck Lever 979d7cc7397SChuck Lever ret = -EINVAL; 980d7cc7397SChuck Lever pcl_for_each_segment(segment, chunk) { 981d7cc7397SChuck Lever struct svc_rdma_segment dummy; 982d7cc7397SChuck Lever 983d7cc7397SChuck Lever if (offset > segment->rs_length) { 984d7cc7397SChuck Lever offset -= segment->rs_length; 985d7cc7397SChuck Lever continue; 986d7cc7397SChuck Lever } 987d7cc7397SChuck Lever 988d7cc7397SChuck Lever dummy.rs_handle = segment->rs_handle; 989d7cc7397SChuck Lever dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; 990d7cc7397SChuck Lever dummy.rs_offset = segment->rs_offset + offset; 991d7cc7397SChuck Lever 992d7cc7397SChuck Lever ret = svc_rdma_build_read_segment(info, &dummy); 993d7cc7397SChuck Lever if (ret < 0) 994d7cc7397SChuck Lever break; 995d7cc7397SChuck Lever 996d7cc7397SChuck Lever info->ri_totalbytes += dummy.rs_length; 997d7cc7397SChuck Lever length -= dummy.rs_length; 998d7cc7397SChuck Lever offset = 0; 999d7cc7397SChuck Lever } 1000d7cc7397SChuck Lever return ret; 1001d7cc7397SChuck Lever } 1002d7cc7397SChuck Lever 1003d7cc7397SChuck Lever /** 1004d7cc7397SChuck Lever * svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message 1005d7cc7397SChuck Lever * @info: context for RDMA Reads 1006d7cc7397SChuck Lever * 1007d7cc7397SChuck Lever * Return values: 1008d7cc7397SChuck Lever * %0: RDMA Read WQEs were successfully built 1009d7cc7397SChuck Lever * %-EINVAL: there were not enough resources to finish 1010d7cc7397SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1011d7cc7397SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1012d7cc7397SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1013d7cc7397SChuck Lever */ 1014d7cc7397SChuck Lever static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info) 1015d7cc7397SChuck Lever { 1016d7cc7397SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 1017d7cc7397SChuck Lever const struct svc_rdma_chunk *call_chunk = 1018d7cc7397SChuck Lever pcl_first_chunk(&head->rc_call_pcl); 1019d7cc7397SChuck Lever const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; 1020d7cc7397SChuck Lever struct svc_rdma_chunk *chunk, *next; 1021d7cc7397SChuck Lever unsigned int start, length; 1022d7cc7397SChuck Lever int ret; 1023d7cc7397SChuck Lever 1024d7cc7397SChuck Lever if (pcl_is_empty(pcl)) 1025d7cc7397SChuck Lever return svc_rdma_build_read_chunk(info, call_chunk); 1026d7cc7397SChuck Lever 1027d7cc7397SChuck Lever start = 0; 1028d7cc7397SChuck Lever chunk = pcl_first_chunk(pcl); 1029d7cc7397SChuck Lever length = chunk->ch_position; 1030d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, start, length); 1031d7cc7397SChuck Lever if (ret < 0) 1032d7cc7397SChuck Lever return ret; 1033d7cc7397SChuck Lever 1034d7cc7397SChuck Lever pcl_for_each_chunk(chunk, pcl) { 1035d7cc7397SChuck Lever ret = svc_rdma_build_read_chunk(info, chunk); 1036d7cc7397SChuck Lever if (ret < 0) 1037d7cc7397SChuck Lever return ret; 1038d7cc7397SChuck Lever 1039d7cc7397SChuck Lever next = pcl_next_chunk(pcl, chunk); 1040d7cc7397SChuck Lever if (!next) 1041d7cc7397SChuck Lever break; 1042d7cc7397SChuck Lever 1043d7cc7397SChuck Lever start += length; 1044d7cc7397SChuck Lever length = next->ch_position - info->ri_totalbytes; 1045d7cc7397SChuck Lever ret = svc_rdma_read_chunk_range(info, call_chunk, 1046d7cc7397SChuck Lever start, length); 1047d7cc7397SChuck Lever if (ret < 0) 1048d7cc7397SChuck Lever return ret; 1049d7cc7397SChuck Lever } 1050d7cc7397SChuck Lever 1051d7cc7397SChuck Lever start += length; 1052d7cc7397SChuck Lever length = call_chunk->ch_length - start; 1053d7cc7397SChuck Lever return svc_rdma_read_chunk_range(info, call_chunk, start, length); 1054d7cc7397SChuck Lever } 1055d7cc7397SChuck Lever 1056d7cc7397SChuck Lever /** 1057d96962e6SChuck Lever * svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message 1058d96962e6SChuck Lever * @info: context for RDMA Reads 1059d96962e6SChuck Lever * 1060d96962e6SChuck Lever * The start of the data lands in the first page just after the 10617d81ee87SChuck Lever * Transport header, and the rest lands in rqstp->rq_arg.pages. 1062026d958bSChuck Lever * 1063026d958bSChuck Lever * Assumptions: 1064d96962e6SChuck Lever * - A PZRC is never sent in an RDMA_MSG message, though it's 1065d96962e6SChuck Lever * allowed by spec. 1066d96962e6SChuck Lever * 1067d96962e6SChuck Lever * Return values: 1068d96962e6SChuck Lever * %0: RDMA Read WQEs were successfully built 1069d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1070d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1071d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1072d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1073026d958bSChuck Lever */ 1074d7cc7397SChuck Lever static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info) 1075026d958bSChuck Lever { 10767d81ee87SChuck Lever struct xdr_buf *buf = &info->ri_rqst->rq_arg; 1077026d958bSChuck Lever int ret; 1078026d958bSChuck Lever 1079d7cc7397SChuck Lever ret = svc_rdma_read_call_chunk(info); 1080026d958bSChuck Lever if (ret < 0) 1081026d958bSChuck Lever goto out; 1082026d958bSChuck Lever 1083bafe9c27SChuck Lever buf->len += info->ri_totalbytes; 1084bafe9c27SChuck Lever buf->buflen += info->ri_totalbytes; 1085026d958bSChuck Lever 10867d81ee87SChuck Lever buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); 1087bafe9c27SChuck Lever buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); 10887d81ee87SChuck Lever buf->pages = &info->ri_rqst->rq_pages[1]; 1089bafe9c27SChuck Lever buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; 1090026d958bSChuck Lever 1091026d958bSChuck Lever out: 1092026d958bSChuck Lever return ret; 1093026d958bSChuck Lever } 1094026d958bSChuck Lever 1095026d958bSChuck Lever /** 1096d96962e6SChuck Lever * svc_rdma_process_read_list - Pull list of Read chunks from the client 1097026d958bSChuck Lever * @rdma: controlling RDMA transport 1098026d958bSChuck Lever * @rqstp: set of pages to use as Read sink buffers 1099026d958bSChuck Lever * @head: pages under I/O collect here 1100026d958bSChuck Lever * 1101d96962e6SChuck Lever * The RPC/RDMA protocol assumes that the upper layer's XDR decoders 1102d96962e6SChuck Lever * pull each Read chunk as they decode an incoming RPC message. 1103026d958bSChuck Lever * 1104d96962e6SChuck Lever * On Linux, however, the server needs to have a fully-constructed RPC 1105d96962e6SChuck Lever * message in rqstp->rq_arg when there is a positive return code from 1106d96962e6SChuck Lever * ->xpo_recvfrom. So the Read list is safety-checked immediately when 1107d96962e6SChuck Lever * it is received, then here the whole Read list is pulled all at once. 1108d96962e6SChuck Lever * The ingress RPC message is fully reconstructed once all associated 1109d96962e6SChuck Lever * RDMA Reads have completed. 1110d96962e6SChuck Lever * 1111d96962e6SChuck Lever * Return values: 1112d96962e6SChuck Lever * %1: all needed RDMA Reads were posted successfully, 1113d96962e6SChuck Lever * %-EINVAL: client provided too many chunks or segments, 1114d96962e6SChuck Lever * %-ENOMEM: rdma_rw context pool was exhausted, 1115d96962e6SChuck Lever * %-ENOTCONN: posting failed (connection is lost), 1116d96962e6SChuck Lever * %-EIO: rdma_rw initialization failed (DMA mapping, etc). 1117026d958bSChuck Lever */ 1118d96962e6SChuck Lever int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, 1119d96962e6SChuck Lever struct svc_rqst *rqstp, 1120d96962e6SChuck Lever struct svc_rdma_recv_ctxt *head) 1121026d958bSChuck Lever { 1122026d958bSChuck Lever struct svc_rdma_read_info *info; 1123b704be09SChuck Lever struct svc_rdma_chunk_ctxt *cc; 1124026d958bSChuck Lever int ret; 1125026d958bSChuck Lever 1126026d958bSChuck Lever info = svc_rdma_read_info_alloc(rdma); 1127026d958bSChuck Lever if (!info) 1128026d958bSChuck Lever return -ENOMEM; 1129b704be09SChuck Lever cc = &info->ri_cc; 1130d96962e6SChuck Lever info->ri_rqst = rqstp; 1131026d958bSChuck Lever info->ri_readctxt = head; 11323316f063SChuck Lever info->ri_pageno = 0; 11333316f063SChuck Lever info->ri_pageoff = 0; 1134bafe9c27SChuck Lever info->ri_totalbytes = 0; 1135026d958bSChuck Lever 1136d7cc7397SChuck Lever if (pcl_is_empty(&head->rc_call_pcl)) { 1137d7cc7397SChuck Lever if (head->rc_read_pcl.cl_count == 1) 1138d7cc7397SChuck Lever ret = svc_rdma_read_data_item(info); 1139026d958bSChuck Lever else 1140d7cc7397SChuck Lever ret = svc_rdma_read_multiple_chunks(info); 1141d7cc7397SChuck Lever } else 1142d96962e6SChuck Lever ret = svc_rdma_read_special(info); 1143026d958bSChuck Lever if (ret < 0) 114407d0ff3bSChuck Lever goto out_err; 1145026d958bSChuck Lever 1146b704be09SChuck Lever trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount); 11477d81ee87SChuck Lever init_completion(&cc->cc_done); 1148b704be09SChuck Lever ret = svc_rdma_post_chunk_ctxt(cc); 1149026d958bSChuck Lever if (ret < 0) 115007d0ff3bSChuck Lever goto out_err; 11517d81ee87SChuck Lever 11527d81ee87SChuck Lever ret = 1; 11537d81ee87SChuck Lever wait_for_completion(&cc->cc_done); 11547d81ee87SChuck Lever if (cc->cc_status != IB_WC_SUCCESS) 11557d81ee87SChuck Lever ret = -EIO; 11567d81ee87SChuck Lever 11577d81ee87SChuck Lever /* rq_respages starts after the last arg page */ 11587d81ee87SChuck Lever rqstp->rq_respages = &rqstp->rq_pages[head->rc_page_count]; 11597d81ee87SChuck Lever rqstp->rq_next_page = rqstp->rq_respages + 1; 11607d81ee87SChuck Lever 11617d81ee87SChuck Lever /* Ensure svc_rdma_recv_ctxt_put() does not try to release pages */ 11627d81ee87SChuck Lever head->rc_page_count = 0; 116307d0ff3bSChuck Lever 116407d0ff3bSChuck Lever out_err: 1165026d958bSChuck Lever svc_rdma_read_info_free(info); 1166026d958bSChuck Lever return ret; 1167026d958bSChuck Lever } 1168