1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2f13193f5SChuck Lever /* 3ecf85b23SChuck Lever * Copyright (c) 2016-2018 Oracle. All rights reserved. 4f13193f5SChuck Lever * 5f13193f5SChuck Lever * Use the core R/W API to move RPC-over-RDMA Read and Write chunks. 6f13193f5SChuck Lever */ 7f13193f5SChuck Lever 898895edbSChuck Lever #include <rdma/rw.h> 998895edbSChuck Lever 10f13193f5SChuck Lever #include <linux/sunrpc/rpc_rdma.h> 11f13193f5SChuck Lever #include <linux/sunrpc/svc_rdma.h> 12f13193f5SChuck Lever #include <linux/sunrpc/debug.h> 13f13193f5SChuck Lever 1498895edbSChuck Lever #include "xprt_rdma.h" 1598895edbSChuck Lever #include <trace/events/rpcrdma.h> 16f13193f5SChuck Lever 17f13193f5SChuck Lever #define RPCDBG_FACILITY RPCDBG_SVCXPRT 18f13193f5SChuck Lever 19026d958bSChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); 20026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); 21026d958bSChuck Lever 22f13193f5SChuck Lever /* Each R/W context contains state for one chain of RDMA Read or 23f13193f5SChuck Lever * Write Work Requests. 24f13193f5SChuck Lever * 25f13193f5SChuck Lever * Each WR chain handles a single contiguous server-side buffer, 26f13193f5SChuck Lever * because scatterlist entries after the first have to start on 27f13193f5SChuck Lever * page alignment. xdr_buf iovecs cannot guarantee alignment. 28f13193f5SChuck Lever * 29f13193f5SChuck Lever * Each WR chain handles only one R_key. Each RPC-over-RDMA segment 30f13193f5SChuck Lever * from a client may contain a unique R_key, so each WR chain moves 31f13193f5SChuck Lever * up to one segment at a time. 32f13193f5SChuck Lever * 33f13193f5SChuck Lever * The scatterlist makes this data structure over 4KB in size. To 34f13193f5SChuck Lever * make it less likely to fail, and to handle the allocation for 35f13193f5SChuck Lever * smaller I/O requests without disabling bottom-halves, these 36f13193f5SChuck Lever * contexts are created on demand, but cached and reused until the 37f13193f5SChuck Lever * controlling svcxprt_rdma is destroyed. 38f13193f5SChuck Lever */ 39f13193f5SChuck Lever struct svc_rdma_rw_ctxt { 40f13193f5SChuck Lever struct list_head rw_list; 41f13193f5SChuck Lever struct rdma_rw_ctx rw_ctx; 42f13193f5SChuck Lever int rw_nents; 43f13193f5SChuck Lever struct sg_table rw_sg_table; 44f13193f5SChuck Lever struct scatterlist rw_first_sgl[0]; 45f13193f5SChuck Lever }; 46f13193f5SChuck Lever 47f13193f5SChuck Lever static inline struct svc_rdma_rw_ctxt * 48f13193f5SChuck Lever svc_rdma_next_ctxt(struct list_head *list) 49f13193f5SChuck Lever { 50f13193f5SChuck Lever return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt, 51f13193f5SChuck Lever rw_list); 52f13193f5SChuck Lever } 53f13193f5SChuck Lever 54f13193f5SChuck Lever static struct svc_rdma_rw_ctxt * 55f13193f5SChuck Lever svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) 56f13193f5SChuck Lever { 57f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 58f13193f5SChuck Lever 59f13193f5SChuck Lever spin_lock(&rdma->sc_rw_ctxt_lock); 60f13193f5SChuck Lever 61f13193f5SChuck Lever ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts); 62f13193f5SChuck Lever if (ctxt) { 63f13193f5SChuck Lever list_del(&ctxt->rw_list); 64f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 65f13193f5SChuck Lever } else { 66f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 6714cfbd94SGustavo A. R. Silva ctxt = kmalloc(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), 68f13193f5SChuck Lever GFP_KERNEL); 69f13193f5SChuck Lever if (!ctxt) 70f13193f5SChuck Lever goto out; 71f13193f5SChuck Lever INIT_LIST_HEAD(&ctxt->rw_list); 72f13193f5SChuck Lever } 73f13193f5SChuck Lever 74f13193f5SChuck Lever ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; 75f13193f5SChuck Lever if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, 76f13193f5SChuck Lever ctxt->rw_sg_table.sgl)) { 77f13193f5SChuck Lever kfree(ctxt); 78f13193f5SChuck Lever ctxt = NULL; 79f13193f5SChuck Lever } 80f13193f5SChuck Lever out: 81f13193f5SChuck Lever return ctxt; 82f13193f5SChuck Lever } 83f13193f5SChuck Lever 84f13193f5SChuck Lever static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, 85f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 86f13193f5SChuck Lever { 87f13193f5SChuck Lever sg_free_table_chained(&ctxt->rw_sg_table, true); 88f13193f5SChuck Lever 89f13193f5SChuck Lever spin_lock(&rdma->sc_rw_ctxt_lock); 90f13193f5SChuck Lever list_add(&ctxt->rw_list, &rdma->sc_rw_ctxts); 91f13193f5SChuck Lever spin_unlock(&rdma->sc_rw_ctxt_lock); 92f13193f5SChuck Lever } 93f13193f5SChuck Lever 94f13193f5SChuck Lever /** 95f13193f5SChuck Lever * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts 96f13193f5SChuck Lever * @rdma: transport about to be destroyed 97f13193f5SChuck Lever * 98f13193f5SChuck Lever */ 99f13193f5SChuck Lever void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) 100f13193f5SChuck Lever { 101f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 102f13193f5SChuck Lever 103f13193f5SChuck Lever while ((ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts)) != NULL) { 104f13193f5SChuck Lever list_del(&ctxt->rw_list); 105f13193f5SChuck Lever kfree(ctxt); 106f13193f5SChuck Lever } 107f13193f5SChuck Lever } 108f13193f5SChuck Lever 109f13193f5SChuck Lever /* A chunk context tracks all I/O for moving one Read or Write 110f13193f5SChuck Lever * chunk. This is a a set of rdma_rw's that handle data movement 111f13193f5SChuck Lever * for all segments of one chunk. 112f13193f5SChuck Lever * 113f13193f5SChuck Lever * These are small, acquired with a single allocator call, and 114f13193f5SChuck Lever * no more than one is needed per chunk. They are allocated on 115f13193f5SChuck Lever * demand, and not cached. 116f13193f5SChuck Lever */ 117f13193f5SChuck Lever struct svc_rdma_chunk_ctxt { 118f13193f5SChuck Lever struct ib_cqe cc_cqe; 119f13193f5SChuck Lever struct svcxprt_rdma *cc_rdma; 120f13193f5SChuck Lever struct list_head cc_rwctxts; 121f13193f5SChuck Lever int cc_sqecount; 122f13193f5SChuck Lever }; 123f13193f5SChuck Lever 124f13193f5SChuck Lever static void svc_rdma_cc_init(struct svcxprt_rdma *rdma, 12535a30fc3SChuck Lever struct svc_rdma_chunk_ctxt *cc) 126f13193f5SChuck Lever { 127f13193f5SChuck Lever cc->cc_rdma = rdma; 128f13193f5SChuck Lever svc_xprt_get(&rdma->sc_xprt); 129f13193f5SChuck Lever 130f13193f5SChuck Lever INIT_LIST_HEAD(&cc->cc_rwctxts); 131f13193f5SChuck Lever cc->cc_sqecount = 0; 132f13193f5SChuck Lever } 133f13193f5SChuck Lever 13435a30fc3SChuck Lever static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc, 13535a30fc3SChuck Lever enum dma_data_direction dir) 136f13193f5SChuck Lever { 137f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 138f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 139f13193f5SChuck Lever 140f13193f5SChuck Lever while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) { 141f13193f5SChuck Lever list_del(&ctxt->rw_list); 142f13193f5SChuck Lever 143f13193f5SChuck Lever rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp, 144f13193f5SChuck Lever rdma->sc_port_num, ctxt->rw_sg_table.sgl, 14535a30fc3SChuck Lever ctxt->rw_nents, dir); 146f13193f5SChuck Lever svc_rdma_put_rw_ctxt(rdma, ctxt); 147f13193f5SChuck Lever } 148f13193f5SChuck Lever svc_xprt_put(&rdma->sc_xprt); 149f13193f5SChuck Lever } 150f13193f5SChuck Lever 151f13193f5SChuck Lever /* State for sending a Write or Reply chunk. 152f13193f5SChuck Lever * - Tracks progress of writing one chunk over all its segments 153f13193f5SChuck Lever * - Stores arguments for the SGL constructor functions 154f13193f5SChuck Lever */ 155f13193f5SChuck Lever struct svc_rdma_write_info { 156f13193f5SChuck Lever /* write state of this chunk */ 157f13193f5SChuck Lever unsigned int wi_seg_off; 158f13193f5SChuck Lever unsigned int wi_seg_no; 159f13193f5SChuck Lever unsigned int wi_nsegs; 160f13193f5SChuck Lever __be32 *wi_segs; 161f13193f5SChuck Lever 162f13193f5SChuck Lever /* SGL constructor arguments */ 163f13193f5SChuck Lever struct xdr_buf *wi_xdr; 164f13193f5SChuck Lever unsigned char *wi_base; 165f13193f5SChuck Lever unsigned int wi_next_off; 166f13193f5SChuck Lever 167f13193f5SChuck Lever struct svc_rdma_chunk_ctxt wi_cc; 168f13193f5SChuck Lever }; 169f13193f5SChuck Lever 170f13193f5SChuck Lever static struct svc_rdma_write_info * 171f13193f5SChuck Lever svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, __be32 *chunk) 172f13193f5SChuck Lever { 173f13193f5SChuck Lever struct svc_rdma_write_info *info; 174f13193f5SChuck Lever 175f13193f5SChuck Lever info = kmalloc(sizeof(*info), GFP_KERNEL); 176f13193f5SChuck Lever if (!info) 177f13193f5SChuck Lever return info; 178f13193f5SChuck Lever 179f13193f5SChuck Lever info->wi_seg_off = 0; 180f13193f5SChuck Lever info->wi_seg_no = 0; 181f13193f5SChuck Lever info->wi_nsegs = be32_to_cpup(++chunk); 182f13193f5SChuck Lever info->wi_segs = ++chunk; 18335a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->wi_cc); 184026d958bSChuck Lever info->wi_cc.cc_cqe.done = svc_rdma_write_done; 185f13193f5SChuck Lever return info; 186f13193f5SChuck Lever } 187f13193f5SChuck Lever 188f13193f5SChuck Lever static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) 189f13193f5SChuck Lever { 19035a30fc3SChuck Lever svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE); 191f13193f5SChuck Lever kfree(info); 192f13193f5SChuck Lever } 193f13193f5SChuck Lever 194f13193f5SChuck Lever /** 195f13193f5SChuck Lever * svc_rdma_write_done - Write chunk completion 196f13193f5SChuck Lever * @cq: controlling Completion Queue 197f13193f5SChuck Lever * @wc: Work Completion 198f13193f5SChuck Lever * 199f13193f5SChuck Lever * Pages under I/O are freed by a subsequent Send completion. 200f13193f5SChuck Lever */ 201f13193f5SChuck Lever static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 202f13193f5SChuck Lever { 203f13193f5SChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 204f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = 205f13193f5SChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 206f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 207f13193f5SChuck Lever struct svc_rdma_write_info *info = 208f13193f5SChuck Lever container_of(cc, struct svc_rdma_write_info, wi_cc); 209f13193f5SChuck Lever 210bd2abef3SChuck Lever trace_svcrdma_wc_write(wc); 211bd2abef3SChuck Lever 212f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 213f13193f5SChuck Lever wake_up(&rdma->sc_send_wait); 214f13193f5SChuck Lever 2158820bcaaSChuck Lever if (unlikely(wc->status != IB_WC_SUCCESS)) 216f13193f5SChuck Lever set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); 217f13193f5SChuck Lever 218f13193f5SChuck Lever svc_rdma_write_info_free(info); 219f13193f5SChuck Lever } 220f13193f5SChuck Lever 221026d958bSChuck Lever /* State for pulling a Read chunk. 222026d958bSChuck Lever */ 223026d958bSChuck Lever struct svc_rdma_read_info { 224ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *ri_readctxt; 225026d958bSChuck Lever unsigned int ri_position; 226026d958bSChuck Lever unsigned int ri_pageno; 227026d958bSChuck Lever unsigned int ri_pageoff; 228026d958bSChuck Lever unsigned int ri_chunklen; 229026d958bSChuck Lever 230026d958bSChuck Lever struct svc_rdma_chunk_ctxt ri_cc; 231026d958bSChuck Lever }; 232026d958bSChuck Lever 233026d958bSChuck Lever static struct svc_rdma_read_info * 234026d958bSChuck Lever svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma) 235026d958bSChuck Lever { 236026d958bSChuck Lever struct svc_rdma_read_info *info; 237026d958bSChuck Lever 238026d958bSChuck Lever info = kmalloc(sizeof(*info), GFP_KERNEL); 239026d958bSChuck Lever if (!info) 240026d958bSChuck Lever return info; 241026d958bSChuck Lever 24235a30fc3SChuck Lever svc_rdma_cc_init(rdma, &info->ri_cc); 243026d958bSChuck Lever info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done; 244026d958bSChuck Lever return info; 245026d958bSChuck Lever } 246026d958bSChuck Lever 247026d958bSChuck Lever static void svc_rdma_read_info_free(struct svc_rdma_read_info *info) 248026d958bSChuck Lever { 24935a30fc3SChuck Lever svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE); 250026d958bSChuck Lever kfree(info); 251026d958bSChuck Lever } 252026d958bSChuck Lever 253026d958bSChuck Lever /** 254026d958bSChuck Lever * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx 255026d958bSChuck Lever * @cq: controlling Completion Queue 256026d958bSChuck Lever * @wc: Work Completion 257026d958bSChuck Lever * 258026d958bSChuck Lever */ 259026d958bSChuck Lever static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) 260026d958bSChuck Lever { 261026d958bSChuck Lever struct ib_cqe *cqe = wc->wr_cqe; 262026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = 263026d958bSChuck Lever container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); 264026d958bSChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 265026d958bSChuck Lever struct svc_rdma_read_info *info = 266026d958bSChuck Lever container_of(cc, struct svc_rdma_read_info, ri_cc); 267026d958bSChuck Lever 268bd2abef3SChuck Lever trace_svcrdma_wc_read(wc); 269bd2abef3SChuck Lever 270026d958bSChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 271026d958bSChuck Lever wake_up(&rdma->sc_send_wait); 272026d958bSChuck Lever 273026d958bSChuck Lever if (unlikely(wc->status != IB_WC_SUCCESS)) { 274026d958bSChuck Lever set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); 2751e5f4160SChuck Lever svc_rdma_recv_ctxt_put(rdma, info->ri_readctxt); 276026d958bSChuck Lever } else { 277026d958bSChuck Lever spin_lock(&rdma->sc_rq_dto_lock); 278ecf85b23SChuck Lever list_add_tail(&info->ri_readctxt->rc_list, 279026d958bSChuck Lever &rdma->sc_read_complete_q); 28095503d29SJ. Bruce Fields /* Note the unlock pairs with the smp_rmb in svc_xprt_ready: */ 28195503d29SJ. Bruce Fields set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags); 282026d958bSChuck Lever spin_unlock(&rdma->sc_rq_dto_lock); 283026d958bSChuck Lever 284026d958bSChuck Lever svc_xprt_enqueue(&rdma->sc_xprt); 285026d958bSChuck Lever } 286026d958bSChuck Lever 287026d958bSChuck Lever svc_rdma_read_info_free(info); 288026d958bSChuck Lever } 289026d958bSChuck Lever 290f13193f5SChuck Lever /* This function sleeps when the transport's Send Queue is congested. 291f13193f5SChuck Lever * 292f13193f5SChuck Lever * Assumptions: 293f13193f5SChuck Lever * - If ib_post_send() succeeds, only one completion is expected, 294f13193f5SChuck Lever * even if one or more WRs are flushed. This is true when posting 295f13193f5SChuck Lever * an rdma_rw_ctx or when posting a single signaled WR. 296f13193f5SChuck Lever */ 297f13193f5SChuck Lever static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) 298f13193f5SChuck Lever { 299f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 300f13193f5SChuck Lever struct svc_xprt *xprt = &rdma->sc_xprt; 301d34ac5cdSBart Van Assche struct ib_send_wr *first_wr; 302d34ac5cdSBart Van Assche const struct ib_send_wr *bad_wr; 303f13193f5SChuck Lever struct list_head *tmp; 304f13193f5SChuck Lever struct ib_cqe *cqe; 305f13193f5SChuck Lever int ret; 306f13193f5SChuck Lever 307107c1d0aSChuck Lever if (cc->cc_sqecount > rdma->sc_sq_depth) 308107c1d0aSChuck Lever return -EINVAL; 309107c1d0aSChuck Lever 310f13193f5SChuck Lever first_wr = NULL; 311f13193f5SChuck Lever cqe = &cc->cc_cqe; 312f13193f5SChuck Lever list_for_each(tmp, &cc->cc_rwctxts) { 313f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 314f13193f5SChuck Lever 315f13193f5SChuck Lever ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list); 316f13193f5SChuck Lever first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp, 317f13193f5SChuck Lever rdma->sc_port_num, cqe, first_wr); 318f13193f5SChuck Lever cqe = NULL; 319f13193f5SChuck Lever } 320f13193f5SChuck Lever 321f13193f5SChuck Lever do { 322f13193f5SChuck Lever if (atomic_sub_return(cc->cc_sqecount, 323f13193f5SChuck Lever &rdma->sc_sq_avail) > 0) { 3245d85a822SJason Gunthorpe ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); 325bd2abef3SChuck Lever trace_svcrdma_post_rw(&cc->cc_cqe, 326bd2abef3SChuck Lever cc->cc_sqecount, ret); 327f13193f5SChuck Lever if (ret) 328f13193f5SChuck Lever break; 329f13193f5SChuck Lever return 0; 330f13193f5SChuck Lever } 331f13193f5SChuck Lever 332bd2abef3SChuck Lever trace_svcrdma_sq_full(rdma); 333f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 334f13193f5SChuck Lever wait_event(rdma->sc_send_wait, 335f13193f5SChuck Lever atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); 336bd2abef3SChuck Lever trace_svcrdma_sq_retry(rdma); 337f13193f5SChuck Lever } while (1); 338f13193f5SChuck Lever 339f13193f5SChuck Lever set_bit(XPT_CLOSE, &xprt->xpt_flags); 340f13193f5SChuck Lever 341f13193f5SChuck Lever /* If even one was posted, there will be a completion. */ 342f13193f5SChuck Lever if (bad_wr != first_wr) 343f13193f5SChuck Lever return 0; 344f13193f5SChuck Lever 345f13193f5SChuck Lever atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); 346f13193f5SChuck Lever wake_up(&rdma->sc_send_wait); 347f13193f5SChuck Lever return -ENOTCONN; 348f13193f5SChuck Lever } 349f13193f5SChuck Lever 350f13193f5SChuck Lever /* Build and DMA-map an SGL that covers one kvec in an xdr_buf 351f13193f5SChuck Lever */ 352f13193f5SChuck Lever static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, 353f13193f5SChuck Lever unsigned int len, 354f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 355f13193f5SChuck Lever { 356f13193f5SChuck Lever struct scatterlist *sg = ctxt->rw_sg_table.sgl; 357f13193f5SChuck Lever 358f13193f5SChuck Lever sg_set_buf(&sg[0], info->wi_base, len); 359f13193f5SChuck Lever info->wi_base += len; 360f13193f5SChuck Lever 361f13193f5SChuck Lever ctxt->rw_nents = 1; 362f13193f5SChuck Lever } 363f13193f5SChuck Lever 364f13193f5SChuck Lever /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. 365f13193f5SChuck Lever */ 366f13193f5SChuck Lever static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, 367f13193f5SChuck Lever unsigned int remaining, 368f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt) 369f13193f5SChuck Lever { 370f13193f5SChuck Lever unsigned int sge_no, sge_bytes, page_off, page_no; 371f13193f5SChuck Lever struct xdr_buf *xdr = info->wi_xdr; 372f13193f5SChuck Lever struct scatterlist *sg; 373f13193f5SChuck Lever struct page **page; 374f13193f5SChuck Lever 37591b022ecSChuck Lever page_off = info->wi_next_off + xdr->page_base; 37691b022ecSChuck Lever page_no = page_off >> PAGE_SHIFT; 37791b022ecSChuck Lever page_off = offset_in_page(page_off); 378f13193f5SChuck Lever page = xdr->pages + page_no; 379f13193f5SChuck Lever info->wi_next_off += remaining; 380f13193f5SChuck Lever sg = ctxt->rw_sg_table.sgl; 381f13193f5SChuck Lever sge_no = 0; 382f13193f5SChuck Lever do { 383f13193f5SChuck Lever sge_bytes = min_t(unsigned int, remaining, 384f13193f5SChuck Lever PAGE_SIZE - page_off); 385f13193f5SChuck Lever sg_set_page(sg, *page, sge_bytes, page_off); 386f13193f5SChuck Lever 387f13193f5SChuck Lever remaining -= sge_bytes; 388f13193f5SChuck Lever sg = sg_next(sg); 389f13193f5SChuck Lever page_off = 0; 390f13193f5SChuck Lever sge_no++; 391f13193f5SChuck Lever page++; 392f13193f5SChuck Lever } while (remaining); 393f13193f5SChuck Lever 394f13193f5SChuck Lever ctxt->rw_nents = sge_no; 395f13193f5SChuck Lever } 396f13193f5SChuck Lever 397f13193f5SChuck Lever /* Construct RDMA Write WRs to send a portion of an xdr_buf containing 398f13193f5SChuck Lever * an RPC Reply. 399f13193f5SChuck Lever */ 400f13193f5SChuck Lever static int 401f13193f5SChuck Lever svc_rdma_build_writes(struct svc_rdma_write_info *info, 402f13193f5SChuck Lever void (*constructor)(struct svc_rdma_write_info *info, 403f13193f5SChuck Lever unsigned int len, 404f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt), 405f13193f5SChuck Lever unsigned int remaining) 406f13193f5SChuck Lever { 407f13193f5SChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; 408f13193f5SChuck Lever struct svcxprt_rdma *rdma = cc->cc_rdma; 409f13193f5SChuck Lever struct svc_rdma_rw_ctxt *ctxt; 410f13193f5SChuck Lever __be32 *seg; 411f13193f5SChuck Lever int ret; 412f13193f5SChuck Lever 413f13193f5SChuck Lever seg = info->wi_segs + info->wi_seg_no * rpcrdma_segment_maxsz; 414f13193f5SChuck Lever do { 415f13193f5SChuck Lever unsigned int write_len; 416f13193f5SChuck Lever u32 seg_length, seg_handle; 417f13193f5SChuck Lever u64 seg_offset; 418f13193f5SChuck Lever 419f13193f5SChuck Lever if (info->wi_seg_no >= info->wi_nsegs) 420f13193f5SChuck Lever goto out_overflow; 421f13193f5SChuck Lever 422f13193f5SChuck Lever seg_handle = be32_to_cpup(seg); 423f13193f5SChuck Lever seg_length = be32_to_cpup(seg + 1); 424f13193f5SChuck Lever xdr_decode_hyper(seg + 2, &seg_offset); 425f13193f5SChuck Lever seg_offset += info->wi_seg_off; 426f13193f5SChuck Lever 427f13193f5SChuck Lever write_len = min(remaining, seg_length - info->wi_seg_off); 428f13193f5SChuck Lever ctxt = svc_rdma_get_rw_ctxt(rdma, 429f13193f5SChuck Lever (write_len >> PAGE_SHIFT) + 2); 430f13193f5SChuck Lever if (!ctxt) 431f13193f5SChuck Lever goto out_noctx; 432f13193f5SChuck Lever 433f13193f5SChuck Lever constructor(info, write_len, ctxt); 434f13193f5SChuck Lever ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, 435f13193f5SChuck Lever rdma->sc_port_num, ctxt->rw_sg_table.sgl, 436f13193f5SChuck Lever ctxt->rw_nents, 0, seg_offset, 437f13193f5SChuck Lever seg_handle, DMA_TO_DEVICE); 438f13193f5SChuck Lever if (ret < 0) 439f13193f5SChuck Lever goto out_initerr; 440f13193f5SChuck Lever 44198895edbSChuck Lever trace_svcrdma_encode_wseg(seg_handle, write_len, seg_offset); 442f13193f5SChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 443f13193f5SChuck Lever cc->cc_sqecount += ret; 444f13193f5SChuck Lever if (write_len == seg_length - info->wi_seg_off) { 445f13193f5SChuck Lever seg += 4; 446f13193f5SChuck Lever info->wi_seg_no++; 447f13193f5SChuck Lever info->wi_seg_off = 0; 448f13193f5SChuck Lever } else { 449f13193f5SChuck Lever info->wi_seg_off += write_len; 450f13193f5SChuck Lever } 451f13193f5SChuck Lever remaining -= write_len; 452f13193f5SChuck Lever } while (remaining); 453f13193f5SChuck Lever 454f13193f5SChuck Lever return 0; 455f13193f5SChuck Lever 456f13193f5SChuck Lever out_overflow: 457f13193f5SChuck Lever dprintk("svcrdma: inadequate space in Write chunk (%u)\n", 458f13193f5SChuck Lever info->wi_nsegs); 459f13193f5SChuck Lever return -E2BIG; 460f13193f5SChuck Lever 461f13193f5SChuck Lever out_noctx: 462f13193f5SChuck Lever dprintk("svcrdma: no R/W ctxs available\n"); 463f13193f5SChuck Lever return -ENOMEM; 464f13193f5SChuck Lever 465f13193f5SChuck Lever out_initerr: 466f13193f5SChuck Lever svc_rdma_put_rw_ctxt(rdma, ctxt); 467bd2abef3SChuck Lever trace_svcrdma_dma_map_rwctx(rdma, ret); 468f13193f5SChuck Lever return -EIO; 469f13193f5SChuck Lever } 470f13193f5SChuck Lever 471f13193f5SChuck Lever /* Send one of an xdr_buf's kvecs by itself. To send a Reply 472f13193f5SChuck Lever * chunk, the whole RPC Reply is written back to the client. 473f13193f5SChuck Lever * This function writes either the head or tail of the xdr_buf 474f13193f5SChuck Lever * containing the Reply. 475f13193f5SChuck Lever */ 476f13193f5SChuck Lever static int svc_rdma_send_xdr_kvec(struct svc_rdma_write_info *info, 477f13193f5SChuck Lever struct kvec *vec) 478f13193f5SChuck Lever { 479f13193f5SChuck Lever info->wi_base = vec->iov_base; 480f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_vec_to_sg, 481f13193f5SChuck Lever vec->iov_len); 482f13193f5SChuck Lever } 483f13193f5SChuck Lever 484f13193f5SChuck Lever /* Send an xdr_buf's page list by itself. A Write chunk is 485f13193f5SChuck Lever * just the page list. a Reply chunk is the head, page list, 486f13193f5SChuck Lever * and tail. This function is shared between the two types 487f13193f5SChuck Lever * of chunk. 488f13193f5SChuck Lever */ 489f13193f5SChuck Lever static int svc_rdma_send_xdr_pagelist(struct svc_rdma_write_info *info, 490f13193f5SChuck Lever struct xdr_buf *xdr) 491f13193f5SChuck Lever { 492f13193f5SChuck Lever info->wi_xdr = xdr; 493f13193f5SChuck Lever info->wi_next_off = 0; 494f13193f5SChuck Lever return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg, 495f13193f5SChuck Lever xdr->page_len); 496f13193f5SChuck Lever } 497f13193f5SChuck Lever 498f13193f5SChuck Lever /** 499f13193f5SChuck Lever * svc_rdma_send_write_chunk - Write all segments in a Write chunk 500f13193f5SChuck Lever * @rdma: controlling RDMA transport 501f13193f5SChuck Lever * @wr_ch: Write chunk provided by client 502f13193f5SChuck Lever * @xdr: xdr_buf containing the data payload 503f13193f5SChuck Lever * 504f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 505f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Write chunk, 506107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 507f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 508f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 509f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 510f13193f5SChuck Lever */ 511f13193f5SChuck Lever int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, __be32 *wr_ch, 512f13193f5SChuck Lever struct xdr_buf *xdr) 513f13193f5SChuck Lever { 514f13193f5SChuck Lever struct svc_rdma_write_info *info; 515f13193f5SChuck Lever int ret; 516f13193f5SChuck Lever 517f13193f5SChuck Lever if (!xdr->page_len) 518f13193f5SChuck Lever return 0; 519f13193f5SChuck Lever 520f13193f5SChuck Lever info = svc_rdma_write_info_alloc(rdma, wr_ch); 521f13193f5SChuck Lever if (!info) 522f13193f5SChuck Lever return -ENOMEM; 523f13193f5SChuck Lever 524f13193f5SChuck Lever ret = svc_rdma_send_xdr_pagelist(info, xdr); 525f13193f5SChuck Lever if (ret < 0) 526f13193f5SChuck Lever goto out_err; 527f13193f5SChuck Lever 528f13193f5SChuck Lever ret = svc_rdma_post_chunk_ctxt(&info->wi_cc); 529f13193f5SChuck Lever if (ret < 0) 530f13193f5SChuck Lever goto out_err; 53198895edbSChuck Lever 53298895edbSChuck Lever trace_svcrdma_encode_write(xdr->page_len); 533f13193f5SChuck Lever return xdr->page_len; 534f13193f5SChuck Lever 535f13193f5SChuck Lever out_err: 536f13193f5SChuck Lever svc_rdma_write_info_free(info); 537f13193f5SChuck Lever return ret; 538f13193f5SChuck Lever } 539f13193f5SChuck Lever 540f13193f5SChuck Lever /** 541f13193f5SChuck Lever * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk 542f13193f5SChuck Lever * @rdma: controlling RDMA transport 543f13193f5SChuck Lever * @rp_ch: Reply chunk provided by client 544f13193f5SChuck Lever * @writelist: true if client provided a Write list 545f13193f5SChuck Lever * @xdr: xdr_buf containing an RPC Reply 546f13193f5SChuck Lever * 547f13193f5SChuck Lever * Returns a non-negative number of bytes the chunk consumed, or 548f13193f5SChuck Lever * %-E2BIG if the payload was larger than the Reply chunk, 549107c1d0aSChuck Lever * %-EINVAL if client provided too many segments, 550f13193f5SChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 551f13193f5SChuck Lever * %-ENOTCONN if posting failed (connection is lost), 552f13193f5SChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 553f13193f5SChuck Lever */ 554f13193f5SChuck Lever int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, __be32 *rp_ch, 555f13193f5SChuck Lever bool writelist, struct xdr_buf *xdr) 556f13193f5SChuck Lever { 557f13193f5SChuck Lever struct svc_rdma_write_info *info; 558f13193f5SChuck Lever int consumed, ret; 559f13193f5SChuck Lever 560f13193f5SChuck Lever info = svc_rdma_write_info_alloc(rdma, rp_ch); 561f13193f5SChuck Lever if (!info) 562f13193f5SChuck Lever return -ENOMEM; 563f13193f5SChuck Lever 564f13193f5SChuck Lever ret = svc_rdma_send_xdr_kvec(info, &xdr->head[0]); 565f13193f5SChuck Lever if (ret < 0) 566f13193f5SChuck Lever goto out_err; 567f13193f5SChuck Lever consumed = xdr->head[0].iov_len; 568f13193f5SChuck Lever 569f13193f5SChuck Lever /* Send the page list in the Reply chunk only if the 570f13193f5SChuck Lever * client did not provide Write chunks. 571f13193f5SChuck Lever */ 572f13193f5SChuck Lever if (!writelist && xdr->page_len) { 573f13193f5SChuck Lever ret = svc_rdma_send_xdr_pagelist(info, xdr); 574f13193f5SChuck Lever if (ret < 0) 575f13193f5SChuck Lever goto out_err; 576f13193f5SChuck Lever consumed += xdr->page_len; 577f13193f5SChuck Lever } 578f13193f5SChuck Lever 579f13193f5SChuck Lever if (xdr->tail[0].iov_len) { 580f13193f5SChuck Lever ret = svc_rdma_send_xdr_kvec(info, &xdr->tail[0]); 581f13193f5SChuck Lever if (ret < 0) 582f13193f5SChuck Lever goto out_err; 583f13193f5SChuck Lever consumed += xdr->tail[0].iov_len; 584f13193f5SChuck Lever } 585f13193f5SChuck Lever 586f13193f5SChuck Lever ret = svc_rdma_post_chunk_ctxt(&info->wi_cc); 587f13193f5SChuck Lever if (ret < 0) 588f13193f5SChuck Lever goto out_err; 58998895edbSChuck Lever 59098895edbSChuck Lever trace_svcrdma_encode_reply(consumed); 591f13193f5SChuck Lever return consumed; 592f13193f5SChuck Lever 593f13193f5SChuck Lever out_err: 594f13193f5SChuck Lever svc_rdma_write_info_free(info); 595f13193f5SChuck Lever return ret; 596f13193f5SChuck Lever } 597026d958bSChuck Lever 598026d958bSChuck Lever static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, 599026d958bSChuck Lever struct svc_rqst *rqstp, 600026d958bSChuck Lever u32 rkey, u32 len, u64 offset) 601026d958bSChuck Lever { 602ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 603026d958bSChuck Lever struct svc_rdma_chunk_ctxt *cc = &info->ri_cc; 604026d958bSChuck Lever struct svc_rdma_rw_ctxt *ctxt; 605026d958bSChuck Lever unsigned int sge_no, seg_len; 606026d958bSChuck Lever struct scatterlist *sg; 607026d958bSChuck Lever int ret; 608026d958bSChuck Lever 609026d958bSChuck Lever sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; 610026d958bSChuck Lever ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); 611026d958bSChuck Lever if (!ctxt) 612026d958bSChuck Lever goto out_noctx; 613026d958bSChuck Lever ctxt->rw_nents = sge_no; 614026d958bSChuck Lever 615026d958bSChuck Lever sg = ctxt->rw_sg_table.sgl; 616026d958bSChuck Lever for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) { 617026d958bSChuck Lever seg_len = min_t(unsigned int, len, 618026d958bSChuck Lever PAGE_SIZE - info->ri_pageoff); 619026d958bSChuck Lever 620ecf85b23SChuck Lever head->rc_arg.pages[info->ri_pageno] = 621026d958bSChuck Lever rqstp->rq_pages[info->ri_pageno]; 622026d958bSChuck Lever if (!info->ri_pageoff) 623ecf85b23SChuck Lever head->rc_page_count++; 624026d958bSChuck Lever 625026d958bSChuck Lever sg_set_page(sg, rqstp->rq_pages[info->ri_pageno], 626026d958bSChuck Lever seg_len, info->ri_pageoff); 627026d958bSChuck Lever sg = sg_next(sg); 628026d958bSChuck Lever 629026d958bSChuck Lever info->ri_pageoff += seg_len; 630026d958bSChuck Lever if (info->ri_pageoff == PAGE_SIZE) { 631026d958bSChuck Lever info->ri_pageno++; 632026d958bSChuck Lever info->ri_pageoff = 0; 633026d958bSChuck Lever } 634026d958bSChuck Lever len -= seg_len; 635026d958bSChuck Lever 636026d958bSChuck Lever /* Safety check */ 637026d958bSChuck Lever if (len && 638026d958bSChuck Lever &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end) 639026d958bSChuck Lever goto out_overrun; 640026d958bSChuck Lever } 641026d958bSChuck Lever 642026d958bSChuck Lever ret = rdma_rw_ctx_init(&ctxt->rw_ctx, cc->cc_rdma->sc_qp, 643026d958bSChuck Lever cc->cc_rdma->sc_port_num, 644026d958bSChuck Lever ctxt->rw_sg_table.sgl, ctxt->rw_nents, 645026d958bSChuck Lever 0, offset, rkey, DMA_FROM_DEVICE); 646026d958bSChuck Lever if (ret < 0) 647026d958bSChuck Lever goto out_initerr; 648026d958bSChuck Lever 649026d958bSChuck Lever list_add(&ctxt->rw_list, &cc->cc_rwctxts); 650026d958bSChuck Lever cc->cc_sqecount += ret; 651026d958bSChuck Lever return 0; 652026d958bSChuck Lever 653026d958bSChuck Lever out_noctx: 654026d958bSChuck Lever dprintk("svcrdma: no R/W ctxs available\n"); 655026d958bSChuck Lever return -ENOMEM; 656026d958bSChuck Lever 657026d958bSChuck Lever out_overrun: 658026d958bSChuck Lever dprintk("svcrdma: request overruns rq_pages\n"); 659026d958bSChuck Lever return -EINVAL; 660026d958bSChuck Lever 661026d958bSChuck Lever out_initerr: 662bd2abef3SChuck Lever trace_svcrdma_dma_map_rwctx(cc->cc_rdma, ret); 663026d958bSChuck Lever svc_rdma_put_rw_ctxt(cc->cc_rdma, ctxt); 664026d958bSChuck Lever return -EIO; 665026d958bSChuck Lever } 666026d958bSChuck Lever 6677075a867SChuck Lever /* Walk the segments in the Read chunk starting at @p and construct 6687075a867SChuck Lever * RDMA Read operations to pull the chunk to the server. 6697075a867SChuck Lever */ 670026d958bSChuck Lever static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, 671026d958bSChuck Lever struct svc_rdma_read_info *info, 672026d958bSChuck Lever __be32 *p) 673026d958bSChuck Lever { 67407d0ff3bSChuck Lever unsigned int i; 675026d958bSChuck Lever int ret; 676026d958bSChuck Lever 6777075a867SChuck Lever ret = -EINVAL; 678026d958bSChuck Lever info->ri_chunklen = 0; 6797075a867SChuck Lever while (*p++ != xdr_zero && be32_to_cpup(p++) == info->ri_position) { 680026d958bSChuck Lever u32 rs_handle, rs_length; 681026d958bSChuck Lever u64 rs_offset; 682026d958bSChuck Lever 683026d958bSChuck Lever rs_handle = be32_to_cpup(p++); 684026d958bSChuck Lever rs_length = be32_to_cpup(p++); 685026d958bSChuck Lever p = xdr_decode_hyper(p, &rs_offset); 686026d958bSChuck Lever 687026d958bSChuck Lever ret = svc_rdma_build_read_segment(info, rqstp, 688026d958bSChuck Lever rs_handle, rs_length, 689026d958bSChuck Lever rs_offset); 690026d958bSChuck Lever if (ret < 0) 691026d958bSChuck Lever break; 692026d958bSChuck Lever 69398895edbSChuck Lever trace_svcrdma_encode_rseg(rs_handle, rs_length, rs_offset); 694026d958bSChuck Lever info->ri_chunklen += rs_length; 695026d958bSChuck Lever } 696026d958bSChuck Lever 69707d0ff3bSChuck Lever /* Pages under I/O have been copied to head->rc_pages. 69807d0ff3bSChuck Lever * Prevent their premature release by svc_xprt_release() . 69907d0ff3bSChuck Lever */ 70007d0ff3bSChuck Lever for (i = 0; i < info->ri_readctxt->rc_page_count; i++) 70107d0ff3bSChuck Lever rqstp->rq_pages[i] = NULL; 70207d0ff3bSChuck Lever 703026d958bSChuck Lever return ret; 704026d958bSChuck Lever } 705026d958bSChuck Lever 706026d958bSChuck Lever /* Construct RDMA Reads to pull over a normal Read chunk. The chunk 707ecf85b23SChuck Lever * data lands in the page list of head->rc_arg.pages. 708026d958bSChuck Lever * 709ecf85b23SChuck Lever * Currently NFSD does not look at the head->rc_arg.tail[0] iovec. 710026d958bSChuck Lever * Therefore, XDR round-up of the Read chunk and trailing 711026d958bSChuck Lever * inline content must both be added at the end of the pagelist. 712026d958bSChuck Lever */ 713026d958bSChuck Lever static int svc_rdma_build_normal_read_chunk(struct svc_rqst *rqstp, 714026d958bSChuck Lever struct svc_rdma_read_info *info, 715026d958bSChuck Lever __be32 *p) 716026d958bSChuck Lever { 717ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 718026d958bSChuck Lever int ret; 719026d958bSChuck Lever 720026d958bSChuck Lever ret = svc_rdma_build_read_chunk(rqstp, info, p); 721026d958bSChuck Lever if (ret < 0) 722026d958bSChuck Lever goto out; 723026d958bSChuck Lever 72498895edbSChuck Lever trace_svcrdma_encode_read(info->ri_chunklen, info->ri_position); 72598895edbSChuck Lever 7263316f063SChuck Lever head->rc_hdr_count = 0; 7273316f063SChuck Lever 728193bcb7bSChuck Lever /* Split the Receive buffer between the head and tail 729193bcb7bSChuck Lever * buffers at Read chunk's position. XDR roundup of the 730193bcb7bSChuck Lever * chunk is not included in either the pagelist or in 731193bcb7bSChuck Lever * the tail. 732026d958bSChuck Lever */ 733ecf85b23SChuck Lever head->rc_arg.tail[0].iov_base = 734ecf85b23SChuck Lever head->rc_arg.head[0].iov_base + info->ri_position; 735ecf85b23SChuck Lever head->rc_arg.tail[0].iov_len = 736ecf85b23SChuck Lever head->rc_arg.head[0].iov_len - info->ri_position; 737ecf85b23SChuck Lever head->rc_arg.head[0].iov_len = info->ri_position; 738026d958bSChuck Lever 739175e0310SChuck Lever /* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2). 740193bcb7bSChuck Lever * 741175e0310SChuck Lever * If the client already rounded up the chunk length, the 742175e0310SChuck Lever * length does not change. Otherwise, the length of the page 743175e0310SChuck Lever * list is increased to include XDR round-up. 744175e0310SChuck Lever * 745175e0310SChuck Lever * Currently these chunks always start at page offset 0, 746175e0310SChuck Lever * thus the rounded-up length never crosses a page boundary. 747026d958bSChuck Lever */ 748175e0310SChuck Lever info->ri_chunklen = XDR_QUADLEN(info->ri_chunklen) << 2; 749026d958bSChuck Lever 750ecf85b23SChuck Lever head->rc_arg.page_len = info->ri_chunklen; 751ecf85b23SChuck Lever head->rc_arg.len += info->ri_chunklen; 752ecf85b23SChuck Lever head->rc_arg.buflen += info->ri_chunklen; 753026d958bSChuck Lever 754026d958bSChuck Lever out: 755026d958bSChuck Lever return ret; 756026d958bSChuck Lever } 757026d958bSChuck Lever 758026d958bSChuck Lever /* Construct RDMA Reads to pull over a Position Zero Read chunk. 759026d958bSChuck Lever * The start of the data lands in the first page just after 760026d958bSChuck Lever * the Transport header, and the rest lands in the page list of 761ecf85b23SChuck Lever * head->rc_arg.pages. 762026d958bSChuck Lever * 763026d958bSChuck Lever * Assumptions: 764026d958bSChuck Lever * - A PZRC has an XDR-aligned length (no implicit round-up). 765026d958bSChuck Lever * - There can be no trailing inline content (IOW, we assume 766026d958bSChuck Lever * a PZRC is never sent in an RDMA_MSG message, though it's 767026d958bSChuck Lever * allowed by spec). 768026d958bSChuck Lever */ 769026d958bSChuck Lever static int svc_rdma_build_pz_read_chunk(struct svc_rqst *rqstp, 770026d958bSChuck Lever struct svc_rdma_read_info *info, 771026d958bSChuck Lever __be32 *p) 772026d958bSChuck Lever { 773ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head = info->ri_readctxt; 774026d958bSChuck Lever int ret; 775026d958bSChuck Lever 776026d958bSChuck Lever ret = svc_rdma_build_read_chunk(rqstp, info, p); 777026d958bSChuck Lever if (ret < 0) 778026d958bSChuck Lever goto out; 779026d958bSChuck Lever 78098895edbSChuck Lever trace_svcrdma_encode_pzr(info->ri_chunklen); 78198895edbSChuck Lever 782ecf85b23SChuck Lever head->rc_arg.len += info->ri_chunklen; 783ecf85b23SChuck Lever head->rc_arg.buflen += info->ri_chunklen; 784026d958bSChuck Lever 7853316f063SChuck Lever head->rc_hdr_count = 1; 7863316f063SChuck Lever head->rc_arg.head[0].iov_base = page_address(head->rc_pages[0]); 7873316f063SChuck Lever head->rc_arg.head[0].iov_len = min_t(size_t, PAGE_SIZE, 7883316f063SChuck Lever info->ri_chunklen); 7893316f063SChuck Lever 7903316f063SChuck Lever head->rc_arg.page_len = info->ri_chunklen - 7913316f063SChuck Lever head->rc_arg.head[0].iov_len; 792026d958bSChuck Lever 793026d958bSChuck Lever out: 794026d958bSChuck Lever return ret; 795026d958bSChuck Lever } 796026d958bSChuck Lever 797026d958bSChuck Lever /** 798026d958bSChuck Lever * svc_rdma_recv_read_chunk - Pull a Read chunk from the client 799026d958bSChuck Lever * @rdma: controlling RDMA transport 800026d958bSChuck Lever * @rqstp: set of pages to use as Read sink buffers 801026d958bSChuck Lever * @head: pages under I/O collect here 802026d958bSChuck Lever * @p: pointer to start of Read chunk 803026d958bSChuck Lever * 804026d958bSChuck Lever * Returns: 805026d958bSChuck Lever * %0 if all needed RDMA Reads were posted successfully, 806026d958bSChuck Lever * %-EINVAL if client provided too many segments, 807026d958bSChuck Lever * %-ENOMEM if rdma_rw context pool was exhausted, 808026d958bSChuck Lever * %-ENOTCONN if posting failed (connection is lost), 809026d958bSChuck Lever * %-EIO if rdma_rw initialization failed (DMA mapping, etc). 810026d958bSChuck Lever * 811026d958bSChuck Lever * Assumptions: 812026d958bSChuck Lever * - All Read segments in @p have the same Position value. 813026d958bSChuck Lever */ 814026d958bSChuck Lever int svc_rdma_recv_read_chunk(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, 815ecf85b23SChuck Lever struct svc_rdma_recv_ctxt *head, __be32 *p) 816026d958bSChuck Lever { 817026d958bSChuck Lever struct svc_rdma_read_info *info; 818026d958bSChuck Lever int ret; 819026d958bSChuck Lever 820026d958bSChuck Lever /* The request (with page list) is constructed in 821ecf85b23SChuck Lever * head->rc_arg. Pages involved with RDMA Read I/O are 822026d958bSChuck Lever * transferred there. 823026d958bSChuck Lever */ 824ecf85b23SChuck Lever head->rc_arg.head[0] = rqstp->rq_arg.head[0]; 825ecf85b23SChuck Lever head->rc_arg.tail[0] = rqstp->rq_arg.tail[0]; 826ecf85b23SChuck Lever head->rc_arg.pages = head->rc_pages; 827ecf85b23SChuck Lever head->rc_arg.page_base = 0; 828ecf85b23SChuck Lever head->rc_arg.page_len = 0; 829ecf85b23SChuck Lever head->rc_arg.len = rqstp->rq_arg.len; 830ecf85b23SChuck Lever head->rc_arg.buflen = rqstp->rq_arg.buflen; 831026d958bSChuck Lever 832026d958bSChuck Lever info = svc_rdma_read_info_alloc(rdma); 833026d958bSChuck Lever if (!info) 834026d958bSChuck Lever return -ENOMEM; 835026d958bSChuck Lever info->ri_readctxt = head; 8363316f063SChuck Lever info->ri_pageno = 0; 8373316f063SChuck Lever info->ri_pageoff = 0; 838026d958bSChuck Lever 839026d958bSChuck Lever info->ri_position = be32_to_cpup(p + 1); 840026d958bSChuck Lever if (info->ri_position) 841026d958bSChuck Lever ret = svc_rdma_build_normal_read_chunk(rqstp, info, p); 842026d958bSChuck Lever else 843026d958bSChuck Lever ret = svc_rdma_build_pz_read_chunk(rqstp, info, p); 844026d958bSChuck Lever if (ret < 0) 84507d0ff3bSChuck Lever goto out_err; 846026d958bSChuck Lever 847026d958bSChuck Lever ret = svc_rdma_post_chunk_ctxt(&info->ri_cc); 848026d958bSChuck Lever if (ret < 0) 84907d0ff3bSChuck Lever goto out_err; 85007d0ff3bSChuck Lever return 0; 85107d0ff3bSChuck Lever 85207d0ff3bSChuck Lever out_err: 853026d958bSChuck Lever svc_rdma_read_info_free(info); 854026d958bSChuck Lever return ret; 855026d958bSChuck Lever } 856