19cb83748SJack Wang // SPDX-License-Identifier: GPL-2.0-or-later 29cb83748SJack Wang /* 39cb83748SJack Wang * RDMA Transport Layer 49cb83748SJack Wang * 59cb83748SJack Wang * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. 69cb83748SJack Wang * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. 79cb83748SJack Wang * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. 89cb83748SJack Wang */ 99cb83748SJack Wang 109cb83748SJack Wang #undef pr_fmt 119cb83748SJack Wang #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt 129cb83748SJack Wang 139cb83748SJack Wang #include <linux/module.h> 149cb83748SJack Wang #include <linux/mempool.h> 159cb83748SJack Wang 169cb83748SJack Wang #include "rtrs-srv.h" 179cb83748SJack Wang #include "rtrs-log.h" 188094ba0aSLeon Romanovsky #include <rdma/ib_cm.h> 19558d52b2SMd Haris Iqbal #include <rdma/ib_verbs.h> 209cb83748SJack Wang 219cb83748SJack Wang MODULE_DESCRIPTION("RDMA Transport Server"); 229cb83748SJack Wang MODULE_LICENSE("GPL"); 239cb83748SJack Wang 249cb83748SJack Wang /* Must be power of 2, see mask from mr->page_size in ib_sg_to_pages() */ 259cb83748SJack Wang #define DEFAULT_MAX_CHUNK_SIZE (128 << 10) 269cb83748SJack Wang #define DEFAULT_SESS_QUEUE_DEPTH 512 279cb83748SJack Wang #define MAX_HDR_SIZE PAGE_SIZE 289cb83748SJack Wang 299cb83748SJack Wang /* We guarantee to serve 10 paths at least */ 309cb83748SJack Wang #define CHUNK_POOL_SZ 10 319cb83748SJack Wang 329cb83748SJack Wang static struct rtrs_rdma_dev_pd dev_pd; 339cb83748SJack Wang static mempool_t *chunk_pool; 349cb83748SJack Wang struct class *rtrs_dev_class; 35558d52b2SMd Haris Iqbal static struct rtrs_srv_ib_ctx ib_ctx; 369cb83748SJack Wang 379cb83748SJack Wang static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE; 389cb83748SJack Wang static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH; 399cb83748SJack Wang 409cb83748SJack Wang static bool always_invalidate = true; 419cb83748SJack Wang module_param(always_invalidate, bool, 0444); 429cb83748SJack Wang MODULE_PARM_DESC(always_invalidate, 439cb83748SJack Wang "Invalidate memory registration for contiguous memory regions before accessing."); 449cb83748SJack Wang 459cb83748SJack Wang module_param_named(max_chunk_size, max_chunk_size, int, 0444); 469cb83748SJack Wang MODULE_PARM_DESC(max_chunk_size, 479cb83748SJack Wang "Max size for each IO request, when change the unit is in byte (default: " 489cb83748SJack Wang __stringify(DEFAULT_MAX_CHUNK_SIZE) "KB)"); 499cb83748SJack Wang 509cb83748SJack Wang module_param_named(sess_queue_depth, sess_queue_depth, int, 0444); 519cb83748SJack Wang MODULE_PARM_DESC(sess_queue_depth, 529cb83748SJack Wang "Number of buffers for pending I/O requests to allocate per session. Maximum: " 539cb83748SJack Wang __stringify(MAX_SESS_QUEUE_DEPTH) " (default: " 549cb83748SJack Wang __stringify(DEFAULT_SESS_QUEUE_DEPTH) ")"); 559cb83748SJack Wang 569cb83748SJack Wang static cpumask_t cq_affinity_mask = { CPU_BITS_ALL }; 579cb83748SJack Wang 589cb83748SJack Wang static struct workqueue_struct *rtrs_wq; 599cb83748SJack Wang 609cb83748SJack Wang static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) 619cb83748SJack Wang { 629cb83748SJack Wang return container_of(c, struct rtrs_srv_con, c); 639cb83748SJack Wang } 649cb83748SJack Wang 659cb83748SJack Wang static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s) 669cb83748SJack Wang { 679cb83748SJack Wang return container_of(s, struct rtrs_srv_sess, s); 689cb83748SJack Wang } 699cb83748SJack Wang 70*0aedfb69SGuoqing Jiang static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, 719cb83748SJack Wang enum rtrs_srv_state new_state) 729cb83748SJack Wang { 739cb83748SJack Wang enum rtrs_srv_state old_state; 749cb83748SJack Wang bool changed = false; 759cb83748SJack Wang 76*0aedfb69SGuoqing Jiang spin_lock_irq(&sess->state_lock); 779cb83748SJack Wang old_state = sess->state; 789cb83748SJack Wang switch (new_state) { 799cb83748SJack Wang case RTRS_SRV_CONNECTED: 80485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CONNECTING) 819cb83748SJack Wang changed = true; 829cb83748SJack Wang break; 839cb83748SJack Wang case RTRS_SRV_CLOSING: 84485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CONNECTING || 85485f2fb1SGuoqing Jiang old_state == RTRS_SRV_CONNECTED) 869cb83748SJack Wang changed = true; 879cb83748SJack Wang break; 889cb83748SJack Wang case RTRS_SRV_CLOSED: 89485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CLOSING) 909cb83748SJack Wang changed = true; 919cb83748SJack Wang break; 929cb83748SJack Wang default: 939cb83748SJack Wang break; 949cb83748SJack Wang } 959cb83748SJack Wang if (changed) 969cb83748SJack Wang sess->state = new_state; 979cb83748SJack Wang spin_unlock_irq(&sess->state_lock); 989cb83748SJack Wang 999cb83748SJack Wang return changed; 1009cb83748SJack Wang } 1019cb83748SJack Wang 1029cb83748SJack Wang static void free_id(struct rtrs_srv_op *id) 1039cb83748SJack Wang { 1049cb83748SJack Wang if (!id) 1059cb83748SJack Wang return; 1069cb83748SJack Wang kfree(id); 1079cb83748SJack Wang } 1089cb83748SJack Wang 1099cb83748SJack Wang static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess) 1109cb83748SJack Wang { 1119cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1129cb83748SJack Wang int i; 1139cb83748SJack Wang 1149cb83748SJack Wang WARN_ON(atomic_read(&sess->ids_inflight)); 1159cb83748SJack Wang if (sess->ops_ids) { 1169cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 1179cb83748SJack Wang free_id(sess->ops_ids[i]); 1189cb83748SJack Wang kfree(sess->ops_ids); 1199cb83748SJack Wang sess->ops_ids = NULL; 1209cb83748SJack Wang } 1219cb83748SJack Wang } 1229cb83748SJack Wang 1239cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc); 1249cb83748SJack Wang 1259cb83748SJack Wang static struct ib_cqe io_comp_cqe = { 1269cb83748SJack Wang .done = rtrs_srv_rdma_done 1279cb83748SJack Wang }; 1289cb83748SJack Wang 1299cb83748SJack Wang static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) 1309cb83748SJack Wang { 1319cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1329cb83748SJack Wang struct rtrs_srv_op *id; 1339cb83748SJack Wang int i; 1349cb83748SJack Wang 1359cb83748SJack Wang sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids), 1369cb83748SJack Wang GFP_KERNEL); 1379cb83748SJack Wang if (!sess->ops_ids) 1389cb83748SJack Wang goto err; 1399cb83748SJack Wang 1409cb83748SJack Wang for (i = 0; i < srv->queue_depth; ++i) { 1419cb83748SJack Wang id = kzalloc(sizeof(*id), GFP_KERNEL); 1429cb83748SJack Wang if (!id) 1439cb83748SJack Wang goto err; 1449cb83748SJack Wang 1459cb83748SJack Wang sess->ops_ids[i] = id; 1469cb83748SJack Wang } 1479cb83748SJack Wang init_waitqueue_head(&sess->ids_waitq); 1489cb83748SJack Wang atomic_set(&sess->ids_inflight, 0); 1499cb83748SJack Wang 1509cb83748SJack Wang return 0; 1519cb83748SJack Wang 1529cb83748SJack Wang err: 1539cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 1549cb83748SJack Wang return -ENOMEM; 1559cb83748SJack Wang } 1569cb83748SJack Wang 1579cb83748SJack Wang static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess) 1589cb83748SJack Wang { 1599cb83748SJack Wang atomic_inc(&sess->ids_inflight); 1609cb83748SJack Wang } 1619cb83748SJack Wang 1629cb83748SJack Wang static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess) 1639cb83748SJack Wang { 1649cb83748SJack Wang if (atomic_dec_and_test(&sess->ids_inflight)) 1659cb83748SJack Wang wake_up(&sess->ids_waitq); 1669cb83748SJack Wang } 1679cb83748SJack Wang 1689cb83748SJack Wang static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess) 1699cb83748SJack Wang { 1709cb83748SJack Wang wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight)); 1719cb83748SJack Wang } 1729cb83748SJack Wang 1739cb83748SJack Wang 1749cb83748SJack Wang static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc) 1759cb83748SJack Wang { 1763b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 1779cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 1789cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 1799cb83748SJack Wang 1809cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 1819cb83748SJack Wang rtrs_err(s, "REG MR failed: %s\n", 1829cb83748SJack Wang ib_wc_status_msg(wc->status)); 1839cb83748SJack Wang close_sess(sess); 1849cb83748SJack Wang return; 1859cb83748SJack Wang } 1869cb83748SJack Wang } 1879cb83748SJack Wang 1889cb83748SJack Wang static struct ib_cqe local_reg_cqe = { 1899cb83748SJack Wang .done = rtrs_srv_reg_mr_done 1909cb83748SJack Wang }; 1919cb83748SJack Wang 1929cb83748SJack Wang static int rdma_write_sg(struct rtrs_srv_op *id) 1939cb83748SJack Wang { 1949cb83748SJack Wang struct rtrs_sess *s = id->con->c.sess; 1959cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 1969cb83748SJack Wang dma_addr_t dma_addr = sess->dma_addr[id->msg_id]; 1979cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 1989cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 199e6daa8f6SJack Wang struct ib_send_wr inv_wr; 200e6daa8f6SJack Wang struct ib_rdma_wr imm_wr; 2019cb83748SJack Wang struct ib_rdma_wr *wr = NULL; 2029cb83748SJack Wang enum ib_send_flags flags; 2039cb83748SJack Wang size_t sg_cnt; 2049cb83748SJack Wang int err, offset; 2059cb83748SJack Wang bool need_inval; 2069cb83748SJack Wang u32 rkey = 0; 2079cb83748SJack Wang struct ib_reg_wr rwr; 2089cb83748SJack Wang struct ib_sge *plist; 2099cb83748SJack Wang struct ib_sge list; 2109cb83748SJack Wang 2119cb83748SJack Wang sg_cnt = le16_to_cpu(id->rd_msg->sg_cnt); 2129cb83748SJack Wang need_inval = le16_to_cpu(id->rd_msg->flags) & RTRS_MSG_NEED_INVAL_F; 2139cb83748SJack Wang if (unlikely(sg_cnt != 1)) 2149cb83748SJack Wang return -EINVAL; 2159cb83748SJack Wang 2169cb83748SJack Wang offset = 0; 2179cb83748SJack Wang 2189cb83748SJack Wang wr = &id->tx_wr; 2199cb83748SJack Wang plist = &id->tx_sg; 2209cb83748SJack Wang plist->addr = dma_addr + offset; 2219cb83748SJack Wang plist->length = le32_to_cpu(id->rd_msg->desc[0].len); 2229cb83748SJack Wang 2239cb83748SJack Wang /* WR will fail with length error 2249cb83748SJack Wang * if this is 0 2259cb83748SJack Wang */ 2269cb83748SJack Wang if (unlikely(plist->length == 0)) { 2279cb83748SJack Wang rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); 2289cb83748SJack Wang return -EINVAL; 2299cb83748SJack Wang } 2309cb83748SJack Wang 2319cb83748SJack Wang plist->lkey = sess->s.dev->ib_pd->local_dma_lkey; 2329cb83748SJack Wang offset += plist->length; 2339cb83748SJack Wang 2349cb83748SJack Wang wr->wr.sg_list = plist; 2359cb83748SJack Wang wr->wr.num_sge = 1; 2369cb83748SJack Wang wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); 2379cb83748SJack Wang wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); 2389cb83748SJack Wang if (rkey == 0) 2399cb83748SJack Wang rkey = wr->rkey; 2409cb83748SJack Wang else 2419cb83748SJack Wang /* Only one key is actually used */ 2429cb83748SJack Wang WARN_ON_ONCE(rkey != wr->rkey); 2439cb83748SJack Wang 2449cb83748SJack Wang wr->wr.opcode = IB_WR_RDMA_WRITE; 2458537f2deSJack Wang wr->wr.wr_cqe = &io_comp_cqe; 2469cb83748SJack Wang wr->wr.ex.imm_data = 0; 2479cb83748SJack Wang wr->wr.send_flags = 0; 2489cb83748SJack Wang 2499cb83748SJack Wang if (need_inval && always_invalidate) { 2509cb83748SJack Wang wr->wr.next = &rwr.wr; 2519cb83748SJack Wang rwr.wr.next = &inv_wr; 252e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 2539cb83748SJack Wang } else if (always_invalidate) { 2549cb83748SJack Wang wr->wr.next = &rwr.wr; 255e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 2569cb83748SJack Wang } else if (need_inval) { 2579cb83748SJack Wang wr->wr.next = &inv_wr; 258e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 2599cb83748SJack Wang } else { 260e6daa8f6SJack Wang wr->wr.next = &imm_wr.wr; 2619cb83748SJack Wang } 2629cb83748SJack Wang /* 2639cb83748SJack Wang * From time to time we have to post signaled sends, 2649cb83748SJack Wang * or send queue will fill up and only QP reset can help. 2659cb83748SJack Wang */ 2669cb83748SJack Wang flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ? 2679cb83748SJack Wang 0 : IB_SEND_SIGNALED; 2689cb83748SJack Wang 2699cb83748SJack Wang if (need_inval) { 2709cb83748SJack Wang inv_wr.sg_list = NULL; 2719cb83748SJack Wang inv_wr.num_sge = 0; 2729cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 2738537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 2749cb83748SJack Wang inv_wr.send_flags = 0; 2759cb83748SJack Wang inv_wr.ex.invalidate_rkey = rkey; 2769cb83748SJack Wang } 2779cb83748SJack Wang 278e6daa8f6SJack Wang imm_wr.wr.next = NULL; 2799cb83748SJack Wang if (always_invalidate) { 2809cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 2819cb83748SJack Wang 2829cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 2839cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 2848537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 2859cb83748SJack Wang rwr.wr.num_sge = 0; 2869cb83748SJack Wang rwr.mr = srv_mr->mr; 2879cb83748SJack Wang rwr.wr.send_flags = 0; 2889cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 2899cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 2909cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 2919cb83748SJack Wang msg = srv_mr->iu->buf; 2929cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 2939cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 2949cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 2959cb83748SJack Wang 2969cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 2979cb83748SJack Wang list.length = sizeof(*msg); 2989cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 299e6daa8f6SJack Wang imm_wr.wr.sg_list = &list; 300e6daa8f6SJack Wang imm_wr.wr.num_sge = 1; 301e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; 3029cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 3039cb83748SJack Wang srv_mr->iu->dma_addr, 3049cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 3059cb83748SJack Wang } else { 306e6daa8f6SJack Wang imm_wr.wr.sg_list = NULL; 307e6daa8f6SJack Wang imm_wr.wr.num_sge = 0; 308e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 3099cb83748SJack Wang } 310e6daa8f6SJack Wang imm_wr.wr.send_flags = flags; 311e6daa8f6SJack Wang imm_wr.wr.ex.imm_data = cpu_to_be32(rtrs_to_io_rsp_imm(id->msg_id, 3129cb83748SJack Wang 0, need_inval)); 3139cb83748SJack Wang 314e6daa8f6SJack Wang imm_wr.wr.wr_cqe = &io_comp_cqe; 3159cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr, 3169cb83748SJack Wang offset, DMA_BIDIRECTIONAL); 3179cb83748SJack Wang 3189cb83748SJack Wang err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL); 3199cb83748SJack Wang if (unlikely(err)) 3209cb83748SJack Wang rtrs_err(s, 3219cb83748SJack Wang "Posting RDMA-Write-Request to QP failed, err: %d\n", 3229cb83748SJack Wang err); 3239cb83748SJack Wang 3249cb83748SJack Wang return err; 3259cb83748SJack Wang } 3269cb83748SJack Wang 3279cb83748SJack Wang /** 3289cb83748SJack Wang * send_io_resp_imm() - respond to client with empty IMM on failed READ/WRITE 3299cb83748SJack Wang * requests or on successful WRITE request. 3309cb83748SJack Wang * @con: the connection to send back result 3319cb83748SJack Wang * @id: the id associated with the IO 3329cb83748SJack Wang * @errno: the error number of the IO. 3339cb83748SJack Wang * 3349cb83748SJack Wang * Return 0 on success, errno otherwise. 3359cb83748SJack Wang */ 3369cb83748SJack Wang static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, 3379cb83748SJack Wang int errno) 3389cb83748SJack Wang { 3399cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 3409cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 341e6daa8f6SJack Wang struct ib_send_wr inv_wr, *wr = NULL; 342e6daa8f6SJack Wang struct ib_rdma_wr imm_wr; 3439cb83748SJack Wang struct ib_reg_wr rwr; 3449cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 3459cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 3469cb83748SJack Wang bool need_inval = false; 3479cb83748SJack Wang enum ib_send_flags flags; 3489cb83748SJack Wang u32 imm; 3499cb83748SJack Wang int err; 3509cb83748SJack Wang 3519cb83748SJack Wang if (id->dir == READ) { 3529cb83748SJack Wang struct rtrs_msg_rdma_read *rd_msg = id->rd_msg; 3539cb83748SJack Wang size_t sg_cnt; 3549cb83748SJack Wang 3559cb83748SJack Wang need_inval = le16_to_cpu(rd_msg->flags) & 3569cb83748SJack Wang RTRS_MSG_NEED_INVAL_F; 3579cb83748SJack Wang sg_cnt = le16_to_cpu(rd_msg->sg_cnt); 3589cb83748SJack Wang 3599cb83748SJack Wang if (need_inval) { 3609cb83748SJack Wang if (likely(sg_cnt)) { 3618537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 3629cb83748SJack Wang inv_wr.sg_list = NULL; 3639cb83748SJack Wang inv_wr.num_sge = 0; 3649cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 3659cb83748SJack Wang inv_wr.send_flags = 0; 3669cb83748SJack Wang /* Only one key is actually used */ 3679cb83748SJack Wang inv_wr.ex.invalidate_rkey = 3689cb83748SJack Wang le32_to_cpu(rd_msg->desc[0].key); 3699cb83748SJack Wang } else { 3709cb83748SJack Wang WARN_ON_ONCE(1); 3719cb83748SJack Wang need_inval = false; 3729cb83748SJack Wang } 3739cb83748SJack Wang } 3749cb83748SJack Wang } 3759cb83748SJack Wang 3769cb83748SJack Wang if (need_inval && always_invalidate) { 3779cb83748SJack Wang wr = &inv_wr; 3789cb83748SJack Wang inv_wr.next = &rwr.wr; 379e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 3809cb83748SJack Wang } else if (always_invalidate) { 3819cb83748SJack Wang wr = &rwr.wr; 382e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 3839cb83748SJack Wang } else if (need_inval) { 3849cb83748SJack Wang wr = &inv_wr; 385e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 3869cb83748SJack Wang } else { 387e6daa8f6SJack Wang wr = &imm_wr.wr; 3889cb83748SJack Wang } 3899cb83748SJack Wang /* 3909cb83748SJack Wang * From time to time we have to post signalled sends, 3919cb83748SJack Wang * or send queue will fill up and only QP reset can help. 3929cb83748SJack Wang */ 3939cb83748SJack Wang flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ? 3949cb83748SJack Wang 0 : IB_SEND_SIGNALED; 3959cb83748SJack Wang imm = rtrs_to_io_rsp_imm(id->msg_id, errno, need_inval); 396e6daa8f6SJack Wang imm_wr.wr.next = NULL; 3979cb83748SJack Wang if (always_invalidate) { 3989cb83748SJack Wang struct ib_sge list; 3999cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 4009cb83748SJack Wang 4019cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 402e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 4039cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 4048537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 4059cb83748SJack Wang rwr.wr.num_sge = 0; 4069cb83748SJack Wang rwr.wr.send_flags = 0; 4079cb83748SJack Wang rwr.mr = srv_mr->mr; 4089cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 4099cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 4109cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 4119cb83748SJack Wang msg = srv_mr->iu->buf; 4129cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 4139cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 4149cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 4159cb83748SJack Wang 4169cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 4179cb83748SJack Wang list.length = sizeof(*msg); 4189cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 419e6daa8f6SJack Wang imm_wr.wr.sg_list = &list; 420e6daa8f6SJack Wang imm_wr.wr.num_sge = 1; 421e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; 4229cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 4239cb83748SJack Wang srv_mr->iu->dma_addr, 4249cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 4259cb83748SJack Wang } else { 426e6daa8f6SJack Wang imm_wr.wr.sg_list = NULL; 427e6daa8f6SJack Wang imm_wr.wr.num_sge = 0; 428e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 4299cb83748SJack Wang } 430e6daa8f6SJack Wang imm_wr.wr.send_flags = flags; 431e6daa8f6SJack Wang imm_wr.wr.wr_cqe = &io_comp_cqe; 4329cb83748SJack Wang 433e6daa8f6SJack Wang imm_wr.wr.ex.imm_data = cpu_to_be32(imm); 4349cb83748SJack Wang 4359cb83748SJack Wang err = ib_post_send(id->con->c.qp, wr, NULL); 4369cb83748SJack Wang if (unlikely(err)) 4379cb83748SJack Wang rtrs_err_rl(s, "Posting RDMA-Reply to QP failed, err: %d\n", 4389cb83748SJack Wang err); 4399cb83748SJack Wang 4409cb83748SJack Wang return err; 4419cb83748SJack Wang } 4429cb83748SJack Wang 4439cb83748SJack Wang void close_sess(struct rtrs_srv_sess *sess) 4449cb83748SJack Wang { 445ffea6ad1SGuoqing Jiang if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING)) 4469cb83748SJack Wang queue_work(rtrs_wq, &sess->close_work); 4479cb83748SJack Wang WARN_ON(sess->state != RTRS_SRV_CLOSING); 4489cb83748SJack Wang } 4499cb83748SJack Wang 4509cb83748SJack Wang static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) 4519cb83748SJack Wang { 4529cb83748SJack Wang switch (state) { 4539cb83748SJack Wang case RTRS_SRV_CONNECTING: 4549cb83748SJack Wang return "RTRS_SRV_CONNECTING"; 4559cb83748SJack Wang case RTRS_SRV_CONNECTED: 4569cb83748SJack Wang return "RTRS_SRV_CONNECTED"; 4579cb83748SJack Wang case RTRS_SRV_CLOSING: 4589cb83748SJack Wang return "RTRS_SRV_CLOSING"; 4599cb83748SJack Wang case RTRS_SRV_CLOSED: 4609cb83748SJack Wang return "RTRS_SRV_CLOSED"; 4619cb83748SJack Wang default: 4629cb83748SJack Wang return "UNKNOWN"; 4639cb83748SJack Wang } 4649cb83748SJack Wang } 4659cb83748SJack Wang 4669cb83748SJack Wang /** 4679cb83748SJack Wang * rtrs_srv_resp_rdma() - Finish an RDMA request 4689cb83748SJack Wang * 4699cb83748SJack Wang * @id: Internal RTRS operation identifier 4709cb83748SJack Wang * @status: Response Code sent to the other side for this operation. 4719cb83748SJack Wang * 0 = success, <=0 error 4729cb83748SJack Wang * Context: any 4739cb83748SJack Wang * 4749cb83748SJack Wang * Finish a RDMA operation. A message is sent to the client and the 4759cb83748SJack Wang * corresponding memory areas will be released. 4769cb83748SJack Wang */ 4779cb83748SJack Wang bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) 4789cb83748SJack Wang { 4799cb83748SJack Wang struct rtrs_srv_sess *sess; 4809cb83748SJack Wang struct rtrs_srv_con *con; 4819cb83748SJack Wang struct rtrs_sess *s; 4829cb83748SJack Wang int err; 4839cb83748SJack Wang 4849cb83748SJack Wang if (WARN_ON(!id)) 4859cb83748SJack Wang return true; 4869cb83748SJack Wang 4879cb83748SJack Wang con = id->con; 4889cb83748SJack Wang s = con->c.sess; 4899cb83748SJack Wang sess = to_srv_sess(s); 4909cb83748SJack Wang 4919cb83748SJack Wang id->status = status; 4929cb83748SJack Wang 4939cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 4949cb83748SJack Wang rtrs_err_rl(s, 49542cdc190SGioh Kim "Sending I/O response failed, session %s is disconnected, sess state %s\n", 49642cdc190SGioh Kim kobject_name(&sess->kobj), 4979cb83748SJack Wang rtrs_srv_state_str(sess->state)); 4989cb83748SJack Wang goto out; 4999cb83748SJack Wang } 5009cb83748SJack Wang if (always_invalidate) { 5019cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id]; 5029cb83748SJack Wang 5039cb83748SJack Wang ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); 5049cb83748SJack Wang } 5059cb83748SJack Wang if (unlikely(atomic_sub_return(1, 5069cb83748SJack Wang &con->sq_wr_avail) < 0)) { 50742cdc190SGioh Kim rtrs_err(s, "IB send queue full: sess=%s cid=%d\n", 50842cdc190SGioh Kim kobject_name(&sess->kobj), 50942cdc190SGioh Kim con->c.cid); 5109cb83748SJack Wang atomic_add(1, &con->sq_wr_avail); 5119cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 5129cb83748SJack Wang list_add_tail(&id->wait_list, &con->rsp_wr_wait_list); 5139cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 5149cb83748SJack Wang return false; 5159cb83748SJack Wang } 5169cb83748SJack Wang 5179cb83748SJack Wang if (status || id->dir == WRITE || !id->rd_msg->sg_cnt) 5189cb83748SJack Wang err = send_io_resp_imm(con, id, status); 5199cb83748SJack Wang else 5209cb83748SJack Wang err = rdma_write_sg(id); 5219cb83748SJack Wang 5229cb83748SJack Wang if (unlikely(err)) { 52342cdc190SGioh Kim rtrs_err_rl(s, "IO response failed: %d: sess=%s\n", err, 52442cdc190SGioh Kim kobject_name(&sess->kobj)); 5259cb83748SJack Wang close_sess(sess); 5269cb83748SJack Wang } 5279cb83748SJack Wang out: 5289cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 5299cb83748SJack Wang return true; 5309cb83748SJack Wang } 5319cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_resp_rdma); 5329cb83748SJack Wang 5339cb83748SJack Wang /** 5349cb83748SJack Wang * rtrs_srv_set_sess_priv() - Set private pointer in rtrs_srv. 5359cb83748SJack Wang * @srv: Session pointer 5369cb83748SJack Wang * @priv: The private pointer that is associated with the session. 5379cb83748SJack Wang */ 5389cb83748SJack Wang void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv) 5399cb83748SJack Wang { 5409cb83748SJack Wang srv->priv = priv; 5419cb83748SJack Wang } 5429cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_set_sess_priv); 5439cb83748SJack Wang 5449cb83748SJack Wang static void unmap_cont_bufs(struct rtrs_srv_sess *sess) 5459cb83748SJack Wang { 5469cb83748SJack Wang int i; 5479cb83748SJack Wang 5489cb83748SJack Wang for (i = 0; i < sess->mrs_num; i++) { 5499cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 5509cb83748SJack Wang 5519cb83748SJack Wang srv_mr = &sess->mrs[i]; 5528bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 5539cb83748SJack Wang ib_dereg_mr(srv_mr->mr); 5549cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl, 5559cb83748SJack Wang srv_mr->sgt.nents, DMA_BIDIRECTIONAL); 5569cb83748SJack Wang sg_free_table(&srv_mr->sgt); 5579cb83748SJack Wang } 5589cb83748SJack Wang kfree(sess->mrs); 5599cb83748SJack Wang } 5609cb83748SJack Wang 5619cb83748SJack Wang static int map_cont_bufs(struct rtrs_srv_sess *sess) 5629cb83748SJack Wang { 5639cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 5649cb83748SJack Wang struct rtrs_sess *ss = &sess->s; 5659cb83748SJack Wang int i, mri, err, mrs_num; 5669cb83748SJack Wang unsigned int chunk_bits; 5679cb83748SJack Wang int chunks_per_mr = 1; 5689cb83748SJack Wang 5699cb83748SJack Wang /* 5709cb83748SJack Wang * Here we map queue_depth chunks to MR. Firstly we have to 5719cb83748SJack Wang * figure out how many chunks can we map per MR. 5729cb83748SJack Wang */ 5739cb83748SJack Wang if (always_invalidate) { 5749cb83748SJack Wang /* 5759cb83748SJack Wang * in order to do invalidate for each chunks of memory, we needs 5769cb83748SJack Wang * more memory regions. 5779cb83748SJack Wang */ 5789cb83748SJack Wang mrs_num = srv->queue_depth; 5799cb83748SJack Wang } else { 5809cb83748SJack Wang chunks_per_mr = 5819cb83748SJack Wang sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; 5829cb83748SJack Wang mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr); 5839cb83748SJack Wang chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num); 5849cb83748SJack Wang } 5859cb83748SJack Wang 5869cb83748SJack Wang sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL); 5879cb83748SJack Wang if (!sess->mrs) 5889cb83748SJack Wang return -ENOMEM; 5899cb83748SJack Wang 5909cb83748SJack Wang sess->mrs_num = mrs_num; 5919cb83748SJack Wang 5929cb83748SJack Wang for (mri = 0; mri < mrs_num; mri++) { 5939cb83748SJack Wang struct rtrs_srv_mr *srv_mr = &sess->mrs[mri]; 5949cb83748SJack Wang struct sg_table *sgt = &srv_mr->sgt; 5959cb83748SJack Wang struct scatterlist *s; 5969cb83748SJack Wang struct ib_mr *mr; 5979cb83748SJack Wang int nr, chunks; 5989cb83748SJack Wang 5999cb83748SJack Wang chunks = chunks_per_mr * mri; 6009cb83748SJack Wang if (!always_invalidate) 6019cb83748SJack Wang chunks_per_mr = min_t(int, chunks_per_mr, 6029cb83748SJack Wang srv->queue_depth - chunks); 6039cb83748SJack Wang 6049cb83748SJack Wang err = sg_alloc_table(sgt, chunks_per_mr, GFP_KERNEL); 6059cb83748SJack Wang if (err) 6069cb83748SJack Wang goto err; 6079cb83748SJack Wang 6089cb83748SJack Wang for_each_sg(sgt->sgl, s, chunks_per_mr, i) 6099cb83748SJack Wang sg_set_page(s, srv->chunks[chunks + i], 6109cb83748SJack Wang max_chunk_size, 0); 6119cb83748SJack Wang 6129cb83748SJack Wang nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl, 6139cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6149cb83748SJack Wang if (nr < sgt->nents) { 6159cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6169cb83748SJack Wang goto free_sg; 6179cb83748SJack Wang } 6189cb83748SJack Wang mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, 6199cb83748SJack Wang sgt->nents); 6209cb83748SJack Wang if (IS_ERR(mr)) { 6219cb83748SJack Wang err = PTR_ERR(mr); 6229cb83748SJack Wang goto unmap_sg; 6239cb83748SJack Wang } 6249cb83748SJack Wang nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents, 6259cb83748SJack Wang NULL, max_chunk_size); 626b386cd65SDan Carpenter if (nr < 0 || nr < sgt->nents) { 6279cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6289cb83748SJack Wang goto dereg_mr; 6299cb83748SJack Wang } 6309cb83748SJack Wang 6319cb83748SJack Wang if (always_invalidate) { 6329cb83748SJack Wang srv_mr->iu = rtrs_iu_alloc(1, 6339cb83748SJack Wang sizeof(struct rtrs_msg_rkey_rsp), 6349cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 6359cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_rdma_done); 6369cb83748SJack Wang if (!srv_mr->iu) { 6376b31afceSWei Yongjun err = -ENOMEM; 6386b31afceSWei Yongjun rtrs_err(ss, "rtrs_iu_alloc(), err: %d\n", err); 639f77c4839SGuoqing Jiang goto dereg_mr; 6409cb83748SJack Wang } 6419cb83748SJack Wang } 6429cb83748SJack Wang /* Eventually dma addr for each chunk can be cached */ 6439cb83748SJack Wang for_each_sg(sgt->sgl, s, sgt->orig_nents, i) 6449cb83748SJack Wang sess->dma_addr[chunks + i] = sg_dma_address(s); 6459cb83748SJack Wang 6469cb83748SJack Wang ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 6479cb83748SJack Wang srv_mr->mr = mr; 6489cb83748SJack Wang 6499cb83748SJack Wang continue; 6509cb83748SJack Wang err: 6519cb83748SJack Wang while (mri--) { 6529cb83748SJack Wang srv_mr = &sess->mrs[mri]; 6539cb83748SJack Wang sgt = &srv_mr->sgt; 6549cb83748SJack Wang mr = srv_mr->mr; 6558bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 6569cb83748SJack Wang dereg_mr: 6579cb83748SJack Wang ib_dereg_mr(mr); 6589cb83748SJack Wang unmap_sg: 6599cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl, 6609cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6619cb83748SJack Wang free_sg: 6629cb83748SJack Wang sg_free_table(sgt); 6639cb83748SJack Wang } 6649cb83748SJack Wang kfree(sess->mrs); 6659cb83748SJack Wang 6669cb83748SJack Wang return err; 6679cb83748SJack Wang } 6689cb83748SJack Wang 6699cb83748SJack Wang chunk_bits = ilog2(srv->queue_depth - 1) + 1; 6709cb83748SJack Wang sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); 6719cb83748SJack Wang 6729cb83748SJack Wang return 0; 6739cb83748SJack Wang } 6749cb83748SJack Wang 6759cb83748SJack Wang static void rtrs_srv_hb_err_handler(struct rtrs_con *c) 6769cb83748SJack Wang { 6779cb83748SJack Wang close_sess(to_srv_sess(c->sess)); 6789cb83748SJack Wang } 6799cb83748SJack Wang 6809cb83748SJack Wang static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess) 6819cb83748SJack Wang { 6829cb83748SJack Wang rtrs_init_hb(&sess->s, &io_comp_cqe, 6839cb83748SJack Wang RTRS_HB_INTERVAL_MS, 6849cb83748SJack Wang RTRS_HB_MISSED_MAX, 6859cb83748SJack Wang rtrs_srv_hb_err_handler, 6869cb83748SJack Wang rtrs_wq); 6879cb83748SJack Wang } 6889cb83748SJack Wang 6899cb83748SJack Wang static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess) 6909cb83748SJack Wang { 6919cb83748SJack Wang rtrs_start_hb(&sess->s); 6929cb83748SJack Wang } 6939cb83748SJack Wang 6949cb83748SJack Wang static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess) 6959cb83748SJack Wang { 6969cb83748SJack Wang rtrs_stop_hb(&sess->s); 6979cb83748SJack Wang } 6989cb83748SJack Wang 6999cb83748SJack Wang static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) 7009cb83748SJack Wang { 7013b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 7029cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7039cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7049cb83748SJack Wang struct rtrs_iu *iu; 7059cb83748SJack Wang 7069cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 7078bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 7089cb83748SJack Wang 7099cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 7109cb83748SJack Wang rtrs_err(s, "Sess info response send failed: %s\n", 7119cb83748SJack Wang ib_wc_status_msg(wc->status)); 7129cb83748SJack Wang close_sess(sess); 7139cb83748SJack Wang return; 7149cb83748SJack Wang } 7159cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_SEND); 7169cb83748SJack Wang } 7179cb83748SJack Wang 7189cb83748SJack Wang static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) 7199cb83748SJack Wang { 7209cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7219cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7229cb83748SJack Wang int up; 7239cb83748SJack Wang 7249cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7259cb83748SJack Wang up = ++srv->paths_up; 7269cb83748SJack Wang if (up == 1) 7279cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL); 7289cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7299cb83748SJack Wang 7309cb83748SJack Wang /* Mark session as established */ 7319cb83748SJack Wang sess->established = true; 7329cb83748SJack Wang } 7339cb83748SJack Wang 7349cb83748SJack Wang static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) 7359cb83748SJack Wang { 7369cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7379cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7389cb83748SJack Wang 7399cb83748SJack Wang if (!sess->established) 7409cb83748SJack Wang return; 7419cb83748SJack Wang 7429cb83748SJack Wang sess->established = false; 7439cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7449cb83748SJack Wang WARN_ON(!srv->paths_up); 7459cb83748SJack Wang if (--srv->paths_up == 0) 7469cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv); 7479cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7489cb83748SJack Wang } 7499cb83748SJack Wang 7509cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess); 7519cb83748SJack Wang 7529cb83748SJack Wang static int process_info_req(struct rtrs_srv_con *con, 7539cb83748SJack Wang struct rtrs_msg_info_req *msg) 7549cb83748SJack Wang { 7559cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7569cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7579cb83748SJack Wang struct ib_send_wr *reg_wr = NULL; 7589cb83748SJack Wang struct rtrs_msg_info_rsp *rsp; 7599cb83748SJack Wang struct rtrs_iu *tx_iu; 7609cb83748SJack Wang struct ib_reg_wr *rwr; 7619cb83748SJack Wang int mri, err; 7629cb83748SJack Wang size_t tx_sz; 7639cb83748SJack Wang 7649cb83748SJack Wang err = post_recv_sess(sess); 7659cb83748SJack Wang if (unlikely(err)) { 7669cb83748SJack Wang rtrs_err(s, "post_recv_sess(), err: %d\n", err); 7679cb83748SJack Wang return err; 7689cb83748SJack Wang } 7699cb83748SJack Wang rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL); 7709cb83748SJack Wang if (unlikely(!rwr)) 7719cb83748SJack Wang return -ENOMEM; 7722d612f0dSDima Stepanov strscpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname)); 7739cb83748SJack Wang 7749cb83748SJack Wang tx_sz = sizeof(*rsp); 7759cb83748SJack Wang tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num; 7769cb83748SJack Wang tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev, 7779cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_info_rsp_done); 7789cb83748SJack Wang if (unlikely(!tx_iu)) { 7799cb83748SJack Wang err = -ENOMEM; 7809cb83748SJack Wang goto rwr_free; 7819cb83748SJack Wang } 7829cb83748SJack Wang 7839cb83748SJack Wang rsp = tx_iu->buf; 7849cb83748SJack Wang rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP); 7859cb83748SJack Wang rsp->sg_cnt = cpu_to_le16(sess->mrs_num); 7869cb83748SJack Wang 7879cb83748SJack Wang for (mri = 0; mri < sess->mrs_num; mri++) { 7889cb83748SJack Wang struct ib_mr *mr = sess->mrs[mri].mr; 7899cb83748SJack Wang 7909cb83748SJack Wang rsp->desc[mri].addr = cpu_to_le64(mr->iova); 7919cb83748SJack Wang rsp->desc[mri].key = cpu_to_le32(mr->rkey); 7929cb83748SJack Wang rsp->desc[mri].len = cpu_to_le32(mr->length); 7939cb83748SJack Wang 7949cb83748SJack Wang /* 7959cb83748SJack Wang * Fill in reg MR request and chain them *backwards* 7969cb83748SJack Wang */ 7979cb83748SJack Wang rwr[mri].wr.next = mri ? &rwr[mri - 1].wr : NULL; 7989cb83748SJack Wang rwr[mri].wr.opcode = IB_WR_REG_MR; 7999cb83748SJack Wang rwr[mri].wr.wr_cqe = &local_reg_cqe; 8009cb83748SJack Wang rwr[mri].wr.num_sge = 0; 801e8ae7ddbSJack Wang rwr[mri].wr.send_flags = 0; 8029cb83748SJack Wang rwr[mri].mr = mr; 8039cb83748SJack Wang rwr[mri].key = mr->rkey; 8049cb83748SJack Wang rwr[mri].access = (IB_ACCESS_LOCAL_WRITE | 8059cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 8069cb83748SJack Wang reg_wr = &rwr[mri].wr; 8079cb83748SJack Wang } 8089cb83748SJack Wang 8099cb83748SJack Wang err = rtrs_srv_create_sess_files(sess); 8109cb83748SJack Wang if (unlikely(err)) 8119cb83748SJack Wang goto iu_free; 8129cb83748SJack Wang kobject_get(&sess->kobj); 8139cb83748SJack Wang get_device(&sess->srv->dev); 8149cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED); 8159cb83748SJack Wang rtrs_srv_start_hb(sess); 8169cb83748SJack Wang 8179cb83748SJack Wang /* 8189cb83748SJack Wang * We do not account number of established connections at the current 8199cb83748SJack Wang * moment, we rely on the client, which should send info request when 8209cb83748SJack Wang * all connections are successfully established. Thus, simply notify 8219cb83748SJack Wang * listener with a proper event if we are the first path. 8229cb83748SJack Wang */ 8239cb83748SJack Wang rtrs_srv_sess_up(sess); 8249cb83748SJack Wang 8259cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, 8269cb83748SJack Wang tx_iu->size, DMA_TO_DEVICE); 8279cb83748SJack Wang 8289cb83748SJack Wang /* Send info response */ 8299cb83748SJack Wang err = rtrs_iu_post_send(&con->c, tx_iu, tx_sz, reg_wr); 8309cb83748SJack Wang if (unlikely(err)) { 8319cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err); 8329cb83748SJack Wang iu_free: 8338bd372acSGioh Kim rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); 8349cb83748SJack Wang } 8359cb83748SJack Wang rwr_free: 8369cb83748SJack Wang kfree(rwr); 8379cb83748SJack Wang 8389cb83748SJack Wang return err; 8399cb83748SJack Wang } 8409cb83748SJack Wang 8419cb83748SJack Wang static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) 8429cb83748SJack Wang { 8433b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 8449cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8459cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 8469cb83748SJack Wang struct rtrs_msg_info_req *msg; 8479cb83748SJack Wang struct rtrs_iu *iu; 8489cb83748SJack Wang int err; 8499cb83748SJack Wang 8509cb83748SJack Wang WARN_ON(con->c.cid); 8519cb83748SJack Wang 8529cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 8539cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 8549cb83748SJack Wang rtrs_err(s, "Sess info request receive failed: %s\n", 8559cb83748SJack Wang ib_wc_status_msg(wc->status)); 8569cb83748SJack Wang goto close; 8579cb83748SJack Wang } 8589cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_RECV); 8599cb83748SJack Wang 8609cb83748SJack Wang if (unlikely(wc->byte_len < sizeof(*msg))) { 8619cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: size %d\n", 8629cb83748SJack Wang wc->byte_len); 8639cb83748SJack Wang goto close; 8649cb83748SJack Wang } 8659cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, 8669cb83748SJack Wang iu->size, DMA_FROM_DEVICE); 8679cb83748SJack Wang msg = iu->buf; 8689cb83748SJack Wang if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ)) { 8699cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: type %d\n", 8709cb83748SJack Wang le16_to_cpu(msg->type)); 8719cb83748SJack Wang goto close; 8729cb83748SJack Wang } 8739cb83748SJack Wang err = process_info_req(con, msg); 8749cb83748SJack Wang if (unlikely(err)) 8759cb83748SJack Wang goto close; 8769cb83748SJack Wang 8779cb83748SJack Wang out: 8788bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 8799cb83748SJack Wang return; 8809cb83748SJack Wang close: 8819cb83748SJack Wang close_sess(sess); 8829cb83748SJack Wang goto out; 8839cb83748SJack Wang } 8849cb83748SJack Wang 8859cb83748SJack Wang static int post_recv_info_req(struct rtrs_srv_con *con) 8869cb83748SJack Wang { 8879cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8889cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 8899cb83748SJack Wang struct rtrs_iu *rx_iu; 8909cb83748SJack Wang int err; 8919cb83748SJack Wang 8929cb83748SJack Wang rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), 8939cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 8949cb83748SJack Wang DMA_FROM_DEVICE, rtrs_srv_info_req_done); 8959cb83748SJack Wang if (unlikely(!rx_iu)) 8969cb83748SJack Wang return -ENOMEM; 8979cb83748SJack Wang /* Prepare for getting info response */ 8989cb83748SJack Wang err = rtrs_iu_post_recv(&con->c, rx_iu); 8999cb83748SJack Wang if (unlikely(err)) { 9009cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err); 9018bd372acSGioh Kim rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); 9029cb83748SJack Wang return err; 9039cb83748SJack Wang } 9049cb83748SJack Wang 9059cb83748SJack Wang return 0; 9069cb83748SJack Wang } 9079cb83748SJack Wang 9089cb83748SJack Wang static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) 9099cb83748SJack Wang { 9109cb83748SJack Wang int i, err; 9119cb83748SJack Wang 9129cb83748SJack Wang for (i = 0; i < q_size; i++) { 9139cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 9149cb83748SJack Wang if (unlikely(err)) 9159cb83748SJack Wang return err; 9169cb83748SJack Wang } 9179cb83748SJack Wang 9189cb83748SJack Wang return 0; 9199cb83748SJack Wang } 9209cb83748SJack Wang 9219cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess) 9229cb83748SJack Wang { 9239cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9249cb83748SJack Wang struct rtrs_sess *s = &sess->s; 9259cb83748SJack Wang size_t q_size; 9269cb83748SJack Wang int err, cid; 9279cb83748SJack Wang 9289cb83748SJack Wang for (cid = 0; cid < sess->s.con_num; cid++) { 9299cb83748SJack Wang if (cid == 0) 9309cb83748SJack Wang q_size = SERVICE_CON_QUEUE_DEPTH; 9319cb83748SJack Wang else 9329cb83748SJack Wang q_size = srv->queue_depth; 9339cb83748SJack Wang 9349cb83748SJack Wang err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size); 9359cb83748SJack Wang if (unlikely(err)) { 9369cb83748SJack Wang rtrs_err(s, "post_recv_io(), err: %d\n", err); 9379cb83748SJack Wang return err; 9389cb83748SJack Wang } 9399cb83748SJack Wang } 9409cb83748SJack Wang 9419cb83748SJack Wang return 0; 9429cb83748SJack Wang } 9439cb83748SJack Wang 9449cb83748SJack Wang static void process_read(struct rtrs_srv_con *con, 9459cb83748SJack Wang struct rtrs_msg_rdma_read *msg, 9469cb83748SJack Wang u32 buf_id, u32 off) 9479cb83748SJack Wang { 9489cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9499cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9509cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9519cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 9529cb83748SJack Wang struct rtrs_srv_op *id; 9539cb83748SJack Wang 9549cb83748SJack Wang size_t usr_len, data_len; 9559cb83748SJack Wang void *data; 9569cb83748SJack Wang int ret; 9579cb83748SJack Wang 9589cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 9599cb83748SJack Wang rtrs_err_rl(s, 9609cb83748SJack Wang "Processing read request failed, session is disconnected, sess state %s\n", 9619cb83748SJack Wang rtrs_srv_state_str(sess->state)); 9629cb83748SJack Wang return; 9639cb83748SJack Wang } 9649cb83748SJack Wang if (unlikely(msg->sg_cnt != 1 && msg->sg_cnt != 0)) { 9659cb83748SJack Wang rtrs_err_rl(s, 9669cb83748SJack Wang "Processing read request failed, invalid message\n"); 9679cb83748SJack Wang return; 9689cb83748SJack Wang } 9699cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 9709cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, READ); 9719cb83748SJack Wang id = sess->ops_ids[buf_id]; 9729cb83748SJack Wang id->con = con; 9739cb83748SJack Wang id->dir = READ; 9749cb83748SJack Wang id->msg_id = buf_id; 9759cb83748SJack Wang id->rd_msg = msg; 9769cb83748SJack Wang usr_len = le16_to_cpu(msg->usr_len); 9779cb83748SJack Wang data_len = off - usr_len; 9789cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 979c81cba85SGioh Kim ret = ctx->ops.rdma_ev(srv->priv, id, READ, data, data_len, 9809cb83748SJack Wang data + data_len, usr_len); 9819cb83748SJack Wang 9829cb83748SJack Wang if (unlikely(ret)) { 9839cb83748SJack Wang rtrs_err_rl(s, 9849cb83748SJack Wang "Processing read request failed, user module cb reported for msg_id %d, err: %d\n", 9859cb83748SJack Wang buf_id, ret); 9869cb83748SJack Wang goto send_err_msg; 9879cb83748SJack Wang } 9889cb83748SJack Wang 9899cb83748SJack Wang return; 9909cb83748SJack Wang 9919cb83748SJack Wang send_err_msg: 9929cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 9939cb83748SJack Wang if (ret < 0) { 9949cb83748SJack Wang rtrs_err_rl(s, 9959cb83748SJack Wang "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n", 9969cb83748SJack Wang buf_id, ret); 9979cb83748SJack Wang close_sess(sess); 9989cb83748SJack Wang } 9999cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10009cb83748SJack Wang } 10019cb83748SJack Wang 10029cb83748SJack Wang static void process_write(struct rtrs_srv_con *con, 10039cb83748SJack Wang struct rtrs_msg_rdma_write *req, 10049cb83748SJack Wang u32 buf_id, u32 off) 10059cb83748SJack Wang { 10069cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10079cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10089cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 10099cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 10109cb83748SJack Wang struct rtrs_srv_op *id; 10119cb83748SJack Wang 10129cb83748SJack Wang size_t data_len, usr_len; 10139cb83748SJack Wang void *data; 10149cb83748SJack Wang int ret; 10159cb83748SJack Wang 10169cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 10179cb83748SJack Wang rtrs_err_rl(s, 10189cb83748SJack Wang "Processing write request failed, session is disconnected, sess state %s\n", 10199cb83748SJack Wang rtrs_srv_state_str(sess->state)); 10209cb83748SJack Wang return; 10219cb83748SJack Wang } 10229cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 10239cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, WRITE); 10249cb83748SJack Wang id = sess->ops_ids[buf_id]; 10259cb83748SJack Wang id->con = con; 10269cb83748SJack Wang id->dir = WRITE; 10279cb83748SJack Wang id->msg_id = buf_id; 10289cb83748SJack Wang 10299cb83748SJack Wang usr_len = le16_to_cpu(req->usr_len); 10309cb83748SJack Wang data_len = off - usr_len; 10319cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 1032c81cba85SGioh Kim ret = ctx->ops.rdma_ev(srv->priv, id, WRITE, data, data_len, 10339cb83748SJack Wang data + data_len, usr_len); 10349cb83748SJack Wang if (unlikely(ret)) { 10359cb83748SJack Wang rtrs_err_rl(s, 10369cb83748SJack Wang "Processing write request failed, user module callback reports err: %d\n", 10379cb83748SJack Wang ret); 10389cb83748SJack Wang goto send_err_msg; 10399cb83748SJack Wang } 10409cb83748SJack Wang 10419cb83748SJack Wang return; 10429cb83748SJack Wang 10439cb83748SJack Wang send_err_msg: 10449cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10459cb83748SJack Wang if (ret < 0) { 10469cb83748SJack Wang rtrs_err_rl(s, 10479cb83748SJack Wang "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n", 10489cb83748SJack Wang buf_id, ret); 10499cb83748SJack Wang close_sess(sess); 10509cb83748SJack Wang } 10519cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10529cb83748SJack Wang } 10539cb83748SJack Wang 10549cb83748SJack Wang static void process_io_req(struct rtrs_srv_con *con, void *msg, 10559cb83748SJack Wang u32 id, u32 off) 10569cb83748SJack Wang { 10579cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10589cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10599cb83748SJack Wang struct rtrs_msg_rdma_hdr *hdr; 10609cb83748SJack Wang unsigned int type; 10619cb83748SJack Wang 10629cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id], 10639cb83748SJack Wang max_chunk_size, DMA_BIDIRECTIONAL); 10649cb83748SJack Wang hdr = msg; 10659cb83748SJack Wang type = le16_to_cpu(hdr->type); 10669cb83748SJack Wang 10679cb83748SJack Wang switch (type) { 10689cb83748SJack Wang case RTRS_MSG_WRITE: 10699cb83748SJack Wang process_write(con, msg, id, off); 10709cb83748SJack Wang break; 10719cb83748SJack Wang case RTRS_MSG_READ: 10729cb83748SJack Wang process_read(con, msg, id, off); 10739cb83748SJack Wang break; 10749cb83748SJack Wang default: 10759cb83748SJack Wang rtrs_err(s, 10769cb83748SJack Wang "Processing I/O request failed, unknown message type received: 0x%02x\n", 10779cb83748SJack Wang type); 10789cb83748SJack Wang goto err; 10799cb83748SJack Wang } 10809cb83748SJack Wang 10819cb83748SJack Wang return; 10829cb83748SJack Wang 10839cb83748SJack Wang err: 10849cb83748SJack Wang close_sess(sess); 10859cb83748SJack Wang } 10869cb83748SJack Wang 10879cb83748SJack Wang static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) 10889cb83748SJack Wang { 10899cb83748SJack Wang struct rtrs_srv_mr *mr = 10909cb83748SJack Wang container_of(wc->wr_cqe, typeof(*mr), inv_cqe); 10913b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 10929cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10939cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10949cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 10959cb83748SJack Wang u32 msg_id, off; 10969cb83748SJack Wang void *data; 10979cb83748SJack Wang 10989cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 10999cb83748SJack Wang rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n", 11009cb83748SJack Wang ib_wc_status_msg(wc->status)); 11019cb83748SJack Wang close_sess(sess); 11029cb83748SJack Wang } 11039cb83748SJack Wang msg_id = mr->msg_id; 11049cb83748SJack Wang off = mr->msg_off; 11059cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 11069cb83748SJack Wang process_io_req(con, data, msg_id, off); 11079cb83748SJack Wang } 11089cb83748SJack Wang 11099cb83748SJack Wang static int rtrs_srv_inv_rkey(struct rtrs_srv_con *con, 11109cb83748SJack Wang struct rtrs_srv_mr *mr) 11119cb83748SJack Wang { 11129cb83748SJack Wang struct ib_send_wr wr = { 11139cb83748SJack Wang .opcode = IB_WR_LOCAL_INV, 11149cb83748SJack Wang .wr_cqe = &mr->inv_cqe, 11159cb83748SJack Wang .send_flags = IB_SEND_SIGNALED, 11169cb83748SJack Wang .ex.invalidate_rkey = mr->mr->rkey, 11179cb83748SJack Wang }; 11189cb83748SJack Wang mr->inv_cqe.done = rtrs_srv_inv_rkey_done; 11199cb83748SJack Wang 11209cb83748SJack Wang return ib_post_send(con->c.qp, &wr, NULL); 11219cb83748SJack Wang } 11229cb83748SJack Wang 11239cb83748SJack Wang static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) 11249cb83748SJack Wang { 11259cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11269cb83748SJack Wang while (!list_empty(&con->rsp_wr_wait_list)) { 11279cb83748SJack Wang struct rtrs_srv_op *id; 11289cb83748SJack Wang int ret; 11299cb83748SJack Wang 11309cb83748SJack Wang id = list_entry(con->rsp_wr_wait_list.next, 11319cb83748SJack Wang struct rtrs_srv_op, wait_list); 11329cb83748SJack Wang list_del(&id->wait_list); 11339cb83748SJack Wang 11349cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11359cb83748SJack Wang ret = rtrs_srv_resp_rdma(id, id->status); 11369cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11379cb83748SJack Wang 11389cb83748SJack Wang if (!ret) { 11399cb83748SJack Wang list_add(&id->wait_list, &con->rsp_wr_wait_list); 11409cb83748SJack Wang break; 11419cb83748SJack Wang } 11429cb83748SJack Wang } 11439cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11449cb83748SJack Wang } 11459cb83748SJack Wang 11469cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) 11479cb83748SJack Wang { 11483b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 11499cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11509cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11519cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11529cb83748SJack Wang u32 imm_type, imm_payload; 11539cb83748SJack Wang int err; 11549cb83748SJack Wang 11559cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11569cb83748SJack Wang if (wc->status != IB_WC_WR_FLUSH_ERR) { 11579cb83748SJack Wang rtrs_err(s, 11589cb83748SJack Wang "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n", 11599cb83748SJack Wang ib_wc_status_msg(wc->status), wc->wr_cqe, 11609cb83748SJack Wang wc->opcode, wc->vendor_err, wc->byte_len); 11619cb83748SJack Wang close_sess(sess); 11629cb83748SJack Wang } 11639cb83748SJack Wang return; 11649cb83748SJack Wang } 11659cb83748SJack Wang 11669cb83748SJack Wang switch (wc->opcode) { 11679cb83748SJack Wang case IB_WC_RECV_RDMA_WITH_IMM: 11689cb83748SJack Wang /* 11699cb83748SJack Wang * post_recv() RDMA write completions of IO reqs (read/write) 11709cb83748SJack Wang * and hb 11719cb83748SJack Wang */ 11729cb83748SJack Wang if (WARN_ON(wc->wr_cqe != &io_comp_cqe)) 11739cb83748SJack Wang return; 11749cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 11759cb83748SJack Wang if (unlikely(err)) { 11769cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); 11779cb83748SJack Wang close_sess(sess); 11789cb83748SJack Wang break; 11799cb83748SJack Wang } 11809cb83748SJack Wang rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), 11819cb83748SJack Wang &imm_type, &imm_payload); 11829cb83748SJack Wang if (likely(imm_type == RTRS_IO_REQ_IMM)) { 11839cb83748SJack Wang u32 msg_id, off; 11849cb83748SJack Wang void *data; 11859cb83748SJack Wang 11869cb83748SJack Wang msg_id = imm_payload >> sess->mem_bits; 11879cb83748SJack Wang off = imm_payload & ((1 << sess->mem_bits) - 1); 1188bf1d8edbSDan Carpenter if (unlikely(msg_id >= srv->queue_depth || 1189bf1d8edbSDan Carpenter off >= max_chunk_size)) { 11909cb83748SJack Wang rtrs_err(s, "Wrong msg_id %u, off %u\n", 11919cb83748SJack Wang msg_id, off); 11929cb83748SJack Wang close_sess(sess); 11939cb83748SJack Wang return; 11949cb83748SJack Wang } 11959cb83748SJack Wang if (always_invalidate) { 11969cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[msg_id]; 11979cb83748SJack Wang 11989cb83748SJack Wang mr->msg_off = off; 11999cb83748SJack Wang mr->msg_id = msg_id; 12009cb83748SJack Wang err = rtrs_srv_inv_rkey(con, mr); 12019cb83748SJack Wang if (unlikely(err)) { 12029cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", 12039cb83748SJack Wang err); 12049cb83748SJack Wang close_sess(sess); 12059cb83748SJack Wang break; 12069cb83748SJack Wang } 12079cb83748SJack Wang } else { 12089cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 12099cb83748SJack Wang process_io_req(con, data, msg_id, off); 12109cb83748SJack Wang } 12119cb83748SJack Wang } else if (imm_type == RTRS_HB_MSG_IMM) { 12129cb83748SJack Wang WARN_ON(con->c.cid); 12139cb83748SJack Wang rtrs_send_hb_ack(&sess->s); 12149cb83748SJack Wang } else if (imm_type == RTRS_HB_ACK_IMM) { 12159cb83748SJack Wang WARN_ON(con->c.cid); 12169cb83748SJack Wang sess->s.hb_missed_cnt = 0; 12179cb83748SJack Wang } else { 12189cb83748SJack Wang rtrs_wrn(s, "Unknown IMM type %u\n", imm_type); 12199cb83748SJack Wang } 12209cb83748SJack Wang break; 12219cb83748SJack Wang case IB_WC_RDMA_WRITE: 12229cb83748SJack Wang case IB_WC_SEND: 12239cb83748SJack Wang /* 12249cb83748SJack Wang * post_send() RDMA write completions of IO reqs (read/write) 12259cb83748SJack Wang */ 12269cb83748SJack Wang atomic_add(srv->queue_depth, &con->sq_wr_avail); 12279cb83748SJack Wang 12289cb83748SJack Wang if (unlikely(!list_empty_careful(&con->rsp_wr_wait_list))) 12299cb83748SJack Wang rtrs_rdma_process_wr_wait_list(con); 12309cb83748SJack Wang 12319cb83748SJack Wang break; 12329cb83748SJack Wang default: 12339cb83748SJack Wang rtrs_wrn(s, "Unexpected WC type: %d\n", wc->opcode); 12349cb83748SJack Wang return; 12359cb83748SJack Wang } 12369cb83748SJack Wang } 12379cb83748SJack Wang 12389cb83748SJack Wang /** 12399cb83748SJack Wang * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname. 12409cb83748SJack Wang * @srv: Session 12419cb83748SJack Wang * @sessname: Sessname buffer 12429cb83748SJack Wang * @len: Length of sessname buffer 12439cb83748SJack Wang */ 12449cb83748SJack Wang int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) 12459cb83748SJack Wang { 12469cb83748SJack Wang struct rtrs_srv_sess *sess; 12479cb83748SJack Wang int err = -ENOTCONN; 12489cb83748SJack Wang 12499cb83748SJack Wang mutex_lock(&srv->paths_mutex); 12509cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 12519cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTED) 12529cb83748SJack Wang continue; 12532d612f0dSDima Stepanov strscpy(sessname, sess->s.sessname, 12549cb83748SJack Wang min_t(size_t, sizeof(sess->s.sessname), len)); 12559cb83748SJack Wang err = 0; 12569cb83748SJack Wang break; 12579cb83748SJack Wang } 12589cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 12599cb83748SJack Wang 12609cb83748SJack Wang return err; 12619cb83748SJack Wang } 12629cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_sess_name); 12639cb83748SJack Wang 12649cb83748SJack Wang /** 12659cb83748SJack Wang * rtrs_srv_get_sess_qdepth() - Get rtrs_srv qdepth. 12669cb83748SJack Wang * @srv: Session 12679cb83748SJack Wang */ 12689cb83748SJack Wang int rtrs_srv_get_queue_depth(struct rtrs_srv *srv) 12699cb83748SJack Wang { 12709cb83748SJack Wang return srv->queue_depth; 12719cb83748SJack Wang } 12729cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_queue_depth); 12739cb83748SJack Wang 12749cb83748SJack Wang static int find_next_bit_ring(struct rtrs_srv_sess *sess) 12759cb83748SJack Wang { 12769cb83748SJack Wang struct ib_device *ib_dev = sess->s.dev->ib_dev; 12779cb83748SJack Wang int v; 12789cb83748SJack Wang 12799cb83748SJack Wang v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask); 12809cb83748SJack Wang if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors) 12819cb83748SJack Wang v = cpumask_first(&cq_affinity_mask); 12829cb83748SJack Wang return v; 12839cb83748SJack Wang } 12849cb83748SJack Wang 12859cb83748SJack Wang static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess) 12869cb83748SJack Wang { 12879cb83748SJack Wang sess->cur_cq_vector = find_next_bit_ring(sess); 12889cb83748SJack Wang 12899cb83748SJack Wang return sess->cur_cq_vector; 12909cb83748SJack Wang } 12919cb83748SJack Wang 129239c2d639SMd Haris Iqbal static void rtrs_srv_dev_release(struct device *dev) 129339c2d639SMd Haris Iqbal { 129439c2d639SMd Haris Iqbal struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev); 129539c2d639SMd Haris Iqbal 129639c2d639SMd Haris Iqbal kfree(srv); 129739c2d639SMd Haris Iqbal } 129839c2d639SMd Haris Iqbal 12999cb83748SJack Wang static void free_srv(struct rtrs_srv *srv) 13009cb83748SJack Wang { 13019cb83748SJack Wang int i; 13029cb83748SJack Wang 13039cb83748SJack Wang WARN_ON(refcount_read(&srv->refcount)); 13049cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 13059cb83748SJack Wang mempool_free(srv->chunks[i], chunk_pool); 13069cb83748SJack Wang kfree(srv->chunks); 13079cb83748SJack Wang mutex_destroy(&srv->paths_mutex); 13089cb83748SJack Wang mutex_destroy(&srv->paths_ev_mutex); 13099cb83748SJack Wang /* last put to release the srv structure */ 13109cb83748SJack Wang put_device(&srv->dev); 13119cb83748SJack Wang } 13129cb83748SJack Wang 13139cb83748SJack Wang static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx, 131403e9b33aSMd Haris Iqbal const uuid_t *paths_uuid, 131503e9b33aSMd Haris Iqbal bool first_conn) 13169cb83748SJack Wang { 13179cb83748SJack Wang struct rtrs_srv *srv; 1318d715ff8aSGuoqing Jiang int i; 13199cb83748SJack Wang 13209cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 1321d715ff8aSGuoqing Jiang list_for_each_entry(srv, &ctx->srv_list, ctx_list) { 1322d715ff8aSGuoqing Jiang if (uuid_equal(&srv->paths_uuid, paths_uuid) && 1323d715ff8aSGuoqing Jiang refcount_inc_not_zero(&srv->refcount)) { 1324d715ff8aSGuoqing Jiang mutex_unlock(&ctx->srv_mutex); 1325d715ff8aSGuoqing Jiang return srv; 1326d715ff8aSGuoqing Jiang } 1327d715ff8aSGuoqing Jiang } 1328ed408529SJack Wang mutex_unlock(&ctx->srv_mutex); 132903e9b33aSMd Haris Iqbal /* 133003e9b33aSMd Haris Iqbal * If this request is not the first connection request from the 133103e9b33aSMd Haris Iqbal * client for this session then fail and return error. 133203e9b33aSMd Haris Iqbal */ 13336564b110SMd Haris Iqbal if (!first_conn) { 13346564b110SMd Haris Iqbal pr_err_ratelimited("Error: Not the first connection request for this session\n"); 133503e9b33aSMd Haris Iqbal return ERR_PTR(-ENXIO); 13366564b110SMd Haris Iqbal } 1337d715ff8aSGuoqing Jiang 1338d715ff8aSGuoqing Jiang /* need to allocate a new srv */ 1339d715ff8aSGuoqing Jiang srv = kzalloc(sizeof(*srv), GFP_KERNEL); 1340ed408529SJack Wang if (!srv) 134103e9b33aSMd Haris Iqbal return ERR_PTR(-ENOMEM); 1342d715ff8aSGuoqing Jiang 1343d715ff8aSGuoqing Jiang INIT_LIST_HEAD(&srv->paths_list); 1344d715ff8aSGuoqing Jiang mutex_init(&srv->paths_mutex); 1345d715ff8aSGuoqing Jiang mutex_init(&srv->paths_ev_mutex); 1346d715ff8aSGuoqing Jiang uuid_copy(&srv->paths_uuid, paths_uuid); 1347d715ff8aSGuoqing Jiang srv->queue_depth = sess_queue_depth; 1348d715ff8aSGuoqing Jiang srv->ctx = ctx; 1349d715ff8aSGuoqing Jiang device_initialize(&srv->dev); 1350d715ff8aSGuoqing Jiang srv->dev.release = rtrs_srv_dev_release; 13519cb83748SJack Wang 1352d715ff8aSGuoqing Jiang srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks), 1353d715ff8aSGuoqing Jiang GFP_KERNEL); 1354d715ff8aSGuoqing Jiang if (!srv->chunks) 1355d715ff8aSGuoqing Jiang goto err_free_srv; 1356d715ff8aSGuoqing Jiang 1357d715ff8aSGuoqing Jiang for (i = 0; i < srv->queue_depth; i++) { 1358d715ff8aSGuoqing Jiang srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL); 1359d715ff8aSGuoqing Jiang if (!srv->chunks[i]) 1360d715ff8aSGuoqing Jiang goto err_free_chunks; 1361d715ff8aSGuoqing Jiang } 1362d715ff8aSGuoqing Jiang refcount_set(&srv->refcount, 1); 1363ed408529SJack Wang mutex_lock(&ctx->srv_mutex); 1364ed408529SJack Wang list_add(&srv->ctx_list, &ctx->srv_list); 1365ed408529SJack Wang mutex_unlock(&ctx->srv_mutex); 1366d715ff8aSGuoqing Jiang 13679cb83748SJack Wang return srv; 1368d715ff8aSGuoqing Jiang 1369d715ff8aSGuoqing Jiang err_free_chunks: 1370d715ff8aSGuoqing Jiang while (i--) 1371d715ff8aSGuoqing Jiang mempool_free(srv->chunks[i], chunk_pool); 1372d715ff8aSGuoqing Jiang kfree(srv->chunks); 1373d715ff8aSGuoqing Jiang 1374d715ff8aSGuoqing Jiang err_free_srv: 1375d715ff8aSGuoqing Jiang kfree(srv); 137603e9b33aSMd Haris Iqbal return ERR_PTR(-ENOMEM); 13779cb83748SJack Wang } 13789cb83748SJack Wang 13799cb83748SJack Wang static void put_srv(struct rtrs_srv *srv) 13809cb83748SJack Wang { 13819cb83748SJack Wang if (refcount_dec_and_test(&srv->refcount)) { 13829cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 13839cb83748SJack Wang 13849cb83748SJack Wang WARN_ON(srv->dev.kobj.state_in_sysfs); 13859cb83748SJack Wang 13869cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 13879cb83748SJack Wang list_del(&srv->ctx_list); 13889cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 13899cb83748SJack Wang free_srv(srv); 13909cb83748SJack Wang } 13919cb83748SJack Wang } 13929cb83748SJack Wang 13939cb83748SJack Wang static void __add_path_to_srv(struct rtrs_srv *srv, 13949cb83748SJack Wang struct rtrs_srv_sess *sess) 13959cb83748SJack Wang { 13969cb83748SJack Wang list_add_tail(&sess->s.entry, &srv->paths_list); 13979cb83748SJack Wang srv->paths_num++; 13989cb83748SJack Wang WARN_ON(srv->paths_num >= MAX_PATHS_NUM); 13999cb83748SJack Wang } 14009cb83748SJack Wang 14019cb83748SJack Wang static void del_path_from_srv(struct rtrs_srv_sess *sess) 14029cb83748SJack Wang { 14039cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 14049cb83748SJack Wang 14059cb83748SJack Wang if (WARN_ON(!srv)) 14069cb83748SJack Wang return; 14079cb83748SJack Wang 14089cb83748SJack Wang mutex_lock(&srv->paths_mutex); 14099cb83748SJack Wang list_del(&sess->s.entry); 14109cb83748SJack Wang WARN_ON(!srv->paths_num); 14119cb83748SJack Wang srv->paths_num--; 14129cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 14139cb83748SJack Wang } 14149cb83748SJack Wang 14159cb83748SJack Wang /* return true if addresses are the same, error other wise */ 14169cb83748SJack Wang static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) 14179cb83748SJack Wang { 14189cb83748SJack Wang switch (a->sa_family) { 14199cb83748SJack Wang case AF_IB: 14209cb83748SJack Wang return memcmp(&((struct sockaddr_ib *)a)->sib_addr, 14219cb83748SJack Wang &((struct sockaddr_ib *)b)->sib_addr, 14229cb83748SJack Wang sizeof(struct ib_addr)) && 14239cb83748SJack Wang (b->sa_family == AF_IB); 14249cb83748SJack Wang case AF_INET: 14259cb83748SJack Wang return memcmp(&((struct sockaddr_in *)a)->sin_addr, 14269cb83748SJack Wang &((struct sockaddr_in *)b)->sin_addr, 14279cb83748SJack Wang sizeof(struct in_addr)) && 14289cb83748SJack Wang (b->sa_family == AF_INET); 14299cb83748SJack Wang case AF_INET6: 14309cb83748SJack Wang return memcmp(&((struct sockaddr_in6 *)a)->sin6_addr, 14319cb83748SJack Wang &((struct sockaddr_in6 *)b)->sin6_addr, 14329cb83748SJack Wang sizeof(struct in6_addr)) && 14339cb83748SJack Wang (b->sa_family == AF_INET6); 14349cb83748SJack Wang default: 14359cb83748SJack Wang return -ENOENT; 14369cb83748SJack Wang } 14379cb83748SJack Wang } 14389cb83748SJack Wang 14399cb83748SJack Wang static bool __is_path_w_addr_exists(struct rtrs_srv *srv, 14409cb83748SJack Wang struct rdma_addr *addr) 14419cb83748SJack Wang { 14429cb83748SJack Wang struct rtrs_srv_sess *sess; 14439cb83748SJack Wang 14449cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 14459cb83748SJack Wang if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr, 14469cb83748SJack Wang (struct sockaddr *)&addr->dst_addr) && 14479cb83748SJack Wang !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr, 14489cb83748SJack Wang (struct sockaddr *)&addr->src_addr)) 14499cb83748SJack Wang return true; 14509cb83748SJack Wang 14519cb83748SJack Wang return false; 14529cb83748SJack Wang } 14539cb83748SJack Wang 14549cb83748SJack Wang static void free_sess(struct rtrs_srv_sess *sess) 14559cb83748SJack Wang { 1456f7452a7eSGioh Kim if (sess->kobj.state_in_sysfs) { 1457f7452a7eSGioh Kim kobject_del(&sess->kobj); 14589cb83748SJack Wang kobject_put(&sess->kobj); 1459f7452a7eSGioh Kim } else { 14609cb83748SJack Wang kfree(sess); 14619cb83748SJack Wang } 1462f7452a7eSGioh Kim } 14639cb83748SJack Wang 14649cb83748SJack Wang static void rtrs_srv_close_work(struct work_struct *work) 14659cb83748SJack Wang { 14669cb83748SJack Wang struct rtrs_srv_sess *sess; 14679cb83748SJack Wang struct rtrs_srv_con *con; 14689cb83748SJack Wang int i; 14699cb83748SJack Wang 14709cb83748SJack Wang sess = container_of(work, typeof(*sess), close_work); 14719cb83748SJack Wang 14729cb83748SJack Wang rtrs_srv_destroy_sess_files(sess); 14739cb83748SJack Wang rtrs_srv_stop_hb(sess); 14749cb83748SJack Wang 14759cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 14769cb83748SJack Wang if (!sess->s.con[i]) 14779cb83748SJack Wang continue; 14789cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 14799cb83748SJack Wang rdma_disconnect(con->c.cm_id); 14809cb83748SJack Wang ib_drain_qp(con->c.qp); 14819cb83748SJack Wang } 14829cb83748SJack Wang /* Wait for all inflights */ 14839cb83748SJack Wang rtrs_srv_wait_ops_ids(sess); 14849cb83748SJack Wang 14859cb83748SJack Wang /* Notify upper layer if we are the last path */ 14869cb83748SJack Wang rtrs_srv_sess_down(sess); 14879cb83748SJack Wang 14889cb83748SJack Wang unmap_cont_bufs(sess); 14899cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 14909cb83748SJack Wang 14919cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 14929cb83748SJack Wang if (!sess->s.con[i]) 14939cb83748SJack Wang continue; 14949cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 14959cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 14969cb83748SJack Wang rdma_destroy_id(con->c.cm_id); 14979cb83748SJack Wang kfree(con); 14989cb83748SJack Wang } 14999cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 15009cb83748SJack Wang 15019cb83748SJack Wang del_path_from_srv(sess); 15029cb83748SJack Wang put_srv(sess->srv); 15039cb83748SJack Wang sess->srv = NULL; 15049cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CLOSED); 15059cb83748SJack Wang 15069cb83748SJack Wang kfree(sess->dma_addr); 15079cb83748SJack Wang kfree(sess->s.con); 15089cb83748SJack Wang free_sess(sess); 15099cb83748SJack Wang } 15109cb83748SJack Wang 15119cb83748SJack Wang static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess, 15129cb83748SJack Wang struct rdma_cm_id *cm_id) 15139cb83748SJack Wang { 15149cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15159cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15169cb83748SJack Wang struct rdma_conn_param param; 15179cb83748SJack Wang int err; 15189cb83748SJack Wang 15199cb83748SJack Wang param = (struct rdma_conn_param) { 15209cb83748SJack Wang .rnr_retry_count = 7, 15219cb83748SJack Wang .private_data = &msg, 15229cb83748SJack Wang .private_data_len = sizeof(msg), 15239cb83748SJack Wang }; 15249cb83748SJack Wang 15259cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15269cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15279cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15289cb83748SJack Wang .queue_depth = cpu_to_le16(srv->queue_depth), 15299cb83748SJack Wang .max_io_size = cpu_to_le32(max_chunk_size - MAX_HDR_SIZE), 15309cb83748SJack Wang .max_hdr_size = cpu_to_le32(MAX_HDR_SIZE), 15319cb83748SJack Wang }; 15329cb83748SJack Wang 15339cb83748SJack Wang if (always_invalidate) 15349cb83748SJack Wang msg.flags = cpu_to_le32(RTRS_MSG_NEW_RKEY_F); 15359cb83748SJack Wang 15369cb83748SJack Wang err = rdma_accept(cm_id, ¶m); 15379cb83748SJack Wang if (err) 15389cb83748SJack Wang pr_err("rdma_accept(), err: %d\n", err); 15399cb83748SJack Wang 15409cb83748SJack Wang return err; 15419cb83748SJack Wang } 15429cb83748SJack Wang 15439cb83748SJack Wang static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) 15449cb83748SJack Wang { 15459cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15469cb83748SJack Wang int err; 15479cb83748SJack Wang 15489cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15499cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15509cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15519cb83748SJack Wang .errno = cpu_to_le16(errno), 15529cb83748SJack Wang }; 15539cb83748SJack Wang 15548094ba0aSLeon Romanovsky err = rdma_reject(cm_id, &msg, sizeof(msg), IB_CM_REJ_CONSUMER_DEFINED); 15559cb83748SJack Wang if (err) 15569cb83748SJack Wang pr_err("rdma_reject(), err: %d\n", err); 15579cb83748SJack Wang 15589cb83748SJack Wang /* Bounce errno back */ 15599cb83748SJack Wang return errno; 15609cb83748SJack Wang } 15619cb83748SJack Wang 15629cb83748SJack Wang static struct rtrs_srv_sess * 15639cb83748SJack Wang __find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid) 15649cb83748SJack Wang { 15659cb83748SJack Wang struct rtrs_srv_sess *sess; 15669cb83748SJack Wang 15679cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 15689cb83748SJack Wang if (uuid_equal(&sess->s.uuid, sess_uuid)) 15699cb83748SJack Wang return sess; 15709cb83748SJack Wang } 15719cb83748SJack Wang 15729cb83748SJack Wang return NULL; 15739cb83748SJack Wang } 15749cb83748SJack Wang 15759cb83748SJack Wang static int create_con(struct rtrs_srv_sess *sess, 15769cb83748SJack Wang struct rdma_cm_id *cm_id, 15779cb83748SJack Wang unsigned int cid) 15789cb83748SJack Wang { 15799cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15809cb83748SJack Wang struct rtrs_sess *s = &sess->s; 15819cb83748SJack Wang struct rtrs_srv_con *con; 15829cb83748SJack Wang 15837490fd1fSJack Wang u32 cq_size, wr_queue_size; 15849cb83748SJack Wang int err, cq_vector; 15859cb83748SJack Wang 15869cb83748SJack Wang con = kzalloc(sizeof(*con), GFP_KERNEL); 15879cb83748SJack Wang if (!con) { 15889cb83748SJack Wang err = -ENOMEM; 15899cb83748SJack Wang goto err; 15909cb83748SJack Wang } 15919cb83748SJack Wang 15929cb83748SJack Wang spin_lock_init(&con->rsp_wr_wait_lock); 15939cb83748SJack Wang INIT_LIST_HEAD(&con->rsp_wr_wait_list); 15949cb83748SJack Wang con->c.cm_id = cm_id; 15959cb83748SJack Wang con->c.sess = &sess->s; 15969cb83748SJack Wang con->c.cid = cid; 15976f5d1b30SJack Wang atomic_set(&con->wr_cnt, 1); 15989cb83748SJack Wang 15999cb83748SJack Wang if (con->c.cid == 0) { 16009cb83748SJack Wang /* 16019cb83748SJack Wang * All receive and all send (each requiring invalidate) 16029cb83748SJack Wang * + 2 for drain and heartbeat 16039cb83748SJack Wang */ 16049cb83748SJack Wang wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2; 16059cb83748SJack Wang cq_size = wr_queue_size; 16069cb83748SJack Wang } else { 16079cb83748SJack Wang /* 16089cb83748SJack Wang * If we have all receive requests posted and 16099cb83748SJack Wang * all write requests posted and each read request 16109cb83748SJack Wang * requires an invalidate request + drain 16119cb83748SJack Wang * and qp gets into error state. 16129cb83748SJack Wang */ 16139cb83748SJack Wang cq_size = srv->queue_depth * 3 + 1; 16149cb83748SJack Wang /* 16159cb83748SJack Wang * In theory we might have queue_depth * 32 16169cb83748SJack Wang * outstanding requests if an unsafe global key is used 16179cb83748SJack Wang * and we have queue_depth read requests each consisting 16189cb83748SJack Wang * of 32 different addresses. div 3 for mlx5. 16199cb83748SJack Wang */ 16209cb83748SJack Wang wr_queue_size = sess->s.dev->ib_dev->attrs.max_qp_wr / 3; 16219cb83748SJack Wang } 16229cb83748SJack Wang atomic_set(&con->sq_wr_avail, wr_queue_size); 16239cb83748SJack Wang cq_vector = rtrs_srv_get_next_cq_vector(sess); 16249cb83748SJack Wang 16259cb83748SJack Wang /* TODO: SOFTIRQ can be faster, but be careful with softirq context */ 16269cb83748SJack Wang err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size, 16277490fd1fSJack Wang wr_queue_size, wr_queue_size, 16287490fd1fSJack Wang IB_POLL_WORKQUEUE); 16299cb83748SJack Wang if (err) { 16309cb83748SJack Wang rtrs_err(s, "rtrs_cq_qp_create(), err: %d\n", err); 16319cb83748SJack Wang goto free_con; 16329cb83748SJack Wang } 16339cb83748SJack Wang if (con->c.cid == 0) { 16349cb83748SJack Wang err = post_recv_info_req(con); 16359cb83748SJack Wang if (err) 16369cb83748SJack Wang goto free_cqqp; 16379cb83748SJack Wang } 16389cb83748SJack Wang WARN_ON(sess->s.con[cid]); 16399cb83748SJack Wang sess->s.con[cid] = &con->c; 16409cb83748SJack Wang 16419cb83748SJack Wang /* 16429cb83748SJack Wang * Change context from server to current connection. The other 16439cb83748SJack Wang * way is to use cm_id->qp->qp_context, which does not work on OFED. 16449cb83748SJack Wang */ 16459cb83748SJack Wang cm_id->context = &con->c; 16469cb83748SJack Wang 16479cb83748SJack Wang return 0; 16489cb83748SJack Wang 16499cb83748SJack Wang free_cqqp: 16509cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 16519cb83748SJack Wang free_con: 16529cb83748SJack Wang kfree(con); 16539cb83748SJack Wang 16549cb83748SJack Wang err: 16559cb83748SJack Wang return err; 16569cb83748SJack Wang } 16579cb83748SJack Wang 16589cb83748SJack Wang static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, 16599cb83748SJack Wang struct rdma_cm_id *cm_id, 16609cb83748SJack Wang unsigned int con_num, 16619cb83748SJack Wang unsigned int recon_cnt, 16629cb83748SJack Wang const uuid_t *uuid) 16639cb83748SJack Wang { 16649cb83748SJack Wang struct rtrs_srv_sess *sess; 16659cb83748SJack Wang int err = -ENOMEM; 166688e2f105SGioh Kim char str[NAME_MAX]; 166788e2f105SGioh Kim struct rtrs_addr path; 16689cb83748SJack Wang 16699cb83748SJack Wang if (srv->paths_num >= MAX_PATHS_NUM) { 16709cb83748SJack Wang err = -ECONNRESET; 16719cb83748SJack Wang goto err; 16729cb83748SJack Wang } 16739cb83748SJack Wang if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) { 16749cb83748SJack Wang err = -EEXIST; 16759cb83748SJack Wang pr_err("Path with same addr exists\n"); 16769cb83748SJack Wang goto err; 16779cb83748SJack Wang } 16789cb83748SJack Wang sess = kzalloc(sizeof(*sess), GFP_KERNEL); 16799cb83748SJack Wang if (!sess) 16809cb83748SJack Wang goto err; 16819cb83748SJack Wang 16829cb83748SJack Wang sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); 16839cb83748SJack Wang if (!sess->stats) 16849cb83748SJack Wang goto err_free_sess; 16859cb83748SJack Wang 16869cb83748SJack Wang sess->stats->sess = sess; 16879cb83748SJack Wang 16889cb83748SJack Wang sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr), 16899cb83748SJack Wang GFP_KERNEL); 16909cb83748SJack Wang if (!sess->dma_addr) 16919cb83748SJack Wang goto err_free_stats; 16929cb83748SJack Wang 16939cb83748SJack Wang sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL); 16949cb83748SJack Wang if (!sess->s.con) 16959cb83748SJack Wang goto err_free_dma_addr; 16969cb83748SJack Wang 16979cb83748SJack Wang sess->state = RTRS_SRV_CONNECTING; 16989cb83748SJack Wang sess->srv = srv; 16999cb83748SJack Wang sess->cur_cq_vector = -1; 17009cb83748SJack Wang sess->s.dst_addr = cm_id->route.addr.dst_addr; 17019cb83748SJack Wang sess->s.src_addr = cm_id->route.addr.src_addr; 170288e2f105SGioh Kim 170388e2f105SGioh Kim /* temporary until receiving session-name from client */ 170488e2f105SGioh Kim path.src = &sess->s.src_addr; 170588e2f105SGioh Kim path.dst = &sess->s.dst_addr; 170688e2f105SGioh Kim rtrs_addr_to_str(&path, str, sizeof(str)); 17072d612f0dSDima Stepanov strscpy(sess->s.sessname, str, sizeof(sess->s.sessname)); 170888e2f105SGioh Kim 17099cb83748SJack Wang sess->s.con_num = con_num; 17109cb83748SJack Wang sess->s.recon_cnt = recon_cnt; 17119cb83748SJack Wang uuid_copy(&sess->s.uuid, uuid); 17129cb83748SJack Wang spin_lock_init(&sess->state_lock); 17139cb83748SJack Wang INIT_WORK(&sess->close_work, rtrs_srv_close_work); 17149cb83748SJack Wang rtrs_srv_init_hb(sess); 17159cb83748SJack Wang 17169cb83748SJack Wang sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); 17179cb83748SJack Wang if (!sess->s.dev) { 17189cb83748SJack Wang err = -ENOMEM; 17199cb83748SJack Wang goto err_free_con; 17209cb83748SJack Wang } 17219cb83748SJack Wang err = map_cont_bufs(sess); 17229cb83748SJack Wang if (err) 17239cb83748SJack Wang goto err_put_dev; 17249cb83748SJack Wang 17259cb83748SJack Wang err = rtrs_srv_alloc_ops_ids(sess); 17269cb83748SJack Wang if (err) 17279cb83748SJack Wang goto err_unmap_bufs; 17289cb83748SJack Wang 17299cb83748SJack Wang __add_path_to_srv(srv, sess); 17309cb83748SJack Wang 17319cb83748SJack Wang return sess; 17329cb83748SJack Wang 17339cb83748SJack Wang err_unmap_bufs: 17349cb83748SJack Wang unmap_cont_bufs(sess); 17359cb83748SJack Wang err_put_dev: 17369cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 17379cb83748SJack Wang err_free_con: 17389cb83748SJack Wang kfree(sess->s.con); 17399cb83748SJack Wang err_free_dma_addr: 17409cb83748SJack Wang kfree(sess->dma_addr); 17419cb83748SJack Wang err_free_stats: 17429cb83748SJack Wang kfree(sess->stats); 17439cb83748SJack Wang err_free_sess: 17449cb83748SJack Wang kfree(sess); 17459cb83748SJack Wang err: 17469cb83748SJack Wang return ERR_PTR(err); 17479cb83748SJack Wang } 17489cb83748SJack Wang 17499cb83748SJack Wang static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, 17509cb83748SJack Wang const struct rtrs_msg_conn_req *msg, 17519cb83748SJack Wang size_t len) 17529cb83748SJack Wang { 17539cb83748SJack Wang struct rtrs_srv_ctx *ctx = cm_id->context; 17549cb83748SJack Wang struct rtrs_srv_sess *sess; 17559cb83748SJack Wang struct rtrs_srv *srv; 17569cb83748SJack Wang 17579cb83748SJack Wang u16 version, con_num, cid; 17589cb83748SJack Wang u16 recon_cnt; 1759cfbeb0b9SGuoqing Jiang int err = -ECONNRESET; 17609cb83748SJack Wang 17619cb83748SJack Wang if (len < sizeof(*msg)) { 17629cb83748SJack Wang pr_err("Invalid RTRS connection request\n"); 1763cfbeb0b9SGuoqing Jiang goto reject_w_err; 17649cb83748SJack Wang } 17659cb83748SJack Wang if (le16_to_cpu(msg->magic) != RTRS_MAGIC) { 17669cb83748SJack Wang pr_err("Invalid RTRS magic\n"); 1767cfbeb0b9SGuoqing Jiang goto reject_w_err; 17689cb83748SJack Wang } 17699cb83748SJack Wang version = le16_to_cpu(msg->version); 17709cb83748SJack Wang if (version >> 8 != RTRS_PROTO_VER_MAJOR) { 17719cb83748SJack Wang pr_err("Unsupported major RTRS version: %d, expected %d\n", 17729cb83748SJack Wang version >> 8, RTRS_PROTO_VER_MAJOR); 1773cfbeb0b9SGuoqing Jiang goto reject_w_err; 17749cb83748SJack Wang } 17759cb83748SJack Wang con_num = le16_to_cpu(msg->cid_num); 17769cb83748SJack Wang if (con_num > 4096) { 17779cb83748SJack Wang /* Sanity check */ 17789cb83748SJack Wang pr_err("Too many connections requested: %d\n", con_num); 1779cfbeb0b9SGuoqing Jiang goto reject_w_err; 17809cb83748SJack Wang } 17819cb83748SJack Wang cid = le16_to_cpu(msg->cid); 17829cb83748SJack Wang if (cid >= con_num) { 17839cb83748SJack Wang /* Sanity check */ 17849cb83748SJack Wang pr_err("Incorrect cid: %d >= %d\n", cid, con_num); 1785cfbeb0b9SGuoqing Jiang goto reject_w_err; 17869cb83748SJack Wang } 17879cb83748SJack Wang recon_cnt = le16_to_cpu(msg->recon_cnt); 178803e9b33aSMd Haris Iqbal srv = get_or_create_srv(ctx, &msg->paths_uuid, msg->first_conn); 1789ed408529SJack Wang if (IS_ERR(srv)) { 179003e9b33aSMd Haris Iqbal err = PTR_ERR(srv); 17916564b110SMd Haris Iqbal pr_err("get_or_create_srv(), error %d\n", err); 17929cb83748SJack Wang goto reject_w_err; 17939cb83748SJack Wang } 17949cb83748SJack Wang mutex_lock(&srv->paths_mutex); 17959cb83748SJack Wang sess = __find_sess(srv, &msg->sess_uuid); 17969cb83748SJack Wang if (sess) { 17979cb83748SJack Wang struct rtrs_sess *s = &sess->s; 17989cb83748SJack Wang 17999cb83748SJack Wang /* Session already holds a reference */ 18009cb83748SJack Wang put_srv(srv); 18019cb83748SJack Wang 18029cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTING) { 18039cb83748SJack Wang rtrs_err(s, "Session in wrong state: %s\n", 18049cb83748SJack Wang rtrs_srv_state_str(sess->state)); 18059cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1806cfbeb0b9SGuoqing Jiang goto reject_w_err; 18079cb83748SJack Wang } 18089cb83748SJack Wang /* 18099cb83748SJack Wang * Sanity checks 18109cb83748SJack Wang */ 1811e172037bSMd Haris Iqbal if (con_num != s->con_num || cid >= s->con_num) { 18129cb83748SJack Wang rtrs_err(s, "Incorrect request: %d, %d\n", 18139cb83748SJack Wang cid, con_num); 18149cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1815cfbeb0b9SGuoqing Jiang goto reject_w_err; 18169cb83748SJack Wang } 1817e172037bSMd Haris Iqbal if (s->con[cid]) { 18189cb83748SJack Wang rtrs_err(s, "Connection already exists: %d\n", 18199cb83748SJack Wang cid); 18209cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1821cfbeb0b9SGuoqing Jiang goto reject_w_err; 18229cb83748SJack Wang } 18239cb83748SJack Wang } else { 18249cb83748SJack Wang sess = __alloc_sess(srv, cm_id, con_num, recon_cnt, 18259cb83748SJack Wang &msg->sess_uuid); 18269cb83748SJack Wang if (IS_ERR(sess)) { 18279cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18289cb83748SJack Wang put_srv(srv); 18299cb83748SJack Wang err = PTR_ERR(sess); 18306564b110SMd Haris Iqbal pr_err("RTRS server session allocation failed: %d\n", err); 18319cb83748SJack Wang goto reject_w_err; 18329cb83748SJack Wang } 18339cb83748SJack Wang } 18349cb83748SJack Wang err = create_con(sess, cm_id, cid); 18359cb83748SJack Wang if (err) { 18366564b110SMd Haris Iqbal rtrs_err((&sess->s), "create_con(), error %d\n", err); 18379cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18389cb83748SJack Wang /* 18399cb83748SJack Wang * Since session has other connections we follow normal way 18409cb83748SJack Wang * through workqueue, but still return an error to tell cma.c 18419cb83748SJack Wang * to call rdma_destroy_id() for current connection. 18429cb83748SJack Wang */ 18439cb83748SJack Wang goto close_and_return_err; 18449cb83748SJack Wang } 18459cb83748SJack Wang err = rtrs_rdma_do_accept(sess, cm_id); 18469cb83748SJack Wang if (err) { 18476564b110SMd Haris Iqbal rtrs_err((&sess->s), "rtrs_rdma_do_accept(), error %d\n", err); 18489cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18499cb83748SJack Wang /* 18509cb83748SJack Wang * Since current connection was successfully added to the 18519cb83748SJack Wang * session we follow normal way through workqueue to close the 18529cb83748SJack Wang * session, thus return 0 to tell cma.c we call 18539cb83748SJack Wang * rdma_destroy_id() ourselves. 18549cb83748SJack Wang */ 18559cb83748SJack Wang err = 0; 18569cb83748SJack Wang goto close_and_return_err; 18579cb83748SJack Wang } 18589cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18599cb83748SJack Wang 18609cb83748SJack Wang return 0; 18619cb83748SJack Wang 18629cb83748SJack Wang reject_w_err: 18639cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, err); 18649cb83748SJack Wang 18659cb83748SJack Wang close_and_return_err: 18669cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 186799f0c380SJack Wang close_sess(sess); 18689cb83748SJack Wang 18699cb83748SJack Wang return err; 18709cb83748SJack Wang } 18719cb83748SJack Wang 18729cb83748SJack Wang static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, 18739cb83748SJack Wang struct rdma_cm_event *ev) 18749cb83748SJack Wang { 18759cb83748SJack Wang struct rtrs_srv_sess *sess = NULL; 18769cb83748SJack Wang struct rtrs_sess *s = NULL; 18779cb83748SJack Wang 18789cb83748SJack Wang if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) { 18799cb83748SJack Wang struct rtrs_con *c = cm_id->context; 18809cb83748SJack Wang 18819cb83748SJack Wang s = c->sess; 18829cb83748SJack Wang sess = to_srv_sess(s); 18839cb83748SJack Wang } 18849cb83748SJack Wang 18859cb83748SJack Wang switch (ev->event) { 18869cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_REQUEST: 18879cb83748SJack Wang /* 18889cb83748SJack Wang * In case of error cma.c will destroy cm_id, 18899cb83748SJack Wang * see cma_process_remove() 18909cb83748SJack Wang */ 18919cb83748SJack Wang return rtrs_rdma_connect(cm_id, ev->param.conn.private_data, 18929cb83748SJack Wang ev->param.conn.private_data_len); 18939cb83748SJack Wang case RDMA_CM_EVENT_ESTABLISHED: 18949cb83748SJack Wang /* Nothing here */ 18959cb83748SJack Wang break; 18969cb83748SJack Wang case RDMA_CM_EVENT_REJECTED: 18979cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_ERROR: 18989cb83748SJack Wang case RDMA_CM_EVENT_UNREACHABLE: 18999cb83748SJack Wang rtrs_err(s, "CM error (CM event: %s, err: %d)\n", 19009cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 190157dae8baSGuoqing Jiang fallthrough; 19029cb83748SJack Wang case RDMA_CM_EVENT_DISCONNECTED: 19039cb83748SJack Wang case RDMA_CM_EVENT_ADDR_CHANGE: 19049cb83748SJack Wang case RDMA_CM_EVENT_TIMEWAIT_EXIT: 19059cb83748SJack Wang case RDMA_CM_EVENT_DEVICE_REMOVAL: 19069cb83748SJack Wang close_sess(sess); 19079cb83748SJack Wang break; 19089cb83748SJack Wang default: 19099cb83748SJack Wang pr_err("Ignoring unexpected CM event %s, err %d\n", 19109cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19119cb83748SJack Wang break; 19129cb83748SJack Wang } 19139cb83748SJack Wang 19149cb83748SJack Wang return 0; 19159cb83748SJack Wang } 19169cb83748SJack Wang 19179cb83748SJack Wang static struct rdma_cm_id *rtrs_srv_cm_init(struct rtrs_srv_ctx *ctx, 19189cb83748SJack Wang struct sockaddr *addr, 19199cb83748SJack Wang enum rdma_ucm_port_space ps) 19209cb83748SJack Wang { 19219cb83748SJack Wang struct rdma_cm_id *cm_id; 19229cb83748SJack Wang int ret; 19239cb83748SJack Wang 19249cb83748SJack Wang cm_id = rdma_create_id(&init_net, rtrs_srv_rdma_cm_handler, 19259cb83748SJack Wang ctx, ps, IB_QPT_RC); 19269cb83748SJack Wang if (IS_ERR(cm_id)) { 19279cb83748SJack Wang ret = PTR_ERR(cm_id); 19289cb83748SJack Wang pr_err("Creating id for RDMA connection failed, err: %d\n", 19299cb83748SJack Wang ret); 19309cb83748SJack Wang goto err_out; 19319cb83748SJack Wang } 19329cb83748SJack Wang ret = rdma_bind_addr(cm_id, addr); 19339cb83748SJack Wang if (ret) { 19349cb83748SJack Wang pr_err("Binding RDMA address failed, err: %d\n", ret); 19359cb83748SJack Wang goto err_cm; 19369cb83748SJack Wang } 19379cb83748SJack Wang ret = rdma_listen(cm_id, 64); 19389cb83748SJack Wang if (ret) { 19399cb83748SJack Wang pr_err("Listening on RDMA connection failed, err: %d\n", 19409cb83748SJack Wang ret); 19419cb83748SJack Wang goto err_cm; 19429cb83748SJack Wang } 19439cb83748SJack Wang 19449cb83748SJack Wang return cm_id; 19459cb83748SJack Wang 19469cb83748SJack Wang err_cm: 19479cb83748SJack Wang rdma_destroy_id(cm_id); 19489cb83748SJack Wang err_out: 19499cb83748SJack Wang 19509cb83748SJack Wang return ERR_PTR(ret); 19519cb83748SJack Wang } 19529cb83748SJack Wang 19539cb83748SJack Wang static int rtrs_srv_rdma_init(struct rtrs_srv_ctx *ctx, u16 port) 19549cb83748SJack Wang { 19559cb83748SJack Wang struct sockaddr_in6 sin = { 19569cb83748SJack Wang .sin6_family = AF_INET6, 19579cb83748SJack Wang .sin6_addr = IN6ADDR_ANY_INIT, 19589cb83748SJack Wang .sin6_port = htons(port), 19599cb83748SJack Wang }; 19609cb83748SJack Wang struct sockaddr_ib sib = { 19619cb83748SJack Wang .sib_family = AF_IB, 19629cb83748SJack Wang .sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port), 19639cb83748SJack Wang .sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL), 19649cb83748SJack Wang .sib_pkey = cpu_to_be16(0xffff), 19659cb83748SJack Wang }; 19669cb83748SJack Wang struct rdma_cm_id *cm_ip, *cm_ib; 19679cb83748SJack Wang int ret; 19689cb83748SJack Wang 19699cb83748SJack Wang /* 19709cb83748SJack Wang * We accept both IPoIB and IB connections, so we need to keep 19719cb83748SJack Wang * two cm id's, one for each socket type and port space. 19729cb83748SJack Wang * If the cm initialization of one of the id's fails, we abort 19739cb83748SJack Wang * everything. 19749cb83748SJack Wang */ 19759cb83748SJack Wang cm_ip = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sin, RDMA_PS_TCP); 19769cb83748SJack Wang if (IS_ERR(cm_ip)) 19779cb83748SJack Wang return PTR_ERR(cm_ip); 19789cb83748SJack Wang 19799cb83748SJack Wang cm_ib = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sib, RDMA_PS_IB); 19809cb83748SJack Wang if (IS_ERR(cm_ib)) { 19819cb83748SJack Wang ret = PTR_ERR(cm_ib); 19829cb83748SJack Wang goto free_cm_ip; 19839cb83748SJack Wang } 19849cb83748SJack Wang 19859cb83748SJack Wang ctx->cm_id_ip = cm_ip; 19869cb83748SJack Wang ctx->cm_id_ib = cm_ib; 19879cb83748SJack Wang 19889cb83748SJack Wang return 0; 19899cb83748SJack Wang 19909cb83748SJack Wang free_cm_ip: 19919cb83748SJack Wang rdma_destroy_id(cm_ip); 19929cb83748SJack Wang 19939cb83748SJack Wang return ret; 19949cb83748SJack Wang } 19959cb83748SJack Wang 19969cb83748SJack Wang static struct rtrs_srv_ctx *alloc_srv_ctx(struct rtrs_srv_ops *ops) 19979cb83748SJack Wang { 19989cb83748SJack Wang struct rtrs_srv_ctx *ctx; 19999cb83748SJack Wang 20009cb83748SJack Wang ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 20019cb83748SJack Wang if (!ctx) 20029cb83748SJack Wang return NULL; 20039cb83748SJack Wang 20049cb83748SJack Wang ctx->ops = *ops; 20059cb83748SJack Wang mutex_init(&ctx->srv_mutex); 20069cb83748SJack Wang INIT_LIST_HEAD(&ctx->srv_list); 20079cb83748SJack Wang 20089cb83748SJack Wang return ctx; 20099cb83748SJack Wang } 20109cb83748SJack Wang 20119cb83748SJack Wang static void free_srv_ctx(struct rtrs_srv_ctx *ctx) 20129cb83748SJack Wang { 20139cb83748SJack Wang WARN_ON(!list_empty(&ctx->srv_list)); 20149cb83748SJack Wang mutex_destroy(&ctx->srv_mutex); 20159cb83748SJack Wang kfree(ctx); 20169cb83748SJack Wang } 20179cb83748SJack Wang 2018558d52b2SMd Haris Iqbal static int rtrs_srv_add_one(struct ib_device *device) 2019558d52b2SMd Haris Iqbal { 2020558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2021558d52b2SMd Haris Iqbal int ret = 0; 2022558d52b2SMd Haris Iqbal 2023558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2024558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2025558d52b2SMd Haris Iqbal goto out; 2026558d52b2SMd Haris Iqbal 2027558d52b2SMd Haris Iqbal /* 2028558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will create them 2029558d52b2SMd Haris Iqbal * only once 2030558d52b2SMd Haris Iqbal */ 2031558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2032558d52b2SMd Haris Iqbal ret = rtrs_srv_rdma_init(ctx, ib_ctx.port); 2033558d52b2SMd Haris Iqbal if (ret) { 2034558d52b2SMd Haris Iqbal /* 2035558d52b2SMd Haris Iqbal * We errored out here. 2036558d52b2SMd Haris Iqbal * According to the ib code, if we encounter an error here then the 2037558d52b2SMd Haris Iqbal * error code is ignored, and no more calls to our ops are made. 2038558d52b2SMd Haris Iqbal */ 2039558d52b2SMd Haris Iqbal pr_err("Failed to initialize RDMA connection"); 2040558d52b2SMd Haris Iqbal goto err_out; 2041558d52b2SMd Haris Iqbal } 2042558d52b2SMd Haris Iqbal 2043558d52b2SMd Haris Iqbal out: 2044558d52b2SMd Haris Iqbal /* 2045558d52b2SMd Haris Iqbal * Keep a track on the number of ib devices added 2046558d52b2SMd Haris Iqbal */ 2047558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count++; 2048558d52b2SMd Haris Iqbal 2049558d52b2SMd Haris Iqbal err_out: 2050558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2051558d52b2SMd Haris Iqbal return ret; 2052558d52b2SMd Haris Iqbal } 2053558d52b2SMd Haris Iqbal 2054558d52b2SMd Haris Iqbal static void rtrs_srv_remove_one(struct ib_device *device, void *client_data) 2055558d52b2SMd Haris Iqbal { 2056558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2057558d52b2SMd Haris Iqbal 2058558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2059558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count--; 2060558d52b2SMd Haris Iqbal 2061558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2062558d52b2SMd Haris Iqbal goto out; 2063558d52b2SMd Haris Iqbal 2064558d52b2SMd Haris Iqbal /* 2065558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will remove them 2066558d52b2SMd Haris Iqbal * only once, when the last device is removed 2067558d52b2SMd Haris Iqbal */ 2068558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2069558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ip); 2070558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ib); 2071558d52b2SMd Haris Iqbal 2072558d52b2SMd Haris Iqbal out: 2073558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2074558d52b2SMd Haris Iqbal } 2075558d52b2SMd Haris Iqbal 2076558d52b2SMd Haris Iqbal static struct ib_client rtrs_srv_client = { 2077558d52b2SMd Haris Iqbal .name = "rtrs_server", 2078558d52b2SMd Haris Iqbal .add = rtrs_srv_add_one, 2079558d52b2SMd Haris Iqbal .remove = rtrs_srv_remove_one 2080558d52b2SMd Haris Iqbal }; 2081558d52b2SMd Haris Iqbal 20829cb83748SJack Wang /** 20839cb83748SJack Wang * rtrs_srv_open() - open RTRS server context 20849cb83748SJack Wang * @ops: callback functions 20859cb83748SJack Wang * @port: port to listen on 20869cb83748SJack Wang * 20879cb83748SJack Wang * Creates server context with specified callbacks. 20889cb83748SJack Wang * 20899cb83748SJack Wang * Return a valid pointer on success otherwise PTR_ERR. 20909cb83748SJack Wang */ 20919cb83748SJack Wang struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) 20929cb83748SJack Wang { 20939cb83748SJack Wang struct rtrs_srv_ctx *ctx; 20949cb83748SJack Wang int err; 20959cb83748SJack Wang 20969cb83748SJack Wang ctx = alloc_srv_ctx(ops); 20979cb83748SJack Wang if (!ctx) 20989cb83748SJack Wang return ERR_PTR(-ENOMEM); 20999cb83748SJack Wang 2100558d52b2SMd Haris Iqbal mutex_init(&ib_ctx.ib_dev_mutex); 2101558d52b2SMd Haris Iqbal ib_ctx.srv_ctx = ctx; 2102558d52b2SMd Haris Iqbal ib_ctx.port = port; 2103558d52b2SMd Haris Iqbal 2104558d52b2SMd Haris Iqbal err = ib_register_client(&rtrs_srv_client); 21059cb83748SJack Wang if (err) { 21069cb83748SJack Wang free_srv_ctx(ctx); 21079cb83748SJack Wang return ERR_PTR(err); 21089cb83748SJack Wang } 21099cb83748SJack Wang 21109cb83748SJack Wang return ctx; 21119cb83748SJack Wang } 21129cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_open); 21139cb83748SJack Wang 21149cb83748SJack Wang static void close_sessions(struct rtrs_srv *srv) 21159cb83748SJack Wang { 21169cb83748SJack Wang struct rtrs_srv_sess *sess; 21179cb83748SJack Wang 21189cb83748SJack Wang mutex_lock(&srv->paths_mutex); 21199cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 21209cb83748SJack Wang close_sess(sess); 21219cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 21229cb83748SJack Wang } 21239cb83748SJack Wang 21249cb83748SJack Wang static void close_ctx(struct rtrs_srv_ctx *ctx) 21259cb83748SJack Wang { 21269cb83748SJack Wang struct rtrs_srv *srv; 21279cb83748SJack Wang 21289cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 21299cb83748SJack Wang list_for_each_entry(srv, &ctx->srv_list, ctx_list) 21309cb83748SJack Wang close_sessions(srv); 21319cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 21329cb83748SJack Wang flush_workqueue(rtrs_wq); 21339cb83748SJack Wang } 21349cb83748SJack Wang 21359cb83748SJack Wang /** 21369cb83748SJack Wang * rtrs_srv_close() - close RTRS server context 21379cb83748SJack Wang * @ctx: pointer to server context 21389cb83748SJack Wang * 21399cb83748SJack Wang * Closes RTRS server context with all client sessions. 21409cb83748SJack Wang */ 21419cb83748SJack Wang void rtrs_srv_close(struct rtrs_srv_ctx *ctx) 21429cb83748SJack Wang { 2143558d52b2SMd Haris Iqbal ib_unregister_client(&rtrs_srv_client); 2144558d52b2SMd Haris Iqbal mutex_destroy(&ib_ctx.ib_dev_mutex); 21459cb83748SJack Wang close_ctx(ctx); 21469cb83748SJack Wang free_srv_ctx(ctx); 21479cb83748SJack Wang } 21489cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_close); 21499cb83748SJack Wang 21509cb83748SJack Wang static int check_module_params(void) 21519cb83748SJack Wang { 21529cb83748SJack Wang if (sess_queue_depth < 1 || sess_queue_depth > MAX_SESS_QUEUE_DEPTH) { 21539cb83748SJack Wang pr_err("Invalid sess_queue_depth value %d, has to be >= %d, <= %d.\n", 21549cb83748SJack Wang sess_queue_depth, 1, MAX_SESS_QUEUE_DEPTH); 21559cb83748SJack Wang return -EINVAL; 21569cb83748SJack Wang } 21573f3d0eabSGioh Kim if (max_chunk_size < MIN_CHUNK_SIZE || !is_power_of_2(max_chunk_size)) { 21589cb83748SJack Wang pr_err("Invalid max_chunk_size value %d, has to be >= %d and should be power of two.\n", 21593f3d0eabSGioh Kim max_chunk_size, MIN_CHUNK_SIZE); 21609cb83748SJack Wang return -EINVAL; 21619cb83748SJack Wang } 21629cb83748SJack Wang 21639cb83748SJack Wang /* 21649cb83748SJack Wang * Check if IB immediate data size is enough to hold the mem_id and the 21659cb83748SJack Wang * offset inside the memory chunk 21669cb83748SJack Wang */ 21679cb83748SJack Wang if ((ilog2(sess_queue_depth - 1) + 1) + 21689cb83748SJack Wang (ilog2(max_chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) { 21699cb83748SJack Wang pr_err("RDMA immediate size (%db) not enough to encode %d buffers of size %dB. Reduce 'sess_queue_depth' or 'max_chunk_size' parameters.\n", 21709cb83748SJack Wang MAX_IMM_PAYL_BITS, sess_queue_depth, max_chunk_size); 21719cb83748SJack Wang return -EINVAL; 21729cb83748SJack Wang } 21739cb83748SJack Wang 21749cb83748SJack Wang return 0; 21759cb83748SJack Wang } 21769cb83748SJack Wang 21779cb83748SJack Wang static int __init rtrs_server_init(void) 21789cb83748SJack Wang { 21799cb83748SJack Wang int err; 21809cb83748SJack Wang 21819cb83748SJack Wang pr_info("Loading module %s, proto %s: (max_chunk_size: %d (pure IO %ld, headers %ld) , sess_queue_depth: %d, always_invalidate: %d)\n", 21829cb83748SJack Wang KBUILD_MODNAME, RTRS_PROTO_VER_STRING, 21839cb83748SJack Wang max_chunk_size, max_chunk_size - MAX_HDR_SIZE, MAX_HDR_SIZE, 21849cb83748SJack Wang sess_queue_depth, always_invalidate); 21859cb83748SJack Wang 21869cb83748SJack Wang rtrs_rdma_dev_pd_init(0, &dev_pd); 21879cb83748SJack Wang 21889cb83748SJack Wang err = check_module_params(); 21899cb83748SJack Wang if (err) { 21909cb83748SJack Wang pr_err("Failed to load module, invalid module parameters, err: %d\n", 21919cb83748SJack Wang err); 21929cb83748SJack Wang return err; 21939cb83748SJack Wang } 21949cb83748SJack Wang chunk_pool = mempool_create_page_pool(sess_queue_depth * CHUNK_POOL_SZ, 21959cb83748SJack Wang get_order(max_chunk_size)); 21969cb83748SJack Wang if (!chunk_pool) 21979cb83748SJack Wang return -ENOMEM; 21989cb83748SJack Wang rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server"); 21999cb83748SJack Wang if (IS_ERR(rtrs_dev_class)) { 22009cb83748SJack Wang err = PTR_ERR(rtrs_dev_class); 22019cb83748SJack Wang goto out_chunk_pool; 22029cb83748SJack Wang } 220303ed5a8cSJack Wang rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0); 22046b31afceSWei Yongjun if (!rtrs_wq) { 22056b31afceSWei Yongjun err = -ENOMEM; 22069cb83748SJack Wang goto out_dev_class; 22076b31afceSWei Yongjun } 22089cb83748SJack Wang 22099cb83748SJack Wang return 0; 22109cb83748SJack Wang 22119cb83748SJack Wang out_dev_class: 22129cb83748SJack Wang class_destroy(rtrs_dev_class); 22139cb83748SJack Wang out_chunk_pool: 22149cb83748SJack Wang mempool_destroy(chunk_pool); 22159cb83748SJack Wang 22169cb83748SJack Wang return err; 22179cb83748SJack Wang } 22189cb83748SJack Wang 22199cb83748SJack Wang static void __exit rtrs_server_exit(void) 22209cb83748SJack Wang { 22219cb83748SJack Wang destroy_workqueue(rtrs_wq); 22229cb83748SJack Wang class_destroy(rtrs_dev_class); 22239cb83748SJack Wang mempool_destroy(chunk_pool); 22249cb83748SJack Wang rtrs_rdma_dev_pd_deinit(&dev_pd); 22259cb83748SJack Wang } 22269cb83748SJack Wang 22279cb83748SJack Wang module_init(rtrs_server_init); 22289cb83748SJack Wang module_exit(rtrs_server_exit); 2229