19cb83748SJack Wang // SPDX-License-Identifier: GPL-2.0-or-later 29cb83748SJack Wang /* 39cb83748SJack Wang * RDMA Transport Layer 49cb83748SJack Wang * 59cb83748SJack Wang * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. 69cb83748SJack Wang * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. 79cb83748SJack Wang * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. 89cb83748SJack Wang */ 99cb83748SJack Wang 109cb83748SJack Wang #undef pr_fmt 119cb83748SJack Wang #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt 129cb83748SJack Wang 139cb83748SJack Wang #include <linux/module.h> 149cb83748SJack Wang #include <linux/mempool.h> 159cb83748SJack Wang 169cb83748SJack Wang #include "rtrs-srv.h" 179cb83748SJack Wang #include "rtrs-log.h" 188094ba0aSLeon Romanovsky #include <rdma/ib_cm.h> 19558d52b2SMd Haris Iqbal #include <rdma/ib_verbs.h> 209cb83748SJack Wang 219cb83748SJack Wang MODULE_DESCRIPTION("RDMA Transport Server"); 229cb83748SJack Wang MODULE_LICENSE("GPL"); 239cb83748SJack Wang 249cb83748SJack Wang /* Must be power of 2, see mask from mr->page_size in ib_sg_to_pages() */ 259cb83748SJack Wang #define DEFAULT_MAX_CHUNK_SIZE (128 << 10) 269cb83748SJack Wang #define DEFAULT_SESS_QUEUE_DEPTH 512 279cb83748SJack Wang #define MAX_HDR_SIZE PAGE_SIZE 289cb83748SJack Wang 299cb83748SJack Wang /* We guarantee to serve 10 paths at least */ 309cb83748SJack Wang #define CHUNK_POOL_SZ 10 319cb83748SJack Wang 329cb83748SJack Wang static struct rtrs_rdma_dev_pd dev_pd; 339cb83748SJack Wang static mempool_t *chunk_pool; 349cb83748SJack Wang struct class *rtrs_dev_class; 35558d52b2SMd Haris Iqbal static struct rtrs_srv_ib_ctx ib_ctx; 369cb83748SJack Wang 379cb83748SJack Wang static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE; 389cb83748SJack Wang static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH; 399cb83748SJack Wang 409cb83748SJack Wang static bool always_invalidate = true; 419cb83748SJack Wang module_param(always_invalidate, bool, 0444); 429cb83748SJack Wang MODULE_PARM_DESC(always_invalidate, 439cb83748SJack Wang "Invalidate memory registration for contiguous memory regions before accessing."); 449cb83748SJack Wang 459cb83748SJack Wang module_param_named(max_chunk_size, max_chunk_size, int, 0444); 469cb83748SJack Wang MODULE_PARM_DESC(max_chunk_size, 479cb83748SJack Wang "Max size for each IO request, when change the unit is in byte (default: " 489cb83748SJack Wang __stringify(DEFAULT_MAX_CHUNK_SIZE) "KB)"); 499cb83748SJack Wang 509cb83748SJack Wang module_param_named(sess_queue_depth, sess_queue_depth, int, 0444); 519cb83748SJack Wang MODULE_PARM_DESC(sess_queue_depth, 529cb83748SJack Wang "Number of buffers for pending I/O requests to allocate per session. Maximum: " 539cb83748SJack Wang __stringify(MAX_SESS_QUEUE_DEPTH) " (default: " 549cb83748SJack Wang __stringify(DEFAULT_SESS_QUEUE_DEPTH) ")"); 559cb83748SJack Wang 569cb83748SJack Wang static cpumask_t cq_affinity_mask = { CPU_BITS_ALL }; 579cb83748SJack Wang 589cb83748SJack Wang static struct workqueue_struct *rtrs_wq; 599cb83748SJack Wang 609cb83748SJack Wang static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) 619cb83748SJack Wang { 629cb83748SJack Wang return container_of(c, struct rtrs_srv_con, c); 639cb83748SJack Wang } 649cb83748SJack Wang 659cb83748SJack Wang static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s) 669cb83748SJack Wang { 679cb83748SJack Wang return container_of(s, struct rtrs_srv_sess, s); 689cb83748SJack Wang } 699cb83748SJack Wang 709cb83748SJack Wang static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess, 719cb83748SJack Wang enum rtrs_srv_state new_state) 729cb83748SJack Wang { 739cb83748SJack Wang enum rtrs_srv_state old_state; 749cb83748SJack Wang bool changed = false; 759cb83748SJack Wang 769cb83748SJack Wang lockdep_assert_held(&sess->state_lock); 779cb83748SJack Wang old_state = sess->state; 789cb83748SJack Wang switch (new_state) { 799cb83748SJack Wang case RTRS_SRV_CONNECTED: 80485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CONNECTING) 819cb83748SJack Wang changed = true; 829cb83748SJack Wang break; 839cb83748SJack Wang case RTRS_SRV_CLOSING: 84485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CONNECTING || 85485f2fb1SGuoqing Jiang old_state == RTRS_SRV_CONNECTED) 869cb83748SJack Wang changed = true; 879cb83748SJack Wang break; 889cb83748SJack Wang case RTRS_SRV_CLOSED: 89485f2fb1SGuoqing Jiang if (old_state == RTRS_SRV_CLOSING) 909cb83748SJack Wang changed = true; 919cb83748SJack Wang break; 929cb83748SJack Wang default: 939cb83748SJack Wang break; 949cb83748SJack Wang } 959cb83748SJack Wang if (changed) 969cb83748SJack Wang sess->state = new_state; 979cb83748SJack Wang 989cb83748SJack Wang return changed; 999cb83748SJack Wang } 1009cb83748SJack Wang 101ffea6ad1SGuoqing Jiang static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, 102ffea6ad1SGuoqing Jiang enum rtrs_srv_state new_state) 1039cb83748SJack Wang { 1049cb83748SJack Wang bool changed; 1059cb83748SJack Wang 1069cb83748SJack Wang spin_lock_irq(&sess->state_lock); 1079cb83748SJack Wang changed = __rtrs_srv_change_state(sess, new_state); 1089cb83748SJack Wang spin_unlock_irq(&sess->state_lock); 1099cb83748SJack Wang 1109cb83748SJack Wang return changed; 1119cb83748SJack Wang } 1129cb83748SJack Wang 1139cb83748SJack Wang static void free_id(struct rtrs_srv_op *id) 1149cb83748SJack Wang { 1159cb83748SJack Wang if (!id) 1169cb83748SJack Wang return; 1179cb83748SJack Wang kfree(id); 1189cb83748SJack Wang } 1199cb83748SJack Wang 1209cb83748SJack Wang static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess) 1219cb83748SJack Wang { 1229cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1239cb83748SJack Wang int i; 1249cb83748SJack Wang 1259cb83748SJack Wang WARN_ON(atomic_read(&sess->ids_inflight)); 1269cb83748SJack Wang if (sess->ops_ids) { 1279cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 1289cb83748SJack Wang free_id(sess->ops_ids[i]); 1299cb83748SJack Wang kfree(sess->ops_ids); 1309cb83748SJack Wang sess->ops_ids = NULL; 1319cb83748SJack Wang } 1329cb83748SJack Wang } 1339cb83748SJack Wang 1349cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc); 1359cb83748SJack Wang 1369cb83748SJack Wang static struct ib_cqe io_comp_cqe = { 1379cb83748SJack Wang .done = rtrs_srv_rdma_done 1389cb83748SJack Wang }; 1399cb83748SJack Wang 1409cb83748SJack Wang static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) 1419cb83748SJack Wang { 1429cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1439cb83748SJack Wang struct rtrs_srv_op *id; 1449cb83748SJack Wang int i; 1459cb83748SJack Wang 1469cb83748SJack Wang sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids), 1479cb83748SJack Wang GFP_KERNEL); 1489cb83748SJack Wang if (!sess->ops_ids) 1499cb83748SJack Wang goto err; 1509cb83748SJack Wang 1519cb83748SJack Wang for (i = 0; i < srv->queue_depth; ++i) { 1529cb83748SJack Wang id = kzalloc(sizeof(*id), GFP_KERNEL); 1539cb83748SJack Wang if (!id) 1549cb83748SJack Wang goto err; 1559cb83748SJack Wang 1569cb83748SJack Wang sess->ops_ids[i] = id; 1579cb83748SJack Wang } 1589cb83748SJack Wang init_waitqueue_head(&sess->ids_waitq); 1599cb83748SJack Wang atomic_set(&sess->ids_inflight, 0); 1609cb83748SJack Wang 1619cb83748SJack Wang return 0; 1629cb83748SJack Wang 1639cb83748SJack Wang err: 1649cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 1659cb83748SJack Wang return -ENOMEM; 1669cb83748SJack Wang } 1679cb83748SJack Wang 1689cb83748SJack Wang static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess) 1699cb83748SJack Wang { 1709cb83748SJack Wang atomic_inc(&sess->ids_inflight); 1719cb83748SJack Wang } 1729cb83748SJack Wang 1739cb83748SJack Wang static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess) 1749cb83748SJack Wang { 1759cb83748SJack Wang if (atomic_dec_and_test(&sess->ids_inflight)) 1769cb83748SJack Wang wake_up(&sess->ids_waitq); 1779cb83748SJack Wang } 1789cb83748SJack Wang 1799cb83748SJack Wang static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess) 1809cb83748SJack Wang { 1819cb83748SJack Wang wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight)); 1829cb83748SJack Wang } 1839cb83748SJack Wang 1849cb83748SJack Wang 1859cb83748SJack Wang static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc) 1869cb83748SJack Wang { 1873b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 1889cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 1899cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 1909cb83748SJack Wang 1919cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 1929cb83748SJack Wang rtrs_err(s, "REG MR failed: %s\n", 1939cb83748SJack Wang ib_wc_status_msg(wc->status)); 1949cb83748SJack Wang close_sess(sess); 1959cb83748SJack Wang return; 1969cb83748SJack Wang } 1979cb83748SJack Wang } 1989cb83748SJack Wang 1999cb83748SJack Wang static struct ib_cqe local_reg_cqe = { 2009cb83748SJack Wang .done = rtrs_srv_reg_mr_done 2019cb83748SJack Wang }; 2029cb83748SJack Wang 2039cb83748SJack Wang static int rdma_write_sg(struct rtrs_srv_op *id) 2049cb83748SJack Wang { 2059cb83748SJack Wang struct rtrs_sess *s = id->con->c.sess; 2069cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 2079cb83748SJack Wang dma_addr_t dma_addr = sess->dma_addr[id->msg_id]; 2089cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 2099cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 210e6daa8f6SJack Wang struct ib_send_wr inv_wr; 211e6daa8f6SJack Wang struct ib_rdma_wr imm_wr; 2129cb83748SJack Wang struct ib_rdma_wr *wr = NULL; 2139cb83748SJack Wang enum ib_send_flags flags; 2149cb83748SJack Wang size_t sg_cnt; 2159cb83748SJack Wang int err, offset; 2169cb83748SJack Wang bool need_inval; 2179cb83748SJack Wang u32 rkey = 0; 2189cb83748SJack Wang struct ib_reg_wr rwr; 2199cb83748SJack Wang struct ib_sge *plist; 2209cb83748SJack Wang struct ib_sge list; 2219cb83748SJack Wang 2229cb83748SJack Wang sg_cnt = le16_to_cpu(id->rd_msg->sg_cnt); 2239cb83748SJack Wang need_inval = le16_to_cpu(id->rd_msg->flags) & RTRS_MSG_NEED_INVAL_F; 2249cb83748SJack Wang if (unlikely(sg_cnt != 1)) 2259cb83748SJack Wang return -EINVAL; 2269cb83748SJack Wang 2279cb83748SJack Wang offset = 0; 2289cb83748SJack Wang 2299cb83748SJack Wang wr = &id->tx_wr; 2309cb83748SJack Wang plist = &id->tx_sg; 2319cb83748SJack Wang plist->addr = dma_addr + offset; 2329cb83748SJack Wang plist->length = le32_to_cpu(id->rd_msg->desc[0].len); 2339cb83748SJack Wang 2349cb83748SJack Wang /* WR will fail with length error 2359cb83748SJack Wang * if this is 0 2369cb83748SJack Wang */ 2379cb83748SJack Wang if (unlikely(plist->length == 0)) { 2389cb83748SJack Wang rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); 2399cb83748SJack Wang return -EINVAL; 2409cb83748SJack Wang } 2419cb83748SJack Wang 2429cb83748SJack Wang plist->lkey = sess->s.dev->ib_pd->local_dma_lkey; 2439cb83748SJack Wang offset += plist->length; 2449cb83748SJack Wang 2459cb83748SJack Wang wr->wr.sg_list = plist; 2469cb83748SJack Wang wr->wr.num_sge = 1; 2479cb83748SJack Wang wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); 2489cb83748SJack Wang wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); 2499cb83748SJack Wang if (rkey == 0) 2509cb83748SJack Wang rkey = wr->rkey; 2519cb83748SJack Wang else 2529cb83748SJack Wang /* Only one key is actually used */ 2539cb83748SJack Wang WARN_ON_ONCE(rkey != wr->rkey); 2549cb83748SJack Wang 2559cb83748SJack Wang wr->wr.opcode = IB_WR_RDMA_WRITE; 2568537f2deSJack Wang wr->wr.wr_cqe = &io_comp_cqe; 2579cb83748SJack Wang wr->wr.ex.imm_data = 0; 2589cb83748SJack Wang wr->wr.send_flags = 0; 2599cb83748SJack Wang 2609cb83748SJack Wang if (need_inval && always_invalidate) { 2619cb83748SJack Wang wr->wr.next = &rwr.wr; 2629cb83748SJack Wang rwr.wr.next = &inv_wr; 263e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 2649cb83748SJack Wang } else if (always_invalidate) { 2659cb83748SJack Wang wr->wr.next = &rwr.wr; 266e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 2679cb83748SJack Wang } else if (need_inval) { 2689cb83748SJack Wang wr->wr.next = &inv_wr; 269e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 2709cb83748SJack Wang } else { 271e6daa8f6SJack Wang wr->wr.next = &imm_wr.wr; 2729cb83748SJack Wang } 2739cb83748SJack Wang /* 2749cb83748SJack Wang * From time to time we have to post signaled sends, 2759cb83748SJack Wang * or send queue will fill up and only QP reset can help. 2769cb83748SJack Wang */ 2779cb83748SJack Wang flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ? 2789cb83748SJack Wang 0 : IB_SEND_SIGNALED; 2799cb83748SJack Wang 2809cb83748SJack Wang if (need_inval) { 2819cb83748SJack Wang inv_wr.sg_list = NULL; 2829cb83748SJack Wang inv_wr.num_sge = 0; 2839cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 2848537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 2859cb83748SJack Wang inv_wr.send_flags = 0; 2869cb83748SJack Wang inv_wr.ex.invalidate_rkey = rkey; 2879cb83748SJack Wang } 2889cb83748SJack Wang 289e6daa8f6SJack Wang imm_wr.wr.next = NULL; 2909cb83748SJack Wang if (always_invalidate) { 2919cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 2929cb83748SJack Wang 2939cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 2949cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 2958537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 2969cb83748SJack Wang rwr.wr.num_sge = 0; 2979cb83748SJack Wang rwr.mr = srv_mr->mr; 2989cb83748SJack Wang rwr.wr.send_flags = 0; 2999cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 3009cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 3019cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 3029cb83748SJack Wang msg = srv_mr->iu->buf; 3039cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 3049cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 3059cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 3069cb83748SJack Wang 3079cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 3089cb83748SJack Wang list.length = sizeof(*msg); 3099cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 310e6daa8f6SJack Wang imm_wr.wr.sg_list = &list; 311e6daa8f6SJack Wang imm_wr.wr.num_sge = 1; 312e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; 3139cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 3149cb83748SJack Wang srv_mr->iu->dma_addr, 3159cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 3169cb83748SJack Wang } else { 317e6daa8f6SJack Wang imm_wr.wr.sg_list = NULL; 318e6daa8f6SJack Wang imm_wr.wr.num_sge = 0; 319e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 3209cb83748SJack Wang } 321e6daa8f6SJack Wang imm_wr.wr.send_flags = flags; 322e6daa8f6SJack Wang imm_wr.wr.ex.imm_data = cpu_to_be32(rtrs_to_io_rsp_imm(id->msg_id, 3239cb83748SJack Wang 0, need_inval)); 3249cb83748SJack Wang 325e6daa8f6SJack Wang imm_wr.wr.wr_cqe = &io_comp_cqe; 3269cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr, 3279cb83748SJack Wang offset, DMA_BIDIRECTIONAL); 3289cb83748SJack Wang 3299cb83748SJack Wang err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL); 3309cb83748SJack Wang if (unlikely(err)) 3319cb83748SJack Wang rtrs_err(s, 3329cb83748SJack Wang "Posting RDMA-Write-Request to QP failed, err: %d\n", 3339cb83748SJack Wang err); 3349cb83748SJack Wang 3359cb83748SJack Wang return err; 3369cb83748SJack Wang } 3379cb83748SJack Wang 3389cb83748SJack Wang /** 3399cb83748SJack Wang * send_io_resp_imm() - respond to client with empty IMM on failed READ/WRITE 3409cb83748SJack Wang * requests or on successful WRITE request. 3419cb83748SJack Wang * @con: the connection to send back result 3429cb83748SJack Wang * @id: the id associated with the IO 3439cb83748SJack Wang * @errno: the error number of the IO. 3449cb83748SJack Wang * 3459cb83748SJack Wang * Return 0 on success, errno otherwise. 3469cb83748SJack Wang */ 3479cb83748SJack Wang static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, 3489cb83748SJack Wang int errno) 3499cb83748SJack Wang { 3509cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 3519cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 352e6daa8f6SJack Wang struct ib_send_wr inv_wr, *wr = NULL; 353e6daa8f6SJack Wang struct ib_rdma_wr imm_wr; 3549cb83748SJack Wang struct ib_reg_wr rwr; 3559cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 3569cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 3579cb83748SJack Wang bool need_inval = false; 3589cb83748SJack Wang enum ib_send_flags flags; 3599cb83748SJack Wang u32 imm; 3609cb83748SJack Wang int err; 3619cb83748SJack Wang 3629cb83748SJack Wang if (id->dir == READ) { 3639cb83748SJack Wang struct rtrs_msg_rdma_read *rd_msg = id->rd_msg; 3649cb83748SJack Wang size_t sg_cnt; 3659cb83748SJack Wang 3669cb83748SJack Wang need_inval = le16_to_cpu(rd_msg->flags) & 3679cb83748SJack Wang RTRS_MSG_NEED_INVAL_F; 3689cb83748SJack Wang sg_cnt = le16_to_cpu(rd_msg->sg_cnt); 3699cb83748SJack Wang 3709cb83748SJack Wang if (need_inval) { 3719cb83748SJack Wang if (likely(sg_cnt)) { 3728537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 3739cb83748SJack Wang inv_wr.sg_list = NULL; 3749cb83748SJack Wang inv_wr.num_sge = 0; 3759cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 3769cb83748SJack Wang inv_wr.send_flags = 0; 3779cb83748SJack Wang /* Only one key is actually used */ 3789cb83748SJack Wang inv_wr.ex.invalidate_rkey = 3799cb83748SJack Wang le32_to_cpu(rd_msg->desc[0].key); 3809cb83748SJack Wang } else { 3819cb83748SJack Wang WARN_ON_ONCE(1); 3829cb83748SJack Wang need_inval = false; 3839cb83748SJack Wang } 3849cb83748SJack Wang } 3859cb83748SJack Wang } 3869cb83748SJack Wang 3879cb83748SJack Wang if (need_inval && always_invalidate) { 3889cb83748SJack Wang wr = &inv_wr; 3899cb83748SJack Wang inv_wr.next = &rwr.wr; 390e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 3919cb83748SJack Wang } else if (always_invalidate) { 3929cb83748SJack Wang wr = &rwr.wr; 393e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 3949cb83748SJack Wang } else if (need_inval) { 3959cb83748SJack Wang wr = &inv_wr; 396e6daa8f6SJack Wang inv_wr.next = &imm_wr.wr; 3979cb83748SJack Wang } else { 398e6daa8f6SJack Wang wr = &imm_wr.wr; 3999cb83748SJack Wang } 4009cb83748SJack Wang /* 4019cb83748SJack Wang * From time to time we have to post signalled sends, 4029cb83748SJack Wang * or send queue will fill up and only QP reset can help. 4039cb83748SJack Wang */ 4049cb83748SJack Wang flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ? 4059cb83748SJack Wang 0 : IB_SEND_SIGNALED; 4069cb83748SJack Wang imm = rtrs_to_io_rsp_imm(id->msg_id, errno, need_inval); 407e6daa8f6SJack Wang imm_wr.wr.next = NULL; 4089cb83748SJack Wang if (always_invalidate) { 4099cb83748SJack Wang struct ib_sge list; 4109cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 4119cb83748SJack Wang 4129cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 413e6daa8f6SJack Wang rwr.wr.next = &imm_wr.wr; 4149cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 4158537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 4169cb83748SJack Wang rwr.wr.num_sge = 0; 4179cb83748SJack Wang rwr.wr.send_flags = 0; 4189cb83748SJack Wang rwr.mr = srv_mr->mr; 4199cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 4209cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 4219cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 4229cb83748SJack Wang msg = srv_mr->iu->buf; 4239cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 4249cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 4259cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 4269cb83748SJack Wang 4279cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 4289cb83748SJack Wang list.length = sizeof(*msg); 4299cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 430e6daa8f6SJack Wang imm_wr.wr.sg_list = &list; 431e6daa8f6SJack Wang imm_wr.wr.num_sge = 1; 432e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; 4339cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 4349cb83748SJack Wang srv_mr->iu->dma_addr, 4359cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 4369cb83748SJack Wang } else { 437e6daa8f6SJack Wang imm_wr.wr.sg_list = NULL; 438e6daa8f6SJack Wang imm_wr.wr.num_sge = 0; 439e6daa8f6SJack Wang imm_wr.wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 4409cb83748SJack Wang } 441e6daa8f6SJack Wang imm_wr.wr.send_flags = flags; 442e6daa8f6SJack Wang imm_wr.wr.wr_cqe = &io_comp_cqe; 4439cb83748SJack Wang 444e6daa8f6SJack Wang imm_wr.wr.ex.imm_data = cpu_to_be32(imm); 4459cb83748SJack Wang 4469cb83748SJack Wang err = ib_post_send(id->con->c.qp, wr, NULL); 4479cb83748SJack Wang if (unlikely(err)) 4489cb83748SJack Wang rtrs_err_rl(s, "Posting RDMA-Reply to QP failed, err: %d\n", 4499cb83748SJack Wang err); 4509cb83748SJack Wang 4519cb83748SJack Wang return err; 4529cb83748SJack Wang } 4539cb83748SJack Wang 4549cb83748SJack Wang void close_sess(struct rtrs_srv_sess *sess) 4559cb83748SJack Wang { 456ffea6ad1SGuoqing Jiang if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING)) 4579cb83748SJack Wang queue_work(rtrs_wq, &sess->close_work); 4589cb83748SJack Wang WARN_ON(sess->state != RTRS_SRV_CLOSING); 4599cb83748SJack Wang } 4609cb83748SJack Wang 4619cb83748SJack Wang static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) 4629cb83748SJack Wang { 4639cb83748SJack Wang switch (state) { 4649cb83748SJack Wang case RTRS_SRV_CONNECTING: 4659cb83748SJack Wang return "RTRS_SRV_CONNECTING"; 4669cb83748SJack Wang case RTRS_SRV_CONNECTED: 4679cb83748SJack Wang return "RTRS_SRV_CONNECTED"; 4689cb83748SJack Wang case RTRS_SRV_CLOSING: 4699cb83748SJack Wang return "RTRS_SRV_CLOSING"; 4709cb83748SJack Wang case RTRS_SRV_CLOSED: 4719cb83748SJack Wang return "RTRS_SRV_CLOSED"; 4729cb83748SJack Wang default: 4739cb83748SJack Wang return "UNKNOWN"; 4749cb83748SJack Wang } 4759cb83748SJack Wang } 4769cb83748SJack Wang 4779cb83748SJack Wang /** 4789cb83748SJack Wang * rtrs_srv_resp_rdma() - Finish an RDMA request 4799cb83748SJack Wang * 4809cb83748SJack Wang * @id: Internal RTRS operation identifier 4819cb83748SJack Wang * @status: Response Code sent to the other side for this operation. 4829cb83748SJack Wang * 0 = success, <=0 error 4839cb83748SJack Wang * Context: any 4849cb83748SJack Wang * 4859cb83748SJack Wang * Finish a RDMA operation. A message is sent to the client and the 4869cb83748SJack Wang * corresponding memory areas will be released. 4879cb83748SJack Wang */ 4889cb83748SJack Wang bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) 4899cb83748SJack Wang { 4909cb83748SJack Wang struct rtrs_srv_sess *sess; 4919cb83748SJack Wang struct rtrs_srv_con *con; 4929cb83748SJack Wang struct rtrs_sess *s; 4939cb83748SJack Wang int err; 4949cb83748SJack Wang 4959cb83748SJack Wang if (WARN_ON(!id)) 4969cb83748SJack Wang return true; 4979cb83748SJack Wang 4989cb83748SJack Wang con = id->con; 4999cb83748SJack Wang s = con->c.sess; 5009cb83748SJack Wang sess = to_srv_sess(s); 5019cb83748SJack Wang 5029cb83748SJack Wang id->status = status; 5039cb83748SJack Wang 5049cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 5059cb83748SJack Wang rtrs_err_rl(s, 50642cdc190SGioh Kim "Sending I/O response failed, session %s is disconnected, sess state %s\n", 50742cdc190SGioh Kim kobject_name(&sess->kobj), 5089cb83748SJack Wang rtrs_srv_state_str(sess->state)); 5099cb83748SJack Wang goto out; 5109cb83748SJack Wang } 5119cb83748SJack Wang if (always_invalidate) { 5129cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id]; 5139cb83748SJack Wang 5149cb83748SJack Wang ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); 5159cb83748SJack Wang } 5169cb83748SJack Wang if (unlikely(atomic_sub_return(1, 5179cb83748SJack Wang &con->sq_wr_avail) < 0)) { 51842cdc190SGioh Kim rtrs_err(s, "IB send queue full: sess=%s cid=%d\n", 51942cdc190SGioh Kim kobject_name(&sess->kobj), 52042cdc190SGioh Kim con->c.cid); 5219cb83748SJack Wang atomic_add(1, &con->sq_wr_avail); 5229cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 5239cb83748SJack Wang list_add_tail(&id->wait_list, &con->rsp_wr_wait_list); 5249cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 5259cb83748SJack Wang return false; 5269cb83748SJack Wang } 5279cb83748SJack Wang 5289cb83748SJack Wang if (status || id->dir == WRITE || !id->rd_msg->sg_cnt) 5299cb83748SJack Wang err = send_io_resp_imm(con, id, status); 5309cb83748SJack Wang else 5319cb83748SJack Wang err = rdma_write_sg(id); 5329cb83748SJack Wang 5339cb83748SJack Wang if (unlikely(err)) { 53442cdc190SGioh Kim rtrs_err_rl(s, "IO response failed: %d: sess=%s\n", err, 53542cdc190SGioh Kim kobject_name(&sess->kobj)); 5369cb83748SJack Wang close_sess(sess); 5379cb83748SJack Wang } 5389cb83748SJack Wang out: 5399cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 5409cb83748SJack Wang return true; 5419cb83748SJack Wang } 5429cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_resp_rdma); 5439cb83748SJack Wang 5449cb83748SJack Wang /** 5459cb83748SJack Wang * rtrs_srv_set_sess_priv() - Set private pointer in rtrs_srv. 5469cb83748SJack Wang * @srv: Session pointer 5479cb83748SJack Wang * @priv: The private pointer that is associated with the session. 5489cb83748SJack Wang */ 5499cb83748SJack Wang void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv) 5509cb83748SJack Wang { 5519cb83748SJack Wang srv->priv = priv; 5529cb83748SJack Wang } 5539cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_set_sess_priv); 5549cb83748SJack Wang 5559cb83748SJack Wang static void unmap_cont_bufs(struct rtrs_srv_sess *sess) 5569cb83748SJack Wang { 5579cb83748SJack Wang int i; 5589cb83748SJack Wang 5599cb83748SJack Wang for (i = 0; i < sess->mrs_num; i++) { 5609cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 5619cb83748SJack Wang 5629cb83748SJack Wang srv_mr = &sess->mrs[i]; 5638bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 5649cb83748SJack Wang ib_dereg_mr(srv_mr->mr); 5659cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl, 5669cb83748SJack Wang srv_mr->sgt.nents, DMA_BIDIRECTIONAL); 5679cb83748SJack Wang sg_free_table(&srv_mr->sgt); 5689cb83748SJack Wang } 5699cb83748SJack Wang kfree(sess->mrs); 5709cb83748SJack Wang } 5719cb83748SJack Wang 5729cb83748SJack Wang static int map_cont_bufs(struct rtrs_srv_sess *sess) 5739cb83748SJack Wang { 5749cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 5759cb83748SJack Wang struct rtrs_sess *ss = &sess->s; 5769cb83748SJack Wang int i, mri, err, mrs_num; 5779cb83748SJack Wang unsigned int chunk_bits; 5789cb83748SJack Wang int chunks_per_mr = 1; 5799cb83748SJack Wang 5809cb83748SJack Wang /* 5819cb83748SJack Wang * Here we map queue_depth chunks to MR. Firstly we have to 5829cb83748SJack Wang * figure out how many chunks can we map per MR. 5839cb83748SJack Wang */ 5849cb83748SJack Wang if (always_invalidate) { 5859cb83748SJack Wang /* 5869cb83748SJack Wang * in order to do invalidate for each chunks of memory, we needs 5879cb83748SJack Wang * more memory regions. 5889cb83748SJack Wang */ 5899cb83748SJack Wang mrs_num = srv->queue_depth; 5909cb83748SJack Wang } else { 5919cb83748SJack Wang chunks_per_mr = 5929cb83748SJack Wang sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; 5939cb83748SJack Wang mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr); 5949cb83748SJack Wang chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num); 5959cb83748SJack Wang } 5969cb83748SJack Wang 5979cb83748SJack Wang sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL); 5989cb83748SJack Wang if (!sess->mrs) 5999cb83748SJack Wang return -ENOMEM; 6009cb83748SJack Wang 6019cb83748SJack Wang sess->mrs_num = mrs_num; 6029cb83748SJack Wang 6039cb83748SJack Wang for (mri = 0; mri < mrs_num; mri++) { 6049cb83748SJack Wang struct rtrs_srv_mr *srv_mr = &sess->mrs[mri]; 6059cb83748SJack Wang struct sg_table *sgt = &srv_mr->sgt; 6069cb83748SJack Wang struct scatterlist *s; 6079cb83748SJack Wang struct ib_mr *mr; 6089cb83748SJack Wang int nr, chunks; 6099cb83748SJack Wang 6109cb83748SJack Wang chunks = chunks_per_mr * mri; 6119cb83748SJack Wang if (!always_invalidate) 6129cb83748SJack Wang chunks_per_mr = min_t(int, chunks_per_mr, 6139cb83748SJack Wang srv->queue_depth - chunks); 6149cb83748SJack Wang 6159cb83748SJack Wang err = sg_alloc_table(sgt, chunks_per_mr, GFP_KERNEL); 6169cb83748SJack Wang if (err) 6179cb83748SJack Wang goto err; 6189cb83748SJack Wang 6199cb83748SJack Wang for_each_sg(sgt->sgl, s, chunks_per_mr, i) 6209cb83748SJack Wang sg_set_page(s, srv->chunks[chunks + i], 6219cb83748SJack Wang max_chunk_size, 0); 6229cb83748SJack Wang 6239cb83748SJack Wang nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl, 6249cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6259cb83748SJack Wang if (nr < sgt->nents) { 6269cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6279cb83748SJack Wang goto free_sg; 6289cb83748SJack Wang } 6299cb83748SJack Wang mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, 6309cb83748SJack Wang sgt->nents); 6319cb83748SJack Wang if (IS_ERR(mr)) { 6329cb83748SJack Wang err = PTR_ERR(mr); 6339cb83748SJack Wang goto unmap_sg; 6349cb83748SJack Wang } 6359cb83748SJack Wang nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents, 6369cb83748SJack Wang NULL, max_chunk_size); 637b386cd65SDan Carpenter if (nr < 0 || nr < sgt->nents) { 6389cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6399cb83748SJack Wang goto dereg_mr; 6409cb83748SJack Wang } 6419cb83748SJack Wang 6429cb83748SJack Wang if (always_invalidate) { 6439cb83748SJack Wang srv_mr->iu = rtrs_iu_alloc(1, 6449cb83748SJack Wang sizeof(struct rtrs_msg_rkey_rsp), 6459cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 6469cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_rdma_done); 6479cb83748SJack Wang if (!srv_mr->iu) { 6486b31afceSWei Yongjun err = -ENOMEM; 6496b31afceSWei Yongjun rtrs_err(ss, "rtrs_iu_alloc(), err: %d\n", err); 650f77c4839SGuoqing Jiang goto dereg_mr; 6519cb83748SJack Wang } 6529cb83748SJack Wang } 6539cb83748SJack Wang /* Eventually dma addr for each chunk can be cached */ 6549cb83748SJack Wang for_each_sg(sgt->sgl, s, sgt->orig_nents, i) 6559cb83748SJack Wang sess->dma_addr[chunks + i] = sg_dma_address(s); 6569cb83748SJack Wang 6579cb83748SJack Wang ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 6589cb83748SJack Wang srv_mr->mr = mr; 6599cb83748SJack Wang 6609cb83748SJack Wang continue; 6619cb83748SJack Wang err: 6629cb83748SJack Wang while (mri--) { 6639cb83748SJack Wang srv_mr = &sess->mrs[mri]; 6649cb83748SJack Wang sgt = &srv_mr->sgt; 6659cb83748SJack Wang mr = srv_mr->mr; 6668bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 6679cb83748SJack Wang dereg_mr: 6689cb83748SJack Wang ib_dereg_mr(mr); 6699cb83748SJack Wang unmap_sg: 6709cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl, 6719cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6729cb83748SJack Wang free_sg: 6739cb83748SJack Wang sg_free_table(sgt); 6749cb83748SJack Wang } 6759cb83748SJack Wang kfree(sess->mrs); 6769cb83748SJack Wang 6779cb83748SJack Wang return err; 6789cb83748SJack Wang } 6799cb83748SJack Wang 6809cb83748SJack Wang chunk_bits = ilog2(srv->queue_depth - 1) + 1; 6819cb83748SJack Wang sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); 6829cb83748SJack Wang 6839cb83748SJack Wang return 0; 6849cb83748SJack Wang } 6859cb83748SJack Wang 6869cb83748SJack Wang static void rtrs_srv_hb_err_handler(struct rtrs_con *c) 6879cb83748SJack Wang { 6889cb83748SJack Wang close_sess(to_srv_sess(c->sess)); 6899cb83748SJack Wang } 6909cb83748SJack Wang 6919cb83748SJack Wang static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess) 6929cb83748SJack Wang { 6939cb83748SJack Wang rtrs_init_hb(&sess->s, &io_comp_cqe, 6949cb83748SJack Wang RTRS_HB_INTERVAL_MS, 6959cb83748SJack Wang RTRS_HB_MISSED_MAX, 6969cb83748SJack Wang rtrs_srv_hb_err_handler, 6979cb83748SJack Wang rtrs_wq); 6989cb83748SJack Wang } 6999cb83748SJack Wang 7009cb83748SJack Wang static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess) 7019cb83748SJack Wang { 7029cb83748SJack Wang rtrs_start_hb(&sess->s); 7039cb83748SJack Wang } 7049cb83748SJack Wang 7059cb83748SJack Wang static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess) 7069cb83748SJack Wang { 7079cb83748SJack Wang rtrs_stop_hb(&sess->s); 7089cb83748SJack Wang } 7099cb83748SJack Wang 7109cb83748SJack Wang static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) 7119cb83748SJack Wang { 7123b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 7139cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7149cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7159cb83748SJack Wang struct rtrs_iu *iu; 7169cb83748SJack Wang 7179cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 7188bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 7199cb83748SJack Wang 7209cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 7219cb83748SJack Wang rtrs_err(s, "Sess info response send failed: %s\n", 7229cb83748SJack Wang ib_wc_status_msg(wc->status)); 7239cb83748SJack Wang close_sess(sess); 7249cb83748SJack Wang return; 7259cb83748SJack Wang } 7269cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_SEND); 7279cb83748SJack Wang } 7289cb83748SJack Wang 7299cb83748SJack Wang static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) 7309cb83748SJack Wang { 7319cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7329cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7339cb83748SJack Wang int up; 7349cb83748SJack Wang 7359cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7369cb83748SJack Wang up = ++srv->paths_up; 7379cb83748SJack Wang if (up == 1) 7389cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL); 7399cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7409cb83748SJack Wang 7419cb83748SJack Wang /* Mark session as established */ 7429cb83748SJack Wang sess->established = true; 7439cb83748SJack Wang } 7449cb83748SJack Wang 7459cb83748SJack Wang static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) 7469cb83748SJack Wang { 7479cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7489cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7499cb83748SJack Wang 7509cb83748SJack Wang if (!sess->established) 7519cb83748SJack Wang return; 7529cb83748SJack Wang 7539cb83748SJack Wang sess->established = false; 7549cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7559cb83748SJack Wang WARN_ON(!srv->paths_up); 7569cb83748SJack Wang if (--srv->paths_up == 0) 7579cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv); 7589cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7599cb83748SJack Wang } 7609cb83748SJack Wang 7619cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess); 7629cb83748SJack Wang 7639cb83748SJack Wang static int process_info_req(struct rtrs_srv_con *con, 7649cb83748SJack Wang struct rtrs_msg_info_req *msg) 7659cb83748SJack Wang { 7669cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7679cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7689cb83748SJack Wang struct ib_send_wr *reg_wr = NULL; 7699cb83748SJack Wang struct rtrs_msg_info_rsp *rsp; 7709cb83748SJack Wang struct rtrs_iu *tx_iu; 7719cb83748SJack Wang struct ib_reg_wr *rwr; 7729cb83748SJack Wang int mri, err; 7739cb83748SJack Wang size_t tx_sz; 7749cb83748SJack Wang 7759cb83748SJack Wang err = post_recv_sess(sess); 7769cb83748SJack Wang if (unlikely(err)) { 7779cb83748SJack Wang rtrs_err(s, "post_recv_sess(), err: %d\n", err); 7789cb83748SJack Wang return err; 7799cb83748SJack Wang } 7809cb83748SJack Wang rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL); 7819cb83748SJack Wang if (unlikely(!rwr)) 7829cb83748SJack Wang return -ENOMEM; 783*2d612f0dSDima Stepanov strscpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname)); 7849cb83748SJack Wang 7859cb83748SJack Wang tx_sz = sizeof(*rsp); 7869cb83748SJack Wang tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num; 7879cb83748SJack Wang tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev, 7889cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_info_rsp_done); 7899cb83748SJack Wang if (unlikely(!tx_iu)) { 7909cb83748SJack Wang err = -ENOMEM; 7919cb83748SJack Wang goto rwr_free; 7929cb83748SJack Wang } 7939cb83748SJack Wang 7949cb83748SJack Wang rsp = tx_iu->buf; 7959cb83748SJack Wang rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP); 7969cb83748SJack Wang rsp->sg_cnt = cpu_to_le16(sess->mrs_num); 7979cb83748SJack Wang 7989cb83748SJack Wang for (mri = 0; mri < sess->mrs_num; mri++) { 7999cb83748SJack Wang struct ib_mr *mr = sess->mrs[mri].mr; 8009cb83748SJack Wang 8019cb83748SJack Wang rsp->desc[mri].addr = cpu_to_le64(mr->iova); 8029cb83748SJack Wang rsp->desc[mri].key = cpu_to_le32(mr->rkey); 8039cb83748SJack Wang rsp->desc[mri].len = cpu_to_le32(mr->length); 8049cb83748SJack Wang 8059cb83748SJack Wang /* 8069cb83748SJack Wang * Fill in reg MR request and chain them *backwards* 8079cb83748SJack Wang */ 8089cb83748SJack Wang rwr[mri].wr.next = mri ? &rwr[mri - 1].wr : NULL; 8099cb83748SJack Wang rwr[mri].wr.opcode = IB_WR_REG_MR; 8109cb83748SJack Wang rwr[mri].wr.wr_cqe = &local_reg_cqe; 8119cb83748SJack Wang rwr[mri].wr.num_sge = 0; 812e8ae7ddbSJack Wang rwr[mri].wr.send_flags = 0; 8139cb83748SJack Wang rwr[mri].mr = mr; 8149cb83748SJack Wang rwr[mri].key = mr->rkey; 8159cb83748SJack Wang rwr[mri].access = (IB_ACCESS_LOCAL_WRITE | 8169cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 8179cb83748SJack Wang reg_wr = &rwr[mri].wr; 8189cb83748SJack Wang } 8199cb83748SJack Wang 8209cb83748SJack Wang err = rtrs_srv_create_sess_files(sess); 8219cb83748SJack Wang if (unlikely(err)) 8229cb83748SJack Wang goto iu_free; 8239cb83748SJack Wang kobject_get(&sess->kobj); 8249cb83748SJack Wang get_device(&sess->srv->dev); 8259cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED); 8269cb83748SJack Wang rtrs_srv_start_hb(sess); 8279cb83748SJack Wang 8289cb83748SJack Wang /* 8299cb83748SJack Wang * We do not account number of established connections at the current 8309cb83748SJack Wang * moment, we rely on the client, which should send info request when 8319cb83748SJack Wang * all connections are successfully established. Thus, simply notify 8329cb83748SJack Wang * listener with a proper event if we are the first path. 8339cb83748SJack Wang */ 8349cb83748SJack Wang rtrs_srv_sess_up(sess); 8359cb83748SJack Wang 8369cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, 8379cb83748SJack Wang tx_iu->size, DMA_TO_DEVICE); 8389cb83748SJack Wang 8399cb83748SJack Wang /* Send info response */ 8409cb83748SJack Wang err = rtrs_iu_post_send(&con->c, tx_iu, tx_sz, reg_wr); 8419cb83748SJack Wang if (unlikely(err)) { 8429cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err); 8439cb83748SJack Wang iu_free: 8448bd372acSGioh Kim rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); 8459cb83748SJack Wang } 8469cb83748SJack Wang rwr_free: 8479cb83748SJack Wang kfree(rwr); 8489cb83748SJack Wang 8499cb83748SJack Wang return err; 8509cb83748SJack Wang } 8519cb83748SJack Wang 8529cb83748SJack Wang static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) 8539cb83748SJack Wang { 8543b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 8559cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8569cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 8579cb83748SJack Wang struct rtrs_msg_info_req *msg; 8589cb83748SJack Wang struct rtrs_iu *iu; 8599cb83748SJack Wang int err; 8609cb83748SJack Wang 8619cb83748SJack Wang WARN_ON(con->c.cid); 8629cb83748SJack Wang 8639cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 8649cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 8659cb83748SJack Wang rtrs_err(s, "Sess info request receive failed: %s\n", 8669cb83748SJack Wang ib_wc_status_msg(wc->status)); 8679cb83748SJack Wang goto close; 8689cb83748SJack Wang } 8699cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_RECV); 8709cb83748SJack Wang 8719cb83748SJack Wang if (unlikely(wc->byte_len < sizeof(*msg))) { 8729cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: size %d\n", 8739cb83748SJack Wang wc->byte_len); 8749cb83748SJack Wang goto close; 8759cb83748SJack Wang } 8769cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, 8779cb83748SJack Wang iu->size, DMA_FROM_DEVICE); 8789cb83748SJack Wang msg = iu->buf; 8799cb83748SJack Wang if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ)) { 8809cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: type %d\n", 8819cb83748SJack Wang le16_to_cpu(msg->type)); 8829cb83748SJack Wang goto close; 8839cb83748SJack Wang } 8849cb83748SJack Wang err = process_info_req(con, msg); 8859cb83748SJack Wang if (unlikely(err)) 8869cb83748SJack Wang goto close; 8879cb83748SJack Wang 8889cb83748SJack Wang out: 8898bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 8909cb83748SJack Wang return; 8919cb83748SJack Wang close: 8929cb83748SJack Wang close_sess(sess); 8939cb83748SJack Wang goto out; 8949cb83748SJack Wang } 8959cb83748SJack Wang 8969cb83748SJack Wang static int post_recv_info_req(struct rtrs_srv_con *con) 8979cb83748SJack Wang { 8989cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8999cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9009cb83748SJack Wang struct rtrs_iu *rx_iu; 9019cb83748SJack Wang int err; 9029cb83748SJack Wang 9039cb83748SJack Wang rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), 9049cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 9059cb83748SJack Wang DMA_FROM_DEVICE, rtrs_srv_info_req_done); 9069cb83748SJack Wang if (unlikely(!rx_iu)) 9079cb83748SJack Wang return -ENOMEM; 9089cb83748SJack Wang /* Prepare for getting info response */ 9099cb83748SJack Wang err = rtrs_iu_post_recv(&con->c, rx_iu); 9109cb83748SJack Wang if (unlikely(err)) { 9119cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err); 9128bd372acSGioh Kim rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); 9139cb83748SJack Wang return err; 9149cb83748SJack Wang } 9159cb83748SJack Wang 9169cb83748SJack Wang return 0; 9179cb83748SJack Wang } 9189cb83748SJack Wang 9199cb83748SJack Wang static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) 9209cb83748SJack Wang { 9219cb83748SJack Wang int i, err; 9229cb83748SJack Wang 9239cb83748SJack Wang for (i = 0; i < q_size; i++) { 9249cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 9259cb83748SJack Wang if (unlikely(err)) 9269cb83748SJack Wang return err; 9279cb83748SJack Wang } 9289cb83748SJack Wang 9299cb83748SJack Wang return 0; 9309cb83748SJack Wang } 9319cb83748SJack Wang 9329cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess) 9339cb83748SJack Wang { 9349cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9359cb83748SJack Wang struct rtrs_sess *s = &sess->s; 9369cb83748SJack Wang size_t q_size; 9379cb83748SJack Wang int err, cid; 9389cb83748SJack Wang 9399cb83748SJack Wang for (cid = 0; cid < sess->s.con_num; cid++) { 9409cb83748SJack Wang if (cid == 0) 9419cb83748SJack Wang q_size = SERVICE_CON_QUEUE_DEPTH; 9429cb83748SJack Wang else 9439cb83748SJack Wang q_size = srv->queue_depth; 9449cb83748SJack Wang 9459cb83748SJack Wang err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size); 9469cb83748SJack Wang if (unlikely(err)) { 9479cb83748SJack Wang rtrs_err(s, "post_recv_io(), err: %d\n", err); 9489cb83748SJack Wang return err; 9499cb83748SJack Wang } 9509cb83748SJack Wang } 9519cb83748SJack Wang 9529cb83748SJack Wang return 0; 9539cb83748SJack Wang } 9549cb83748SJack Wang 9559cb83748SJack Wang static void process_read(struct rtrs_srv_con *con, 9569cb83748SJack Wang struct rtrs_msg_rdma_read *msg, 9579cb83748SJack Wang u32 buf_id, u32 off) 9589cb83748SJack Wang { 9599cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9609cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9619cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9629cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 9639cb83748SJack Wang struct rtrs_srv_op *id; 9649cb83748SJack Wang 9659cb83748SJack Wang size_t usr_len, data_len; 9669cb83748SJack Wang void *data; 9679cb83748SJack Wang int ret; 9689cb83748SJack Wang 9699cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 9709cb83748SJack Wang rtrs_err_rl(s, 9719cb83748SJack Wang "Processing read request failed, session is disconnected, sess state %s\n", 9729cb83748SJack Wang rtrs_srv_state_str(sess->state)); 9739cb83748SJack Wang return; 9749cb83748SJack Wang } 9759cb83748SJack Wang if (unlikely(msg->sg_cnt != 1 && msg->sg_cnt != 0)) { 9769cb83748SJack Wang rtrs_err_rl(s, 9779cb83748SJack Wang "Processing read request failed, invalid message\n"); 9789cb83748SJack Wang return; 9799cb83748SJack Wang } 9809cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 9819cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, READ); 9829cb83748SJack Wang id = sess->ops_ids[buf_id]; 9839cb83748SJack Wang id->con = con; 9849cb83748SJack Wang id->dir = READ; 9859cb83748SJack Wang id->msg_id = buf_id; 9869cb83748SJack Wang id->rd_msg = msg; 9879cb83748SJack Wang usr_len = le16_to_cpu(msg->usr_len); 9889cb83748SJack Wang data_len = off - usr_len; 9899cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 990c81cba85SGioh Kim ret = ctx->ops.rdma_ev(srv->priv, id, READ, data, data_len, 9919cb83748SJack Wang data + data_len, usr_len); 9929cb83748SJack Wang 9939cb83748SJack Wang if (unlikely(ret)) { 9949cb83748SJack Wang rtrs_err_rl(s, 9959cb83748SJack Wang "Processing read request failed, user module cb reported for msg_id %d, err: %d\n", 9969cb83748SJack Wang buf_id, ret); 9979cb83748SJack Wang goto send_err_msg; 9989cb83748SJack Wang } 9999cb83748SJack Wang 10009cb83748SJack Wang return; 10019cb83748SJack Wang 10029cb83748SJack Wang send_err_msg: 10039cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10049cb83748SJack Wang if (ret < 0) { 10059cb83748SJack Wang rtrs_err_rl(s, 10069cb83748SJack Wang "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n", 10079cb83748SJack Wang buf_id, ret); 10089cb83748SJack Wang close_sess(sess); 10099cb83748SJack Wang } 10109cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10119cb83748SJack Wang } 10129cb83748SJack Wang 10139cb83748SJack Wang static void process_write(struct rtrs_srv_con *con, 10149cb83748SJack Wang struct rtrs_msg_rdma_write *req, 10159cb83748SJack Wang u32 buf_id, u32 off) 10169cb83748SJack Wang { 10179cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10189cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10199cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 10209cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 10219cb83748SJack Wang struct rtrs_srv_op *id; 10229cb83748SJack Wang 10239cb83748SJack Wang size_t data_len, usr_len; 10249cb83748SJack Wang void *data; 10259cb83748SJack Wang int ret; 10269cb83748SJack Wang 10279cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 10289cb83748SJack Wang rtrs_err_rl(s, 10299cb83748SJack Wang "Processing write request failed, session is disconnected, sess state %s\n", 10309cb83748SJack Wang rtrs_srv_state_str(sess->state)); 10319cb83748SJack Wang return; 10329cb83748SJack Wang } 10339cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 10349cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, WRITE); 10359cb83748SJack Wang id = sess->ops_ids[buf_id]; 10369cb83748SJack Wang id->con = con; 10379cb83748SJack Wang id->dir = WRITE; 10389cb83748SJack Wang id->msg_id = buf_id; 10399cb83748SJack Wang 10409cb83748SJack Wang usr_len = le16_to_cpu(req->usr_len); 10419cb83748SJack Wang data_len = off - usr_len; 10429cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 1043c81cba85SGioh Kim ret = ctx->ops.rdma_ev(srv->priv, id, WRITE, data, data_len, 10449cb83748SJack Wang data + data_len, usr_len); 10459cb83748SJack Wang if (unlikely(ret)) { 10469cb83748SJack Wang rtrs_err_rl(s, 10479cb83748SJack Wang "Processing write request failed, user module callback reports err: %d\n", 10489cb83748SJack Wang ret); 10499cb83748SJack Wang goto send_err_msg; 10509cb83748SJack Wang } 10519cb83748SJack Wang 10529cb83748SJack Wang return; 10539cb83748SJack Wang 10549cb83748SJack Wang send_err_msg: 10559cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10569cb83748SJack Wang if (ret < 0) { 10579cb83748SJack Wang rtrs_err_rl(s, 10589cb83748SJack Wang "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n", 10599cb83748SJack Wang buf_id, ret); 10609cb83748SJack Wang close_sess(sess); 10619cb83748SJack Wang } 10629cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10639cb83748SJack Wang } 10649cb83748SJack Wang 10659cb83748SJack Wang static void process_io_req(struct rtrs_srv_con *con, void *msg, 10669cb83748SJack Wang u32 id, u32 off) 10679cb83748SJack Wang { 10689cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10699cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10709cb83748SJack Wang struct rtrs_msg_rdma_hdr *hdr; 10719cb83748SJack Wang unsigned int type; 10729cb83748SJack Wang 10739cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id], 10749cb83748SJack Wang max_chunk_size, DMA_BIDIRECTIONAL); 10759cb83748SJack Wang hdr = msg; 10769cb83748SJack Wang type = le16_to_cpu(hdr->type); 10779cb83748SJack Wang 10789cb83748SJack Wang switch (type) { 10799cb83748SJack Wang case RTRS_MSG_WRITE: 10809cb83748SJack Wang process_write(con, msg, id, off); 10819cb83748SJack Wang break; 10829cb83748SJack Wang case RTRS_MSG_READ: 10839cb83748SJack Wang process_read(con, msg, id, off); 10849cb83748SJack Wang break; 10859cb83748SJack Wang default: 10869cb83748SJack Wang rtrs_err(s, 10879cb83748SJack Wang "Processing I/O request failed, unknown message type received: 0x%02x\n", 10889cb83748SJack Wang type); 10899cb83748SJack Wang goto err; 10909cb83748SJack Wang } 10919cb83748SJack Wang 10929cb83748SJack Wang return; 10939cb83748SJack Wang 10949cb83748SJack Wang err: 10959cb83748SJack Wang close_sess(sess); 10969cb83748SJack Wang } 10979cb83748SJack Wang 10989cb83748SJack Wang static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) 10999cb83748SJack Wang { 11009cb83748SJack Wang struct rtrs_srv_mr *mr = 11019cb83748SJack Wang container_of(wc->wr_cqe, typeof(*mr), inv_cqe); 11023b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 11039cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11049cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11059cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11069cb83748SJack Wang u32 msg_id, off; 11079cb83748SJack Wang void *data; 11089cb83748SJack Wang 11099cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11109cb83748SJack Wang rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n", 11119cb83748SJack Wang ib_wc_status_msg(wc->status)); 11129cb83748SJack Wang close_sess(sess); 11139cb83748SJack Wang } 11149cb83748SJack Wang msg_id = mr->msg_id; 11159cb83748SJack Wang off = mr->msg_off; 11169cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 11179cb83748SJack Wang process_io_req(con, data, msg_id, off); 11189cb83748SJack Wang } 11199cb83748SJack Wang 11209cb83748SJack Wang static int rtrs_srv_inv_rkey(struct rtrs_srv_con *con, 11219cb83748SJack Wang struct rtrs_srv_mr *mr) 11229cb83748SJack Wang { 11239cb83748SJack Wang struct ib_send_wr wr = { 11249cb83748SJack Wang .opcode = IB_WR_LOCAL_INV, 11259cb83748SJack Wang .wr_cqe = &mr->inv_cqe, 11269cb83748SJack Wang .send_flags = IB_SEND_SIGNALED, 11279cb83748SJack Wang .ex.invalidate_rkey = mr->mr->rkey, 11289cb83748SJack Wang }; 11299cb83748SJack Wang mr->inv_cqe.done = rtrs_srv_inv_rkey_done; 11309cb83748SJack Wang 11319cb83748SJack Wang return ib_post_send(con->c.qp, &wr, NULL); 11329cb83748SJack Wang } 11339cb83748SJack Wang 11349cb83748SJack Wang static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) 11359cb83748SJack Wang { 11369cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11379cb83748SJack Wang while (!list_empty(&con->rsp_wr_wait_list)) { 11389cb83748SJack Wang struct rtrs_srv_op *id; 11399cb83748SJack Wang int ret; 11409cb83748SJack Wang 11419cb83748SJack Wang id = list_entry(con->rsp_wr_wait_list.next, 11429cb83748SJack Wang struct rtrs_srv_op, wait_list); 11439cb83748SJack Wang list_del(&id->wait_list); 11449cb83748SJack Wang 11459cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11469cb83748SJack Wang ret = rtrs_srv_resp_rdma(id, id->status); 11479cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11489cb83748SJack Wang 11499cb83748SJack Wang if (!ret) { 11509cb83748SJack Wang list_add(&id->wait_list, &con->rsp_wr_wait_list); 11519cb83748SJack Wang break; 11529cb83748SJack Wang } 11539cb83748SJack Wang } 11549cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11559cb83748SJack Wang } 11569cb83748SJack Wang 11579cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) 11589cb83748SJack Wang { 11593b89e92cSJack Wang struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); 11609cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11619cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11629cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11639cb83748SJack Wang u32 imm_type, imm_payload; 11649cb83748SJack Wang int err; 11659cb83748SJack Wang 11669cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11679cb83748SJack Wang if (wc->status != IB_WC_WR_FLUSH_ERR) { 11689cb83748SJack Wang rtrs_err(s, 11699cb83748SJack Wang "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n", 11709cb83748SJack Wang ib_wc_status_msg(wc->status), wc->wr_cqe, 11719cb83748SJack Wang wc->opcode, wc->vendor_err, wc->byte_len); 11729cb83748SJack Wang close_sess(sess); 11739cb83748SJack Wang } 11749cb83748SJack Wang return; 11759cb83748SJack Wang } 11769cb83748SJack Wang 11779cb83748SJack Wang switch (wc->opcode) { 11789cb83748SJack Wang case IB_WC_RECV_RDMA_WITH_IMM: 11799cb83748SJack Wang /* 11809cb83748SJack Wang * post_recv() RDMA write completions of IO reqs (read/write) 11819cb83748SJack Wang * and hb 11829cb83748SJack Wang */ 11839cb83748SJack Wang if (WARN_ON(wc->wr_cqe != &io_comp_cqe)) 11849cb83748SJack Wang return; 11859cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 11869cb83748SJack Wang if (unlikely(err)) { 11879cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); 11889cb83748SJack Wang close_sess(sess); 11899cb83748SJack Wang break; 11909cb83748SJack Wang } 11919cb83748SJack Wang rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), 11929cb83748SJack Wang &imm_type, &imm_payload); 11939cb83748SJack Wang if (likely(imm_type == RTRS_IO_REQ_IMM)) { 11949cb83748SJack Wang u32 msg_id, off; 11959cb83748SJack Wang void *data; 11969cb83748SJack Wang 11979cb83748SJack Wang msg_id = imm_payload >> sess->mem_bits; 11989cb83748SJack Wang off = imm_payload & ((1 << sess->mem_bits) - 1); 1199bf1d8edbSDan Carpenter if (unlikely(msg_id >= srv->queue_depth || 1200bf1d8edbSDan Carpenter off >= max_chunk_size)) { 12019cb83748SJack Wang rtrs_err(s, "Wrong msg_id %u, off %u\n", 12029cb83748SJack Wang msg_id, off); 12039cb83748SJack Wang close_sess(sess); 12049cb83748SJack Wang return; 12059cb83748SJack Wang } 12069cb83748SJack Wang if (always_invalidate) { 12079cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[msg_id]; 12089cb83748SJack Wang 12099cb83748SJack Wang mr->msg_off = off; 12109cb83748SJack Wang mr->msg_id = msg_id; 12119cb83748SJack Wang err = rtrs_srv_inv_rkey(con, mr); 12129cb83748SJack Wang if (unlikely(err)) { 12139cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", 12149cb83748SJack Wang err); 12159cb83748SJack Wang close_sess(sess); 12169cb83748SJack Wang break; 12179cb83748SJack Wang } 12189cb83748SJack Wang } else { 12199cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 12209cb83748SJack Wang process_io_req(con, data, msg_id, off); 12219cb83748SJack Wang } 12229cb83748SJack Wang } else if (imm_type == RTRS_HB_MSG_IMM) { 12239cb83748SJack Wang WARN_ON(con->c.cid); 12249cb83748SJack Wang rtrs_send_hb_ack(&sess->s); 12259cb83748SJack Wang } else if (imm_type == RTRS_HB_ACK_IMM) { 12269cb83748SJack Wang WARN_ON(con->c.cid); 12279cb83748SJack Wang sess->s.hb_missed_cnt = 0; 12289cb83748SJack Wang } else { 12299cb83748SJack Wang rtrs_wrn(s, "Unknown IMM type %u\n", imm_type); 12309cb83748SJack Wang } 12319cb83748SJack Wang break; 12329cb83748SJack Wang case IB_WC_RDMA_WRITE: 12339cb83748SJack Wang case IB_WC_SEND: 12349cb83748SJack Wang /* 12359cb83748SJack Wang * post_send() RDMA write completions of IO reqs (read/write) 12369cb83748SJack Wang */ 12379cb83748SJack Wang atomic_add(srv->queue_depth, &con->sq_wr_avail); 12389cb83748SJack Wang 12399cb83748SJack Wang if (unlikely(!list_empty_careful(&con->rsp_wr_wait_list))) 12409cb83748SJack Wang rtrs_rdma_process_wr_wait_list(con); 12419cb83748SJack Wang 12429cb83748SJack Wang break; 12439cb83748SJack Wang default: 12449cb83748SJack Wang rtrs_wrn(s, "Unexpected WC type: %d\n", wc->opcode); 12459cb83748SJack Wang return; 12469cb83748SJack Wang } 12479cb83748SJack Wang } 12489cb83748SJack Wang 12499cb83748SJack Wang /** 12509cb83748SJack Wang * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname. 12519cb83748SJack Wang * @srv: Session 12529cb83748SJack Wang * @sessname: Sessname buffer 12539cb83748SJack Wang * @len: Length of sessname buffer 12549cb83748SJack Wang */ 12559cb83748SJack Wang int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) 12569cb83748SJack Wang { 12579cb83748SJack Wang struct rtrs_srv_sess *sess; 12589cb83748SJack Wang int err = -ENOTCONN; 12599cb83748SJack Wang 12609cb83748SJack Wang mutex_lock(&srv->paths_mutex); 12619cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 12629cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTED) 12639cb83748SJack Wang continue; 1264*2d612f0dSDima Stepanov strscpy(sessname, sess->s.sessname, 12659cb83748SJack Wang min_t(size_t, sizeof(sess->s.sessname), len)); 12669cb83748SJack Wang err = 0; 12679cb83748SJack Wang break; 12689cb83748SJack Wang } 12699cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 12709cb83748SJack Wang 12719cb83748SJack Wang return err; 12729cb83748SJack Wang } 12739cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_sess_name); 12749cb83748SJack Wang 12759cb83748SJack Wang /** 12769cb83748SJack Wang * rtrs_srv_get_sess_qdepth() - Get rtrs_srv qdepth. 12779cb83748SJack Wang * @srv: Session 12789cb83748SJack Wang */ 12799cb83748SJack Wang int rtrs_srv_get_queue_depth(struct rtrs_srv *srv) 12809cb83748SJack Wang { 12819cb83748SJack Wang return srv->queue_depth; 12829cb83748SJack Wang } 12839cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_queue_depth); 12849cb83748SJack Wang 12859cb83748SJack Wang static int find_next_bit_ring(struct rtrs_srv_sess *sess) 12869cb83748SJack Wang { 12879cb83748SJack Wang struct ib_device *ib_dev = sess->s.dev->ib_dev; 12889cb83748SJack Wang int v; 12899cb83748SJack Wang 12909cb83748SJack Wang v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask); 12919cb83748SJack Wang if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors) 12929cb83748SJack Wang v = cpumask_first(&cq_affinity_mask); 12939cb83748SJack Wang return v; 12949cb83748SJack Wang } 12959cb83748SJack Wang 12969cb83748SJack Wang static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess) 12979cb83748SJack Wang { 12989cb83748SJack Wang sess->cur_cq_vector = find_next_bit_ring(sess); 12999cb83748SJack Wang 13009cb83748SJack Wang return sess->cur_cq_vector; 13019cb83748SJack Wang } 13029cb83748SJack Wang 130339c2d639SMd Haris Iqbal static void rtrs_srv_dev_release(struct device *dev) 130439c2d639SMd Haris Iqbal { 130539c2d639SMd Haris Iqbal struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev); 130639c2d639SMd Haris Iqbal 130739c2d639SMd Haris Iqbal kfree(srv); 130839c2d639SMd Haris Iqbal } 130939c2d639SMd Haris Iqbal 13109cb83748SJack Wang static void free_srv(struct rtrs_srv *srv) 13119cb83748SJack Wang { 13129cb83748SJack Wang int i; 13139cb83748SJack Wang 13149cb83748SJack Wang WARN_ON(refcount_read(&srv->refcount)); 13159cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 13169cb83748SJack Wang mempool_free(srv->chunks[i], chunk_pool); 13179cb83748SJack Wang kfree(srv->chunks); 13189cb83748SJack Wang mutex_destroy(&srv->paths_mutex); 13199cb83748SJack Wang mutex_destroy(&srv->paths_ev_mutex); 13209cb83748SJack Wang /* last put to release the srv structure */ 13219cb83748SJack Wang put_device(&srv->dev); 13229cb83748SJack Wang } 13239cb83748SJack Wang 13249cb83748SJack Wang static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx, 132503e9b33aSMd Haris Iqbal const uuid_t *paths_uuid, 132603e9b33aSMd Haris Iqbal bool first_conn) 13279cb83748SJack Wang { 13289cb83748SJack Wang struct rtrs_srv *srv; 1329d715ff8aSGuoqing Jiang int i; 13309cb83748SJack Wang 13319cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 1332d715ff8aSGuoqing Jiang list_for_each_entry(srv, &ctx->srv_list, ctx_list) { 1333d715ff8aSGuoqing Jiang if (uuid_equal(&srv->paths_uuid, paths_uuid) && 1334d715ff8aSGuoqing Jiang refcount_inc_not_zero(&srv->refcount)) { 1335d715ff8aSGuoqing Jiang mutex_unlock(&ctx->srv_mutex); 1336d715ff8aSGuoqing Jiang return srv; 1337d715ff8aSGuoqing Jiang } 1338d715ff8aSGuoqing Jiang } 1339ed408529SJack Wang mutex_unlock(&ctx->srv_mutex); 134003e9b33aSMd Haris Iqbal /* 134103e9b33aSMd Haris Iqbal * If this request is not the first connection request from the 134203e9b33aSMd Haris Iqbal * client for this session then fail and return error. 134303e9b33aSMd Haris Iqbal */ 13446564b110SMd Haris Iqbal if (!first_conn) { 13456564b110SMd Haris Iqbal pr_err_ratelimited("Error: Not the first connection request for this session\n"); 134603e9b33aSMd Haris Iqbal return ERR_PTR(-ENXIO); 13476564b110SMd Haris Iqbal } 1348d715ff8aSGuoqing Jiang 1349d715ff8aSGuoqing Jiang /* need to allocate a new srv */ 1350d715ff8aSGuoqing Jiang srv = kzalloc(sizeof(*srv), GFP_KERNEL); 1351ed408529SJack Wang if (!srv) 135203e9b33aSMd Haris Iqbal return ERR_PTR(-ENOMEM); 1353d715ff8aSGuoqing Jiang 1354d715ff8aSGuoqing Jiang INIT_LIST_HEAD(&srv->paths_list); 1355d715ff8aSGuoqing Jiang mutex_init(&srv->paths_mutex); 1356d715ff8aSGuoqing Jiang mutex_init(&srv->paths_ev_mutex); 1357d715ff8aSGuoqing Jiang uuid_copy(&srv->paths_uuid, paths_uuid); 1358d715ff8aSGuoqing Jiang srv->queue_depth = sess_queue_depth; 1359d715ff8aSGuoqing Jiang srv->ctx = ctx; 1360d715ff8aSGuoqing Jiang device_initialize(&srv->dev); 1361d715ff8aSGuoqing Jiang srv->dev.release = rtrs_srv_dev_release; 13629cb83748SJack Wang 1363d715ff8aSGuoqing Jiang srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks), 1364d715ff8aSGuoqing Jiang GFP_KERNEL); 1365d715ff8aSGuoqing Jiang if (!srv->chunks) 1366d715ff8aSGuoqing Jiang goto err_free_srv; 1367d715ff8aSGuoqing Jiang 1368d715ff8aSGuoqing Jiang for (i = 0; i < srv->queue_depth; i++) { 1369d715ff8aSGuoqing Jiang srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL); 1370d715ff8aSGuoqing Jiang if (!srv->chunks[i]) 1371d715ff8aSGuoqing Jiang goto err_free_chunks; 1372d715ff8aSGuoqing Jiang } 1373d715ff8aSGuoqing Jiang refcount_set(&srv->refcount, 1); 1374ed408529SJack Wang mutex_lock(&ctx->srv_mutex); 1375ed408529SJack Wang list_add(&srv->ctx_list, &ctx->srv_list); 1376ed408529SJack Wang mutex_unlock(&ctx->srv_mutex); 1377d715ff8aSGuoqing Jiang 13789cb83748SJack Wang return srv; 1379d715ff8aSGuoqing Jiang 1380d715ff8aSGuoqing Jiang err_free_chunks: 1381d715ff8aSGuoqing Jiang while (i--) 1382d715ff8aSGuoqing Jiang mempool_free(srv->chunks[i], chunk_pool); 1383d715ff8aSGuoqing Jiang kfree(srv->chunks); 1384d715ff8aSGuoqing Jiang 1385d715ff8aSGuoqing Jiang err_free_srv: 1386d715ff8aSGuoqing Jiang kfree(srv); 138703e9b33aSMd Haris Iqbal return ERR_PTR(-ENOMEM); 13889cb83748SJack Wang } 13899cb83748SJack Wang 13909cb83748SJack Wang static void put_srv(struct rtrs_srv *srv) 13919cb83748SJack Wang { 13929cb83748SJack Wang if (refcount_dec_and_test(&srv->refcount)) { 13939cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 13949cb83748SJack Wang 13959cb83748SJack Wang WARN_ON(srv->dev.kobj.state_in_sysfs); 13969cb83748SJack Wang 13979cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 13989cb83748SJack Wang list_del(&srv->ctx_list); 13999cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 14009cb83748SJack Wang free_srv(srv); 14019cb83748SJack Wang } 14029cb83748SJack Wang } 14039cb83748SJack Wang 14049cb83748SJack Wang static void __add_path_to_srv(struct rtrs_srv *srv, 14059cb83748SJack Wang struct rtrs_srv_sess *sess) 14069cb83748SJack Wang { 14079cb83748SJack Wang list_add_tail(&sess->s.entry, &srv->paths_list); 14089cb83748SJack Wang srv->paths_num++; 14099cb83748SJack Wang WARN_ON(srv->paths_num >= MAX_PATHS_NUM); 14109cb83748SJack Wang } 14119cb83748SJack Wang 14129cb83748SJack Wang static void del_path_from_srv(struct rtrs_srv_sess *sess) 14139cb83748SJack Wang { 14149cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 14159cb83748SJack Wang 14169cb83748SJack Wang if (WARN_ON(!srv)) 14179cb83748SJack Wang return; 14189cb83748SJack Wang 14199cb83748SJack Wang mutex_lock(&srv->paths_mutex); 14209cb83748SJack Wang list_del(&sess->s.entry); 14219cb83748SJack Wang WARN_ON(!srv->paths_num); 14229cb83748SJack Wang srv->paths_num--; 14239cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 14249cb83748SJack Wang } 14259cb83748SJack Wang 14269cb83748SJack Wang /* return true if addresses are the same, error other wise */ 14279cb83748SJack Wang static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) 14289cb83748SJack Wang { 14299cb83748SJack Wang switch (a->sa_family) { 14309cb83748SJack Wang case AF_IB: 14319cb83748SJack Wang return memcmp(&((struct sockaddr_ib *)a)->sib_addr, 14329cb83748SJack Wang &((struct sockaddr_ib *)b)->sib_addr, 14339cb83748SJack Wang sizeof(struct ib_addr)) && 14349cb83748SJack Wang (b->sa_family == AF_IB); 14359cb83748SJack Wang case AF_INET: 14369cb83748SJack Wang return memcmp(&((struct sockaddr_in *)a)->sin_addr, 14379cb83748SJack Wang &((struct sockaddr_in *)b)->sin_addr, 14389cb83748SJack Wang sizeof(struct in_addr)) && 14399cb83748SJack Wang (b->sa_family == AF_INET); 14409cb83748SJack Wang case AF_INET6: 14419cb83748SJack Wang return memcmp(&((struct sockaddr_in6 *)a)->sin6_addr, 14429cb83748SJack Wang &((struct sockaddr_in6 *)b)->sin6_addr, 14439cb83748SJack Wang sizeof(struct in6_addr)) && 14449cb83748SJack Wang (b->sa_family == AF_INET6); 14459cb83748SJack Wang default: 14469cb83748SJack Wang return -ENOENT; 14479cb83748SJack Wang } 14489cb83748SJack Wang } 14499cb83748SJack Wang 14509cb83748SJack Wang static bool __is_path_w_addr_exists(struct rtrs_srv *srv, 14519cb83748SJack Wang struct rdma_addr *addr) 14529cb83748SJack Wang { 14539cb83748SJack Wang struct rtrs_srv_sess *sess; 14549cb83748SJack Wang 14559cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 14569cb83748SJack Wang if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr, 14579cb83748SJack Wang (struct sockaddr *)&addr->dst_addr) && 14589cb83748SJack Wang !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr, 14599cb83748SJack Wang (struct sockaddr *)&addr->src_addr)) 14609cb83748SJack Wang return true; 14619cb83748SJack Wang 14629cb83748SJack Wang return false; 14639cb83748SJack Wang } 14649cb83748SJack Wang 14659cb83748SJack Wang static void free_sess(struct rtrs_srv_sess *sess) 14669cb83748SJack Wang { 1467f7452a7eSGioh Kim if (sess->kobj.state_in_sysfs) { 1468f7452a7eSGioh Kim kobject_del(&sess->kobj); 14699cb83748SJack Wang kobject_put(&sess->kobj); 1470f7452a7eSGioh Kim } else { 14719cb83748SJack Wang kfree(sess); 14729cb83748SJack Wang } 1473f7452a7eSGioh Kim } 14749cb83748SJack Wang 14759cb83748SJack Wang static void rtrs_srv_close_work(struct work_struct *work) 14769cb83748SJack Wang { 14779cb83748SJack Wang struct rtrs_srv_sess *sess; 14789cb83748SJack Wang struct rtrs_srv_con *con; 14799cb83748SJack Wang int i; 14809cb83748SJack Wang 14819cb83748SJack Wang sess = container_of(work, typeof(*sess), close_work); 14829cb83748SJack Wang 14839cb83748SJack Wang rtrs_srv_destroy_sess_files(sess); 14849cb83748SJack Wang rtrs_srv_stop_hb(sess); 14859cb83748SJack Wang 14869cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 14879cb83748SJack Wang if (!sess->s.con[i]) 14889cb83748SJack Wang continue; 14899cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 14909cb83748SJack Wang rdma_disconnect(con->c.cm_id); 14919cb83748SJack Wang ib_drain_qp(con->c.qp); 14929cb83748SJack Wang } 14939cb83748SJack Wang /* Wait for all inflights */ 14949cb83748SJack Wang rtrs_srv_wait_ops_ids(sess); 14959cb83748SJack Wang 14969cb83748SJack Wang /* Notify upper layer if we are the last path */ 14979cb83748SJack Wang rtrs_srv_sess_down(sess); 14989cb83748SJack Wang 14999cb83748SJack Wang unmap_cont_bufs(sess); 15009cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 15019cb83748SJack Wang 15029cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 15039cb83748SJack Wang if (!sess->s.con[i]) 15049cb83748SJack Wang continue; 15059cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 15069cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 15079cb83748SJack Wang rdma_destroy_id(con->c.cm_id); 15089cb83748SJack Wang kfree(con); 15099cb83748SJack Wang } 15109cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 15119cb83748SJack Wang 15129cb83748SJack Wang del_path_from_srv(sess); 15139cb83748SJack Wang put_srv(sess->srv); 15149cb83748SJack Wang sess->srv = NULL; 15159cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CLOSED); 15169cb83748SJack Wang 15179cb83748SJack Wang kfree(sess->dma_addr); 15189cb83748SJack Wang kfree(sess->s.con); 15199cb83748SJack Wang free_sess(sess); 15209cb83748SJack Wang } 15219cb83748SJack Wang 15229cb83748SJack Wang static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess, 15239cb83748SJack Wang struct rdma_cm_id *cm_id) 15249cb83748SJack Wang { 15259cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15269cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15279cb83748SJack Wang struct rdma_conn_param param; 15289cb83748SJack Wang int err; 15299cb83748SJack Wang 15309cb83748SJack Wang param = (struct rdma_conn_param) { 15319cb83748SJack Wang .rnr_retry_count = 7, 15329cb83748SJack Wang .private_data = &msg, 15339cb83748SJack Wang .private_data_len = sizeof(msg), 15349cb83748SJack Wang }; 15359cb83748SJack Wang 15369cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15379cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15389cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15399cb83748SJack Wang .queue_depth = cpu_to_le16(srv->queue_depth), 15409cb83748SJack Wang .max_io_size = cpu_to_le32(max_chunk_size - MAX_HDR_SIZE), 15419cb83748SJack Wang .max_hdr_size = cpu_to_le32(MAX_HDR_SIZE), 15429cb83748SJack Wang }; 15439cb83748SJack Wang 15449cb83748SJack Wang if (always_invalidate) 15459cb83748SJack Wang msg.flags = cpu_to_le32(RTRS_MSG_NEW_RKEY_F); 15469cb83748SJack Wang 15479cb83748SJack Wang err = rdma_accept(cm_id, ¶m); 15489cb83748SJack Wang if (err) 15499cb83748SJack Wang pr_err("rdma_accept(), err: %d\n", err); 15509cb83748SJack Wang 15519cb83748SJack Wang return err; 15529cb83748SJack Wang } 15539cb83748SJack Wang 15549cb83748SJack Wang static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) 15559cb83748SJack Wang { 15569cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15579cb83748SJack Wang int err; 15589cb83748SJack Wang 15599cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15609cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15619cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15629cb83748SJack Wang .errno = cpu_to_le16(errno), 15639cb83748SJack Wang }; 15649cb83748SJack Wang 15658094ba0aSLeon Romanovsky err = rdma_reject(cm_id, &msg, sizeof(msg), IB_CM_REJ_CONSUMER_DEFINED); 15669cb83748SJack Wang if (err) 15679cb83748SJack Wang pr_err("rdma_reject(), err: %d\n", err); 15689cb83748SJack Wang 15699cb83748SJack Wang /* Bounce errno back */ 15709cb83748SJack Wang return errno; 15719cb83748SJack Wang } 15729cb83748SJack Wang 15739cb83748SJack Wang static struct rtrs_srv_sess * 15749cb83748SJack Wang __find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid) 15759cb83748SJack Wang { 15769cb83748SJack Wang struct rtrs_srv_sess *sess; 15779cb83748SJack Wang 15789cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 15799cb83748SJack Wang if (uuid_equal(&sess->s.uuid, sess_uuid)) 15809cb83748SJack Wang return sess; 15819cb83748SJack Wang } 15829cb83748SJack Wang 15839cb83748SJack Wang return NULL; 15849cb83748SJack Wang } 15859cb83748SJack Wang 15869cb83748SJack Wang static int create_con(struct rtrs_srv_sess *sess, 15879cb83748SJack Wang struct rdma_cm_id *cm_id, 15889cb83748SJack Wang unsigned int cid) 15899cb83748SJack Wang { 15909cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15919cb83748SJack Wang struct rtrs_sess *s = &sess->s; 15929cb83748SJack Wang struct rtrs_srv_con *con; 15939cb83748SJack Wang 15947490fd1fSJack Wang u32 cq_size, wr_queue_size; 15959cb83748SJack Wang int err, cq_vector; 15969cb83748SJack Wang 15979cb83748SJack Wang con = kzalloc(sizeof(*con), GFP_KERNEL); 15989cb83748SJack Wang if (!con) { 15999cb83748SJack Wang err = -ENOMEM; 16009cb83748SJack Wang goto err; 16019cb83748SJack Wang } 16029cb83748SJack Wang 16039cb83748SJack Wang spin_lock_init(&con->rsp_wr_wait_lock); 16049cb83748SJack Wang INIT_LIST_HEAD(&con->rsp_wr_wait_list); 16059cb83748SJack Wang con->c.cm_id = cm_id; 16069cb83748SJack Wang con->c.sess = &sess->s; 16079cb83748SJack Wang con->c.cid = cid; 16086f5d1b30SJack Wang atomic_set(&con->wr_cnt, 1); 16099cb83748SJack Wang 16109cb83748SJack Wang if (con->c.cid == 0) { 16119cb83748SJack Wang /* 16129cb83748SJack Wang * All receive and all send (each requiring invalidate) 16139cb83748SJack Wang * + 2 for drain and heartbeat 16149cb83748SJack Wang */ 16159cb83748SJack Wang wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2; 16169cb83748SJack Wang cq_size = wr_queue_size; 16179cb83748SJack Wang } else { 16189cb83748SJack Wang /* 16199cb83748SJack Wang * If we have all receive requests posted and 16209cb83748SJack Wang * all write requests posted and each read request 16219cb83748SJack Wang * requires an invalidate request + drain 16229cb83748SJack Wang * and qp gets into error state. 16239cb83748SJack Wang */ 16249cb83748SJack Wang cq_size = srv->queue_depth * 3 + 1; 16259cb83748SJack Wang /* 16269cb83748SJack Wang * In theory we might have queue_depth * 32 16279cb83748SJack Wang * outstanding requests if an unsafe global key is used 16289cb83748SJack Wang * and we have queue_depth read requests each consisting 16299cb83748SJack Wang * of 32 different addresses. div 3 for mlx5. 16309cb83748SJack Wang */ 16319cb83748SJack Wang wr_queue_size = sess->s.dev->ib_dev->attrs.max_qp_wr / 3; 16329cb83748SJack Wang } 16339cb83748SJack Wang atomic_set(&con->sq_wr_avail, wr_queue_size); 16349cb83748SJack Wang cq_vector = rtrs_srv_get_next_cq_vector(sess); 16359cb83748SJack Wang 16369cb83748SJack Wang /* TODO: SOFTIRQ can be faster, but be careful with softirq context */ 16379cb83748SJack Wang err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size, 16387490fd1fSJack Wang wr_queue_size, wr_queue_size, 16397490fd1fSJack Wang IB_POLL_WORKQUEUE); 16409cb83748SJack Wang if (err) { 16419cb83748SJack Wang rtrs_err(s, "rtrs_cq_qp_create(), err: %d\n", err); 16429cb83748SJack Wang goto free_con; 16439cb83748SJack Wang } 16449cb83748SJack Wang if (con->c.cid == 0) { 16459cb83748SJack Wang err = post_recv_info_req(con); 16469cb83748SJack Wang if (err) 16479cb83748SJack Wang goto free_cqqp; 16489cb83748SJack Wang } 16499cb83748SJack Wang WARN_ON(sess->s.con[cid]); 16509cb83748SJack Wang sess->s.con[cid] = &con->c; 16519cb83748SJack Wang 16529cb83748SJack Wang /* 16539cb83748SJack Wang * Change context from server to current connection. The other 16549cb83748SJack Wang * way is to use cm_id->qp->qp_context, which does not work on OFED. 16559cb83748SJack Wang */ 16569cb83748SJack Wang cm_id->context = &con->c; 16579cb83748SJack Wang 16589cb83748SJack Wang return 0; 16599cb83748SJack Wang 16609cb83748SJack Wang free_cqqp: 16619cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 16629cb83748SJack Wang free_con: 16639cb83748SJack Wang kfree(con); 16649cb83748SJack Wang 16659cb83748SJack Wang err: 16669cb83748SJack Wang return err; 16679cb83748SJack Wang } 16689cb83748SJack Wang 16699cb83748SJack Wang static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, 16709cb83748SJack Wang struct rdma_cm_id *cm_id, 16719cb83748SJack Wang unsigned int con_num, 16729cb83748SJack Wang unsigned int recon_cnt, 16739cb83748SJack Wang const uuid_t *uuid) 16749cb83748SJack Wang { 16759cb83748SJack Wang struct rtrs_srv_sess *sess; 16769cb83748SJack Wang int err = -ENOMEM; 167788e2f105SGioh Kim char str[NAME_MAX]; 167888e2f105SGioh Kim struct rtrs_addr path; 16799cb83748SJack Wang 16809cb83748SJack Wang if (srv->paths_num >= MAX_PATHS_NUM) { 16819cb83748SJack Wang err = -ECONNRESET; 16829cb83748SJack Wang goto err; 16839cb83748SJack Wang } 16849cb83748SJack Wang if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) { 16859cb83748SJack Wang err = -EEXIST; 16869cb83748SJack Wang pr_err("Path with same addr exists\n"); 16879cb83748SJack Wang goto err; 16889cb83748SJack Wang } 16899cb83748SJack Wang sess = kzalloc(sizeof(*sess), GFP_KERNEL); 16909cb83748SJack Wang if (!sess) 16919cb83748SJack Wang goto err; 16929cb83748SJack Wang 16939cb83748SJack Wang sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); 16949cb83748SJack Wang if (!sess->stats) 16959cb83748SJack Wang goto err_free_sess; 16969cb83748SJack Wang 16979cb83748SJack Wang sess->stats->sess = sess; 16989cb83748SJack Wang 16999cb83748SJack Wang sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr), 17009cb83748SJack Wang GFP_KERNEL); 17019cb83748SJack Wang if (!sess->dma_addr) 17029cb83748SJack Wang goto err_free_stats; 17039cb83748SJack Wang 17049cb83748SJack Wang sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL); 17059cb83748SJack Wang if (!sess->s.con) 17069cb83748SJack Wang goto err_free_dma_addr; 17079cb83748SJack Wang 17089cb83748SJack Wang sess->state = RTRS_SRV_CONNECTING; 17099cb83748SJack Wang sess->srv = srv; 17109cb83748SJack Wang sess->cur_cq_vector = -1; 17119cb83748SJack Wang sess->s.dst_addr = cm_id->route.addr.dst_addr; 17129cb83748SJack Wang sess->s.src_addr = cm_id->route.addr.src_addr; 171388e2f105SGioh Kim 171488e2f105SGioh Kim /* temporary until receiving session-name from client */ 171588e2f105SGioh Kim path.src = &sess->s.src_addr; 171688e2f105SGioh Kim path.dst = &sess->s.dst_addr; 171788e2f105SGioh Kim rtrs_addr_to_str(&path, str, sizeof(str)); 1718*2d612f0dSDima Stepanov strscpy(sess->s.sessname, str, sizeof(sess->s.sessname)); 171988e2f105SGioh Kim 17209cb83748SJack Wang sess->s.con_num = con_num; 17219cb83748SJack Wang sess->s.recon_cnt = recon_cnt; 17229cb83748SJack Wang uuid_copy(&sess->s.uuid, uuid); 17239cb83748SJack Wang spin_lock_init(&sess->state_lock); 17249cb83748SJack Wang INIT_WORK(&sess->close_work, rtrs_srv_close_work); 17259cb83748SJack Wang rtrs_srv_init_hb(sess); 17269cb83748SJack Wang 17279cb83748SJack Wang sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); 17289cb83748SJack Wang if (!sess->s.dev) { 17299cb83748SJack Wang err = -ENOMEM; 17309cb83748SJack Wang goto err_free_con; 17319cb83748SJack Wang } 17329cb83748SJack Wang err = map_cont_bufs(sess); 17339cb83748SJack Wang if (err) 17349cb83748SJack Wang goto err_put_dev; 17359cb83748SJack Wang 17369cb83748SJack Wang err = rtrs_srv_alloc_ops_ids(sess); 17379cb83748SJack Wang if (err) 17389cb83748SJack Wang goto err_unmap_bufs; 17399cb83748SJack Wang 17409cb83748SJack Wang __add_path_to_srv(srv, sess); 17419cb83748SJack Wang 17429cb83748SJack Wang return sess; 17439cb83748SJack Wang 17449cb83748SJack Wang err_unmap_bufs: 17459cb83748SJack Wang unmap_cont_bufs(sess); 17469cb83748SJack Wang err_put_dev: 17479cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 17489cb83748SJack Wang err_free_con: 17499cb83748SJack Wang kfree(sess->s.con); 17509cb83748SJack Wang err_free_dma_addr: 17519cb83748SJack Wang kfree(sess->dma_addr); 17529cb83748SJack Wang err_free_stats: 17539cb83748SJack Wang kfree(sess->stats); 17549cb83748SJack Wang err_free_sess: 17559cb83748SJack Wang kfree(sess); 17569cb83748SJack Wang err: 17579cb83748SJack Wang return ERR_PTR(err); 17589cb83748SJack Wang } 17599cb83748SJack Wang 17609cb83748SJack Wang static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, 17619cb83748SJack Wang const struct rtrs_msg_conn_req *msg, 17629cb83748SJack Wang size_t len) 17639cb83748SJack Wang { 17649cb83748SJack Wang struct rtrs_srv_ctx *ctx = cm_id->context; 17659cb83748SJack Wang struct rtrs_srv_sess *sess; 17669cb83748SJack Wang struct rtrs_srv *srv; 17679cb83748SJack Wang 17689cb83748SJack Wang u16 version, con_num, cid; 17699cb83748SJack Wang u16 recon_cnt; 1770cfbeb0b9SGuoqing Jiang int err = -ECONNRESET; 17719cb83748SJack Wang 17729cb83748SJack Wang if (len < sizeof(*msg)) { 17739cb83748SJack Wang pr_err("Invalid RTRS connection request\n"); 1774cfbeb0b9SGuoqing Jiang goto reject_w_err; 17759cb83748SJack Wang } 17769cb83748SJack Wang if (le16_to_cpu(msg->magic) != RTRS_MAGIC) { 17779cb83748SJack Wang pr_err("Invalid RTRS magic\n"); 1778cfbeb0b9SGuoqing Jiang goto reject_w_err; 17799cb83748SJack Wang } 17809cb83748SJack Wang version = le16_to_cpu(msg->version); 17819cb83748SJack Wang if (version >> 8 != RTRS_PROTO_VER_MAJOR) { 17829cb83748SJack Wang pr_err("Unsupported major RTRS version: %d, expected %d\n", 17839cb83748SJack Wang version >> 8, RTRS_PROTO_VER_MAJOR); 1784cfbeb0b9SGuoqing Jiang goto reject_w_err; 17859cb83748SJack Wang } 17869cb83748SJack Wang con_num = le16_to_cpu(msg->cid_num); 17879cb83748SJack Wang if (con_num > 4096) { 17889cb83748SJack Wang /* Sanity check */ 17899cb83748SJack Wang pr_err("Too many connections requested: %d\n", con_num); 1790cfbeb0b9SGuoqing Jiang goto reject_w_err; 17919cb83748SJack Wang } 17929cb83748SJack Wang cid = le16_to_cpu(msg->cid); 17939cb83748SJack Wang if (cid >= con_num) { 17949cb83748SJack Wang /* Sanity check */ 17959cb83748SJack Wang pr_err("Incorrect cid: %d >= %d\n", cid, con_num); 1796cfbeb0b9SGuoqing Jiang goto reject_w_err; 17979cb83748SJack Wang } 17989cb83748SJack Wang recon_cnt = le16_to_cpu(msg->recon_cnt); 179903e9b33aSMd Haris Iqbal srv = get_or_create_srv(ctx, &msg->paths_uuid, msg->first_conn); 1800ed408529SJack Wang if (IS_ERR(srv)) { 180103e9b33aSMd Haris Iqbal err = PTR_ERR(srv); 18026564b110SMd Haris Iqbal pr_err("get_or_create_srv(), error %d\n", err); 18039cb83748SJack Wang goto reject_w_err; 18049cb83748SJack Wang } 18059cb83748SJack Wang mutex_lock(&srv->paths_mutex); 18069cb83748SJack Wang sess = __find_sess(srv, &msg->sess_uuid); 18079cb83748SJack Wang if (sess) { 18089cb83748SJack Wang struct rtrs_sess *s = &sess->s; 18099cb83748SJack Wang 18109cb83748SJack Wang /* Session already holds a reference */ 18119cb83748SJack Wang put_srv(srv); 18129cb83748SJack Wang 18139cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTING) { 18149cb83748SJack Wang rtrs_err(s, "Session in wrong state: %s\n", 18159cb83748SJack Wang rtrs_srv_state_str(sess->state)); 18169cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1817cfbeb0b9SGuoqing Jiang goto reject_w_err; 18189cb83748SJack Wang } 18199cb83748SJack Wang /* 18209cb83748SJack Wang * Sanity checks 18219cb83748SJack Wang */ 1822e172037bSMd Haris Iqbal if (con_num != s->con_num || cid >= s->con_num) { 18239cb83748SJack Wang rtrs_err(s, "Incorrect request: %d, %d\n", 18249cb83748SJack Wang cid, con_num); 18259cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1826cfbeb0b9SGuoqing Jiang goto reject_w_err; 18279cb83748SJack Wang } 1828e172037bSMd Haris Iqbal if (s->con[cid]) { 18299cb83748SJack Wang rtrs_err(s, "Connection already exists: %d\n", 18309cb83748SJack Wang cid); 18319cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 1832cfbeb0b9SGuoqing Jiang goto reject_w_err; 18339cb83748SJack Wang } 18349cb83748SJack Wang } else { 18359cb83748SJack Wang sess = __alloc_sess(srv, cm_id, con_num, recon_cnt, 18369cb83748SJack Wang &msg->sess_uuid); 18379cb83748SJack Wang if (IS_ERR(sess)) { 18389cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18399cb83748SJack Wang put_srv(srv); 18409cb83748SJack Wang err = PTR_ERR(sess); 18416564b110SMd Haris Iqbal pr_err("RTRS server session allocation failed: %d\n", err); 18429cb83748SJack Wang goto reject_w_err; 18439cb83748SJack Wang } 18449cb83748SJack Wang } 18459cb83748SJack Wang err = create_con(sess, cm_id, cid); 18469cb83748SJack Wang if (err) { 18476564b110SMd Haris Iqbal rtrs_err((&sess->s), "create_con(), error %d\n", err); 18489cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18499cb83748SJack Wang /* 18509cb83748SJack Wang * Since session has other connections we follow normal way 18519cb83748SJack Wang * through workqueue, but still return an error to tell cma.c 18529cb83748SJack Wang * to call rdma_destroy_id() for current connection. 18539cb83748SJack Wang */ 18549cb83748SJack Wang goto close_and_return_err; 18559cb83748SJack Wang } 18569cb83748SJack Wang err = rtrs_rdma_do_accept(sess, cm_id); 18579cb83748SJack Wang if (err) { 18586564b110SMd Haris Iqbal rtrs_err((&sess->s), "rtrs_rdma_do_accept(), error %d\n", err); 18599cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18609cb83748SJack Wang /* 18619cb83748SJack Wang * Since current connection was successfully added to the 18629cb83748SJack Wang * session we follow normal way through workqueue to close the 18639cb83748SJack Wang * session, thus return 0 to tell cma.c we call 18649cb83748SJack Wang * rdma_destroy_id() ourselves. 18659cb83748SJack Wang */ 18669cb83748SJack Wang err = 0; 18679cb83748SJack Wang goto close_and_return_err; 18689cb83748SJack Wang } 18699cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18709cb83748SJack Wang 18719cb83748SJack Wang return 0; 18729cb83748SJack Wang 18739cb83748SJack Wang reject_w_err: 18749cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, err); 18759cb83748SJack Wang 18769cb83748SJack Wang close_and_return_err: 18779cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 187899f0c380SJack Wang close_sess(sess); 18799cb83748SJack Wang 18809cb83748SJack Wang return err; 18819cb83748SJack Wang } 18829cb83748SJack Wang 18839cb83748SJack Wang static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, 18849cb83748SJack Wang struct rdma_cm_event *ev) 18859cb83748SJack Wang { 18869cb83748SJack Wang struct rtrs_srv_sess *sess = NULL; 18879cb83748SJack Wang struct rtrs_sess *s = NULL; 18889cb83748SJack Wang 18899cb83748SJack Wang if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) { 18909cb83748SJack Wang struct rtrs_con *c = cm_id->context; 18919cb83748SJack Wang 18929cb83748SJack Wang s = c->sess; 18939cb83748SJack Wang sess = to_srv_sess(s); 18949cb83748SJack Wang } 18959cb83748SJack Wang 18969cb83748SJack Wang switch (ev->event) { 18979cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_REQUEST: 18989cb83748SJack Wang /* 18999cb83748SJack Wang * In case of error cma.c will destroy cm_id, 19009cb83748SJack Wang * see cma_process_remove() 19019cb83748SJack Wang */ 19029cb83748SJack Wang return rtrs_rdma_connect(cm_id, ev->param.conn.private_data, 19039cb83748SJack Wang ev->param.conn.private_data_len); 19049cb83748SJack Wang case RDMA_CM_EVENT_ESTABLISHED: 19059cb83748SJack Wang /* Nothing here */ 19069cb83748SJack Wang break; 19079cb83748SJack Wang case RDMA_CM_EVENT_REJECTED: 19089cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_ERROR: 19099cb83748SJack Wang case RDMA_CM_EVENT_UNREACHABLE: 19109cb83748SJack Wang rtrs_err(s, "CM error (CM event: %s, err: %d)\n", 19119cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 191257dae8baSGuoqing Jiang fallthrough; 19139cb83748SJack Wang case RDMA_CM_EVENT_DISCONNECTED: 19149cb83748SJack Wang case RDMA_CM_EVENT_ADDR_CHANGE: 19159cb83748SJack Wang case RDMA_CM_EVENT_TIMEWAIT_EXIT: 19169cb83748SJack Wang case RDMA_CM_EVENT_DEVICE_REMOVAL: 19179cb83748SJack Wang close_sess(sess); 19189cb83748SJack Wang break; 19199cb83748SJack Wang default: 19209cb83748SJack Wang pr_err("Ignoring unexpected CM event %s, err %d\n", 19219cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19229cb83748SJack Wang break; 19239cb83748SJack Wang } 19249cb83748SJack Wang 19259cb83748SJack Wang return 0; 19269cb83748SJack Wang } 19279cb83748SJack Wang 19289cb83748SJack Wang static struct rdma_cm_id *rtrs_srv_cm_init(struct rtrs_srv_ctx *ctx, 19299cb83748SJack Wang struct sockaddr *addr, 19309cb83748SJack Wang enum rdma_ucm_port_space ps) 19319cb83748SJack Wang { 19329cb83748SJack Wang struct rdma_cm_id *cm_id; 19339cb83748SJack Wang int ret; 19349cb83748SJack Wang 19359cb83748SJack Wang cm_id = rdma_create_id(&init_net, rtrs_srv_rdma_cm_handler, 19369cb83748SJack Wang ctx, ps, IB_QPT_RC); 19379cb83748SJack Wang if (IS_ERR(cm_id)) { 19389cb83748SJack Wang ret = PTR_ERR(cm_id); 19399cb83748SJack Wang pr_err("Creating id for RDMA connection failed, err: %d\n", 19409cb83748SJack Wang ret); 19419cb83748SJack Wang goto err_out; 19429cb83748SJack Wang } 19439cb83748SJack Wang ret = rdma_bind_addr(cm_id, addr); 19449cb83748SJack Wang if (ret) { 19459cb83748SJack Wang pr_err("Binding RDMA address failed, err: %d\n", ret); 19469cb83748SJack Wang goto err_cm; 19479cb83748SJack Wang } 19489cb83748SJack Wang ret = rdma_listen(cm_id, 64); 19499cb83748SJack Wang if (ret) { 19509cb83748SJack Wang pr_err("Listening on RDMA connection failed, err: %d\n", 19519cb83748SJack Wang ret); 19529cb83748SJack Wang goto err_cm; 19539cb83748SJack Wang } 19549cb83748SJack Wang 19559cb83748SJack Wang return cm_id; 19569cb83748SJack Wang 19579cb83748SJack Wang err_cm: 19589cb83748SJack Wang rdma_destroy_id(cm_id); 19599cb83748SJack Wang err_out: 19609cb83748SJack Wang 19619cb83748SJack Wang return ERR_PTR(ret); 19629cb83748SJack Wang } 19639cb83748SJack Wang 19649cb83748SJack Wang static int rtrs_srv_rdma_init(struct rtrs_srv_ctx *ctx, u16 port) 19659cb83748SJack Wang { 19669cb83748SJack Wang struct sockaddr_in6 sin = { 19679cb83748SJack Wang .sin6_family = AF_INET6, 19689cb83748SJack Wang .sin6_addr = IN6ADDR_ANY_INIT, 19699cb83748SJack Wang .sin6_port = htons(port), 19709cb83748SJack Wang }; 19719cb83748SJack Wang struct sockaddr_ib sib = { 19729cb83748SJack Wang .sib_family = AF_IB, 19739cb83748SJack Wang .sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port), 19749cb83748SJack Wang .sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL), 19759cb83748SJack Wang .sib_pkey = cpu_to_be16(0xffff), 19769cb83748SJack Wang }; 19779cb83748SJack Wang struct rdma_cm_id *cm_ip, *cm_ib; 19789cb83748SJack Wang int ret; 19799cb83748SJack Wang 19809cb83748SJack Wang /* 19819cb83748SJack Wang * We accept both IPoIB and IB connections, so we need to keep 19829cb83748SJack Wang * two cm id's, one for each socket type and port space. 19839cb83748SJack Wang * If the cm initialization of one of the id's fails, we abort 19849cb83748SJack Wang * everything. 19859cb83748SJack Wang */ 19869cb83748SJack Wang cm_ip = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sin, RDMA_PS_TCP); 19879cb83748SJack Wang if (IS_ERR(cm_ip)) 19889cb83748SJack Wang return PTR_ERR(cm_ip); 19899cb83748SJack Wang 19909cb83748SJack Wang cm_ib = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sib, RDMA_PS_IB); 19919cb83748SJack Wang if (IS_ERR(cm_ib)) { 19929cb83748SJack Wang ret = PTR_ERR(cm_ib); 19939cb83748SJack Wang goto free_cm_ip; 19949cb83748SJack Wang } 19959cb83748SJack Wang 19969cb83748SJack Wang ctx->cm_id_ip = cm_ip; 19979cb83748SJack Wang ctx->cm_id_ib = cm_ib; 19989cb83748SJack Wang 19999cb83748SJack Wang return 0; 20009cb83748SJack Wang 20019cb83748SJack Wang free_cm_ip: 20029cb83748SJack Wang rdma_destroy_id(cm_ip); 20039cb83748SJack Wang 20049cb83748SJack Wang return ret; 20059cb83748SJack Wang } 20069cb83748SJack Wang 20079cb83748SJack Wang static struct rtrs_srv_ctx *alloc_srv_ctx(struct rtrs_srv_ops *ops) 20089cb83748SJack Wang { 20099cb83748SJack Wang struct rtrs_srv_ctx *ctx; 20109cb83748SJack Wang 20119cb83748SJack Wang ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 20129cb83748SJack Wang if (!ctx) 20139cb83748SJack Wang return NULL; 20149cb83748SJack Wang 20159cb83748SJack Wang ctx->ops = *ops; 20169cb83748SJack Wang mutex_init(&ctx->srv_mutex); 20179cb83748SJack Wang INIT_LIST_HEAD(&ctx->srv_list); 20189cb83748SJack Wang 20199cb83748SJack Wang return ctx; 20209cb83748SJack Wang } 20219cb83748SJack Wang 20229cb83748SJack Wang static void free_srv_ctx(struct rtrs_srv_ctx *ctx) 20239cb83748SJack Wang { 20249cb83748SJack Wang WARN_ON(!list_empty(&ctx->srv_list)); 20259cb83748SJack Wang mutex_destroy(&ctx->srv_mutex); 20269cb83748SJack Wang kfree(ctx); 20279cb83748SJack Wang } 20289cb83748SJack Wang 2029558d52b2SMd Haris Iqbal static int rtrs_srv_add_one(struct ib_device *device) 2030558d52b2SMd Haris Iqbal { 2031558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2032558d52b2SMd Haris Iqbal int ret = 0; 2033558d52b2SMd Haris Iqbal 2034558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2035558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2036558d52b2SMd Haris Iqbal goto out; 2037558d52b2SMd Haris Iqbal 2038558d52b2SMd Haris Iqbal /* 2039558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will create them 2040558d52b2SMd Haris Iqbal * only once 2041558d52b2SMd Haris Iqbal */ 2042558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2043558d52b2SMd Haris Iqbal ret = rtrs_srv_rdma_init(ctx, ib_ctx.port); 2044558d52b2SMd Haris Iqbal if (ret) { 2045558d52b2SMd Haris Iqbal /* 2046558d52b2SMd Haris Iqbal * We errored out here. 2047558d52b2SMd Haris Iqbal * According to the ib code, if we encounter an error here then the 2048558d52b2SMd Haris Iqbal * error code is ignored, and no more calls to our ops are made. 2049558d52b2SMd Haris Iqbal */ 2050558d52b2SMd Haris Iqbal pr_err("Failed to initialize RDMA connection"); 2051558d52b2SMd Haris Iqbal goto err_out; 2052558d52b2SMd Haris Iqbal } 2053558d52b2SMd Haris Iqbal 2054558d52b2SMd Haris Iqbal out: 2055558d52b2SMd Haris Iqbal /* 2056558d52b2SMd Haris Iqbal * Keep a track on the number of ib devices added 2057558d52b2SMd Haris Iqbal */ 2058558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count++; 2059558d52b2SMd Haris Iqbal 2060558d52b2SMd Haris Iqbal err_out: 2061558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2062558d52b2SMd Haris Iqbal return ret; 2063558d52b2SMd Haris Iqbal } 2064558d52b2SMd Haris Iqbal 2065558d52b2SMd Haris Iqbal static void rtrs_srv_remove_one(struct ib_device *device, void *client_data) 2066558d52b2SMd Haris Iqbal { 2067558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2068558d52b2SMd Haris Iqbal 2069558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2070558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count--; 2071558d52b2SMd Haris Iqbal 2072558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2073558d52b2SMd Haris Iqbal goto out; 2074558d52b2SMd Haris Iqbal 2075558d52b2SMd Haris Iqbal /* 2076558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will remove them 2077558d52b2SMd Haris Iqbal * only once, when the last device is removed 2078558d52b2SMd Haris Iqbal */ 2079558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2080558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ip); 2081558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ib); 2082558d52b2SMd Haris Iqbal 2083558d52b2SMd Haris Iqbal out: 2084558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2085558d52b2SMd Haris Iqbal } 2086558d52b2SMd Haris Iqbal 2087558d52b2SMd Haris Iqbal static struct ib_client rtrs_srv_client = { 2088558d52b2SMd Haris Iqbal .name = "rtrs_server", 2089558d52b2SMd Haris Iqbal .add = rtrs_srv_add_one, 2090558d52b2SMd Haris Iqbal .remove = rtrs_srv_remove_one 2091558d52b2SMd Haris Iqbal }; 2092558d52b2SMd Haris Iqbal 20939cb83748SJack Wang /** 20949cb83748SJack Wang * rtrs_srv_open() - open RTRS server context 20959cb83748SJack Wang * @ops: callback functions 20969cb83748SJack Wang * @port: port to listen on 20979cb83748SJack Wang * 20989cb83748SJack Wang * Creates server context with specified callbacks. 20999cb83748SJack Wang * 21009cb83748SJack Wang * Return a valid pointer on success otherwise PTR_ERR. 21019cb83748SJack Wang */ 21029cb83748SJack Wang struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) 21039cb83748SJack Wang { 21049cb83748SJack Wang struct rtrs_srv_ctx *ctx; 21059cb83748SJack Wang int err; 21069cb83748SJack Wang 21079cb83748SJack Wang ctx = alloc_srv_ctx(ops); 21089cb83748SJack Wang if (!ctx) 21099cb83748SJack Wang return ERR_PTR(-ENOMEM); 21109cb83748SJack Wang 2111558d52b2SMd Haris Iqbal mutex_init(&ib_ctx.ib_dev_mutex); 2112558d52b2SMd Haris Iqbal ib_ctx.srv_ctx = ctx; 2113558d52b2SMd Haris Iqbal ib_ctx.port = port; 2114558d52b2SMd Haris Iqbal 2115558d52b2SMd Haris Iqbal err = ib_register_client(&rtrs_srv_client); 21169cb83748SJack Wang if (err) { 21179cb83748SJack Wang free_srv_ctx(ctx); 21189cb83748SJack Wang return ERR_PTR(err); 21199cb83748SJack Wang } 21209cb83748SJack Wang 21219cb83748SJack Wang return ctx; 21229cb83748SJack Wang } 21239cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_open); 21249cb83748SJack Wang 21259cb83748SJack Wang static void close_sessions(struct rtrs_srv *srv) 21269cb83748SJack Wang { 21279cb83748SJack Wang struct rtrs_srv_sess *sess; 21289cb83748SJack Wang 21299cb83748SJack Wang mutex_lock(&srv->paths_mutex); 21309cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 21319cb83748SJack Wang close_sess(sess); 21329cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 21339cb83748SJack Wang } 21349cb83748SJack Wang 21359cb83748SJack Wang static void close_ctx(struct rtrs_srv_ctx *ctx) 21369cb83748SJack Wang { 21379cb83748SJack Wang struct rtrs_srv *srv; 21389cb83748SJack Wang 21399cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 21409cb83748SJack Wang list_for_each_entry(srv, &ctx->srv_list, ctx_list) 21419cb83748SJack Wang close_sessions(srv); 21429cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 21439cb83748SJack Wang flush_workqueue(rtrs_wq); 21449cb83748SJack Wang } 21459cb83748SJack Wang 21469cb83748SJack Wang /** 21479cb83748SJack Wang * rtrs_srv_close() - close RTRS server context 21489cb83748SJack Wang * @ctx: pointer to server context 21499cb83748SJack Wang * 21509cb83748SJack Wang * Closes RTRS server context with all client sessions. 21519cb83748SJack Wang */ 21529cb83748SJack Wang void rtrs_srv_close(struct rtrs_srv_ctx *ctx) 21539cb83748SJack Wang { 2154558d52b2SMd Haris Iqbal ib_unregister_client(&rtrs_srv_client); 2155558d52b2SMd Haris Iqbal mutex_destroy(&ib_ctx.ib_dev_mutex); 21569cb83748SJack Wang close_ctx(ctx); 21579cb83748SJack Wang free_srv_ctx(ctx); 21589cb83748SJack Wang } 21599cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_close); 21609cb83748SJack Wang 21619cb83748SJack Wang static int check_module_params(void) 21629cb83748SJack Wang { 21639cb83748SJack Wang if (sess_queue_depth < 1 || sess_queue_depth > MAX_SESS_QUEUE_DEPTH) { 21649cb83748SJack Wang pr_err("Invalid sess_queue_depth value %d, has to be >= %d, <= %d.\n", 21659cb83748SJack Wang sess_queue_depth, 1, MAX_SESS_QUEUE_DEPTH); 21669cb83748SJack Wang return -EINVAL; 21679cb83748SJack Wang } 21683f3d0eabSGioh Kim if (max_chunk_size < MIN_CHUNK_SIZE || !is_power_of_2(max_chunk_size)) { 21699cb83748SJack Wang pr_err("Invalid max_chunk_size value %d, has to be >= %d and should be power of two.\n", 21703f3d0eabSGioh Kim max_chunk_size, MIN_CHUNK_SIZE); 21719cb83748SJack Wang return -EINVAL; 21729cb83748SJack Wang } 21739cb83748SJack Wang 21749cb83748SJack Wang /* 21759cb83748SJack Wang * Check if IB immediate data size is enough to hold the mem_id and the 21769cb83748SJack Wang * offset inside the memory chunk 21779cb83748SJack Wang */ 21789cb83748SJack Wang if ((ilog2(sess_queue_depth - 1) + 1) + 21799cb83748SJack Wang (ilog2(max_chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) { 21809cb83748SJack Wang pr_err("RDMA immediate size (%db) not enough to encode %d buffers of size %dB. Reduce 'sess_queue_depth' or 'max_chunk_size' parameters.\n", 21819cb83748SJack Wang MAX_IMM_PAYL_BITS, sess_queue_depth, max_chunk_size); 21829cb83748SJack Wang return -EINVAL; 21839cb83748SJack Wang } 21849cb83748SJack Wang 21859cb83748SJack Wang return 0; 21869cb83748SJack Wang } 21879cb83748SJack Wang 21889cb83748SJack Wang static int __init rtrs_server_init(void) 21899cb83748SJack Wang { 21909cb83748SJack Wang int err; 21919cb83748SJack Wang 21929cb83748SJack Wang pr_info("Loading module %s, proto %s: (max_chunk_size: %d (pure IO %ld, headers %ld) , sess_queue_depth: %d, always_invalidate: %d)\n", 21939cb83748SJack Wang KBUILD_MODNAME, RTRS_PROTO_VER_STRING, 21949cb83748SJack Wang max_chunk_size, max_chunk_size - MAX_HDR_SIZE, MAX_HDR_SIZE, 21959cb83748SJack Wang sess_queue_depth, always_invalidate); 21969cb83748SJack Wang 21979cb83748SJack Wang rtrs_rdma_dev_pd_init(0, &dev_pd); 21989cb83748SJack Wang 21999cb83748SJack Wang err = check_module_params(); 22009cb83748SJack Wang if (err) { 22019cb83748SJack Wang pr_err("Failed to load module, invalid module parameters, err: %d\n", 22029cb83748SJack Wang err); 22039cb83748SJack Wang return err; 22049cb83748SJack Wang } 22059cb83748SJack Wang chunk_pool = mempool_create_page_pool(sess_queue_depth * CHUNK_POOL_SZ, 22069cb83748SJack Wang get_order(max_chunk_size)); 22079cb83748SJack Wang if (!chunk_pool) 22089cb83748SJack Wang return -ENOMEM; 22099cb83748SJack Wang rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server"); 22109cb83748SJack Wang if (IS_ERR(rtrs_dev_class)) { 22119cb83748SJack Wang err = PTR_ERR(rtrs_dev_class); 22129cb83748SJack Wang goto out_chunk_pool; 22139cb83748SJack Wang } 221403ed5a8cSJack Wang rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0); 22156b31afceSWei Yongjun if (!rtrs_wq) { 22166b31afceSWei Yongjun err = -ENOMEM; 22179cb83748SJack Wang goto out_dev_class; 22186b31afceSWei Yongjun } 22199cb83748SJack Wang 22209cb83748SJack Wang return 0; 22219cb83748SJack Wang 22229cb83748SJack Wang out_dev_class: 22239cb83748SJack Wang class_destroy(rtrs_dev_class); 22249cb83748SJack Wang out_chunk_pool: 22259cb83748SJack Wang mempool_destroy(chunk_pool); 22269cb83748SJack Wang 22279cb83748SJack Wang return err; 22289cb83748SJack Wang } 22299cb83748SJack Wang 22309cb83748SJack Wang static void __exit rtrs_server_exit(void) 22319cb83748SJack Wang { 22329cb83748SJack Wang destroy_workqueue(rtrs_wq); 22339cb83748SJack Wang class_destroy(rtrs_dev_class); 22349cb83748SJack Wang mempool_destroy(chunk_pool); 22359cb83748SJack Wang rtrs_rdma_dev_pd_deinit(&dev_pd); 22369cb83748SJack Wang } 22379cb83748SJack Wang 22389cb83748SJack Wang module_init(rtrs_server_init); 22399cb83748SJack Wang module_exit(rtrs_server_exit); 2240