19cb83748SJack Wang // SPDX-License-Identifier: GPL-2.0-or-later 29cb83748SJack Wang /* 39cb83748SJack Wang * RDMA Transport Layer 49cb83748SJack Wang * 59cb83748SJack Wang * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. 69cb83748SJack Wang * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. 79cb83748SJack Wang * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. 89cb83748SJack Wang */ 99cb83748SJack Wang 109cb83748SJack Wang #undef pr_fmt 119cb83748SJack Wang #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt 129cb83748SJack Wang 139cb83748SJack Wang #include <linux/module.h> 149cb83748SJack Wang #include <linux/mempool.h> 159cb83748SJack Wang 169cb83748SJack Wang #include "rtrs-srv.h" 179cb83748SJack Wang #include "rtrs-log.h" 188094ba0aSLeon Romanovsky #include <rdma/ib_cm.h> 19558d52b2SMd Haris Iqbal #include <rdma/ib_verbs.h> 209cb83748SJack Wang 219cb83748SJack Wang MODULE_DESCRIPTION("RDMA Transport Server"); 229cb83748SJack Wang MODULE_LICENSE("GPL"); 239cb83748SJack Wang 249cb83748SJack Wang /* Must be power of 2, see mask from mr->page_size in ib_sg_to_pages() */ 259cb83748SJack Wang #define DEFAULT_MAX_CHUNK_SIZE (128 << 10) 269cb83748SJack Wang #define DEFAULT_SESS_QUEUE_DEPTH 512 279cb83748SJack Wang #define MAX_HDR_SIZE PAGE_SIZE 289cb83748SJack Wang 299cb83748SJack Wang /* We guarantee to serve 10 paths at least */ 309cb83748SJack Wang #define CHUNK_POOL_SZ 10 319cb83748SJack Wang 329cb83748SJack Wang static struct rtrs_rdma_dev_pd dev_pd; 339cb83748SJack Wang static mempool_t *chunk_pool; 349cb83748SJack Wang struct class *rtrs_dev_class; 35558d52b2SMd Haris Iqbal static struct rtrs_srv_ib_ctx ib_ctx; 369cb83748SJack Wang 379cb83748SJack Wang static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE; 389cb83748SJack Wang static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH; 399cb83748SJack Wang 409cb83748SJack Wang static bool always_invalidate = true; 419cb83748SJack Wang module_param(always_invalidate, bool, 0444); 429cb83748SJack Wang MODULE_PARM_DESC(always_invalidate, 439cb83748SJack Wang "Invalidate memory registration for contiguous memory regions before accessing."); 449cb83748SJack Wang 459cb83748SJack Wang module_param_named(max_chunk_size, max_chunk_size, int, 0444); 469cb83748SJack Wang MODULE_PARM_DESC(max_chunk_size, 479cb83748SJack Wang "Max size for each IO request, when change the unit is in byte (default: " 489cb83748SJack Wang __stringify(DEFAULT_MAX_CHUNK_SIZE) "KB)"); 499cb83748SJack Wang 509cb83748SJack Wang module_param_named(sess_queue_depth, sess_queue_depth, int, 0444); 519cb83748SJack Wang MODULE_PARM_DESC(sess_queue_depth, 529cb83748SJack Wang "Number of buffers for pending I/O requests to allocate per session. Maximum: " 539cb83748SJack Wang __stringify(MAX_SESS_QUEUE_DEPTH) " (default: " 549cb83748SJack Wang __stringify(DEFAULT_SESS_QUEUE_DEPTH) ")"); 559cb83748SJack Wang 569cb83748SJack Wang static cpumask_t cq_affinity_mask = { CPU_BITS_ALL }; 579cb83748SJack Wang 589cb83748SJack Wang static struct workqueue_struct *rtrs_wq; 599cb83748SJack Wang 609cb83748SJack Wang static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) 619cb83748SJack Wang { 629cb83748SJack Wang return container_of(c, struct rtrs_srv_con, c); 639cb83748SJack Wang } 649cb83748SJack Wang 659cb83748SJack Wang static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s) 669cb83748SJack Wang { 679cb83748SJack Wang return container_of(s, struct rtrs_srv_sess, s); 689cb83748SJack Wang } 699cb83748SJack Wang 709cb83748SJack Wang static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess, 719cb83748SJack Wang enum rtrs_srv_state new_state) 729cb83748SJack Wang { 739cb83748SJack Wang enum rtrs_srv_state old_state; 749cb83748SJack Wang bool changed = false; 759cb83748SJack Wang 769cb83748SJack Wang lockdep_assert_held(&sess->state_lock); 779cb83748SJack Wang old_state = sess->state; 789cb83748SJack Wang switch (new_state) { 799cb83748SJack Wang case RTRS_SRV_CONNECTED: 809cb83748SJack Wang switch (old_state) { 819cb83748SJack Wang case RTRS_SRV_CONNECTING: 829cb83748SJack Wang changed = true; 839cb83748SJack Wang fallthrough; 849cb83748SJack Wang default: 859cb83748SJack Wang break; 869cb83748SJack Wang } 879cb83748SJack Wang break; 889cb83748SJack Wang case RTRS_SRV_CLOSING: 899cb83748SJack Wang switch (old_state) { 909cb83748SJack Wang case RTRS_SRV_CONNECTING: 919cb83748SJack Wang case RTRS_SRV_CONNECTED: 929cb83748SJack Wang changed = true; 939cb83748SJack Wang fallthrough; 949cb83748SJack Wang default: 959cb83748SJack Wang break; 969cb83748SJack Wang } 979cb83748SJack Wang break; 989cb83748SJack Wang case RTRS_SRV_CLOSED: 999cb83748SJack Wang switch (old_state) { 1009cb83748SJack Wang case RTRS_SRV_CLOSING: 1019cb83748SJack Wang changed = true; 1029cb83748SJack Wang fallthrough; 1039cb83748SJack Wang default: 1049cb83748SJack Wang break; 1059cb83748SJack Wang } 1069cb83748SJack Wang break; 1079cb83748SJack Wang default: 1089cb83748SJack Wang break; 1099cb83748SJack Wang } 1109cb83748SJack Wang if (changed) 1119cb83748SJack Wang sess->state = new_state; 1129cb83748SJack Wang 1139cb83748SJack Wang return changed; 1149cb83748SJack Wang } 1159cb83748SJack Wang 116ffea6ad1SGuoqing Jiang static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, 117ffea6ad1SGuoqing Jiang enum rtrs_srv_state new_state) 1189cb83748SJack Wang { 1199cb83748SJack Wang bool changed; 1209cb83748SJack Wang 1219cb83748SJack Wang spin_lock_irq(&sess->state_lock); 1229cb83748SJack Wang changed = __rtrs_srv_change_state(sess, new_state); 1239cb83748SJack Wang spin_unlock_irq(&sess->state_lock); 1249cb83748SJack Wang 1259cb83748SJack Wang return changed; 1269cb83748SJack Wang } 1279cb83748SJack Wang 1289cb83748SJack Wang static void free_id(struct rtrs_srv_op *id) 1299cb83748SJack Wang { 1309cb83748SJack Wang if (!id) 1319cb83748SJack Wang return; 1329cb83748SJack Wang kfree(id); 1339cb83748SJack Wang } 1349cb83748SJack Wang 1359cb83748SJack Wang static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess) 1369cb83748SJack Wang { 1379cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1389cb83748SJack Wang int i; 1399cb83748SJack Wang 1409cb83748SJack Wang WARN_ON(atomic_read(&sess->ids_inflight)); 1419cb83748SJack Wang if (sess->ops_ids) { 1429cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 1439cb83748SJack Wang free_id(sess->ops_ids[i]); 1449cb83748SJack Wang kfree(sess->ops_ids); 1459cb83748SJack Wang sess->ops_ids = NULL; 1469cb83748SJack Wang } 1479cb83748SJack Wang } 1489cb83748SJack Wang 1499cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc); 1509cb83748SJack Wang 1519cb83748SJack Wang static struct ib_cqe io_comp_cqe = { 1529cb83748SJack Wang .done = rtrs_srv_rdma_done 1539cb83748SJack Wang }; 1549cb83748SJack Wang 1559cb83748SJack Wang static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) 1569cb83748SJack Wang { 1579cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1589cb83748SJack Wang struct rtrs_srv_op *id; 1599cb83748SJack Wang int i; 1609cb83748SJack Wang 1619cb83748SJack Wang sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids), 1629cb83748SJack Wang GFP_KERNEL); 1639cb83748SJack Wang if (!sess->ops_ids) 1649cb83748SJack Wang goto err; 1659cb83748SJack Wang 1669cb83748SJack Wang for (i = 0; i < srv->queue_depth; ++i) { 1679cb83748SJack Wang id = kzalloc(sizeof(*id), GFP_KERNEL); 1689cb83748SJack Wang if (!id) 1699cb83748SJack Wang goto err; 1709cb83748SJack Wang 1719cb83748SJack Wang sess->ops_ids[i] = id; 1729cb83748SJack Wang } 1739cb83748SJack Wang init_waitqueue_head(&sess->ids_waitq); 1749cb83748SJack Wang atomic_set(&sess->ids_inflight, 0); 1759cb83748SJack Wang 1769cb83748SJack Wang return 0; 1779cb83748SJack Wang 1789cb83748SJack Wang err: 1799cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 1809cb83748SJack Wang return -ENOMEM; 1819cb83748SJack Wang } 1829cb83748SJack Wang 1839cb83748SJack Wang static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess) 1849cb83748SJack Wang { 1859cb83748SJack Wang atomic_inc(&sess->ids_inflight); 1869cb83748SJack Wang } 1879cb83748SJack Wang 1889cb83748SJack Wang static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess) 1899cb83748SJack Wang { 1909cb83748SJack Wang if (atomic_dec_and_test(&sess->ids_inflight)) 1919cb83748SJack Wang wake_up(&sess->ids_waitq); 1929cb83748SJack Wang } 1939cb83748SJack Wang 1949cb83748SJack Wang static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess) 1959cb83748SJack Wang { 1969cb83748SJack Wang wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight)); 1979cb83748SJack Wang } 1989cb83748SJack Wang 1999cb83748SJack Wang 2009cb83748SJack Wang static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc) 2019cb83748SJack Wang { 2029cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 2039cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 2049cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 2059cb83748SJack Wang 2069cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 2079cb83748SJack Wang rtrs_err(s, "REG MR failed: %s\n", 2089cb83748SJack Wang ib_wc_status_msg(wc->status)); 2099cb83748SJack Wang close_sess(sess); 2109cb83748SJack Wang return; 2119cb83748SJack Wang } 2129cb83748SJack Wang } 2139cb83748SJack Wang 2149cb83748SJack Wang static struct ib_cqe local_reg_cqe = { 2159cb83748SJack Wang .done = rtrs_srv_reg_mr_done 2169cb83748SJack Wang }; 2179cb83748SJack Wang 2189cb83748SJack Wang static int rdma_write_sg(struct rtrs_srv_op *id) 2199cb83748SJack Wang { 2209cb83748SJack Wang struct rtrs_sess *s = id->con->c.sess; 2219cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 2229cb83748SJack Wang dma_addr_t dma_addr = sess->dma_addr[id->msg_id]; 2239cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 2249cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 2259cb83748SJack Wang struct ib_send_wr inv_wr, imm_wr; 2269cb83748SJack Wang struct ib_rdma_wr *wr = NULL; 2279cb83748SJack Wang enum ib_send_flags flags; 2289cb83748SJack Wang size_t sg_cnt; 2299cb83748SJack Wang int err, offset; 2309cb83748SJack Wang bool need_inval; 2319cb83748SJack Wang u32 rkey = 0; 2329cb83748SJack Wang struct ib_reg_wr rwr; 2339cb83748SJack Wang struct ib_sge *plist; 2349cb83748SJack Wang struct ib_sge list; 2359cb83748SJack Wang 2369cb83748SJack Wang sg_cnt = le16_to_cpu(id->rd_msg->sg_cnt); 2379cb83748SJack Wang need_inval = le16_to_cpu(id->rd_msg->flags) & RTRS_MSG_NEED_INVAL_F; 2389cb83748SJack Wang if (unlikely(sg_cnt != 1)) 2399cb83748SJack Wang return -EINVAL; 2409cb83748SJack Wang 2419cb83748SJack Wang offset = 0; 2429cb83748SJack Wang 2439cb83748SJack Wang wr = &id->tx_wr; 2449cb83748SJack Wang plist = &id->tx_sg; 2459cb83748SJack Wang plist->addr = dma_addr + offset; 2469cb83748SJack Wang plist->length = le32_to_cpu(id->rd_msg->desc[0].len); 2479cb83748SJack Wang 2489cb83748SJack Wang /* WR will fail with length error 2499cb83748SJack Wang * if this is 0 2509cb83748SJack Wang */ 2519cb83748SJack Wang if (unlikely(plist->length == 0)) { 2529cb83748SJack Wang rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); 2539cb83748SJack Wang return -EINVAL; 2549cb83748SJack Wang } 2559cb83748SJack Wang 2569cb83748SJack Wang plist->lkey = sess->s.dev->ib_pd->local_dma_lkey; 2579cb83748SJack Wang offset += plist->length; 2589cb83748SJack Wang 2599cb83748SJack Wang wr->wr.sg_list = plist; 2609cb83748SJack Wang wr->wr.num_sge = 1; 2619cb83748SJack Wang wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); 2629cb83748SJack Wang wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); 2639cb83748SJack Wang if (rkey == 0) 2649cb83748SJack Wang rkey = wr->rkey; 2659cb83748SJack Wang else 2669cb83748SJack Wang /* Only one key is actually used */ 2679cb83748SJack Wang WARN_ON_ONCE(rkey != wr->rkey); 2689cb83748SJack Wang 2699cb83748SJack Wang wr->wr.opcode = IB_WR_RDMA_WRITE; 2708537f2deSJack Wang wr->wr.wr_cqe = &io_comp_cqe; 2719cb83748SJack Wang wr->wr.ex.imm_data = 0; 2729cb83748SJack Wang wr->wr.send_flags = 0; 2739cb83748SJack Wang 2749cb83748SJack Wang if (need_inval && always_invalidate) { 2759cb83748SJack Wang wr->wr.next = &rwr.wr; 2769cb83748SJack Wang rwr.wr.next = &inv_wr; 2779cb83748SJack Wang inv_wr.next = &imm_wr; 2789cb83748SJack Wang } else if (always_invalidate) { 2799cb83748SJack Wang wr->wr.next = &rwr.wr; 2809cb83748SJack Wang rwr.wr.next = &imm_wr; 2819cb83748SJack Wang } else if (need_inval) { 2829cb83748SJack Wang wr->wr.next = &inv_wr; 2839cb83748SJack Wang inv_wr.next = &imm_wr; 2849cb83748SJack Wang } else { 2859cb83748SJack Wang wr->wr.next = &imm_wr; 2869cb83748SJack Wang } 2879cb83748SJack Wang /* 2889cb83748SJack Wang * From time to time we have to post signaled sends, 2899cb83748SJack Wang * or send queue will fill up and only QP reset can help. 2909cb83748SJack Wang */ 2919cb83748SJack Wang flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ? 2929cb83748SJack Wang 0 : IB_SEND_SIGNALED; 2939cb83748SJack Wang 2949cb83748SJack Wang if (need_inval) { 2959cb83748SJack Wang inv_wr.sg_list = NULL; 2969cb83748SJack Wang inv_wr.num_sge = 0; 2979cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 2988537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 2999cb83748SJack Wang inv_wr.send_flags = 0; 3009cb83748SJack Wang inv_wr.ex.invalidate_rkey = rkey; 3019cb83748SJack Wang } 3029cb83748SJack Wang 3039cb83748SJack Wang imm_wr.next = NULL; 3049cb83748SJack Wang if (always_invalidate) { 3059cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 3069cb83748SJack Wang 3079cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 3089cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 3098537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 3109cb83748SJack Wang rwr.wr.num_sge = 0; 3119cb83748SJack Wang rwr.mr = srv_mr->mr; 3129cb83748SJack Wang rwr.wr.send_flags = 0; 3139cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 3149cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 3159cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 3169cb83748SJack Wang msg = srv_mr->iu->buf; 3179cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 3189cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 3199cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 3209cb83748SJack Wang 3219cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 3229cb83748SJack Wang list.length = sizeof(*msg); 3239cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 3249cb83748SJack Wang imm_wr.sg_list = &list; 3259cb83748SJack Wang imm_wr.num_sge = 1; 3269cb83748SJack Wang imm_wr.opcode = IB_WR_SEND_WITH_IMM; 3279cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 3289cb83748SJack Wang srv_mr->iu->dma_addr, 3299cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 3309cb83748SJack Wang } else { 3319cb83748SJack Wang imm_wr.sg_list = NULL; 3329cb83748SJack Wang imm_wr.num_sge = 0; 3339cb83748SJack Wang imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 3349cb83748SJack Wang } 3359cb83748SJack Wang imm_wr.send_flags = flags; 3369cb83748SJack Wang imm_wr.ex.imm_data = cpu_to_be32(rtrs_to_io_rsp_imm(id->msg_id, 3379cb83748SJack Wang 0, need_inval)); 3389cb83748SJack Wang 3399cb83748SJack Wang imm_wr.wr_cqe = &io_comp_cqe; 3409cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr, 3419cb83748SJack Wang offset, DMA_BIDIRECTIONAL); 3429cb83748SJack Wang 3439cb83748SJack Wang err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL); 3449cb83748SJack Wang if (unlikely(err)) 3459cb83748SJack Wang rtrs_err(s, 3469cb83748SJack Wang "Posting RDMA-Write-Request to QP failed, err: %d\n", 3479cb83748SJack Wang err); 3489cb83748SJack Wang 3499cb83748SJack Wang return err; 3509cb83748SJack Wang } 3519cb83748SJack Wang 3529cb83748SJack Wang /** 3539cb83748SJack Wang * send_io_resp_imm() - respond to client with empty IMM on failed READ/WRITE 3549cb83748SJack Wang * requests or on successful WRITE request. 3559cb83748SJack Wang * @con: the connection to send back result 3569cb83748SJack Wang * @id: the id associated with the IO 3579cb83748SJack Wang * @errno: the error number of the IO. 3589cb83748SJack Wang * 3599cb83748SJack Wang * Return 0 on success, errno otherwise. 3609cb83748SJack Wang */ 3619cb83748SJack Wang static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, 3629cb83748SJack Wang int errno) 3639cb83748SJack Wang { 3649cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 3659cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 3669cb83748SJack Wang struct ib_send_wr inv_wr, imm_wr, *wr = NULL; 3679cb83748SJack Wang struct ib_reg_wr rwr; 3689cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 3699cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 3709cb83748SJack Wang bool need_inval = false; 3719cb83748SJack Wang enum ib_send_flags flags; 3729cb83748SJack Wang u32 imm; 3739cb83748SJack Wang int err; 3749cb83748SJack Wang 3759cb83748SJack Wang if (id->dir == READ) { 3769cb83748SJack Wang struct rtrs_msg_rdma_read *rd_msg = id->rd_msg; 3779cb83748SJack Wang size_t sg_cnt; 3789cb83748SJack Wang 3799cb83748SJack Wang need_inval = le16_to_cpu(rd_msg->flags) & 3809cb83748SJack Wang RTRS_MSG_NEED_INVAL_F; 3819cb83748SJack Wang sg_cnt = le16_to_cpu(rd_msg->sg_cnt); 3829cb83748SJack Wang 3839cb83748SJack Wang if (need_inval) { 3849cb83748SJack Wang if (likely(sg_cnt)) { 3858537f2deSJack Wang inv_wr.wr_cqe = &io_comp_cqe; 3869cb83748SJack Wang inv_wr.sg_list = NULL; 3879cb83748SJack Wang inv_wr.num_sge = 0; 3889cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 3899cb83748SJack Wang inv_wr.send_flags = 0; 3909cb83748SJack Wang /* Only one key is actually used */ 3919cb83748SJack Wang inv_wr.ex.invalidate_rkey = 3929cb83748SJack Wang le32_to_cpu(rd_msg->desc[0].key); 3939cb83748SJack Wang } else { 3949cb83748SJack Wang WARN_ON_ONCE(1); 3959cb83748SJack Wang need_inval = false; 3969cb83748SJack Wang } 3979cb83748SJack Wang } 3989cb83748SJack Wang } 3999cb83748SJack Wang 4009cb83748SJack Wang if (need_inval && always_invalidate) { 4019cb83748SJack Wang wr = &inv_wr; 4029cb83748SJack Wang inv_wr.next = &rwr.wr; 4039cb83748SJack Wang rwr.wr.next = &imm_wr; 4049cb83748SJack Wang } else if (always_invalidate) { 4059cb83748SJack Wang wr = &rwr.wr; 4069cb83748SJack Wang rwr.wr.next = &imm_wr; 4079cb83748SJack Wang } else if (need_inval) { 4089cb83748SJack Wang wr = &inv_wr; 4099cb83748SJack Wang inv_wr.next = &imm_wr; 4109cb83748SJack Wang } else { 4119cb83748SJack Wang wr = &imm_wr; 4129cb83748SJack Wang } 4139cb83748SJack Wang /* 4149cb83748SJack Wang * From time to time we have to post signalled sends, 4159cb83748SJack Wang * or send queue will fill up and only QP reset can help. 4169cb83748SJack Wang */ 4179cb83748SJack Wang flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ? 4189cb83748SJack Wang 0 : IB_SEND_SIGNALED; 4199cb83748SJack Wang imm = rtrs_to_io_rsp_imm(id->msg_id, errno, need_inval); 4209cb83748SJack Wang imm_wr.next = NULL; 4219cb83748SJack Wang if (always_invalidate) { 4229cb83748SJack Wang struct ib_sge list; 4239cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 4249cb83748SJack Wang 4259cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 4269cb83748SJack Wang rwr.wr.next = &imm_wr; 4279cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 4288537f2deSJack Wang rwr.wr.wr_cqe = &local_reg_cqe; 4299cb83748SJack Wang rwr.wr.num_sge = 0; 4309cb83748SJack Wang rwr.wr.send_flags = 0; 4319cb83748SJack Wang rwr.mr = srv_mr->mr; 4329cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 4339cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 4349cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 4359cb83748SJack Wang msg = srv_mr->iu->buf; 4369cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 4379cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 4389cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 4399cb83748SJack Wang 4409cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 4419cb83748SJack Wang list.length = sizeof(*msg); 4429cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 4439cb83748SJack Wang imm_wr.sg_list = &list; 4449cb83748SJack Wang imm_wr.num_sge = 1; 4459cb83748SJack Wang imm_wr.opcode = IB_WR_SEND_WITH_IMM; 4469cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 4479cb83748SJack Wang srv_mr->iu->dma_addr, 4489cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 4499cb83748SJack Wang } else { 4509cb83748SJack Wang imm_wr.sg_list = NULL; 4519cb83748SJack Wang imm_wr.num_sge = 0; 4529cb83748SJack Wang imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 4539cb83748SJack Wang } 4549cb83748SJack Wang imm_wr.send_flags = flags; 4559cb83748SJack Wang imm_wr.wr_cqe = &io_comp_cqe; 4569cb83748SJack Wang 4579cb83748SJack Wang imm_wr.ex.imm_data = cpu_to_be32(imm); 4589cb83748SJack Wang 4599cb83748SJack Wang err = ib_post_send(id->con->c.qp, wr, NULL); 4609cb83748SJack Wang if (unlikely(err)) 4619cb83748SJack Wang rtrs_err_rl(s, "Posting RDMA-Reply to QP failed, err: %d\n", 4629cb83748SJack Wang err); 4639cb83748SJack Wang 4649cb83748SJack Wang return err; 4659cb83748SJack Wang } 4669cb83748SJack Wang 4679cb83748SJack Wang void close_sess(struct rtrs_srv_sess *sess) 4689cb83748SJack Wang { 469ffea6ad1SGuoqing Jiang if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING)) 4709cb83748SJack Wang queue_work(rtrs_wq, &sess->close_work); 4719cb83748SJack Wang WARN_ON(sess->state != RTRS_SRV_CLOSING); 4729cb83748SJack Wang } 4739cb83748SJack Wang 4749cb83748SJack Wang static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) 4759cb83748SJack Wang { 4769cb83748SJack Wang switch (state) { 4779cb83748SJack Wang case RTRS_SRV_CONNECTING: 4789cb83748SJack Wang return "RTRS_SRV_CONNECTING"; 4799cb83748SJack Wang case RTRS_SRV_CONNECTED: 4809cb83748SJack Wang return "RTRS_SRV_CONNECTED"; 4819cb83748SJack Wang case RTRS_SRV_CLOSING: 4829cb83748SJack Wang return "RTRS_SRV_CLOSING"; 4839cb83748SJack Wang case RTRS_SRV_CLOSED: 4849cb83748SJack Wang return "RTRS_SRV_CLOSED"; 4859cb83748SJack Wang default: 4869cb83748SJack Wang return "UNKNOWN"; 4879cb83748SJack Wang } 4889cb83748SJack Wang } 4899cb83748SJack Wang 4909cb83748SJack Wang /** 4919cb83748SJack Wang * rtrs_srv_resp_rdma() - Finish an RDMA request 4929cb83748SJack Wang * 4939cb83748SJack Wang * @id: Internal RTRS operation identifier 4949cb83748SJack Wang * @status: Response Code sent to the other side for this operation. 4959cb83748SJack Wang * 0 = success, <=0 error 4969cb83748SJack Wang * Context: any 4979cb83748SJack Wang * 4989cb83748SJack Wang * Finish a RDMA operation. A message is sent to the client and the 4999cb83748SJack Wang * corresponding memory areas will be released. 5009cb83748SJack Wang */ 5019cb83748SJack Wang bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) 5029cb83748SJack Wang { 5039cb83748SJack Wang struct rtrs_srv_sess *sess; 5049cb83748SJack Wang struct rtrs_srv_con *con; 5059cb83748SJack Wang struct rtrs_sess *s; 5069cb83748SJack Wang int err; 5079cb83748SJack Wang 5089cb83748SJack Wang if (WARN_ON(!id)) 5099cb83748SJack Wang return true; 5109cb83748SJack Wang 5119cb83748SJack Wang con = id->con; 5129cb83748SJack Wang s = con->c.sess; 5139cb83748SJack Wang sess = to_srv_sess(s); 5149cb83748SJack Wang 5159cb83748SJack Wang id->status = status; 5169cb83748SJack Wang 5179cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 5189cb83748SJack Wang rtrs_err_rl(s, 5199cb83748SJack Wang "Sending I/O response failed, session is disconnected, sess state %s\n", 5209cb83748SJack Wang rtrs_srv_state_str(sess->state)); 5219cb83748SJack Wang goto out; 5229cb83748SJack Wang } 5239cb83748SJack Wang if (always_invalidate) { 5249cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id]; 5259cb83748SJack Wang 5269cb83748SJack Wang ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); 5279cb83748SJack Wang } 5289cb83748SJack Wang if (unlikely(atomic_sub_return(1, 5299cb83748SJack Wang &con->sq_wr_avail) < 0)) { 5309cb83748SJack Wang pr_err("IB send queue full\n"); 5319cb83748SJack Wang atomic_add(1, &con->sq_wr_avail); 5329cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 5339cb83748SJack Wang list_add_tail(&id->wait_list, &con->rsp_wr_wait_list); 5349cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 5359cb83748SJack Wang return false; 5369cb83748SJack Wang } 5379cb83748SJack Wang 5389cb83748SJack Wang if (status || id->dir == WRITE || !id->rd_msg->sg_cnt) 5399cb83748SJack Wang err = send_io_resp_imm(con, id, status); 5409cb83748SJack Wang else 5419cb83748SJack Wang err = rdma_write_sg(id); 5429cb83748SJack Wang 5439cb83748SJack Wang if (unlikely(err)) { 5449cb83748SJack Wang rtrs_err_rl(s, "IO response failed: %d\n", err); 5459cb83748SJack Wang close_sess(sess); 5469cb83748SJack Wang } 5479cb83748SJack Wang out: 5489cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 5499cb83748SJack Wang return true; 5509cb83748SJack Wang } 5519cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_resp_rdma); 5529cb83748SJack Wang 5539cb83748SJack Wang /** 5549cb83748SJack Wang * rtrs_srv_set_sess_priv() - Set private pointer in rtrs_srv. 5559cb83748SJack Wang * @srv: Session pointer 5569cb83748SJack Wang * @priv: The private pointer that is associated with the session. 5579cb83748SJack Wang */ 5589cb83748SJack Wang void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv) 5599cb83748SJack Wang { 5609cb83748SJack Wang srv->priv = priv; 5619cb83748SJack Wang } 5629cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_set_sess_priv); 5639cb83748SJack Wang 5649cb83748SJack Wang static void unmap_cont_bufs(struct rtrs_srv_sess *sess) 5659cb83748SJack Wang { 5669cb83748SJack Wang int i; 5679cb83748SJack Wang 5689cb83748SJack Wang for (i = 0; i < sess->mrs_num; i++) { 5699cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 5709cb83748SJack Wang 5719cb83748SJack Wang srv_mr = &sess->mrs[i]; 5728bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 5739cb83748SJack Wang ib_dereg_mr(srv_mr->mr); 5749cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl, 5759cb83748SJack Wang srv_mr->sgt.nents, DMA_BIDIRECTIONAL); 5769cb83748SJack Wang sg_free_table(&srv_mr->sgt); 5779cb83748SJack Wang } 5789cb83748SJack Wang kfree(sess->mrs); 5799cb83748SJack Wang } 5809cb83748SJack Wang 5819cb83748SJack Wang static int map_cont_bufs(struct rtrs_srv_sess *sess) 5829cb83748SJack Wang { 5839cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 5849cb83748SJack Wang struct rtrs_sess *ss = &sess->s; 5859cb83748SJack Wang int i, mri, err, mrs_num; 5869cb83748SJack Wang unsigned int chunk_bits; 5879cb83748SJack Wang int chunks_per_mr = 1; 5889cb83748SJack Wang 5899cb83748SJack Wang /* 5909cb83748SJack Wang * Here we map queue_depth chunks to MR. Firstly we have to 5919cb83748SJack Wang * figure out how many chunks can we map per MR. 5929cb83748SJack Wang */ 5939cb83748SJack Wang if (always_invalidate) { 5949cb83748SJack Wang /* 5959cb83748SJack Wang * in order to do invalidate for each chunks of memory, we needs 5969cb83748SJack Wang * more memory regions. 5979cb83748SJack Wang */ 5989cb83748SJack Wang mrs_num = srv->queue_depth; 5999cb83748SJack Wang } else { 6009cb83748SJack Wang chunks_per_mr = 6019cb83748SJack Wang sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; 6029cb83748SJack Wang mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr); 6039cb83748SJack Wang chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num); 6049cb83748SJack Wang } 6059cb83748SJack Wang 6069cb83748SJack Wang sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL); 6079cb83748SJack Wang if (!sess->mrs) 6089cb83748SJack Wang return -ENOMEM; 6099cb83748SJack Wang 6109cb83748SJack Wang sess->mrs_num = mrs_num; 6119cb83748SJack Wang 6129cb83748SJack Wang for (mri = 0; mri < mrs_num; mri++) { 6139cb83748SJack Wang struct rtrs_srv_mr *srv_mr = &sess->mrs[mri]; 6149cb83748SJack Wang struct sg_table *sgt = &srv_mr->sgt; 6159cb83748SJack Wang struct scatterlist *s; 6169cb83748SJack Wang struct ib_mr *mr; 6179cb83748SJack Wang int nr, chunks; 6189cb83748SJack Wang 6199cb83748SJack Wang chunks = chunks_per_mr * mri; 6209cb83748SJack Wang if (!always_invalidate) 6219cb83748SJack Wang chunks_per_mr = min_t(int, chunks_per_mr, 6229cb83748SJack Wang srv->queue_depth - chunks); 6239cb83748SJack Wang 6249cb83748SJack Wang err = sg_alloc_table(sgt, chunks_per_mr, GFP_KERNEL); 6259cb83748SJack Wang if (err) 6269cb83748SJack Wang goto err; 6279cb83748SJack Wang 6289cb83748SJack Wang for_each_sg(sgt->sgl, s, chunks_per_mr, i) 6299cb83748SJack Wang sg_set_page(s, srv->chunks[chunks + i], 6309cb83748SJack Wang max_chunk_size, 0); 6319cb83748SJack Wang 6329cb83748SJack Wang nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl, 6339cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6349cb83748SJack Wang if (nr < sgt->nents) { 6359cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6369cb83748SJack Wang goto free_sg; 6379cb83748SJack Wang } 6389cb83748SJack Wang mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, 6399cb83748SJack Wang sgt->nents); 6409cb83748SJack Wang if (IS_ERR(mr)) { 6419cb83748SJack Wang err = PTR_ERR(mr); 6429cb83748SJack Wang goto unmap_sg; 6439cb83748SJack Wang } 6449cb83748SJack Wang nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents, 6459cb83748SJack Wang NULL, max_chunk_size); 646b386cd65SDan Carpenter if (nr < 0 || nr < sgt->nents) { 6479cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6489cb83748SJack Wang goto dereg_mr; 6499cb83748SJack Wang } 6509cb83748SJack Wang 6519cb83748SJack Wang if (always_invalidate) { 6529cb83748SJack Wang srv_mr->iu = rtrs_iu_alloc(1, 6539cb83748SJack Wang sizeof(struct rtrs_msg_rkey_rsp), 6549cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 6559cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_rdma_done); 6569cb83748SJack Wang if (!srv_mr->iu) { 6576b31afceSWei Yongjun err = -ENOMEM; 6586b31afceSWei Yongjun rtrs_err(ss, "rtrs_iu_alloc(), err: %d\n", err); 659f77c4839SGuoqing Jiang goto dereg_mr; 6609cb83748SJack Wang } 6619cb83748SJack Wang } 6629cb83748SJack Wang /* Eventually dma addr for each chunk can be cached */ 6639cb83748SJack Wang for_each_sg(sgt->sgl, s, sgt->orig_nents, i) 6649cb83748SJack Wang sess->dma_addr[chunks + i] = sg_dma_address(s); 6659cb83748SJack Wang 6669cb83748SJack Wang ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 6679cb83748SJack Wang srv_mr->mr = mr; 6689cb83748SJack Wang 6699cb83748SJack Wang continue; 6709cb83748SJack Wang err: 6719cb83748SJack Wang while (mri--) { 6729cb83748SJack Wang srv_mr = &sess->mrs[mri]; 6739cb83748SJack Wang sgt = &srv_mr->sgt; 6749cb83748SJack Wang mr = srv_mr->mr; 6758bd372acSGioh Kim rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); 6769cb83748SJack Wang dereg_mr: 6779cb83748SJack Wang ib_dereg_mr(mr); 6789cb83748SJack Wang unmap_sg: 6799cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl, 6809cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6819cb83748SJack Wang free_sg: 6829cb83748SJack Wang sg_free_table(sgt); 6839cb83748SJack Wang } 6849cb83748SJack Wang kfree(sess->mrs); 6859cb83748SJack Wang 6869cb83748SJack Wang return err; 6879cb83748SJack Wang } 6889cb83748SJack Wang 6899cb83748SJack Wang chunk_bits = ilog2(srv->queue_depth - 1) + 1; 6909cb83748SJack Wang sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); 6919cb83748SJack Wang 6929cb83748SJack Wang return 0; 6939cb83748SJack Wang } 6949cb83748SJack Wang 6959cb83748SJack Wang static void rtrs_srv_hb_err_handler(struct rtrs_con *c) 6969cb83748SJack Wang { 6979cb83748SJack Wang close_sess(to_srv_sess(c->sess)); 6989cb83748SJack Wang } 6999cb83748SJack Wang 7009cb83748SJack Wang static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess) 7019cb83748SJack Wang { 7029cb83748SJack Wang rtrs_init_hb(&sess->s, &io_comp_cqe, 7039cb83748SJack Wang RTRS_HB_INTERVAL_MS, 7049cb83748SJack Wang RTRS_HB_MISSED_MAX, 7059cb83748SJack Wang rtrs_srv_hb_err_handler, 7069cb83748SJack Wang rtrs_wq); 7079cb83748SJack Wang } 7089cb83748SJack Wang 7099cb83748SJack Wang static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess) 7109cb83748SJack Wang { 7119cb83748SJack Wang rtrs_start_hb(&sess->s); 7129cb83748SJack Wang } 7139cb83748SJack Wang 7149cb83748SJack Wang static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess) 7159cb83748SJack Wang { 7169cb83748SJack Wang rtrs_stop_hb(&sess->s); 7179cb83748SJack Wang } 7189cb83748SJack Wang 7199cb83748SJack Wang static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) 7209cb83748SJack Wang { 7219cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 7229cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7239cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7249cb83748SJack Wang struct rtrs_iu *iu; 7259cb83748SJack Wang 7269cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 7278bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 7289cb83748SJack Wang 7299cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 7309cb83748SJack Wang rtrs_err(s, "Sess info response send failed: %s\n", 7319cb83748SJack Wang ib_wc_status_msg(wc->status)); 7329cb83748SJack Wang close_sess(sess); 7339cb83748SJack Wang return; 7349cb83748SJack Wang } 7359cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_SEND); 7369cb83748SJack Wang } 7379cb83748SJack Wang 7389cb83748SJack Wang static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) 7399cb83748SJack Wang { 7409cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7419cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7429cb83748SJack Wang int up; 7439cb83748SJack Wang 7449cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7459cb83748SJack Wang up = ++srv->paths_up; 7469cb83748SJack Wang if (up == 1) 7479cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL); 7489cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7499cb83748SJack Wang 7509cb83748SJack Wang /* Mark session as established */ 7519cb83748SJack Wang sess->established = true; 7529cb83748SJack Wang } 7539cb83748SJack Wang 7549cb83748SJack Wang static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) 7559cb83748SJack Wang { 7569cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7579cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7589cb83748SJack Wang 7599cb83748SJack Wang if (!sess->established) 7609cb83748SJack Wang return; 7619cb83748SJack Wang 7629cb83748SJack Wang sess->established = false; 7639cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7649cb83748SJack Wang WARN_ON(!srv->paths_up); 7659cb83748SJack Wang if (--srv->paths_up == 0) 7669cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv); 7679cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7689cb83748SJack Wang } 7699cb83748SJack Wang 7709cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess); 7719cb83748SJack Wang 7729cb83748SJack Wang static int process_info_req(struct rtrs_srv_con *con, 7739cb83748SJack Wang struct rtrs_msg_info_req *msg) 7749cb83748SJack Wang { 7759cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7769cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7779cb83748SJack Wang struct ib_send_wr *reg_wr = NULL; 7789cb83748SJack Wang struct rtrs_msg_info_rsp *rsp; 7799cb83748SJack Wang struct rtrs_iu *tx_iu; 7809cb83748SJack Wang struct ib_reg_wr *rwr; 7819cb83748SJack Wang int mri, err; 7829cb83748SJack Wang size_t tx_sz; 7839cb83748SJack Wang 7849cb83748SJack Wang err = post_recv_sess(sess); 7859cb83748SJack Wang if (unlikely(err)) { 7869cb83748SJack Wang rtrs_err(s, "post_recv_sess(), err: %d\n", err); 7879cb83748SJack Wang return err; 7889cb83748SJack Wang } 7899cb83748SJack Wang rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL); 7909cb83748SJack Wang if (unlikely(!rwr)) 7919cb83748SJack Wang return -ENOMEM; 7929cb83748SJack Wang strlcpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname)); 7939cb83748SJack Wang 7949cb83748SJack Wang tx_sz = sizeof(*rsp); 7959cb83748SJack Wang tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num; 7969cb83748SJack Wang tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev, 7979cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_info_rsp_done); 7989cb83748SJack Wang if (unlikely(!tx_iu)) { 7999cb83748SJack Wang err = -ENOMEM; 8009cb83748SJack Wang goto rwr_free; 8019cb83748SJack Wang } 8029cb83748SJack Wang 8039cb83748SJack Wang rsp = tx_iu->buf; 8049cb83748SJack Wang rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP); 8059cb83748SJack Wang rsp->sg_cnt = cpu_to_le16(sess->mrs_num); 8069cb83748SJack Wang 8079cb83748SJack Wang for (mri = 0; mri < sess->mrs_num; mri++) { 8089cb83748SJack Wang struct ib_mr *mr = sess->mrs[mri].mr; 8099cb83748SJack Wang 8109cb83748SJack Wang rsp->desc[mri].addr = cpu_to_le64(mr->iova); 8119cb83748SJack Wang rsp->desc[mri].key = cpu_to_le32(mr->rkey); 8129cb83748SJack Wang rsp->desc[mri].len = cpu_to_le32(mr->length); 8139cb83748SJack Wang 8149cb83748SJack Wang /* 8159cb83748SJack Wang * Fill in reg MR request and chain them *backwards* 8169cb83748SJack Wang */ 8179cb83748SJack Wang rwr[mri].wr.next = mri ? &rwr[mri - 1].wr : NULL; 8189cb83748SJack Wang rwr[mri].wr.opcode = IB_WR_REG_MR; 8199cb83748SJack Wang rwr[mri].wr.wr_cqe = &local_reg_cqe; 8209cb83748SJack Wang rwr[mri].wr.num_sge = 0; 821e8ae7ddbSJack Wang rwr[mri].wr.send_flags = 0; 8229cb83748SJack Wang rwr[mri].mr = mr; 8239cb83748SJack Wang rwr[mri].key = mr->rkey; 8249cb83748SJack Wang rwr[mri].access = (IB_ACCESS_LOCAL_WRITE | 8259cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 8269cb83748SJack Wang reg_wr = &rwr[mri].wr; 8279cb83748SJack Wang } 8289cb83748SJack Wang 8299cb83748SJack Wang err = rtrs_srv_create_sess_files(sess); 8309cb83748SJack Wang if (unlikely(err)) 8319cb83748SJack Wang goto iu_free; 8329cb83748SJack Wang kobject_get(&sess->kobj); 8339cb83748SJack Wang get_device(&sess->srv->dev); 8349cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED); 8359cb83748SJack Wang rtrs_srv_start_hb(sess); 8369cb83748SJack Wang 8379cb83748SJack Wang /* 8389cb83748SJack Wang * We do not account number of established connections at the current 8399cb83748SJack Wang * moment, we rely on the client, which should send info request when 8409cb83748SJack Wang * all connections are successfully established. Thus, simply notify 8419cb83748SJack Wang * listener with a proper event if we are the first path. 8429cb83748SJack Wang */ 8439cb83748SJack Wang rtrs_srv_sess_up(sess); 8449cb83748SJack Wang 8459cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, 8469cb83748SJack Wang tx_iu->size, DMA_TO_DEVICE); 8479cb83748SJack Wang 8489cb83748SJack Wang /* Send info response */ 8499cb83748SJack Wang err = rtrs_iu_post_send(&con->c, tx_iu, tx_sz, reg_wr); 8509cb83748SJack Wang if (unlikely(err)) { 8519cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err); 8529cb83748SJack Wang iu_free: 8538bd372acSGioh Kim rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); 8549cb83748SJack Wang } 8559cb83748SJack Wang rwr_free: 8569cb83748SJack Wang kfree(rwr); 8579cb83748SJack Wang 8589cb83748SJack Wang return err; 8599cb83748SJack Wang } 8609cb83748SJack Wang 8619cb83748SJack Wang static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) 8629cb83748SJack Wang { 8639cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 8649cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8659cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 8669cb83748SJack Wang struct rtrs_msg_info_req *msg; 8679cb83748SJack Wang struct rtrs_iu *iu; 8689cb83748SJack Wang int err; 8699cb83748SJack Wang 8709cb83748SJack Wang WARN_ON(con->c.cid); 8719cb83748SJack Wang 8729cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 8739cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 8749cb83748SJack Wang rtrs_err(s, "Sess info request receive failed: %s\n", 8759cb83748SJack Wang ib_wc_status_msg(wc->status)); 8769cb83748SJack Wang goto close; 8779cb83748SJack Wang } 8789cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_RECV); 8799cb83748SJack Wang 8809cb83748SJack Wang if (unlikely(wc->byte_len < sizeof(*msg))) { 8819cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: size %d\n", 8829cb83748SJack Wang wc->byte_len); 8839cb83748SJack Wang goto close; 8849cb83748SJack Wang } 8859cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, 8869cb83748SJack Wang iu->size, DMA_FROM_DEVICE); 8879cb83748SJack Wang msg = iu->buf; 8889cb83748SJack Wang if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ)) { 8899cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: type %d\n", 8909cb83748SJack Wang le16_to_cpu(msg->type)); 8919cb83748SJack Wang goto close; 8929cb83748SJack Wang } 8939cb83748SJack Wang err = process_info_req(con, msg); 8949cb83748SJack Wang if (unlikely(err)) 8959cb83748SJack Wang goto close; 8969cb83748SJack Wang 8979cb83748SJack Wang out: 8988bd372acSGioh Kim rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); 8999cb83748SJack Wang return; 9009cb83748SJack Wang close: 9019cb83748SJack Wang close_sess(sess); 9029cb83748SJack Wang goto out; 9039cb83748SJack Wang } 9049cb83748SJack Wang 9059cb83748SJack Wang static int post_recv_info_req(struct rtrs_srv_con *con) 9069cb83748SJack Wang { 9079cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9089cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9099cb83748SJack Wang struct rtrs_iu *rx_iu; 9109cb83748SJack Wang int err; 9119cb83748SJack Wang 9129cb83748SJack Wang rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), 9139cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 9149cb83748SJack Wang DMA_FROM_DEVICE, rtrs_srv_info_req_done); 9159cb83748SJack Wang if (unlikely(!rx_iu)) 9169cb83748SJack Wang return -ENOMEM; 9179cb83748SJack Wang /* Prepare for getting info response */ 9189cb83748SJack Wang err = rtrs_iu_post_recv(&con->c, rx_iu); 9199cb83748SJack Wang if (unlikely(err)) { 9209cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err); 9218bd372acSGioh Kim rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); 9229cb83748SJack Wang return err; 9239cb83748SJack Wang } 9249cb83748SJack Wang 9259cb83748SJack Wang return 0; 9269cb83748SJack Wang } 9279cb83748SJack Wang 9289cb83748SJack Wang static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) 9299cb83748SJack Wang { 9309cb83748SJack Wang int i, err; 9319cb83748SJack Wang 9329cb83748SJack Wang for (i = 0; i < q_size; i++) { 9339cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 9349cb83748SJack Wang if (unlikely(err)) 9359cb83748SJack Wang return err; 9369cb83748SJack Wang } 9379cb83748SJack Wang 9389cb83748SJack Wang return 0; 9399cb83748SJack Wang } 9409cb83748SJack Wang 9419cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess) 9429cb83748SJack Wang { 9439cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9449cb83748SJack Wang struct rtrs_sess *s = &sess->s; 9459cb83748SJack Wang size_t q_size; 9469cb83748SJack Wang int err, cid; 9479cb83748SJack Wang 9489cb83748SJack Wang for (cid = 0; cid < sess->s.con_num; cid++) { 9499cb83748SJack Wang if (cid == 0) 9509cb83748SJack Wang q_size = SERVICE_CON_QUEUE_DEPTH; 9519cb83748SJack Wang else 9529cb83748SJack Wang q_size = srv->queue_depth; 9539cb83748SJack Wang 9549cb83748SJack Wang err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size); 9559cb83748SJack Wang if (unlikely(err)) { 9569cb83748SJack Wang rtrs_err(s, "post_recv_io(), err: %d\n", err); 9579cb83748SJack Wang return err; 9589cb83748SJack Wang } 9599cb83748SJack Wang } 9609cb83748SJack Wang 9619cb83748SJack Wang return 0; 9629cb83748SJack Wang } 9639cb83748SJack Wang 9649cb83748SJack Wang static void process_read(struct rtrs_srv_con *con, 9659cb83748SJack Wang struct rtrs_msg_rdma_read *msg, 9669cb83748SJack Wang u32 buf_id, u32 off) 9679cb83748SJack Wang { 9689cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9699cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9709cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9719cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 9729cb83748SJack Wang struct rtrs_srv_op *id; 9739cb83748SJack Wang 9749cb83748SJack Wang size_t usr_len, data_len; 9759cb83748SJack Wang void *data; 9769cb83748SJack Wang int ret; 9779cb83748SJack Wang 9789cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 9799cb83748SJack Wang rtrs_err_rl(s, 9809cb83748SJack Wang "Processing read request failed, session is disconnected, sess state %s\n", 9819cb83748SJack Wang rtrs_srv_state_str(sess->state)); 9829cb83748SJack Wang return; 9839cb83748SJack Wang } 9849cb83748SJack Wang if (unlikely(msg->sg_cnt != 1 && msg->sg_cnt != 0)) { 9859cb83748SJack Wang rtrs_err_rl(s, 9869cb83748SJack Wang "Processing read request failed, invalid message\n"); 9879cb83748SJack Wang return; 9889cb83748SJack Wang } 9899cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 9909cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, READ); 9919cb83748SJack Wang id = sess->ops_ids[buf_id]; 9929cb83748SJack Wang id->con = con; 9939cb83748SJack Wang id->dir = READ; 9949cb83748SJack Wang id->msg_id = buf_id; 9959cb83748SJack Wang id->rd_msg = msg; 9969cb83748SJack Wang usr_len = le16_to_cpu(msg->usr_len); 9979cb83748SJack Wang data_len = off - usr_len; 9989cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 9999cb83748SJack Wang ret = ctx->ops.rdma_ev(srv, srv->priv, id, READ, data, data_len, 10009cb83748SJack Wang data + data_len, usr_len); 10019cb83748SJack Wang 10029cb83748SJack Wang if (unlikely(ret)) { 10039cb83748SJack Wang rtrs_err_rl(s, 10049cb83748SJack Wang "Processing read request failed, user module cb reported for msg_id %d, err: %d\n", 10059cb83748SJack Wang buf_id, ret); 10069cb83748SJack Wang goto send_err_msg; 10079cb83748SJack Wang } 10089cb83748SJack Wang 10099cb83748SJack Wang return; 10109cb83748SJack Wang 10119cb83748SJack Wang send_err_msg: 10129cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10139cb83748SJack Wang if (ret < 0) { 10149cb83748SJack Wang rtrs_err_rl(s, 10159cb83748SJack Wang "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n", 10169cb83748SJack Wang buf_id, ret); 10179cb83748SJack Wang close_sess(sess); 10189cb83748SJack Wang } 10199cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10209cb83748SJack Wang } 10219cb83748SJack Wang 10229cb83748SJack Wang static void process_write(struct rtrs_srv_con *con, 10239cb83748SJack Wang struct rtrs_msg_rdma_write *req, 10249cb83748SJack Wang u32 buf_id, u32 off) 10259cb83748SJack Wang { 10269cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10279cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10289cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 10299cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 10309cb83748SJack Wang struct rtrs_srv_op *id; 10319cb83748SJack Wang 10329cb83748SJack Wang size_t data_len, usr_len; 10339cb83748SJack Wang void *data; 10349cb83748SJack Wang int ret; 10359cb83748SJack Wang 10369cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 10379cb83748SJack Wang rtrs_err_rl(s, 10389cb83748SJack Wang "Processing write request failed, session is disconnected, sess state %s\n", 10399cb83748SJack Wang rtrs_srv_state_str(sess->state)); 10409cb83748SJack Wang return; 10419cb83748SJack Wang } 10429cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 10439cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, WRITE); 10449cb83748SJack Wang id = sess->ops_ids[buf_id]; 10459cb83748SJack Wang id->con = con; 10469cb83748SJack Wang id->dir = WRITE; 10479cb83748SJack Wang id->msg_id = buf_id; 10489cb83748SJack Wang 10499cb83748SJack Wang usr_len = le16_to_cpu(req->usr_len); 10509cb83748SJack Wang data_len = off - usr_len; 10519cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 10529cb83748SJack Wang ret = ctx->ops.rdma_ev(srv, srv->priv, id, WRITE, data, data_len, 10539cb83748SJack Wang data + data_len, usr_len); 10549cb83748SJack Wang if (unlikely(ret)) { 10559cb83748SJack Wang rtrs_err_rl(s, 10569cb83748SJack Wang "Processing write request failed, user module callback reports err: %d\n", 10579cb83748SJack Wang ret); 10589cb83748SJack Wang goto send_err_msg; 10599cb83748SJack Wang } 10609cb83748SJack Wang 10619cb83748SJack Wang return; 10629cb83748SJack Wang 10639cb83748SJack Wang send_err_msg: 10649cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10659cb83748SJack Wang if (ret < 0) { 10669cb83748SJack Wang rtrs_err_rl(s, 10679cb83748SJack Wang "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n", 10689cb83748SJack Wang buf_id, ret); 10699cb83748SJack Wang close_sess(sess); 10709cb83748SJack Wang } 10719cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10729cb83748SJack Wang } 10739cb83748SJack Wang 10749cb83748SJack Wang static void process_io_req(struct rtrs_srv_con *con, void *msg, 10759cb83748SJack Wang u32 id, u32 off) 10769cb83748SJack Wang { 10779cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10789cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10799cb83748SJack Wang struct rtrs_msg_rdma_hdr *hdr; 10809cb83748SJack Wang unsigned int type; 10819cb83748SJack Wang 10829cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id], 10839cb83748SJack Wang max_chunk_size, DMA_BIDIRECTIONAL); 10849cb83748SJack Wang hdr = msg; 10859cb83748SJack Wang type = le16_to_cpu(hdr->type); 10869cb83748SJack Wang 10879cb83748SJack Wang switch (type) { 10889cb83748SJack Wang case RTRS_MSG_WRITE: 10899cb83748SJack Wang process_write(con, msg, id, off); 10909cb83748SJack Wang break; 10919cb83748SJack Wang case RTRS_MSG_READ: 10929cb83748SJack Wang process_read(con, msg, id, off); 10939cb83748SJack Wang break; 10949cb83748SJack Wang default: 10959cb83748SJack Wang rtrs_err(s, 10969cb83748SJack Wang "Processing I/O request failed, unknown message type received: 0x%02x\n", 10979cb83748SJack Wang type); 10989cb83748SJack Wang goto err; 10999cb83748SJack Wang } 11009cb83748SJack Wang 11019cb83748SJack Wang return; 11029cb83748SJack Wang 11039cb83748SJack Wang err: 11049cb83748SJack Wang close_sess(sess); 11059cb83748SJack Wang } 11069cb83748SJack Wang 11079cb83748SJack Wang static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) 11089cb83748SJack Wang { 11099cb83748SJack Wang struct rtrs_srv_mr *mr = 11109cb83748SJack Wang container_of(wc->wr_cqe, typeof(*mr), inv_cqe); 11119cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 11129cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11139cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11149cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11159cb83748SJack Wang u32 msg_id, off; 11169cb83748SJack Wang void *data; 11179cb83748SJack Wang 11189cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11199cb83748SJack Wang rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n", 11209cb83748SJack Wang ib_wc_status_msg(wc->status)); 11219cb83748SJack Wang close_sess(sess); 11229cb83748SJack Wang } 11239cb83748SJack Wang msg_id = mr->msg_id; 11249cb83748SJack Wang off = mr->msg_off; 11259cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 11269cb83748SJack Wang process_io_req(con, data, msg_id, off); 11279cb83748SJack Wang } 11289cb83748SJack Wang 11299cb83748SJack Wang static int rtrs_srv_inv_rkey(struct rtrs_srv_con *con, 11309cb83748SJack Wang struct rtrs_srv_mr *mr) 11319cb83748SJack Wang { 11329cb83748SJack Wang struct ib_send_wr wr = { 11339cb83748SJack Wang .opcode = IB_WR_LOCAL_INV, 11349cb83748SJack Wang .wr_cqe = &mr->inv_cqe, 11359cb83748SJack Wang .send_flags = IB_SEND_SIGNALED, 11369cb83748SJack Wang .ex.invalidate_rkey = mr->mr->rkey, 11379cb83748SJack Wang }; 11389cb83748SJack Wang mr->inv_cqe.done = rtrs_srv_inv_rkey_done; 11399cb83748SJack Wang 11409cb83748SJack Wang return ib_post_send(con->c.qp, &wr, NULL); 11419cb83748SJack Wang } 11429cb83748SJack Wang 11439cb83748SJack Wang static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) 11449cb83748SJack Wang { 11459cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11469cb83748SJack Wang while (!list_empty(&con->rsp_wr_wait_list)) { 11479cb83748SJack Wang struct rtrs_srv_op *id; 11489cb83748SJack Wang int ret; 11499cb83748SJack Wang 11509cb83748SJack Wang id = list_entry(con->rsp_wr_wait_list.next, 11519cb83748SJack Wang struct rtrs_srv_op, wait_list); 11529cb83748SJack Wang list_del(&id->wait_list); 11539cb83748SJack Wang 11549cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11559cb83748SJack Wang ret = rtrs_srv_resp_rdma(id, id->status); 11569cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11579cb83748SJack Wang 11589cb83748SJack Wang if (!ret) { 11599cb83748SJack Wang list_add(&id->wait_list, &con->rsp_wr_wait_list); 11609cb83748SJack Wang break; 11619cb83748SJack Wang } 11629cb83748SJack Wang } 11639cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11649cb83748SJack Wang } 11659cb83748SJack Wang 11669cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) 11679cb83748SJack Wang { 11689cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 11699cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11709cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11719cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11729cb83748SJack Wang u32 imm_type, imm_payload; 11739cb83748SJack Wang int err; 11749cb83748SJack Wang 11759cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11769cb83748SJack Wang if (wc->status != IB_WC_WR_FLUSH_ERR) { 11779cb83748SJack Wang rtrs_err(s, 11789cb83748SJack Wang "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n", 11799cb83748SJack Wang ib_wc_status_msg(wc->status), wc->wr_cqe, 11809cb83748SJack Wang wc->opcode, wc->vendor_err, wc->byte_len); 11819cb83748SJack Wang close_sess(sess); 11829cb83748SJack Wang } 11839cb83748SJack Wang return; 11849cb83748SJack Wang } 11859cb83748SJack Wang 11869cb83748SJack Wang switch (wc->opcode) { 11879cb83748SJack Wang case IB_WC_RECV_RDMA_WITH_IMM: 11889cb83748SJack Wang /* 11899cb83748SJack Wang * post_recv() RDMA write completions of IO reqs (read/write) 11909cb83748SJack Wang * and hb 11919cb83748SJack Wang */ 11929cb83748SJack Wang if (WARN_ON(wc->wr_cqe != &io_comp_cqe)) 11939cb83748SJack Wang return; 11949cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 11959cb83748SJack Wang if (unlikely(err)) { 11969cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); 11979cb83748SJack Wang close_sess(sess); 11989cb83748SJack Wang break; 11999cb83748SJack Wang } 12009cb83748SJack Wang rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), 12019cb83748SJack Wang &imm_type, &imm_payload); 12029cb83748SJack Wang if (likely(imm_type == RTRS_IO_REQ_IMM)) { 12039cb83748SJack Wang u32 msg_id, off; 12049cb83748SJack Wang void *data; 12059cb83748SJack Wang 12069cb83748SJack Wang msg_id = imm_payload >> sess->mem_bits; 12079cb83748SJack Wang off = imm_payload & ((1 << sess->mem_bits) - 1); 1208bf1d8edbSDan Carpenter if (unlikely(msg_id >= srv->queue_depth || 1209bf1d8edbSDan Carpenter off >= max_chunk_size)) { 12109cb83748SJack Wang rtrs_err(s, "Wrong msg_id %u, off %u\n", 12119cb83748SJack Wang msg_id, off); 12129cb83748SJack Wang close_sess(sess); 12139cb83748SJack Wang return; 12149cb83748SJack Wang } 12159cb83748SJack Wang if (always_invalidate) { 12169cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[msg_id]; 12179cb83748SJack Wang 12189cb83748SJack Wang mr->msg_off = off; 12199cb83748SJack Wang mr->msg_id = msg_id; 12209cb83748SJack Wang err = rtrs_srv_inv_rkey(con, mr); 12219cb83748SJack Wang if (unlikely(err)) { 12229cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", 12239cb83748SJack Wang err); 12249cb83748SJack Wang close_sess(sess); 12259cb83748SJack Wang break; 12269cb83748SJack Wang } 12279cb83748SJack Wang } else { 12289cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 12299cb83748SJack Wang process_io_req(con, data, msg_id, off); 12309cb83748SJack Wang } 12319cb83748SJack Wang } else if (imm_type == RTRS_HB_MSG_IMM) { 12329cb83748SJack Wang WARN_ON(con->c.cid); 12339cb83748SJack Wang rtrs_send_hb_ack(&sess->s); 12349cb83748SJack Wang } else if (imm_type == RTRS_HB_ACK_IMM) { 12359cb83748SJack Wang WARN_ON(con->c.cid); 12369cb83748SJack Wang sess->s.hb_missed_cnt = 0; 12379cb83748SJack Wang } else { 12389cb83748SJack Wang rtrs_wrn(s, "Unknown IMM type %u\n", imm_type); 12399cb83748SJack Wang } 12409cb83748SJack Wang break; 12419cb83748SJack Wang case IB_WC_RDMA_WRITE: 12429cb83748SJack Wang case IB_WC_SEND: 12439cb83748SJack Wang /* 12449cb83748SJack Wang * post_send() RDMA write completions of IO reqs (read/write) 12459cb83748SJack Wang */ 12469cb83748SJack Wang atomic_add(srv->queue_depth, &con->sq_wr_avail); 12479cb83748SJack Wang 12489cb83748SJack Wang if (unlikely(!list_empty_careful(&con->rsp_wr_wait_list))) 12499cb83748SJack Wang rtrs_rdma_process_wr_wait_list(con); 12509cb83748SJack Wang 12519cb83748SJack Wang break; 12529cb83748SJack Wang default: 12539cb83748SJack Wang rtrs_wrn(s, "Unexpected WC type: %d\n", wc->opcode); 12549cb83748SJack Wang return; 12559cb83748SJack Wang } 12569cb83748SJack Wang } 12579cb83748SJack Wang 12589cb83748SJack Wang /** 12599cb83748SJack Wang * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname. 12609cb83748SJack Wang * @srv: Session 12619cb83748SJack Wang * @sessname: Sessname buffer 12629cb83748SJack Wang * @len: Length of sessname buffer 12639cb83748SJack Wang */ 12649cb83748SJack Wang int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) 12659cb83748SJack Wang { 12669cb83748SJack Wang struct rtrs_srv_sess *sess; 12679cb83748SJack Wang int err = -ENOTCONN; 12689cb83748SJack Wang 12699cb83748SJack Wang mutex_lock(&srv->paths_mutex); 12709cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 12719cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTED) 12729cb83748SJack Wang continue; 12739cb83748SJack Wang strlcpy(sessname, sess->s.sessname, 12749cb83748SJack Wang min_t(size_t, sizeof(sess->s.sessname), len)); 12759cb83748SJack Wang err = 0; 12769cb83748SJack Wang break; 12779cb83748SJack Wang } 12789cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 12799cb83748SJack Wang 12809cb83748SJack Wang return err; 12819cb83748SJack Wang } 12829cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_sess_name); 12839cb83748SJack Wang 12849cb83748SJack Wang /** 12859cb83748SJack Wang * rtrs_srv_get_sess_qdepth() - Get rtrs_srv qdepth. 12869cb83748SJack Wang * @srv: Session 12879cb83748SJack Wang */ 12889cb83748SJack Wang int rtrs_srv_get_queue_depth(struct rtrs_srv *srv) 12899cb83748SJack Wang { 12909cb83748SJack Wang return srv->queue_depth; 12919cb83748SJack Wang } 12929cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_queue_depth); 12939cb83748SJack Wang 12949cb83748SJack Wang static int find_next_bit_ring(struct rtrs_srv_sess *sess) 12959cb83748SJack Wang { 12969cb83748SJack Wang struct ib_device *ib_dev = sess->s.dev->ib_dev; 12979cb83748SJack Wang int v; 12989cb83748SJack Wang 12999cb83748SJack Wang v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask); 13009cb83748SJack Wang if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors) 13019cb83748SJack Wang v = cpumask_first(&cq_affinity_mask); 13029cb83748SJack Wang return v; 13039cb83748SJack Wang } 13049cb83748SJack Wang 13059cb83748SJack Wang static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess) 13069cb83748SJack Wang { 13079cb83748SJack Wang sess->cur_cq_vector = find_next_bit_ring(sess); 13089cb83748SJack Wang 13099cb83748SJack Wang return sess->cur_cq_vector; 13109cb83748SJack Wang } 13119cb83748SJack Wang 131239c2d639SMd Haris Iqbal static void rtrs_srv_dev_release(struct device *dev) 131339c2d639SMd Haris Iqbal { 131439c2d639SMd Haris Iqbal struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev); 131539c2d639SMd Haris Iqbal 131639c2d639SMd Haris Iqbal kfree(srv); 131739c2d639SMd Haris Iqbal } 131839c2d639SMd Haris Iqbal 13199cb83748SJack Wang static void free_srv(struct rtrs_srv *srv) 13209cb83748SJack Wang { 13219cb83748SJack Wang int i; 13229cb83748SJack Wang 13239cb83748SJack Wang WARN_ON(refcount_read(&srv->refcount)); 13249cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 13259cb83748SJack Wang mempool_free(srv->chunks[i], chunk_pool); 13269cb83748SJack Wang kfree(srv->chunks); 13279cb83748SJack Wang mutex_destroy(&srv->paths_mutex); 13289cb83748SJack Wang mutex_destroy(&srv->paths_ev_mutex); 13299cb83748SJack Wang /* last put to release the srv structure */ 13309cb83748SJack Wang put_device(&srv->dev); 13319cb83748SJack Wang } 13329cb83748SJack Wang 13339cb83748SJack Wang static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx, 13349cb83748SJack Wang const uuid_t *paths_uuid) 13359cb83748SJack Wang { 13369cb83748SJack Wang struct rtrs_srv *srv; 1337d715ff8aSGuoqing Jiang int i; 13389cb83748SJack Wang 13399cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 1340d715ff8aSGuoqing Jiang list_for_each_entry(srv, &ctx->srv_list, ctx_list) { 1341d715ff8aSGuoqing Jiang if (uuid_equal(&srv->paths_uuid, paths_uuid) && 1342d715ff8aSGuoqing Jiang refcount_inc_not_zero(&srv->refcount)) { 1343d715ff8aSGuoqing Jiang mutex_unlock(&ctx->srv_mutex); 1344d715ff8aSGuoqing Jiang return srv; 1345d715ff8aSGuoqing Jiang } 1346d715ff8aSGuoqing Jiang } 1347d715ff8aSGuoqing Jiang 1348d715ff8aSGuoqing Jiang /* need to allocate a new srv */ 1349d715ff8aSGuoqing Jiang srv = kzalloc(sizeof(*srv), GFP_KERNEL); 1350d715ff8aSGuoqing Jiang if (!srv) { 1351d715ff8aSGuoqing Jiang mutex_unlock(&ctx->srv_mutex); 1352d715ff8aSGuoqing Jiang return NULL; 1353d715ff8aSGuoqing Jiang } 1354d715ff8aSGuoqing Jiang 1355d715ff8aSGuoqing Jiang INIT_LIST_HEAD(&srv->paths_list); 1356d715ff8aSGuoqing Jiang mutex_init(&srv->paths_mutex); 1357d715ff8aSGuoqing Jiang mutex_init(&srv->paths_ev_mutex); 1358d715ff8aSGuoqing Jiang uuid_copy(&srv->paths_uuid, paths_uuid); 1359d715ff8aSGuoqing Jiang srv->queue_depth = sess_queue_depth; 1360d715ff8aSGuoqing Jiang srv->ctx = ctx; 1361d715ff8aSGuoqing Jiang device_initialize(&srv->dev); 1362d715ff8aSGuoqing Jiang srv->dev.release = rtrs_srv_dev_release; 1363d715ff8aSGuoqing Jiang list_add(&srv->ctx_list, &ctx->srv_list); 13649cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 13659cb83748SJack Wang 1366d715ff8aSGuoqing Jiang srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks), 1367d715ff8aSGuoqing Jiang GFP_KERNEL); 1368d715ff8aSGuoqing Jiang if (!srv->chunks) 1369d715ff8aSGuoqing Jiang goto err_free_srv; 1370d715ff8aSGuoqing Jiang 1371d715ff8aSGuoqing Jiang for (i = 0; i < srv->queue_depth; i++) { 1372d715ff8aSGuoqing Jiang srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL); 1373d715ff8aSGuoqing Jiang if (!srv->chunks[i]) 1374d715ff8aSGuoqing Jiang goto err_free_chunks; 1375d715ff8aSGuoqing Jiang } 1376d715ff8aSGuoqing Jiang refcount_set(&srv->refcount, 1); 1377d715ff8aSGuoqing Jiang 13789cb83748SJack Wang return srv; 1379d715ff8aSGuoqing Jiang 1380d715ff8aSGuoqing Jiang err_free_chunks: 1381d715ff8aSGuoqing Jiang while (i--) 1382d715ff8aSGuoqing Jiang mempool_free(srv->chunks[i], chunk_pool); 1383d715ff8aSGuoqing Jiang kfree(srv->chunks); 1384d715ff8aSGuoqing Jiang 1385d715ff8aSGuoqing Jiang err_free_srv: 1386d715ff8aSGuoqing Jiang kfree(srv); 1387d715ff8aSGuoqing Jiang return NULL; 13889cb83748SJack Wang } 13899cb83748SJack Wang 13909cb83748SJack Wang static void put_srv(struct rtrs_srv *srv) 13919cb83748SJack Wang { 13929cb83748SJack Wang if (refcount_dec_and_test(&srv->refcount)) { 13939cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 13949cb83748SJack Wang 13959cb83748SJack Wang WARN_ON(srv->dev.kobj.state_in_sysfs); 13969cb83748SJack Wang 13979cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 13989cb83748SJack Wang list_del(&srv->ctx_list); 13999cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 14009cb83748SJack Wang free_srv(srv); 14019cb83748SJack Wang } 14029cb83748SJack Wang } 14039cb83748SJack Wang 14049cb83748SJack Wang static void __add_path_to_srv(struct rtrs_srv *srv, 14059cb83748SJack Wang struct rtrs_srv_sess *sess) 14069cb83748SJack Wang { 14079cb83748SJack Wang list_add_tail(&sess->s.entry, &srv->paths_list); 14089cb83748SJack Wang srv->paths_num++; 14099cb83748SJack Wang WARN_ON(srv->paths_num >= MAX_PATHS_NUM); 14109cb83748SJack Wang } 14119cb83748SJack Wang 14129cb83748SJack Wang static void del_path_from_srv(struct rtrs_srv_sess *sess) 14139cb83748SJack Wang { 14149cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 14159cb83748SJack Wang 14169cb83748SJack Wang if (WARN_ON(!srv)) 14179cb83748SJack Wang return; 14189cb83748SJack Wang 14199cb83748SJack Wang mutex_lock(&srv->paths_mutex); 14209cb83748SJack Wang list_del(&sess->s.entry); 14219cb83748SJack Wang WARN_ON(!srv->paths_num); 14229cb83748SJack Wang srv->paths_num--; 14239cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 14249cb83748SJack Wang } 14259cb83748SJack Wang 14269cb83748SJack Wang /* return true if addresses are the same, error other wise */ 14279cb83748SJack Wang static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) 14289cb83748SJack Wang { 14299cb83748SJack Wang switch (a->sa_family) { 14309cb83748SJack Wang case AF_IB: 14319cb83748SJack Wang return memcmp(&((struct sockaddr_ib *)a)->sib_addr, 14329cb83748SJack Wang &((struct sockaddr_ib *)b)->sib_addr, 14339cb83748SJack Wang sizeof(struct ib_addr)) && 14349cb83748SJack Wang (b->sa_family == AF_IB); 14359cb83748SJack Wang case AF_INET: 14369cb83748SJack Wang return memcmp(&((struct sockaddr_in *)a)->sin_addr, 14379cb83748SJack Wang &((struct sockaddr_in *)b)->sin_addr, 14389cb83748SJack Wang sizeof(struct in_addr)) && 14399cb83748SJack Wang (b->sa_family == AF_INET); 14409cb83748SJack Wang case AF_INET6: 14419cb83748SJack Wang return memcmp(&((struct sockaddr_in6 *)a)->sin6_addr, 14429cb83748SJack Wang &((struct sockaddr_in6 *)b)->sin6_addr, 14439cb83748SJack Wang sizeof(struct in6_addr)) && 14449cb83748SJack Wang (b->sa_family == AF_INET6); 14459cb83748SJack Wang default: 14469cb83748SJack Wang return -ENOENT; 14479cb83748SJack Wang } 14489cb83748SJack Wang } 14499cb83748SJack Wang 14509cb83748SJack Wang static bool __is_path_w_addr_exists(struct rtrs_srv *srv, 14519cb83748SJack Wang struct rdma_addr *addr) 14529cb83748SJack Wang { 14539cb83748SJack Wang struct rtrs_srv_sess *sess; 14549cb83748SJack Wang 14559cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 14569cb83748SJack Wang if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr, 14579cb83748SJack Wang (struct sockaddr *)&addr->dst_addr) && 14589cb83748SJack Wang !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr, 14599cb83748SJack Wang (struct sockaddr *)&addr->src_addr)) 14609cb83748SJack Wang return true; 14619cb83748SJack Wang 14629cb83748SJack Wang return false; 14639cb83748SJack Wang } 14649cb83748SJack Wang 14659cb83748SJack Wang static void free_sess(struct rtrs_srv_sess *sess) 14669cb83748SJack Wang { 14679cb83748SJack Wang if (sess->kobj.state_in_sysfs) 14689cb83748SJack Wang kobject_put(&sess->kobj); 14699cb83748SJack Wang else 14709cb83748SJack Wang kfree(sess); 14719cb83748SJack Wang } 14729cb83748SJack Wang 14739cb83748SJack Wang static void rtrs_srv_close_work(struct work_struct *work) 14749cb83748SJack Wang { 14759cb83748SJack Wang struct rtrs_srv_sess *sess; 14769cb83748SJack Wang struct rtrs_srv_con *con; 14779cb83748SJack Wang int i; 14789cb83748SJack Wang 14799cb83748SJack Wang sess = container_of(work, typeof(*sess), close_work); 14809cb83748SJack Wang 14819cb83748SJack Wang rtrs_srv_destroy_sess_files(sess); 14829cb83748SJack Wang rtrs_srv_stop_hb(sess); 14839cb83748SJack Wang 14849cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 14859cb83748SJack Wang if (!sess->s.con[i]) 14869cb83748SJack Wang continue; 14879cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 14889cb83748SJack Wang rdma_disconnect(con->c.cm_id); 14899cb83748SJack Wang ib_drain_qp(con->c.qp); 14909cb83748SJack Wang } 14919cb83748SJack Wang /* Wait for all inflights */ 14929cb83748SJack Wang rtrs_srv_wait_ops_ids(sess); 14939cb83748SJack Wang 14949cb83748SJack Wang /* Notify upper layer if we are the last path */ 14959cb83748SJack Wang rtrs_srv_sess_down(sess); 14969cb83748SJack Wang 14979cb83748SJack Wang unmap_cont_bufs(sess); 14989cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 14999cb83748SJack Wang 15009cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 15019cb83748SJack Wang if (!sess->s.con[i]) 15029cb83748SJack Wang continue; 15039cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 15049cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 15059cb83748SJack Wang rdma_destroy_id(con->c.cm_id); 15069cb83748SJack Wang kfree(con); 15079cb83748SJack Wang } 15089cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 15099cb83748SJack Wang 15109cb83748SJack Wang del_path_from_srv(sess); 15119cb83748SJack Wang put_srv(sess->srv); 15129cb83748SJack Wang sess->srv = NULL; 15139cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CLOSED); 15149cb83748SJack Wang 15159cb83748SJack Wang kfree(sess->dma_addr); 15169cb83748SJack Wang kfree(sess->s.con); 15179cb83748SJack Wang free_sess(sess); 15189cb83748SJack Wang } 15199cb83748SJack Wang 15209cb83748SJack Wang static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess, 15219cb83748SJack Wang struct rdma_cm_id *cm_id) 15229cb83748SJack Wang { 15239cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15249cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15259cb83748SJack Wang struct rdma_conn_param param; 15269cb83748SJack Wang int err; 15279cb83748SJack Wang 15289cb83748SJack Wang param = (struct rdma_conn_param) { 15299cb83748SJack Wang .rnr_retry_count = 7, 15309cb83748SJack Wang .private_data = &msg, 15319cb83748SJack Wang .private_data_len = sizeof(msg), 15329cb83748SJack Wang }; 15339cb83748SJack Wang 15349cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15359cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15369cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15379cb83748SJack Wang .queue_depth = cpu_to_le16(srv->queue_depth), 15389cb83748SJack Wang .max_io_size = cpu_to_le32(max_chunk_size - MAX_HDR_SIZE), 15399cb83748SJack Wang .max_hdr_size = cpu_to_le32(MAX_HDR_SIZE), 15409cb83748SJack Wang }; 15419cb83748SJack Wang 15429cb83748SJack Wang if (always_invalidate) 15439cb83748SJack Wang msg.flags = cpu_to_le32(RTRS_MSG_NEW_RKEY_F); 15449cb83748SJack Wang 15459cb83748SJack Wang err = rdma_accept(cm_id, ¶m); 15469cb83748SJack Wang if (err) 15479cb83748SJack Wang pr_err("rdma_accept(), err: %d\n", err); 15489cb83748SJack Wang 15499cb83748SJack Wang return err; 15509cb83748SJack Wang } 15519cb83748SJack Wang 15529cb83748SJack Wang static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) 15539cb83748SJack Wang { 15549cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15559cb83748SJack Wang int err; 15569cb83748SJack Wang 15579cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15589cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15599cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15609cb83748SJack Wang .errno = cpu_to_le16(errno), 15619cb83748SJack Wang }; 15629cb83748SJack Wang 15638094ba0aSLeon Romanovsky err = rdma_reject(cm_id, &msg, sizeof(msg), IB_CM_REJ_CONSUMER_DEFINED); 15649cb83748SJack Wang if (err) 15659cb83748SJack Wang pr_err("rdma_reject(), err: %d\n", err); 15669cb83748SJack Wang 15679cb83748SJack Wang /* Bounce errno back */ 15689cb83748SJack Wang return errno; 15699cb83748SJack Wang } 15709cb83748SJack Wang 15719cb83748SJack Wang static struct rtrs_srv_sess * 15729cb83748SJack Wang __find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid) 15739cb83748SJack Wang { 15749cb83748SJack Wang struct rtrs_srv_sess *sess; 15759cb83748SJack Wang 15769cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 15779cb83748SJack Wang if (uuid_equal(&sess->s.uuid, sess_uuid)) 15789cb83748SJack Wang return sess; 15799cb83748SJack Wang } 15809cb83748SJack Wang 15819cb83748SJack Wang return NULL; 15829cb83748SJack Wang } 15839cb83748SJack Wang 15849cb83748SJack Wang static int create_con(struct rtrs_srv_sess *sess, 15859cb83748SJack Wang struct rdma_cm_id *cm_id, 15869cb83748SJack Wang unsigned int cid) 15879cb83748SJack Wang { 15889cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15899cb83748SJack Wang struct rtrs_sess *s = &sess->s; 15909cb83748SJack Wang struct rtrs_srv_con *con; 15919cb83748SJack Wang 15927490fd1fSJack Wang u32 cq_size, wr_queue_size; 15939cb83748SJack Wang int err, cq_vector; 15949cb83748SJack Wang 15959cb83748SJack Wang con = kzalloc(sizeof(*con), GFP_KERNEL); 15969cb83748SJack Wang if (!con) { 15979cb83748SJack Wang err = -ENOMEM; 15989cb83748SJack Wang goto err; 15999cb83748SJack Wang } 16009cb83748SJack Wang 16019cb83748SJack Wang spin_lock_init(&con->rsp_wr_wait_lock); 16029cb83748SJack Wang INIT_LIST_HEAD(&con->rsp_wr_wait_list); 16039cb83748SJack Wang con->c.cm_id = cm_id; 16049cb83748SJack Wang con->c.sess = &sess->s; 16059cb83748SJack Wang con->c.cid = cid; 1606*6f5d1b30SJack Wang atomic_set(&con->wr_cnt, 1); 16079cb83748SJack Wang 16089cb83748SJack Wang if (con->c.cid == 0) { 16099cb83748SJack Wang /* 16109cb83748SJack Wang * All receive and all send (each requiring invalidate) 16119cb83748SJack Wang * + 2 for drain and heartbeat 16129cb83748SJack Wang */ 16139cb83748SJack Wang wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2; 16149cb83748SJack Wang cq_size = wr_queue_size; 16159cb83748SJack Wang } else { 16169cb83748SJack Wang /* 16179cb83748SJack Wang * If we have all receive requests posted and 16189cb83748SJack Wang * all write requests posted and each read request 16199cb83748SJack Wang * requires an invalidate request + drain 16209cb83748SJack Wang * and qp gets into error state. 16219cb83748SJack Wang */ 16229cb83748SJack Wang cq_size = srv->queue_depth * 3 + 1; 16239cb83748SJack Wang /* 16249cb83748SJack Wang * In theory we might have queue_depth * 32 16259cb83748SJack Wang * outstanding requests if an unsafe global key is used 16269cb83748SJack Wang * and we have queue_depth read requests each consisting 16279cb83748SJack Wang * of 32 different addresses. div 3 for mlx5. 16289cb83748SJack Wang */ 16299cb83748SJack Wang wr_queue_size = sess->s.dev->ib_dev->attrs.max_qp_wr / 3; 16309cb83748SJack Wang } 16319cb83748SJack Wang atomic_set(&con->sq_wr_avail, wr_queue_size); 16329cb83748SJack Wang cq_vector = rtrs_srv_get_next_cq_vector(sess); 16339cb83748SJack Wang 16349cb83748SJack Wang /* TODO: SOFTIRQ can be faster, but be careful with softirq context */ 16359cb83748SJack Wang err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size, 16367490fd1fSJack Wang wr_queue_size, wr_queue_size, 16377490fd1fSJack Wang IB_POLL_WORKQUEUE); 16389cb83748SJack Wang if (err) { 16399cb83748SJack Wang rtrs_err(s, "rtrs_cq_qp_create(), err: %d\n", err); 16409cb83748SJack Wang goto free_con; 16419cb83748SJack Wang } 16429cb83748SJack Wang if (con->c.cid == 0) { 16439cb83748SJack Wang err = post_recv_info_req(con); 16449cb83748SJack Wang if (err) 16459cb83748SJack Wang goto free_cqqp; 16469cb83748SJack Wang } 16479cb83748SJack Wang WARN_ON(sess->s.con[cid]); 16489cb83748SJack Wang sess->s.con[cid] = &con->c; 16499cb83748SJack Wang 16509cb83748SJack Wang /* 16519cb83748SJack Wang * Change context from server to current connection. The other 16529cb83748SJack Wang * way is to use cm_id->qp->qp_context, which does not work on OFED. 16539cb83748SJack Wang */ 16549cb83748SJack Wang cm_id->context = &con->c; 16559cb83748SJack Wang 16569cb83748SJack Wang return 0; 16579cb83748SJack Wang 16589cb83748SJack Wang free_cqqp: 16599cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 16609cb83748SJack Wang free_con: 16619cb83748SJack Wang kfree(con); 16629cb83748SJack Wang 16639cb83748SJack Wang err: 16649cb83748SJack Wang return err; 16659cb83748SJack Wang } 16669cb83748SJack Wang 16679cb83748SJack Wang static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, 16689cb83748SJack Wang struct rdma_cm_id *cm_id, 16699cb83748SJack Wang unsigned int con_num, 16709cb83748SJack Wang unsigned int recon_cnt, 16719cb83748SJack Wang const uuid_t *uuid) 16729cb83748SJack Wang { 16739cb83748SJack Wang struct rtrs_srv_sess *sess; 16749cb83748SJack Wang int err = -ENOMEM; 16759cb83748SJack Wang 16769cb83748SJack Wang if (srv->paths_num >= MAX_PATHS_NUM) { 16779cb83748SJack Wang err = -ECONNRESET; 16789cb83748SJack Wang goto err; 16799cb83748SJack Wang } 16809cb83748SJack Wang if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) { 16819cb83748SJack Wang err = -EEXIST; 16829cb83748SJack Wang pr_err("Path with same addr exists\n"); 16839cb83748SJack Wang goto err; 16849cb83748SJack Wang } 16859cb83748SJack Wang sess = kzalloc(sizeof(*sess), GFP_KERNEL); 16869cb83748SJack Wang if (!sess) 16879cb83748SJack Wang goto err; 16889cb83748SJack Wang 16899cb83748SJack Wang sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); 16909cb83748SJack Wang if (!sess->stats) 16919cb83748SJack Wang goto err_free_sess; 16929cb83748SJack Wang 16939cb83748SJack Wang sess->stats->sess = sess; 16949cb83748SJack Wang 16959cb83748SJack Wang sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr), 16969cb83748SJack Wang GFP_KERNEL); 16979cb83748SJack Wang if (!sess->dma_addr) 16989cb83748SJack Wang goto err_free_stats; 16999cb83748SJack Wang 17009cb83748SJack Wang sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL); 17019cb83748SJack Wang if (!sess->s.con) 17029cb83748SJack Wang goto err_free_dma_addr; 17039cb83748SJack Wang 17049cb83748SJack Wang sess->state = RTRS_SRV_CONNECTING; 17059cb83748SJack Wang sess->srv = srv; 17069cb83748SJack Wang sess->cur_cq_vector = -1; 17079cb83748SJack Wang sess->s.dst_addr = cm_id->route.addr.dst_addr; 17089cb83748SJack Wang sess->s.src_addr = cm_id->route.addr.src_addr; 17099cb83748SJack Wang sess->s.con_num = con_num; 17109cb83748SJack Wang sess->s.recon_cnt = recon_cnt; 17119cb83748SJack Wang uuid_copy(&sess->s.uuid, uuid); 17129cb83748SJack Wang spin_lock_init(&sess->state_lock); 17139cb83748SJack Wang INIT_WORK(&sess->close_work, rtrs_srv_close_work); 17149cb83748SJack Wang rtrs_srv_init_hb(sess); 17159cb83748SJack Wang 17169cb83748SJack Wang sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); 17179cb83748SJack Wang if (!sess->s.dev) { 17189cb83748SJack Wang err = -ENOMEM; 17199cb83748SJack Wang goto err_free_con; 17209cb83748SJack Wang } 17219cb83748SJack Wang err = map_cont_bufs(sess); 17229cb83748SJack Wang if (err) 17239cb83748SJack Wang goto err_put_dev; 17249cb83748SJack Wang 17259cb83748SJack Wang err = rtrs_srv_alloc_ops_ids(sess); 17269cb83748SJack Wang if (err) 17279cb83748SJack Wang goto err_unmap_bufs; 17289cb83748SJack Wang 17299cb83748SJack Wang __add_path_to_srv(srv, sess); 17309cb83748SJack Wang 17319cb83748SJack Wang return sess; 17329cb83748SJack Wang 17339cb83748SJack Wang err_unmap_bufs: 17349cb83748SJack Wang unmap_cont_bufs(sess); 17359cb83748SJack Wang err_put_dev: 17369cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 17379cb83748SJack Wang err_free_con: 17389cb83748SJack Wang kfree(sess->s.con); 17399cb83748SJack Wang err_free_dma_addr: 17409cb83748SJack Wang kfree(sess->dma_addr); 17419cb83748SJack Wang err_free_stats: 17429cb83748SJack Wang kfree(sess->stats); 17439cb83748SJack Wang err_free_sess: 17449cb83748SJack Wang kfree(sess); 17459cb83748SJack Wang err: 17469cb83748SJack Wang return ERR_PTR(err); 17479cb83748SJack Wang } 17489cb83748SJack Wang 17499cb83748SJack Wang static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, 17509cb83748SJack Wang const struct rtrs_msg_conn_req *msg, 17519cb83748SJack Wang size_t len) 17529cb83748SJack Wang { 17539cb83748SJack Wang struct rtrs_srv_ctx *ctx = cm_id->context; 17549cb83748SJack Wang struct rtrs_srv_sess *sess; 17559cb83748SJack Wang struct rtrs_srv *srv; 17569cb83748SJack Wang 17579cb83748SJack Wang u16 version, con_num, cid; 17589cb83748SJack Wang u16 recon_cnt; 17599cb83748SJack Wang int err; 17609cb83748SJack Wang 17619cb83748SJack Wang if (len < sizeof(*msg)) { 17629cb83748SJack Wang pr_err("Invalid RTRS connection request\n"); 17639cb83748SJack Wang goto reject_w_econnreset; 17649cb83748SJack Wang } 17659cb83748SJack Wang if (le16_to_cpu(msg->magic) != RTRS_MAGIC) { 17669cb83748SJack Wang pr_err("Invalid RTRS magic\n"); 17679cb83748SJack Wang goto reject_w_econnreset; 17689cb83748SJack Wang } 17699cb83748SJack Wang version = le16_to_cpu(msg->version); 17709cb83748SJack Wang if (version >> 8 != RTRS_PROTO_VER_MAJOR) { 17719cb83748SJack Wang pr_err("Unsupported major RTRS version: %d, expected %d\n", 17729cb83748SJack Wang version >> 8, RTRS_PROTO_VER_MAJOR); 17739cb83748SJack Wang goto reject_w_econnreset; 17749cb83748SJack Wang } 17759cb83748SJack Wang con_num = le16_to_cpu(msg->cid_num); 17769cb83748SJack Wang if (con_num > 4096) { 17779cb83748SJack Wang /* Sanity check */ 17789cb83748SJack Wang pr_err("Too many connections requested: %d\n", con_num); 17799cb83748SJack Wang goto reject_w_econnreset; 17809cb83748SJack Wang } 17819cb83748SJack Wang cid = le16_to_cpu(msg->cid); 17829cb83748SJack Wang if (cid >= con_num) { 17839cb83748SJack Wang /* Sanity check */ 17849cb83748SJack Wang pr_err("Incorrect cid: %d >= %d\n", cid, con_num); 17859cb83748SJack Wang goto reject_w_econnreset; 17869cb83748SJack Wang } 17879cb83748SJack Wang recon_cnt = le16_to_cpu(msg->recon_cnt); 17889cb83748SJack Wang srv = get_or_create_srv(ctx, &msg->paths_uuid); 1789d715ff8aSGuoqing Jiang /* 1790d715ff8aSGuoqing Jiang * "refcount == 0" happens if a previous thread calls get_or_create_srv 1791d715ff8aSGuoqing Jiang * allocate srv, but chunks of srv are not allocated yet. 1792d715ff8aSGuoqing Jiang */ 1793d715ff8aSGuoqing Jiang if (!srv || refcount_read(&srv->refcount) == 0) { 17949cb83748SJack Wang err = -ENOMEM; 17959cb83748SJack Wang goto reject_w_err; 17969cb83748SJack Wang } 17979cb83748SJack Wang mutex_lock(&srv->paths_mutex); 17989cb83748SJack Wang sess = __find_sess(srv, &msg->sess_uuid); 17999cb83748SJack Wang if (sess) { 18009cb83748SJack Wang struct rtrs_sess *s = &sess->s; 18019cb83748SJack Wang 18029cb83748SJack Wang /* Session already holds a reference */ 18039cb83748SJack Wang put_srv(srv); 18049cb83748SJack Wang 18059cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTING) { 18069cb83748SJack Wang rtrs_err(s, "Session in wrong state: %s\n", 18079cb83748SJack Wang rtrs_srv_state_str(sess->state)); 18089cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18099cb83748SJack Wang goto reject_w_econnreset; 18109cb83748SJack Wang } 18119cb83748SJack Wang /* 18129cb83748SJack Wang * Sanity checks 18139cb83748SJack Wang */ 1814e172037bSMd Haris Iqbal if (con_num != s->con_num || cid >= s->con_num) { 18159cb83748SJack Wang rtrs_err(s, "Incorrect request: %d, %d\n", 18169cb83748SJack Wang cid, con_num); 18179cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18189cb83748SJack Wang goto reject_w_econnreset; 18199cb83748SJack Wang } 1820e172037bSMd Haris Iqbal if (s->con[cid]) { 18219cb83748SJack Wang rtrs_err(s, "Connection already exists: %d\n", 18229cb83748SJack Wang cid); 18239cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18249cb83748SJack Wang goto reject_w_econnreset; 18259cb83748SJack Wang } 18269cb83748SJack Wang } else { 18279cb83748SJack Wang sess = __alloc_sess(srv, cm_id, con_num, recon_cnt, 18289cb83748SJack Wang &msg->sess_uuid); 18299cb83748SJack Wang if (IS_ERR(sess)) { 18309cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18319cb83748SJack Wang put_srv(srv); 18329cb83748SJack Wang err = PTR_ERR(sess); 18339cb83748SJack Wang goto reject_w_err; 18349cb83748SJack Wang } 18359cb83748SJack Wang } 18369cb83748SJack Wang err = create_con(sess, cm_id, cid); 18379cb83748SJack Wang if (err) { 18389cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18399cb83748SJack Wang /* 18409cb83748SJack Wang * Since session has other connections we follow normal way 18419cb83748SJack Wang * through workqueue, but still return an error to tell cma.c 18429cb83748SJack Wang * to call rdma_destroy_id() for current connection. 18439cb83748SJack Wang */ 18449cb83748SJack Wang goto close_and_return_err; 18459cb83748SJack Wang } 18469cb83748SJack Wang err = rtrs_rdma_do_accept(sess, cm_id); 18479cb83748SJack Wang if (err) { 18489cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18499cb83748SJack Wang /* 18509cb83748SJack Wang * Since current connection was successfully added to the 18519cb83748SJack Wang * session we follow normal way through workqueue to close the 18529cb83748SJack Wang * session, thus return 0 to tell cma.c we call 18539cb83748SJack Wang * rdma_destroy_id() ourselves. 18549cb83748SJack Wang */ 18559cb83748SJack Wang err = 0; 18569cb83748SJack Wang goto close_and_return_err; 18579cb83748SJack Wang } 18589cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18599cb83748SJack Wang 18609cb83748SJack Wang return 0; 18619cb83748SJack Wang 18629cb83748SJack Wang reject_w_err: 18639cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, err); 18649cb83748SJack Wang 18659cb83748SJack Wang reject_w_econnreset: 18669cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, -ECONNRESET); 18679cb83748SJack Wang 18689cb83748SJack Wang close_and_return_err: 18699cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 187099f0c380SJack Wang close_sess(sess); 18719cb83748SJack Wang 18729cb83748SJack Wang return err; 18739cb83748SJack Wang } 18749cb83748SJack Wang 18759cb83748SJack Wang static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, 18769cb83748SJack Wang struct rdma_cm_event *ev) 18779cb83748SJack Wang { 18789cb83748SJack Wang struct rtrs_srv_sess *sess = NULL; 18799cb83748SJack Wang struct rtrs_sess *s = NULL; 18809cb83748SJack Wang 18819cb83748SJack Wang if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) { 18829cb83748SJack Wang struct rtrs_con *c = cm_id->context; 18839cb83748SJack Wang 18849cb83748SJack Wang s = c->sess; 18859cb83748SJack Wang sess = to_srv_sess(s); 18869cb83748SJack Wang } 18879cb83748SJack Wang 18889cb83748SJack Wang switch (ev->event) { 18899cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_REQUEST: 18909cb83748SJack Wang /* 18919cb83748SJack Wang * In case of error cma.c will destroy cm_id, 18929cb83748SJack Wang * see cma_process_remove() 18939cb83748SJack Wang */ 18949cb83748SJack Wang return rtrs_rdma_connect(cm_id, ev->param.conn.private_data, 18959cb83748SJack Wang ev->param.conn.private_data_len); 18969cb83748SJack Wang case RDMA_CM_EVENT_ESTABLISHED: 18979cb83748SJack Wang /* Nothing here */ 18989cb83748SJack Wang break; 18999cb83748SJack Wang case RDMA_CM_EVENT_REJECTED: 19009cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_ERROR: 19019cb83748SJack Wang case RDMA_CM_EVENT_UNREACHABLE: 19029cb83748SJack Wang rtrs_err(s, "CM error (CM event: %s, err: %d)\n", 19039cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19049cb83748SJack Wang close_sess(sess); 19059cb83748SJack Wang break; 19069cb83748SJack Wang case RDMA_CM_EVENT_DISCONNECTED: 19079cb83748SJack Wang case RDMA_CM_EVENT_ADDR_CHANGE: 19089cb83748SJack Wang case RDMA_CM_EVENT_TIMEWAIT_EXIT: 19099cb83748SJack Wang close_sess(sess); 19109cb83748SJack Wang break; 19119cb83748SJack Wang case RDMA_CM_EVENT_DEVICE_REMOVAL: 19129cb83748SJack Wang close_sess(sess); 19139cb83748SJack Wang break; 19149cb83748SJack Wang default: 19159cb83748SJack Wang pr_err("Ignoring unexpected CM event %s, err %d\n", 19169cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19179cb83748SJack Wang break; 19189cb83748SJack Wang } 19199cb83748SJack Wang 19209cb83748SJack Wang return 0; 19219cb83748SJack Wang } 19229cb83748SJack Wang 19239cb83748SJack Wang static struct rdma_cm_id *rtrs_srv_cm_init(struct rtrs_srv_ctx *ctx, 19249cb83748SJack Wang struct sockaddr *addr, 19259cb83748SJack Wang enum rdma_ucm_port_space ps) 19269cb83748SJack Wang { 19279cb83748SJack Wang struct rdma_cm_id *cm_id; 19289cb83748SJack Wang int ret; 19299cb83748SJack Wang 19309cb83748SJack Wang cm_id = rdma_create_id(&init_net, rtrs_srv_rdma_cm_handler, 19319cb83748SJack Wang ctx, ps, IB_QPT_RC); 19329cb83748SJack Wang if (IS_ERR(cm_id)) { 19339cb83748SJack Wang ret = PTR_ERR(cm_id); 19349cb83748SJack Wang pr_err("Creating id for RDMA connection failed, err: %d\n", 19359cb83748SJack Wang ret); 19369cb83748SJack Wang goto err_out; 19379cb83748SJack Wang } 19389cb83748SJack Wang ret = rdma_bind_addr(cm_id, addr); 19399cb83748SJack Wang if (ret) { 19409cb83748SJack Wang pr_err("Binding RDMA address failed, err: %d\n", ret); 19419cb83748SJack Wang goto err_cm; 19429cb83748SJack Wang } 19439cb83748SJack Wang ret = rdma_listen(cm_id, 64); 19449cb83748SJack Wang if (ret) { 19459cb83748SJack Wang pr_err("Listening on RDMA connection failed, err: %d\n", 19469cb83748SJack Wang ret); 19479cb83748SJack Wang goto err_cm; 19489cb83748SJack Wang } 19499cb83748SJack Wang 19509cb83748SJack Wang return cm_id; 19519cb83748SJack Wang 19529cb83748SJack Wang err_cm: 19539cb83748SJack Wang rdma_destroy_id(cm_id); 19549cb83748SJack Wang err_out: 19559cb83748SJack Wang 19569cb83748SJack Wang return ERR_PTR(ret); 19579cb83748SJack Wang } 19589cb83748SJack Wang 19599cb83748SJack Wang static int rtrs_srv_rdma_init(struct rtrs_srv_ctx *ctx, u16 port) 19609cb83748SJack Wang { 19619cb83748SJack Wang struct sockaddr_in6 sin = { 19629cb83748SJack Wang .sin6_family = AF_INET6, 19639cb83748SJack Wang .sin6_addr = IN6ADDR_ANY_INIT, 19649cb83748SJack Wang .sin6_port = htons(port), 19659cb83748SJack Wang }; 19669cb83748SJack Wang struct sockaddr_ib sib = { 19679cb83748SJack Wang .sib_family = AF_IB, 19689cb83748SJack Wang .sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port), 19699cb83748SJack Wang .sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL), 19709cb83748SJack Wang .sib_pkey = cpu_to_be16(0xffff), 19719cb83748SJack Wang }; 19729cb83748SJack Wang struct rdma_cm_id *cm_ip, *cm_ib; 19739cb83748SJack Wang int ret; 19749cb83748SJack Wang 19759cb83748SJack Wang /* 19769cb83748SJack Wang * We accept both IPoIB and IB connections, so we need to keep 19779cb83748SJack Wang * two cm id's, one for each socket type and port space. 19789cb83748SJack Wang * If the cm initialization of one of the id's fails, we abort 19799cb83748SJack Wang * everything. 19809cb83748SJack Wang */ 19819cb83748SJack Wang cm_ip = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sin, RDMA_PS_TCP); 19829cb83748SJack Wang if (IS_ERR(cm_ip)) 19839cb83748SJack Wang return PTR_ERR(cm_ip); 19849cb83748SJack Wang 19859cb83748SJack Wang cm_ib = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sib, RDMA_PS_IB); 19869cb83748SJack Wang if (IS_ERR(cm_ib)) { 19879cb83748SJack Wang ret = PTR_ERR(cm_ib); 19889cb83748SJack Wang goto free_cm_ip; 19899cb83748SJack Wang } 19909cb83748SJack Wang 19919cb83748SJack Wang ctx->cm_id_ip = cm_ip; 19929cb83748SJack Wang ctx->cm_id_ib = cm_ib; 19939cb83748SJack Wang 19949cb83748SJack Wang return 0; 19959cb83748SJack Wang 19969cb83748SJack Wang free_cm_ip: 19979cb83748SJack Wang rdma_destroy_id(cm_ip); 19989cb83748SJack Wang 19999cb83748SJack Wang return ret; 20009cb83748SJack Wang } 20019cb83748SJack Wang 20029cb83748SJack Wang static struct rtrs_srv_ctx *alloc_srv_ctx(struct rtrs_srv_ops *ops) 20039cb83748SJack Wang { 20049cb83748SJack Wang struct rtrs_srv_ctx *ctx; 20059cb83748SJack Wang 20069cb83748SJack Wang ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 20079cb83748SJack Wang if (!ctx) 20089cb83748SJack Wang return NULL; 20099cb83748SJack Wang 20109cb83748SJack Wang ctx->ops = *ops; 20119cb83748SJack Wang mutex_init(&ctx->srv_mutex); 20129cb83748SJack Wang INIT_LIST_HEAD(&ctx->srv_list); 20139cb83748SJack Wang 20149cb83748SJack Wang return ctx; 20159cb83748SJack Wang } 20169cb83748SJack Wang 20179cb83748SJack Wang static void free_srv_ctx(struct rtrs_srv_ctx *ctx) 20189cb83748SJack Wang { 20199cb83748SJack Wang WARN_ON(!list_empty(&ctx->srv_list)); 20209cb83748SJack Wang mutex_destroy(&ctx->srv_mutex); 20219cb83748SJack Wang kfree(ctx); 20229cb83748SJack Wang } 20239cb83748SJack Wang 2024558d52b2SMd Haris Iqbal static int rtrs_srv_add_one(struct ib_device *device) 2025558d52b2SMd Haris Iqbal { 2026558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2027558d52b2SMd Haris Iqbal int ret = 0; 2028558d52b2SMd Haris Iqbal 2029558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2030558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2031558d52b2SMd Haris Iqbal goto out; 2032558d52b2SMd Haris Iqbal 2033558d52b2SMd Haris Iqbal /* 2034558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will create them 2035558d52b2SMd Haris Iqbal * only once 2036558d52b2SMd Haris Iqbal */ 2037558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2038558d52b2SMd Haris Iqbal ret = rtrs_srv_rdma_init(ctx, ib_ctx.port); 2039558d52b2SMd Haris Iqbal if (ret) { 2040558d52b2SMd Haris Iqbal /* 2041558d52b2SMd Haris Iqbal * We errored out here. 2042558d52b2SMd Haris Iqbal * According to the ib code, if we encounter an error here then the 2043558d52b2SMd Haris Iqbal * error code is ignored, and no more calls to our ops are made. 2044558d52b2SMd Haris Iqbal */ 2045558d52b2SMd Haris Iqbal pr_err("Failed to initialize RDMA connection"); 2046558d52b2SMd Haris Iqbal goto err_out; 2047558d52b2SMd Haris Iqbal } 2048558d52b2SMd Haris Iqbal 2049558d52b2SMd Haris Iqbal out: 2050558d52b2SMd Haris Iqbal /* 2051558d52b2SMd Haris Iqbal * Keep a track on the number of ib devices added 2052558d52b2SMd Haris Iqbal */ 2053558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count++; 2054558d52b2SMd Haris Iqbal 2055558d52b2SMd Haris Iqbal err_out: 2056558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2057558d52b2SMd Haris Iqbal return ret; 2058558d52b2SMd Haris Iqbal } 2059558d52b2SMd Haris Iqbal 2060558d52b2SMd Haris Iqbal static void rtrs_srv_remove_one(struct ib_device *device, void *client_data) 2061558d52b2SMd Haris Iqbal { 2062558d52b2SMd Haris Iqbal struct rtrs_srv_ctx *ctx; 2063558d52b2SMd Haris Iqbal 2064558d52b2SMd Haris Iqbal mutex_lock(&ib_ctx.ib_dev_mutex); 2065558d52b2SMd Haris Iqbal ib_ctx.ib_dev_count--; 2066558d52b2SMd Haris Iqbal 2067558d52b2SMd Haris Iqbal if (ib_ctx.ib_dev_count) 2068558d52b2SMd Haris Iqbal goto out; 2069558d52b2SMd Haris Iqbal 2070558d52b2SMd Haris Iqbal /* 2071558d52b2SMd Haris Iqbal * Since our CM IDs are NOT bound to any ib device we will remove them 2072558d52b2SMd Haris Iqbal * only once, when the last device is removed 2073558d52b2SMd Haris Iqbal */ 2074558d52b2SMd Haris Iqbal ctx = ib_ctx.srv_ctx; 2075558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ip); 2076558d52b2SMd Haris Iqbal rdma_destroy_id(ctx->cm_id_ib); 2077558d52b2SMd Haris Iqbal 2078558d52b2SMd Haris Iqbal out: 2079558d52b2SMd Haris Iqbal mutex_unlock(&ib_ctx.ib_dev_mutex); 2080558d52b2SMd Haris Iqbal } 2081558d52b2SMd Haris Iqbal 2082558d52b2SMd Haris Iqbal static struct ib_client rtrs_srv_client = { 2083558d52b2SMd Haris Iqbal .name = "rtrs_server", 2084558d52b2SMd Haris Iqbal .add = rtrs_srv_add_one, 2085558d52b2SMd Haris Iqbal .remove = rtrs_srv_remove_one 2086558d52b2SMd Haris Iqbal }; 2087558d52b2SMd Haris Iqbal 20889cb83748SJack Wang /** 20899cb83748SJack Wang * rtrs_srv_open() - open RTRS server context 20909cb83748SJack Wang * @ops: callback functions 20919cb83748SJack Wang * @port: port to listen on 20929cb83748SJack Wang * 20939cb83748SJack Wang * Creates server context with specified callbacks. 20949cb83748SJack Wang * 20959cb83748SJack Wang * Return a valid pointer on success otherwise PTR_ERR. 20969cb83748SJack Wang */ 20979cb83748SJack Wang struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) 20989cb83748SJack Wang { 20999cb83748SJack Wang struct rtrs_srv_ctx *ctx; 21009cb83748SJack Wang int err; 21019cb83748SJack Wang 21029cb83748SJack Wang ctx = alloc_srv_ctx(ops); 21039cb83748SJack Wang if (!ctx) 21049cb83748SJack Wang return ERR_PTR(-ENOMEM); 21059cb83748SJack Wang 2106558d52b2SMd Haris Iqbal mutex_init(&ib_ctx.ib_dev_mutex); 2107558d52b2SMd Haris Iqbal ib_ctx.srv_ctx = ctx; 2108558d52b2SMd Haris Iqbal ib_ctx.port = port; 2109558d52b2SMd Haris Iqbal 2110558d52b2SMd Haris Iqbal err = ib_register_client(&rtrs_srv_client); 21119cb83748SJack Wang if (err) { 21129cb83748SJack Wang free_srv_ctx(ctx); 21139cb83748SJack Wang return ERR_PTR(err); 21149cb83748SJack Wang } 21159cb83748SJack Wang 21169cb83748SJack Wang return ctx; 21179cb83748SJack Wang } 21189cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_open); 21199cb83748SJack Wang 21209cb83748SJack Wang static void close_sessions(struct rtrs_srv *srv) 21219cb83748SJack Wang { 21229cb83748SJack Wang struct rtrs_srv_sess *sess; 21239cb83748SJack Wang 21249cb83748SJack Wang mutex_lock(&srv->paths_mutex); 21259cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 21269cb83748SJack Wang close_sess(sess); 21279cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 21289cb83748SJack Wang } 21299cb83748SJack Wang 21309cb83748SJack Wang static void close_ctx(struct rtrs_srv_ctx *ctx) 21319cb83748SJack Wang { 21329cb83748SJack Wang struct rtrs_srv *srv; 21339cb83748SJack Wang 21349cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 21359cb83748SJack Wang list_for_each_entry(srv, &ctx->srv_list, ctx_list) 21369cb83748SJack Wang close_sessions(srv); 21379cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 21389cb83748SJack Wang flush_workqueue(rtrs_wq); 21399cb83748SJack Wang } 21409cb83748SJack Wang 21419cb83748SJack Wang /** 21429cb83748SJack Wang * rtrs_srv_close() - close RTRS server context 21439cb83748SJack Wang * @ctx: pointer to server context 21449cb83748SJack Wang * 21459cb83748SJack Wang * Closes RTRS server context with all client sessions. 21469cb83748SJack Wang */ 21479cb83748SJack Wang void rtrs_srv_close(struct rtrs_srv_ctx *ctx) 21489cb83748SJack Wang { 2149558d52b2SMd Haris Iqbal ib_unregister_client(&rtrs_srv_client); 2150558d52b2SMd Haris Iqbal mutex_destroy(&ib_ctx.ib_dev_mutex); 21519cb83748SJack Wang close_ctx(ctx); 21529cb83748SJack Wang free_srv_ctx(ctx); 21539cb83748SJack Wang } 21549cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_close); 21559cb83748SJack Wang 21569cb83748SJack Wang static int check_module_params(void) 21579cb83748SJack Wang { 21589cb83748SJack Wang if (sess_queue_depth < 1 || sess_queue_depth > MAX_SESS_QUEUE_DEPTH) { 21599cb83748SJack Wang pr_err("Invalid sess_queue_depth value %d, has to be >= %d, <= %d.\n", 21609cb83748SJack Wang sess_queue_depth, 1, MAX_SESS_QUEUE_DEPTH); 21619cb83748SJack Wang return -EINVAL; 21629cb83748SJack Wang } 21639cb83748SJack Wang if (max_chunk_size < 4096 || !is_power_of_2(max_chunk_size)) { 21649cb83748SJack Wang pr_err("Invalid max_chunk_size value %d, has to be >= %d and should be power of two.\n", 21659cb83748SJack Wang max_chunk_size, 4096); 21669cb83748SJack Wang return -EINVAL; 21679cb83748SJack Wang } 21689cb83748SJack Wang 21699cb83748SJack Wang /* 21709cb83748SJack Wang * Check if IB immediate data size is enough to hold the mem_id and the 21719cb83748SJack Wang * offset inside the memory chunk 21729cb83748SJack Wang */ 21739cb83748SJack Wang if ((ilog2(sess_queue_depth - 1) + 1) + 21749cb83748SJack Wang (ilog2(max_chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) { 21759cb83748SJack Wang pr_err("RDMA immediate size (%db) not enough to encode %d buffers of size %dB. Reduce 'sess_queue_depth' or 'max_chunk_size' parameters.\n", 21769cb83748SJack Wang MAX_IMM_PAYL_BITS, sess_queue_depth, max_chunk_size); 21779cb83748SJack Wang return -EINVAL; 21789cb83748SJack Wang } 21799cb83748SJack Wang 21809cb83748SJack Wang return 0; 21819cb83748SJack Wang } 21829cb83748SJack Wang 21839cb83748SJack Wang static int __init rtrs_server_init(void) 21849cb83748SJack Wang { 21859cb83748SJack Wang int err; 21869cb83748SJack Wang 21879cb83748SJack Wang pr_info("Loading module %s, proto %s: (max_chunk_size: %d (pure IO %ld, headers %ld) , sess_queue_depth: %d, always_invalidate: %d)\n", 21889cb83748SJack Wang KBUILD_MODNAME, RTRS_PROTO_VER_STRING, 21899cb83748SJack Wang max_chunk_size, max_chunk_size - MAX_HDR_SIZE, MAX_HDR_SIZE, 21909cb83748SJack Wang sess_queue_depth, always_invalidate); 21919cb83748SJack Wang 21929cb83748SJack Wang rtrs_rdma_dev_pd_init(0, &dev_pd); 21939cb83748SJack Wang 21949cb83748SJack Wang err = check_module_params(); 21959cb83748SJack Wang if (err) { 21969cb83748SJack Wang pr_err("Failed to load module, invalid module parameters, err: %d\n", 21979cb83748SJack Wang err); 21989cb83748SJack Wang return err; 21999cb83748SJack Wang } 22009cb83748SJack Wang chunk_pool = mempool_create_page_pool(sess_queue_depth * CHUNK_POOL_SZ, 22019cb83748SJack Wang get_order(max_chunk_size)); 22029cb83748SJack Wang if (!chunk_pool) 22039cb83748SJack Wang return -ENOMEM; 22049cb83748SJack Wang rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server"); 22059cb83748SJack Wang if (IS_ERR(rtrs_dev_class)) { 22069cb83748SJack Wang err = PTR_ERR(rtrs_dev_class); 22079cb83748SJack Wang goto out_chunk_pool; 22089cb83748SJack Wang } 220903ed5a8cSJack Wang rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0); 22106b31afceSWei Yongjun if (!rtrs_wq) { 22116b31afceSWei Yongjun err = -ENOMEM; 22129cb83748SJack Wang goto out_dev_class; 22136b31afceSWei Yongjun } 22149cb83748SJack Wang 22159cb83748SJack Wang return 0; 22169cb83748SJack Wang 22179cb83748SJack Wang out_dev_class: 22189cb83748SJack Wang class_destroy(rtrs_dev_class); 22199cb83748SJack Wang out_chunk_pool: 22209cb83748SJack Wang mempool_destroy(chunk_pool); 22219cb83748SJack Wang 22229cb83748SJack Wang return err; 22239cb83748SJack Wang } 22249cb83748SJack Wang 22259cb83748SJack Wang static void __exit rtrs_server_exit(void) 22269cb83748SJack Wang { 22279cb83748SJack Wang destroy_workqueue(rtrs_wq); 22289cb83748SJack Wang class_destroy(rtrs_dev_class); 22299cb83748SJack Wang mempool_destroy(chunk_pool); 22309cb83748SJack Wang rtrs_rdma_dev_pd_deinit(&dev_pd); 22319cb83748SJack Wang } 22329cb83748SJack Wang 22339cb83748SJack Wang module_init(rtrs_server_init); 22349cb83748SJack Wang module_exit(rtrs_server_exit); 2235