19cb83748SJack Wang // SPDX-License-Identifier: GPL-2.0-or-later 29cb83748SJack Wang /* 39cb83748SJack Wang * RDMA Transport Layer 49cb83748SJack Wang * 59cb83748SJack Wang * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. 69cb83748SJack Wang * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. 79cb83748SJack Wang * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. 89cb83748SJack Wang */ 99cb83748SJack Wang 109cb83748SJack Wang #undef pr_fmt 119cb83748SJack Wang #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt 129cb83748SJack Wang 139cb83748SJack Wang #include <linux/module.h> 149cb83748SJack Wang #include <linux/mempool.h> 159cb83748SJack Wang 169cb83748SJack Wang #include "rtrs-srv.h" 179cb83748SJack Wang #include "rtrs-log.h" 189cb83748SJack Wang 199cb83748SJack Wang MODULE_DESCRIPTION("RDMA Transport Server"); 209cb83748SJack Wang MODULE_LICENSE("GPL"); 219cb83748SJack Wang 229cb83748SJack Wang /* Must be power of 2, see mask from mr->page_size in ib_sg_to_pages() */ 239cb83748SJack Wang #define DEFAULT_MAX_CHUNK_SIZE (128 << 10) 249cb83748SJack Wang #define DEFAULT_SESS_QUEUE_DEPTH 512 259cb83748SJack Wang #define MAX_HDR_SIZE PAGE_SIZE 269cb83748SJack Wang 279cb83748SJack Wang /* We guarantee to serve 10 paths at least */ 289cb83748SJack Wang #define CHUNK_POOL_SZ 10 299cb83748SJack Wang 309cb83748SJack Wang static struct rtrs_rdma_dev_pd dev_pd; 319cb83748SJack Wang static mempool_t *chunk_pool; 329cb83748SJack Wang struct class *rtrs_dev_class; 339cb83748SJack Wang 349cb83748SJack Wang static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE; 359cb83748SJack Wang static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH; 369cb83748SJack Wang 379cb83748SJack Wang static bool always_invalidate = true; 389cb83748SJack Wang module_param(always_invalidate, bool, 0444); 399cb83748SJack Wang MODULE_PARM_DESC(always_invalidate, 409cb83748SJack Wang "Invalidate memory registration for contiguous memory regions before accessing."); 419cb83748SJack Wang 429cb83748SJack Wang module_param_named(max_chunk_size, max_chunk_size, int, 0444); 439cb83748SJack Wang MODULE_PARM_DESC(max_chunk_size, 449cb83748SJack Wang "Max size for each IO request, when change the unit is in byte (default: " 459cb83748SJack Wang __stringify(DEFAULT_MAX_CHUNK_SIZE) "KB)"); 469cb83748SJack Wang 479cb83748SJack Wang module_param_named(sess_queue_depth, sess_queue_depth, int, 0444); 489cb83748SJack Wang MODULE_PARM_DESC(sess_queue_depth, 499cb83748SJack Wang "Number of buffers for pending I/O requests to allocate per session. Maximum: " 509cb83748SJack Wang __stringify(MAX_SESS_QUEUE_DEPTH) " (default: " 519cb83748SJack Wang __stringify(DEFAULT_SESS_QUEUE_DEPTH) ")"); 529cb83748SJack Wang 539cb83748SJack Wang static cpumask_t cq_affinity_mask = { CPU_BITS_ALL }; 549cb83748SJack Wang 559cb83748SJack Wang static struct workqueue_struct *rtrs_wq; 569cb83748SJack Wang 579cb83748SJack Wang static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) 589cb83748SJack Wang { 599cb83748SJack Wang return container_of(c, struct rtrs_srv_con, c); 609cb83748SJack Wang } 619cb83748SJack Wang 629cb83748SJack Wang static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s) 639cb83748SJack Wang { 649cb83748SJack Wang return container_of(s, struct rtrs_srv_sess, s); 659cb83748SJack Wang } 669cb83748SJack Wang 679cb83748SJack Wang static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess, 689cb83748SJack Wang enum rtrs_srv_state new_state) 699cb83748SJack Wang { 709cb83748SJack Wang enum rtrs_srv_state old_state; 719cb83748SJack Wang bool changed = false; 729cb83748SJack Wang 739cb83748SJack Wang lockdep_assert_held(&sess->state_lock); 749cb83748SJack Wang old_state = sess->state; 759cb83748SJack Wang switch (new_state) { 769cb83748SJack Wang case RTRS_SRV_CONNECTED: 779cb83748SJack Wang switch (old_state) { 789cb83748SJack Wang case RTRS_SRV_CONNECTING: 799cb83748SJack Wang changed = true; 809cb83748SJack Wang fallthrough; 819cb83748SJack Wang default: 829cb83748SJack Wang break; 839cb83748SJack Wang } 849cb83748SJack Wang break; 859cb83748SJack Wang case RTRS_SRV_CLOSING: 869cb83748SJack Wang switch (old_state) { 879cb83748SJack Wang case RTRS_SRV_CONNECTING: 889cb83748SJack Wang case RTRS_SRV_CONNECTED: 899cb83748SJack Wang changed = true; 909cb83748SJack Wang fallthrough; 919cb83748SJack Wang default: 929cb83748SJack Wang break; 939cb83748SJack Wang } 949cb83748SJack Wang break; 959cb83748SJack Wang case RTRS_SRV_CLOSED: 969cb83748SJack Wang switch (old_state) { 979cb83748SJack Wang case RTRS_SRV_CLOSING: 989cb83748SJack Wang changed = true; 999cb83748SJack Wang fallthrough; 1009cb83748SJack Wang default: 1019cb83748SJack Wang break; 1029cb83748SJack Wang } 1039cb83748SJack Wang break; 1049cb83748SJack Wang default: 1059cb83748SJack Wang break; 1069cb83748SJack Wang } 1079cb83748SJack Wang if (changed) 1089cb83748SJack Wang sess->state = new_state; 1099cb83748SJack Wang 1109cb83748SJack Wang return changed; 1119cb83748SJack Wang } 1129cb83748SJack Wang 1139cb83748SJack Wang static bool rtrs_srv_change_state_get_old(struct rtrs_srv_sess *sess, 1149cb83748SJack Wang enum rtrs_srv_state new_state, 1159cb83748SJack Wang enum rtrs_srv_state *old_state) 1169cb83748SJack Wang { 1179cb83748SJack Wang bool changed; 1189cb83748SJack Wang 1199cb83748SJack Wang spin_lock_irq(&sess->state_lock); 1209cb83748SJack Wang *old_state = sess->state; 1219cb83748SJack Wang changed = __rtrs_srv_change_state(sess, new_state); 1229cb83748SJack Wang spin_unlock_irq(&sess->state_lock); 1239cb83748SJack Wang 1249cb83748SJack Wang return changed; 1259cb83748SJack Wang } 1269cb83748SJack Wang 1279cb83748SJack Wang static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, 1289cb83748SJack Wang enum rtrs_srv_state new_state) 1299cb83748SJack Wang { 1309cb83748SJack Wang enum rtrs_srv_state old_state; 1319cb83748SJack Wang 1329cb83748SJack Wang return rtrs_srv_change_state_get_old(sess, new_state, &old_state); 1339cb83748SJack Wang } 1349cb83748SJack Wang 1359cb83748SJack Wang static void free_id(struct rtrs_srv_op *id) 1369cb83748SJack Wang { 1379cb83748SJack Wang if (!id) 1389cb83748SJack Wang return; 1399cb83748SJack Wang kfree(id); 1409cb83748SJack Wang } 1419cb83748SJack Wang 1429cb83748SJack Wang static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess) 1439cb83748SJack Wang { 1449cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1459cb83748SJack Wang int i; 1469cb83748SJack Wang 1479cb83748SJack Wang WARN_ON(atomic_read(&sess->ids_inflight)); 1489cb83748SJack Wang if (sess->ops_ids) { 1499cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 1509cb83748SJack Wang free_id(sess->ops_ids[i]); 1519cb83748SJack Wang kfree(sess->ops_ids); 1529cb83748SJack Wang sess->ops_ids = NULL; 1539cb83748SJack Wang } 1549cb83748SJack Wang } 1559cb83748SJack Wang 1569cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc); 1579cb83748SJack Wang 1589cb83748SJack Wang static struct ib_cqe io_comp_cqe = { 1599cb83748SJack Wang .done = rtrs_srv_rdma_done 1609cb83748SJack Wang }; 1619cb83748SJack Wang 1629cb83748SJack Wang static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) 1639cb83748SJack Wang { 1649cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 1659cb83748SJack Wang struct rtrs_srv_op *id; 1669cb83748SJack Wang int i; 1679cb83748SJack Wang 1689cb83748SJack Wang sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids), 1699cb83748SJack Wang GFP_KERNEL); 1709cb83748SJack Wang if (!sess->ops_ids) 1719cb83748SJack Wang goto err; 1729cb83748SJack Wang 1739cb83748SJack Wang for (i = 0; i < srv->queue_depth; ++i) { 1749cb83748SJack Wang id = kzalloc(sizeof(*id), GFP_KERNEL); 1759cb83748SJack Wang if (!id) 1769cb83748SJack Wang goto err; 1779cb83748SJack Wang 1789cb83748SJack Wang sess->ops_ids[i] = id; 1799cb83748SJack Wang } 1809cb83748SJack Wang init_waitqueue_head(&sess->ids_waitq); 1819cb83748SJack Wang atomic_set(&sess->ids_inflight, 0); 1829cb83748SJack Wang 1839cb83748SJack Wang return 0; 1849cb83748SJack Wang 1859cb83748SJack Wang err: 1869cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 1879cb83748SJack Wang return -ENOMEM; 1889cb83748SJack Wang } 1899cb83748SJack Wang 1909cb83748SJack Wang static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess) 1919cb83748SJack Wang { 1929cb83748SJack Wang atomic_inc(&sess->ids_inflight); 1939cb83748SJack Wang } 1949cb83748SJack Wang 1959cb83748SJack Wang static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess) 1969cb83748SJack Wang { 1979cb83748SJack Wang if (atomic_dec_and_test(&sess->ids_inflight)) 1989cb83748SJack Wang wake_up(&sess->ids_waitq); 1999cb83748SJack Wang } 2009cb83748SJack Wang 2019cb83748SJack Wang static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess) 2029cb83748SJack Wang { 2039cb83748SJack Wang wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight)); 2049cb83748SJack Wang } 2059cb83748SJack Wang 2069cb83748SJack Wang 2079cb83748SJack Wang static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc) 2089cb83748SJack Wang { 2099cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 2109cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 2119cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 2129cb83748SJack Wang 2139cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 2149cb83748SJack Wang rtrs_err(s, "REG MR failed: %s\n", 2159cb83748SJack Wang ib_wc_status_msg(wc->status)); 2169cb83748SJack Wang close_sess(sess); 2179cb83748SJack Wang return; 2189cb83748SJack Wang } 2199cb83748SJack Wang } 2209cb83748SJack Wang 2219cb83748SJack Wang static struct ib_cqe local_reg_cqe = { 2229cb83748SJack Wang .done = rtrs_srv_reg_mr_done 2239cb83748SJack Wang }; 2249cb83748SJack Wang 2259cb83748SJack Wang static int rdma_write_sg(struct rtrs_srv_op *id) 2269cb83748SJack Wang { 2279cb83748SJack Wang struct rtrs_sess *s = id->con->c.sess; 2289cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 2299cb83748SJack Wang dma_addr_t dma_addr = sess->dma_addr[id->msg_id]; 2309cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 2319cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 2329cb83748SJack Wang struct ib_send_wr inv_wr, imm_wr; 2339cb83748SJack Wang struct ib_rdma_wr *wr = NULL; 2349cb83748SJack Wang enum ib_send_flags flags; 2359cb83748SJack Wang size_t sg_cnt; 2369cb83748SJack Wang int err, offset; 2379cb83748SJack Wang bool need_inval; 2389cb83748SJack Wang u32 rkey = 0; 2399cb83748SJack Wang struct ib_reg_wr rwr; 2409cb83748SJack Wang struct ib_sge *plist; 2419cb83748SJack Wang struct ib_sge list; 2429cb83748SJack Wang 2439cb83748SJack Wang sg_cnt = le16_to_cpu(id->rd_msg->sg_cnt); 2449cb83748SJack Wang need_inval = le16_to_cpu(id->rd_msg->flags) & RTRS_MSG_NEED_INVAL_F; 2459cb83748SJack Wang if (unlikely(sg_cnt != 1)) 2469cb83748SJack Wang return -EINVAL; 2479cb83748SJack Wang 2489cb83748SJack Wang offset = 0; 2499cb83748SJack Wang 2509cb83748SJack Wang wr = &id->tx_wr; 2519cb83748SJack Wang plist = &id->tx_sg; 2529cb83748SJack Wang plist->addr = dma_addr + offset; 2539cb83748SJack Wang plist->length = le32_to_cpu(id->rd_msg->desc[0].len); 2549cb83748SJack Wang 2559cb83748SJack Wang /* WR will fail with length error 2569cb83748SJack Wang * if this is 0 2579cb83748SJack Wang */ 2589cb83748SJack Wang if (unlikely(plist->length == 0)) { 2599cb83748SJack Wang rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); 2609cb83748SJack Wang return -EINVAL; 2619cb83748SJack Wang } 2629cb83748SJack Wang 2639cb83748SJack Wang plist->lkey = sess->s.dev->ib_pd->local_dma_lkey; 2649cb83748SJack Wang offset += plist->length; 2659cb83748SJack Wang 2669cb83748SJack Wang wr->wr.sg_list = plist; 2679cb83748SJack Wang wr->wr.num_sge = 1; 2689cb83748SJack Wang wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); 2699cb83748SJack Wang wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); 2709cb83748SJack Wang if (rkey == 0) 2719cb83748SJack Wang rkey = wr->rkey; 2729cb83748SJack Wang else 2739cb83748SJack Wang /* Only one key is actually used */ 2749cb83748SJack Wang WARN_ON_ONCE(rkey != wr->rkey); 2759cb83748SJack Wang 2769cb83748SJack Wang wr->wr.opcode = IB_WR_RDMA_WRITE; 2779cb83748SJack Wang wr->wr.ex.imm_data = 0; 2789cb83748SJack Wang wr->wr.send_flags = 0; 2799cb83748SJack Wang 2809cb83748SJack Wang if (need_inval && always_invalidate) { 2819cb83748SJack Wang wr->wr.next = &rwr.wr; 2829cb83748SJack Wang rwr.wr.next = &inv_wr; 2839cb83748SJack Wang inv_wr.next = &imm_wr; 2849cb83748SJack Wang } else if (always_invalidate) { 2859cb83748SJack Wang wr->wr.next = &rwr.wr; 2869cb83748SJack Wang rwr.wr.next = &imm_wr; 2879cb83748SJack Wang } else if (need_inval) { 2889cb83748SJack Wang wr->wr.next = &inv_wr; 2899cb83748SJack Wang inv_wr.next = &imm_wr; 2909cb83748SJack Wang } else { 2919cb83748SJack Wang wr->wr.next = &imm_wr; 2929cb83748SJack Wang } 2939cb83748SJack Wang /* 2949cb83748SJack Wang * From time to time we have to post signaled sends, 2959cb83748SJack Wang * or send queue will fill up and only QP reset can help. 2969cb83748SJack Wang */ 2979cb83748SJack Wang flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ? 2989cb83748SJack Wang 0 : IB_SEND_SIGNALED; 2999cb83748SJack Wang 3009cb83748SJack Wang if (need_inval) { 3019cb83748SJack Wang inv_wr.sg_list = NULL; 3029cb83748SJack Wang inv_wr.num_sge = 0; 3039cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 3049cb83748SJack Wang inv_wr.send_flags = 0; 3059cb83748SJack Wang inv_wr.ex.invalidate_rkey = rkey; 3069cb83748SJack Wang } 3079cb83748SJack Wang 3089cb83748SJack Wang imm_wr.next = NULL; 3099cb83748SJack Wang if (always_invalidate) { 3109cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 3119cb83748SJack Wang 3129cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 3139cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 3149cb83748SJack Wang rwr.wr.num_sge = 0; 3159cb83748SJack Wang rwr.mr = srv_mr->mr; 3169cb83748SJack Wang rwr.wr.send_flags = 0; 3179cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 3189cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 3199cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 3209cb83748SJack Wang msg = srv_mr->iu->buf; 3219cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 3229cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 3239cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 3249cb83748SJack Wang 3259cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 3269cb83748SJack Wang list.length = sizeof(*msg); 3279cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 3289cb83748SJack Wang imm_wr.sg_list = &list; 3299cb83748SJack Wang imm_wr.num_sge = 1; 3309cb83748SJack Wang imm_wr.opcode = IB_WR_SEND_WITH_IMM; 3319cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 3329cb83748SJack Wang srv_mr->iu->dma_addr, 3339cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 3349cb83748SJack Wang } else { 3359cb83748SJack Wang imm_wr.sg_list = NULL; 3369cb83748SJack Wang imm_wr.num_sge = 0; 3379cb83748SJack Wang imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 3389cb83748SJack Wang } 3399cb83748SJack Wang imm_wr.send_flags = flags; 3409cb83748SJack Wang imm_wr.ex.imm_data = cpu_to_be32(rtrs_to_io_rsp_imm(id->msg_id, 3419cb83748SJack Wang 0, need_inval)); 3429cb83748SJack Wang 3439cb83748SJack Wang imm_wr.wr_cqe = &io_comp_cqe; 3449cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr, 3459cb83748SJack Wang offset, DMA_BIDIRECTIONAL); 3469cb83748SJack Wang 3479cb83748SJack Wang err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL); 3489cb83748SJack Wang if (unlikely(err)) 3499cb83748SJack Wang rtrs_err(s, 3509cb83748SJack Wang "Posting RDMA-Write-Request to QP failed, err: %d\n", 3519cb83748SJack Wang err); 3529cb83748SJack Wang 3539cb83748SJack Wang return err; 3549cb83748SJack Wang } 3559cb83748SJack Wang 3569cb83748SJack Wang /** 3579cb83748SJack Wang * send_io_resp_imm() - respond to client with empty IMM on failed READ/WRITE 3589cb83748SJack Wang * requests or on successful WRITE request. 3599cb83748SJack Wang * @con: the connection to send back result 3609cb83748SJack Wang * @id: the id associated with the IO 3619cb83748SJack Wang * @errno: the error number of the IO. 3629cb83748SJack Wang * 3639cb83748SJack Wang * Return 0 on success, errno otherwise. 3649cb83748SJack Wang */ 3659cb83748SJack Wang static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, 3669cb83748SJack Wang int errno) 3679cb83748SJack Wang { 3689cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 3699cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 3709cb83748SJack Wang struct ib_send_wr inv_wr, imm_wr, *wr = NULL; 3719cb83748SJack Wang struct ib_reg_wr rwr; 3729cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 3739cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 3749cb83748SJack Wang bool need_inval = false; 3759cb83748SJack Wang enum ib_send_flags flags; 3769cb83748SJack Wang u32 imm; 3779cb83748SJack Wang int err; 3789cb83748SJack Wang 3799cb83748SJack Wang if (id->dir == READ) { 3809cb83748SJack Wang struct rtrs_msg_rdma_read *rd_msg = id->rd_msg; 3819cb83748SJack Wang size_t sg_cnt; 3829cb83748SJack Wang 3839cb83748SJack Wang need_inval = le16_to_cpu(rd_msg->flags) & 3849cb83748SJack Wang RTRS_MSG_NEED_INVAL_F; 3859cb83748SJack Wang sg_cnt = le16_to_cpu(rd_msg->sg_cnt); 3869cb83748SJack Wang 3879cb83748SJack Wang if (need_inval) { 3889cb83748SJack Wang if (likely(sg_cnt)) { 3899cb83748SJack Wang inv_wr.sg_list = NULL; 3909cb83748SJack Wang inv_wr.num_sge = 0; 3919cb83748SJack Wang inv_wr.opcode = IB_WR_SEND_WITH_INV; 3929cb83748SJack Wang inv_wr.send_flags = 0; 3939cb83748SJack Wang /* Only one key is actually used */ 3949cb83748SJack Wang inv_wr.ex.invalidate_rkey = 3959cb83748SJack Wang le32_to_cpu(rd_msg->desc[0].key); 3969cb83748SJack Wang } else { 3979cb83748SJack Wang WARN_ON_ONCE(1); 3989cb83748SJack Wang need_inval = false; 3999cb83748SJack Wang } 4009cb83748SJack Wang } 4019cb83748SJack Wang } 4029cb83748SJack Wang 4039cb83748SJack Wang if (need_inval && always_invalidate) { 4049cb83748SJack Wang wr = &inv_wr; 4059cb83748SJack Wang inv_wr.next = &rwr.wr; 4069cb83748SJack Wang rwr.wr.next = &imm_wr; 4079cb83748SJack Wang } else if (always_invalidate) { 4089cb83748SJack Wang wr = &rwr.wr; 4099cb83748SJack Wang rwr.wr.next = &imm_wr; 4109cb83748SJack Wang } else if (need_inval) { 4119cb83748SJack Wang wr = &inv_wr; 4129cb83748SJack Wang inv_wr.next = &imm_wr; 4139cb83748SJack Wang } else { 4149cb83748SJack Wang wr = &imm_wr; 4159cb83748SJack Wang } 4169cb83748SJack Wang /* 4179cb83748SJack Wang * From time to time we have to post signalled sends, 4189cb83748SJack Wang * or send queue will fill up and only QP reset can help. 4199cb83748SJack Wang */ 4209cb83748SJack Wang flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ? 4219cb83748SJack Wang 0 : IB_SEND_SIGNALED; 4229cb83748SJack Wang imm = rtrs_to_io_rsp_imm(id->msg_id, errno, need_inval); 4239cb83748SJack Wang imm_wr.next = NULL; 4249cb83748SJack Wang if (always_invalidate) { 4259cb83748SJack Wang struct ib_sge list; 4269cb83748SJack Wang struct rtrs_msg_rkey_rsp *msg; 4279cb83748SJack Wang 4289cb83748SJack Wang srv_mr = &sess->mrs[id->msg_id]; 4299cb83748SJack Wang rwr.wr.next = &imm_wr; 4309cb83748SJack Wang rwr.wr.opcode = IB_WR_REG_MR; 4319cb83748SJack Wang rwr.wr.num_sge = 0; 4329cb83748SJack Wang rwr.wr.send_flags = 0; 4339cb83748SJack Wang rwr.mr = srv_mr->mr; 4349cb83748SJack Wang rwr.key = srv_mr->mr->rkey; 4359cb83748SJack Wang rwr.access = (IB_ACCESS_LOCAL_WRITE | 4369cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 4379cb83748SJack Wang msg = srv_mr->iu->buf; 4389cb83748SJack Wang msg->buf_id = cpu_to_le16(id->msg_id); 4399cb83748SJack Wang msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP); 4409cb83748SJack Wang msg->rkey = cpu_to_le32(srv_mr->mr->rkey); 4419cb83748SJack Wang 4429cb83748SJack Wang list.addr = srv_mr->iu->dma_addr; 4439cb83748SJack Wang list.length = sizeof(*msg); 4449cb83748SJack Wang list.lkey = sess->s.dev->ib_pd->local_dma_lkey; 4459cb83748SJack Wang imm_wr.sg_list = &list; 4469cb83748SJack Wang imm_wr.num_sge = 1; 4479cb83748SJack Wang imm_wr.opcode = IB_WR_SEND_WITH_IMM; 4489cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, 4499cb83748SJack Wang srv_mr->iu->dma_addr, 4509cb83748SJack Wang srv_mr->iu->size, DMA_TO_DEVICE); 4519cb83748SJack Wang } else { 4529cb83748SJack Wang imm_wr.sg_list = NULL; 4539cb83748SJack Wang imm_wr.num_sge = 0; 4549cb83748SJack Wang imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM; 4559cb83748SJack Wang } 4569cb83748SJack Wang imm_wr.send_flags = flags; 4579cb83748SJack Wang imm_wr.wr_cqe = &io_comp_cqe; 4589cb83748SJack Wang 4599cb83748SJack Wang imm_wr.ex.imm_data = cpu_to_be32(imm); 4609cb83748SJack Wang 4619cb83748SJack Wang err = ib_post_send(id->con->c.qp, wr, NULL); 4629cb83748SJack Wang if (unlikely(err)) 4639cb83748SJack Wang rtrs_err_rl(s, "Posting RDMA-Reply to QP failed, err: %d\n", 4649cb83748SJack Wang err); 4659cb83748SJack Wang 4669cb83748SJack Wang return err; 4679cb83748SJack Wang } 4689cb83748SJack Wang 4699cb83748SJack Wang void close_sess(struct rtrs_srv_sess *sess) 4709cb83748SJack Wang { 4719cb83748SJack Wang enum rtrs_srv_state old_state; 4729cb83748SJack Wang 4739cb83748SJack Wang if (rtrs_srv_change_state_get_old(sess, RTRS_SRV_CLOSING, 4749cb83748SJack Wang &old_state)) 4759cb83748SJack Wang queue_work(rtrs_wq, &sess->close_work); 4769cb83748SJack Wang WARN_ON(sess->state != RTRS_SRV_CLOSING); 4779cb83748SJack Wang } 4789cb83748SJack Wang 4799cb83748SJack Wang static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) 4809cb83748SJack Wang { 4819cb83748SJack Wang switch (state) { 4829cb83748SJack Wang case RTRS_SRV_CONNECTING: 4839cb83748SJack Wang return "RTRS_SRV_CONNECTING"; 4849cb83748SJack Wang case RTRS_SRV_CONNECTED: 4859cb83748SJack Wang return "RTRS_SRV_CONNECTED"; 4869cb83748SJack Wang case RTRS_SRV_CLOSING: 4879cb83748SJack Wang return "RTRS_SRV_CLOSING"; 4889cb83748SJack Wang case RTRS_SRV_CLOSED: 4899cb83748SJack Wang return "RTRS_SRV_CLOSED"; 4909cb83748SJack Wang default: 4919cb83748SJack Wang return "UNKNOWN"; 4929cb83748SJack Wang } 4939cb83748SJack Wang } 4949cb83748SJack Wang 4959cb83748SJack Wang /** 4969cb83748SJack Wang * rtrs_srv_resp_rdma() - Finish an RDMA request 4979cb83748SJack Wang * 4989cb83748SJack Wang * @id: Internal RTRS operation identifier 4999cb83748SJack Wang * @status: Response Code sent to the other side for this operation. 5009cb83748SJack Wang * 0 = success, <=0 error 5019cb83748SJack Wang * Context: any 5029cb83748SJack Wang * 5039cb83748SJack Wang * Finish a RDMA operation. A message is sent to the client and the 5049cb83748SJack Wang * corresponding memory areas will be released. 5059cb83748SJack Wang */ 5069cb83748SJack Wang bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) 5079cb83748SJack Wang { 5089cb83748SJack Wang struct rtrs_srv_sess *sess; 5099cb83748SJack Wang struct rtrs_srv_con *con; 5109cb83748SJack Wang struct rtrs_sess *s; 5119cb83748SJack Wang int err; 5129cb83748SJack Wang 5139cb83748SJack Wang if (WARN_ON(!id)) 5149cb83748SJack Wang return true; 5159cb83748SJack Wang 5169cb83748SJack Wang con = id->con; 5179cb83748SJack Wang s = con->c.sess; 5189cb83748SJack Wang sess = to_srv_sess(s); 5199cb83748SJack Wang 5209cb83748SJack Wang id->status = status; 5219cb83748SJack Wang 5229cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 5239cb83748SJack Wang rtrs_err_rl(s, 5249cb83748SJack Wang "Sending I/O response failed, session is disconnected, sess state %s\n", 5259cb83748SJack Wang rtrs_srv_state_str(sess->state)); 5269cb83748SJack Wang goto out; 5279cb83748SJack Wang } 5289cb83748SJack Wang if (always_invalidate) { 5299cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id]; 5309cb83748SJack Wang 5319cb83748SJack Wang ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); 5329cb83748SJack Wang } 5339cb83748SJack Wang if (unlikely(atomic_sub_return(1, 5349cb83748SJack Wang &con->sq_wr_avail) < 0)) { 5359cb83748SJack Wang pr_err("IB send queue full\n"); 5369cb83748SJack Wang atomic_add(1, &con->sq_wr_avail); 5379cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 5389cb83748SJack Wang list_add_tail(&id->wait_list, &con->rsp_wr_wait_list); 5399cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 5409cb83748SJack Wang return false; 5419cb83748SJack Wang } 5429cb83748SJack Wang 5439cb83748SJack Wang if (status || id->dir == WRITE || !id->rd_msg->sg_cnt) 5449cb83748SJack Wang err = send_io_resp_imm(con, id, status); 5459cb83748SJack Wang else 5469cb83748SJack Wang err = rdma_write_sg(id); 5479cb83748SJack Wang 5489cb83748SJack Wang if (unlikely(err)) { 5499cb83748SJack Wang rtrs_err_rl(s, "IO response failed: %d\n", err); 5509cb83748SJack Wang close_sess(sess); 5519cb83748SJack Wang } 5529cb83748SJack Wang out: 5539cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 5549cb83748SJack Wang return true; 5559cb83748SJack Wang } 5569cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_resp_rdma); 5579cb83748SJack Wang 5589cb83748SJack Wang /** 5599cb83748SJack Wang * rtrs_srv_set_sess_priv() - Set private pointer in rtrs_srv. 5609cb83748SJack Wang * @srv: Session pointer 5619cb83748SJack Wang * @priv: The private pointer that is associated with the session. 5629cb83748SJack Wang */ 5639cb83748SJack Wang void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv) 5649cb83748SJack Wang { 5659cb83748SJack Wang srv->priv = priv; 5669cb83748SJack Wang } 5679cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_set_sess_priv); 5689cb83748SJack Wang 5699cb83748SJack Wang static void unmap_cont_bufs(struct rtrs_srv_sess *sess) 5709cb83748SJack Wang { 5719cb83748SJack Wang int i; 5729cb83748SJack Wang 5739cb83748SJack Wang for (i = 0; i < sess->mrs_num; i++) { 5749cb83748SJack Wang struct rtrs_srv_mr *srv_mr; 5759cb83748SJack Wang 5769cb83748SJack Wang srv_mr = &sess->mrs[i]; 5779cb83748SJack Wang rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE, 5789cb83748SJack Wang sess->s.dev->ib_dev, 1); 5799cb83748SJack Wang ib_dereg_mr(srv_mr->mr); 5809cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl, 5819cb83748SJack Wang srv_mr->sgt.nents, DMA_BIDIRECTIONAL); 5829cb83748SJack Wang sg_free_table(&srv_mr->sgt); 5839cb83748SJack Wang } 5849cb83748SJack Wang kfree(sess->mrs); 5859cb83748SJack Wang } 5869cb83748SJack Wang 5879cb83748SJack Wang static int map_cont_bufs(struct rtrs_srv_sess *sess) 5889cb83748SJack Wang { 5899cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 5909cb83748SJack Wang struct rtrs_sess *ss = &sess->s; 5919cb83748SJack Wang int i, mri, err, mrs_num; 5929cb83748SJack Wang unsigned int chunk_bits; 5939cb83748SJack Wang int chunks_per_mr = 1; 5949cb83748SJack Wang 5959cb83748SJack Wang /* 5969cb83748SJack Wang * Here we map queue_depth chunks to MR. Firstly we have to 5979cb83748SJack Wang * figure out how many chunks can we map per MR. 5989cb83748SJack Wang */ 5999cb83748SJack Wang if (always_invalidate) { 6009cb83748SJack Wang /* 6019cb83748SJack Wang * in order to do invalidate for each chunks of memory, we needs 6029cb83748SJack Wang * more memory regions. 6039cb83748SJack Wang */ 6049cb83748SJack Wang mrs_num = srv->queue_depth; 6059cb83748SJack Wang } else { 6069cb83748SJack Wang chunks_per_mr = 6079cb83748SJack Wang sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; 6089cb83748SJack Wang mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr); 6099cb83748SJack Wang chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num); 6109cb83748SJack Wang } 6119cb83748SJack Wang 6129cb83748SJack Wang sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL); 6139cb83748SJack Wang if (!sess->mrs) 6149cb83748SJack Wang return -ENOMEM; 6159cb83748SJack Wang 6169cb83748SJack Wang sess->mrs_num = mrs_num; 6179cb83748SJack Wang 6189cb83748SJack Wang for (mri = 0; mri < mrs_num; mri++) { 6199cb83748SJack Wang struct rtrs_srv_mr *srv_mr = &sess->mrs[mri]; 6209cb83748SJack Wang struct sg_table *sgt = &srv_mr->sgt; 6219cb83748SJack Wang struct scatterlist *s; 6229cb83748SJack Wang struct ib_mr *mr; 6239cb83748SJack Wang int nr, chunks; 6249cb83748SJack Wang 6259cb83748SJack Wang chunks = chunks_per_mr * mri; 6269cb83748SJack Wang if (!always_invalidate) 6279cb83748SJack Wang chunks_per_mr = min_t(int, chunks_per_mr, 6289cb83748SJack Wang srv->queue_depth - chunks); 6299cb83748SJack Wang 6309cb83748SJack Wang err = sg_alloc_table(sgt, chunks_per_mr, GFP_KERNEL); 6319cb83748SJack Wang if (err) 6329cb83748SJack Wang goto err; 6339cb83748SJack Wang 6349cb83748SJack Wang for_each_sg(sgt->sgl, s, chunks_per_mr, i) 6359cb83748SJack Wang sg_set_page(s, srv->chunks[chunks + i], 6369cb83748SJack Wang max_chunk_size, 0); 6379cb83748SJack Wang 6389cb83748SJack Wang nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl, 6399cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6409cb83748SJack Wang if (nr < sgt->nents) { 6419cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6429cb83748SJack Wang goto free_sg; 6439cb83748SJack Wang } 6449cb83748SJack Wang mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, 6459cb83748SJack Wang sgt->nents); 6469cb83748SJack Wang if (IS_ERR(mr)) { 6479cb83748SJack Wang err = PTR_ERR(mr); 6489cb83748SJack Wang goto unmap_sg; 6499cb83748SJack Wang } 6509cb83748SJack Wang nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents, 6519cb83748SJack Wang NULL, max_chunk_size); 652b386cd65SDan Carpenter if (nr < 0 || nr < sgt->nents) { 6539cb83748SJack Wang err = nr < 0 ? nr : -EINVAL; 6549cb83748SJack Wang goto dereg_mr; 6559cb83748SJack Wang } 6569cb83748SJack Wang 6579cb83748SJack Wang if (always_invalidate) { 6589cb83748SJack Wang srv_mr->iu = rtrs_iu_alloc(1, 6599cb83748SJack Wang sizeof(struct rtrs_msg_rkey_rsp), 6609cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 6619cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_rdma_done); 6629cb83748SJack Wang if (!srv_mr->iu) { 6639cb83748SJack Wang rtrs_err(ss, "rtrs_iu_alloc(), err: %d\n", 6649cb83748SJack Wang -ENOMEM); 6659cb83748SJack Wang goto free_iu; 6669cb83748SJack Wang } 6679cb83748SJack Wang } 6689cb83748SJack Wang /* Eventually dma addr for each chunk can be cached */ 6699cb83748SJack Wang for_each_sg(sgt->sgl, s, sgt->orig_nents, i) 6709cb83748SJack Wang sess->dma_addr[chunks + i] = sg_dma_address(s); 6719cb83748SJack Wang 6729cb83748SJack Wang ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 6739cb83748SJack Wang srv_mr->mr = mr; 6749cb83748SJack Wang 6759cb83748SJack Wang continue; 6769cb83748SJack Wang err: 6779cb83748SJack Wang while (mri--) { 6789cb83748SJack Wang srv_mr = &sess->mrs[mri]; 6799cb83748SJack Wang sgt = &srv_mr->sgt; 6809cb83748SJack Wang mr = srv_mr->mr; 6819cb83748SJack Wang free_iu: 6829cb83748SJack Wang rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE, 6839cb83748SJack Wang sess->s.dev->ib_dev, 1); 6849cb83748SJack Wang dereg_mr: 6859cb83748SJack Wang ib_dereg_mr(mr); 6869cb83748SJack Wang unmap_sg: 6879cb83748SJack Wang ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl, 6889cb83748SJack Wang sgt->nents, DMA_BIDIRECTIONAL); 6899cb83748SJack Wang free_sg: 6909cb83748SJack Wang sg_free_table(sgt); 6919cb83748SJack Wang } 6929cb83748SJack Wang kfree(sess->mrs); 6939cb83748SJack Wang 6949cb83748SJack Wang return err; 6959cb83748SJack Wang } 6969cb83748SJack Wang 6979cb83748SJack Wang chunk_bits = ilog2(srv->queue_depth - 1) + 1; 6989cb83748SJack Wang sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); 6999cb83748SJack Wang 7009cb83748SJack Wang return 0; 7019cb83748SJack Wang } 7029cb83748SJack Wang 7039cb83748SJack Wang static void rtrs_srv_hb_err_handler(struct rtrs_con *c) 7049cb83748SJack Wang { 7059cb83748SJack Wang close_sess(to_srv_sess(c->sess)); 7069cb83748SJack Wang } 7079cb83748SJack Wang 7089cb83748SJack Wang static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess) 7099cb83748SJack Wang { 7109cb83748SJack Wang rtrs_init_hb(&sess->s, &io_comp_cqe, 7119cb83748SJack Wang RTRS_HB_INTERVAL_MS, 7129cb83748SJack Wang RTRS_HB_MISSED_MAX, 7139cb83748SJack Wang rtrs_srv_hb_err_handler, 7149cb83748SJack Wang rtrs_wq); 7159cb83748SJack Wang } 7169cb83748SJack Wang 7179cb83748SJack Wang static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess) 7189cb83748SJack Wang { 7199cb83748SJack Wang rtrs_start_hb(&sess->s); 7209cb83748SJack Wang } 7219cb83748SJack Wang 7229cb83748SJack Wang static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess) 7239cb83748SJack Wang { 7249cb83748SJack Wang rtrs_stop_hb(&sess->s); 7259cb83748SJack Wang } 7269cb83748SJack Wang 7279cb83748SJack Wang static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) 7289cb83748SJack Wang { 7299cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 7309cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7319cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7329cb83748SJack Wang struct rtrs_iu *iu; 7339cb83748SJack Wang 7349cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 7359cb83748SJack Wang rtrs_iu_free(iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1); 7369cb83748SJack Wang 7379cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 7389cb83748SJack Wang rtrs_err(s, "Sess info response send failed: %s\n", 7399cb83748SJack Wang ib_wc_status_msg(wc->status)); 7409cb83748SJack Wang close_sess(sess); 7419cb83748SJack Wang return; 7429cb83748SJack Wang } 7439cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_SEND); 7449cb83748SJack Wang } 7459cb83748SJack Wang 7469cb83748SJack Wang static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) 7479cb83748SJack Wang { 7489cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7499cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7509cb83748SJack Wang int up; 7519cb83748SJack Wang 7529cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7539cb83748SJack Wang up = ++srv->paths_up; 7549cb83748SJack Wang if (up == 1) 7559cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL); 7569cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7579cb83748SJack Wang 7589cb83748SJack Wang /* Mark session as established */ 7599cb83748SJack Wang sess->established = true; 7609cb83748SJack Wang } 7619cb83748SJack Wang 7629cb83748SJack Wang static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) 7639cb83748SJack Wang { 7649cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 7659cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 7669cb83748SJack Wang 7679cb83748SJack Wang if (!sess->established) 7689cb83748SJack Wang return; 7699cb83748SJack Wang 7709cb83748SJack Wang sess->established = false; 7719cb83748SJack Wang mutex_lock(&srv->paths_ev_mutex); 7729cb83748SJack Wang WARN_ON(!srv->paths_up); 7739cb83748SJack Wang if (--srv->paths_up == 0) 7749cb83748SJack Wang ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv); 7759cb83748SJack Wang mutex_unlock(&srv->paths_ev_mutex); 7769cb83748SJack Wang } 7779cb83748SJack Wang 7789cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess); 7799cb83748SJack Wang 7809cb83748SJack Wang static int process_info_req(struct rtrs_srv_con *con, 7819cb83748SJack Wang struct rtrs_msg_info_req *msg) 7829cb83748SJack Wang { 7839cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 7849cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 7859cb83748SJack Wang struct ib_send_wr *reg_wr = NULL; 7869cb83748SJack Wang struct rtrs_msg_info_rsp *rsp; 7879cb83748SJack Wang struct rtrs_iu *tx_iu; 7889cb83748SJack Wang struct ib_reg_wr *rwr; 7899cb83748SJack Wang int mri, err; 7909cb83748SJack Wang size_t tx_sz; 7919cb83748SJack Wang 7929cb83748SJack Wang err = post_recv_sess(sess); 7939cb83748SJack Wang if (unlikely(err)) { 7949cb83748SJack Wang rtrs_err(s, "post_recv_sess(), err: %d\n", err); 7959cb83748SJack Wang return err; 7969cb83748SJack Wang } 7979cb83748SJack Wang rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL); 7989cb83748SJack Wang if (unlikely(!rwr)) 7999cb83748SJack Wang return -ENOMEM; 8009cb83748SJack Wang strlcpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname)); 8019cb83748SJack Wang 8029cb83748SJack Wang tx_sz = sizeof(*rsp); 8039cb83748SJack Wang tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num; 8049cb83748SJack Wang tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev, 8059cb83748SJack Wang DMA_TO_DEVICE, rtrs_srv_info_rsp_done); 8069cb83748SJack Wang if (unlikely(!tx_iu)) { 8079cb83748SJack Wang err = -ENOMEM; 8089cb83748SJack Wang goto rwr_free; 8099cb83748SJack Wang } 8109cb83748SJack Wang 8119cb83748SJack Wang rsp = tx_iu->buf; 8129cb83748SJack Wang rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP); 8139cb83748SJack Wang rsp->sg_cnt = cpu_to_le16(sess->mrs_num); 8149cb83748SJack Wang 8159cb83748SJack Wang for (mri = 0; mri < sess->mrs_num; mri++) { 8169cb83748SJack Wang struct ib_mr *mr = sess->mrs[mri].mr; 8179cb83748SJack Wang 8189cb83748SJack Wang rsp->desc[mri].addr = cpu_to_le64(mr->iova); 8199cb83748SJack Wang rsp->desc[mri].key = cpu_to_le32(mr->rkey); 8209cb83748SJack Wang rsp->desc[mri].len = cpu_to_le32(mr->length); 8219cb83748SJack Wang 8229cb83748SJack Wang /* 8239cb83748SJack Wang * Fill in reg MR request and chain them *backwards* 8249cb83748SJack Wang */ 8259cb83748SJack Wang rwr[mri].wr.next = mri ? &rwr[mri - 1].wr : NULL; 8269cb83748SJack Wang rwr[mri].wr.opcode = IB_WR_REG_MR; 8279cb83748SJack Wang rwr[mri].wr.wr_cqe = &local_reg_cqe; 8289cb83748SJack Wang rwr[mri].wr.num_sge = 0; 8299cb83748SJack Wang rwr[mri].wr.send_flags = mri ? 0 : IB_SEND_SIGNALED; 8309cb83748SJack Wang rwr[mri].mr = mr; 8319cb83748SJack Wang rwr[mri].key = mr->rkey; 8329cb83748SJack Wang rwr[mri].access = (IB_ACCESS_LOCAL_WRITE | 8339cb83748SJack Wang IB_ACCESS_REMOTE_WRITE); 8349cb83748SJack Wang reg_wr = &rwr[mri].wr; 8359cb83748SJack Wang } 8369cb83748SJack Wang 8379cb83748SJack Wang err = rtrs_srv_create_sess_files(sess); 8389cb83748SJack Wang if (unlikely(err)) 8399cb83748SJack Wang goto iu_free; 8409cb83748SJack Wang kobject_get(&sess->kobj); 8419cb83748SJack Wang get_device(&sess->srv->dev); 8429cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED); 8439cb83748SJack Wang rtrs_srv_start_hb(sess); 8449cb83748SJack Wang 8459cb83748SJack Wang /* 8469cb83748SJack Wang * We do not account number of established connections at the current 8479cb83748SJack Wang * moment, we rely on the client, which should send info request when 8489cb83748SJack Wang * all connections are successfully established. Thus, simply notify 8499cb83748SJack Wang * listener with a proper event if we are the first path. 8509cb83748SJack Wang */ 8519cb83748SJack Wang rtrs_srv_sess_up(sess); 8529cb83748SJack Wang 8539cb83748SJack Wang ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, 8549cb83748SJack Wang tx_iu->size, DMA_TO_DEVICE); 8559cb83748SJack Wang 8569cb83748SJack Wang /* Send info response */ 8579cb83748SJack Wang err = rtrs_iu_post_send(&con->c, tx_iu, tx_sz, reg_wr); 8589cb83748SJack Wang if (unlikely(err)) { 8599cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err); 8609cb83748SJack Wang iu_free: 8619cb83748SJack Wang rtrs_iu_free(tx_iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1); 8629cb83748SJack Wang } 8639cb83748SJack Wang rwr_free: 8649cb83748SJack Wang kfree(rwr); 8659cb83748SJack Wang 8669cb83748SJack Wang return err; 8679cb83748SJack Wang } 8689cb83748SJack Wang 8699cb83748SJack Wang static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) 8709cb83748SJack Wang { 8719cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 8729cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 8739cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 8749cb83748SJack Wang struct rtrs_msg_info_req *msg; 8759cb83748SJack Wang struct rtrs_iu *iu; 8769cb83748SJack Wang int err; 8779cb83748SJack Wang 8789cb83748SJack Wang WARN_ON(con->c.cid); 8799cb83748SJack Wang 8809cb83748SJack Wang iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); 8819cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 8829cb83748SJack Wang rtrs_err(s, "Sess info request receive failed: %s\n", 8839cb83748SJack Wang ib_wc_status_msg(wc->status)); 8849cb83748SJack Wang goto close; 8859cb83748SJack Wang } 8869cb83748SJack Wang WARN_ON(wc->opcode != IB_WC_RECV); 8879cb83748SJack Wang 8889cb83748SJack Wang if (unlikely(wc->byte_len < sizeof(*msg))) { 8899cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: size %d\n", 8909cb83748SJack Wang wc->byte_len); 8919cb83748SJack Wang goto close; 8929cb83748SJack Wang } 8939cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, 8949cb83748SJack Wang iu->size, DMA_FROM_DEVICE); 8959cb83748SJack Wang msg = iu->buf; 8969cb83748SJack Wang if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ)) { 8979cb83748SJack Wang rtrs_err(s, "Sess info request is malformed: type %d\n", 8989cb83748SJack Wang le16_to_cpu(msg->type)); 8999cb83748SJack Wang goto close; 9009cb83748SJack Wang } 9019cb83748SJack Wang err = process_info_req(con, msg); 9029cb83748SJack Wang if (unlikely(err)) 9039cb83748SJack Wang goto close; 9049cb83748SJack Wang 9059cb83748SJack Wang out: 9069cb83748SJack Wang rtrs_iu_free(iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1); 9079cb83748SJack Wang return; 9089cb83748SJack Wang close: 9099cb83748SJack Wang close_sess(sess); 9109cb83748SJack Wang goto out; 9119cb83748SJack Wang } 9129cb83748SJack Wang 9139cb83748SJack Wang static int post_recv_info_req(struct rtrs_srv_con *con) 9149cb83748SJack Wang { 9159cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9169cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9179cb83748SJack Wang struct rtrs_iu *rx_iu; 9189cb83748SJack Wang int err; 9199cb83748SJack Wang 9209cb83748SJack Wang rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), 9219cb83748SJack Wang GFP_KERNEL, sess->s.dev->ib_dev, 9229cb83748SJack Wang DMA_FROM_DEVICE, rtrs_srv_info_req_done); 9239cb83748SJack Wang if (unlikely(!rx_iu)) 9249cb83748SJack Wang return -ENOMEM; 9259cb83748SJack Wang /* Prepare for getting info response */ 9269cb83748SJack Wang err = rtrs_iu_post_recv(&con->c, rx_iu); 9279cb83748SJack Wang if (unlikely(err)) { 9289cb83748SJack Wang rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err); 9299cb83748SJack Wang rtrs_iu_free(rx_iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1); 9309cb83748SJack Wang return err; 9319cb83748SJack Wang } 9329cb83748SJack Wang 9339cb83748SJack Wang return 0; 9349cb83748SJack Wang } 9359cb83748SJack Wang 9369cb83748SJack Wang static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) 9379cb83748SJack Wang { 9389cb83748SJack Wang int i, err; 9399cb83748SJack Wang 9409cb83748SJack Wang for (i = 0; i < q_size; i++) { 9419cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 9429cb83748SJack Wang if (unlikely(err)) 9439cb83748SJack Wang return err; 9449cb83748SJack Wang } 9459cb83748SJack Wang 9469cb83748SJack Wang return 0; 9479cb83748SJack Wang } 9489cb83748SJack Wang 9499cb83748SJack Wang static int post_recv_sess(struct rtrs_srv_sess *sess) 9509cb83748SJack Wang { 9519cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9529cb83748SJack Wang struct rtrs_sess *s = &sess->s; 9539cb83748SJack Wang size_t q_size; 9549cb83748SJack Wang int err, cid; 9559cb83748SJack Wang 9569cb83748SJack Wang for (cid = 0; cid < sess->s.con_num; cid++) { 9579cb83748SJack Wang if (cid == 0) 9589cb83748SJack Wang q_size = SERVICE_CON_QUEUE_DEPTH; 9599cb83748SJack Wang else 9609cb83748SJack Wang q_size = srv->queue_depth; 9619cb83748SJack Wang 9629cb83748SJack Wang err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size); 9639cb83748SJack Wang if (unlikely(err)) { 9649cb83748SJack Wang rtrs_err(s, "post_recv_io(), err: %d\n", err); 9659cb83748SJack Wang return err; 9669cb83748SJack Wang } 9679cb83748SJack Wang } 9689cb83748SJack Wang 9699cb83748SJack Wang return 0; 9709cb83748SJack Wang } 9719cb83748SJack Wang 9729cb83748SJack Wang static void process_read(struct rtrs_srv_con *con, 9739cb83748SJack Wang struct rtrs_msg_rdma_read *msg, 9749cb83748SJack Wang u32 buf_id, u32 off) 9759cb83748SJack Wang { 9769cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 9779cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 9789cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 9799cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 9809cb83748SJack Wang struct rtrs_srv_op *id; 9819cb83748SJack Wang 9829cb83748SJack Wang size_t usr_len, data_len; 9839cb83748SJack Wang void *data; 9849cb83748SJack Wang int ret; 9859cb83748SJack Wang 9869cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 9879cb83748SJack Wang rtrs_err_rl(s, 9889cb83748SJack Wang "Processing read request failed, session is disconnected, sess state %s\n", 9899cb83748SJack Wang rtrs_srv_state_str(sess->state)); 9909cb83748SJack Wang return; 9919cb83748SJack Wang } 9929cb83748SJack Wang if (unlikely(msg->sg_cnt != 1 && msg->sg_cnt != 0)) { 9939cb83748SJack Wang rtrs_err_rl(s, 9949cb83748SJack Wang "Processing read request failed, invalid message\n"); 9959cb83748SJack Wang return; 9969cb83748SJack Wang } 9979cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 9989cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, READ); 9999cb83748SJack Wang id = sess->ops_ids[buf_id]; 10009cb83748SJack Wang id->con = con; 10019cb83748SJack Wang id->dir = READ; 10029cb83748SJack Wang id->msg_id = buf_id; 10039cb83748SJack Wang id->rd_msg = msg; 10049cb83748SJack Wang usr_len = le16_to_cpu(msg->usr_len); 10059cb83748SJack Wang data_len = off - usr_len; 10069cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 10079cb83748SJack Wang ret = ctx->ops.rdma_ev(srv, srv->priv, id, READ, data, data_len, 10089cb83748SJack Wang data + data_len, usr_len); 10099cb83748SJack Wang 10109cb83748SJack Wang if (unlikely(ret)) { 10119cb83748SJack Wang rtrs_err_rl(s, 10129cb83748SJack Wang "Processing read request failed, user module cb reported for msg_id %d, err: %d\n", 10139cb83748SJack Wang buf_id, ret); 10149cb83748SJack Wang goto send_err_msg; 10159cb83748SJack Wang } 10169cb83748SJack Wang 10179cb83748SJack Wang return; 10189cb83748SJack Wang 10199cb83748SJack Wang send_err_msg: 10209cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10219cb83748SJack Wang if (ret < 0) { 10229cb83748SJack Wang rtrs_err_rl(s, 10239cb83748SJack Wang "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n", 10249cb83748SJack Wang buf_id, ret); 10259cb83748SJack Wang close_sess(sess); 10269cb83748SJack Wang } 10279cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10289cb83748SJack Wang } 10299cb83748SJack Wang 10309cb83748SJack Wang static void process_write(struct rtrs_srv_con *con, 10319cb83748SJack Wang struct rtrs_msg_rdma_write *req, 10329cb83748SJack Wang u32 buf_id, u32 off) 10339cb83748SJack Wang { 10349cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10359cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10369cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 10379cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 10389cb83748SJack Wang struct rtrs_srv_op *id; 10399cb83748SJack Wang 10409cb83748SJack Wang size_t data_len, usr_len; 10419cb83748SJack Wang void *data; 10429cb83748SJack Wang int ret; 10439cb83748SJack Wang 10449cb83748SJack Wang if (unlikely(sess->state != RTRS_SRV_CONNECTED)) { 10459cb83748SJack Wang rtrs_err_rl(s, 10469cb83748SJack Wang "Processing write request failed, session is disconnected, sess state %s\n", 10479cb83748SJack Wang rtrs_srv_state_str(sess->state)); 10489cb83748SJack Wang return; 10499cb83748SJack Wang } 10509cb83748SJack Wang rtrs_srv_get_ops_ids(sess); 10519cb83748SJack Wang rtrs_srv_update_rdma_stats(sess->stats, off, WRITE); 10529cb83748SJack Wang id = sess->ops_ids[buf_id]; 10539cb83748SJack Wang id->con = con; 10549cb83748SJack Wang id->dir = WRITE; 10559cb83748SJack Wang id->msg_id = buf_id; 10569cb83748SJack Wang 10579cb83748SJack Wang usr_len = le16_to_cpu(req->usr_len); 10589cb83748SJack Wang data_len = off - usr_len; 10599cb83748SJack Wang data = page_address(srv->chunks[buf_id]); 10609cb83748SJack Wang ret = ctx->ops.rdma_ev(srv, srv->priv, id, WRITE, data, data_len, 10619cb83748SJack Wang data + data_len, usr_len); 10629cb83748SJack Wang if (unlikely(ret)) { 10639cb83748SJack Wang rtrs_err_rl(s, 10649cb83748SJack Wang "Processing write request failed, user module callback reports err: %d\n", 10659cb83748SJack Wang ret); 10669cb83748SJack Wang goto send_err_msg; 10679cb83748SJack Wang } 10689cb83748SJack Wang 10699cb83748SJack Wang return; 10709cb83748SJack Wang 10719cb83748SJack Wang send_err_msg: 10729cb83748SJack Wang ret = send_io_resp_imm(con, id, ret); 10739cb83748SJack Wang if (ret < 0) { 10749cb83748SJack Wang rtrs_err_rl(s, 10759cb83748SJack Wang "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n", 10769cb83748SJack Wang buf_id, ret); 10779cb83748SJack Wang close_sess(sess); 10789cb83748SJack Wang } 10799cb83748SJack Wang rtrs_srv_put_ops_ids(sess); 10809cb83748SJack Wang } 10819cb83748SJack Wang 10829cb83748SJack Wang static void process_io_req(struct rtrs_srv_con *con, void *msg, 10839cb83748SJack Wang u32 id, u32 off) 10849cb83748SJack Wang { 10859cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 10869cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 10879cb83748SJack Wang struct rtrs_msg_rdma_hdr *hdr; 10889cb83748SJack Wang unsigned int type; 10899cb83748SJack Wang 10909cb83748SJack Wang ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id], 10919cb83748SJack Wang max_chunk_size, DMA_BIDIRECTIONAL); 10929cb83748SJack Wang hdr = msg; 10939cb83748SJack Wang type = le16_to_cpu(hdr->type); 10949cb83748SJack Wang 10959cb83748SJack Wang switch (type) { 10969cb83748SJack Wang case RTRS_MSG_WRITE: 10979cb83748SJack Wang process_write(con, msg, id, off); 10989cb83748SJack Wang break; 10999cb83748SJack Wang case RTRS_MSG_READ: 11009cb83748SJack Wang process_read(con, msg, id, off); 11019cb83748SJack Wang break; 11029cb83748SJack Wang default: 11039cb83748SJack Wang rtrs_err(s, 11049cb83748SJack Wang "Processing I/O request failed, unknown message type received: 0x%02x\n", 11059cb83748SJack Wang type); 11069cb83748SJack Wang goto err; 11079cb83748SJack Wang } 11089cb83748SJack Wang 11099cb83748SJack Wang return; 11109cb83748SJack Wang 11119cb83748SJack Wang err: 11129cb83748SJack Wang close_sess(sess); 11139cb83748SJack Wang } 11149cb83748SJack Wang 11159cb83748SJack Wang static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) 11169cb83748SJack Wang { 11179cb83748SJack Wang struct rtrs_srv_mr *mr = 11189cb83748SJack Wang container_of(wc->wr_cqe, typeof(*mr), inv_cqe); 11199cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 11209cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11219cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11229cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11239cb83748SJack Wang u32 msg_id, off; 11249cb83748SJack Wang void *data; 11259cb83748SJack Wang 11269cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11279cb83748SJack Wang rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n", 11289cb83748SJack Wang ib_wc_status_msg(wc->status)); 11299cb83748SJack Wang close_sess(sess); 11309cb83748SJack Wang } 11319cb83748SJack Wang msg_id = mr->msg_id; 11329cb83748SJack Wang off = mr->msg_off; 11339cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 11349cb83748SJack Wang process_io_req(con, data, msg_id, off); 11359cb83748SJack Wang } 11369cb83748SJack Wang 11379cb83748SJack Wang static int rtrs_srv_inv_rkey(struct rtrs_srv_con *con, 11389cb83748SJack Wang struct rtrs_srv_mr *mr) 11399cb83748SJack Wang { 11409cb83748SJack Wang struct ib_send_wr wr = { 11419cb83748SJack Wang .opcode = IB_WR_LOCAL_INV, 11429cb83748SJack Wang .wr_cqe = &mr->inv_cqe, 11439cb83748SJack Wang .send_flags = IB_SEND_SIGNALED, 11449cb83748SJack Wang .ex.invalidate_rkey = mr->mr->rkey, 11459cb83748SJack Wang }; 11469cb83748SJack Wang mr->inv_cqe.done = rtrs_srv_inv_rkey_done; 11479cb83748SJack Wang 11489cb83748SJack Wang return ib_post_send(con->c.qp, &wr, NULL); 11499cb83748SJack Wang } 11509cb83748SJack Wang 11519cb83748SJack Wang static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) 11529cb83748SJack Wang { 11539cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11549cb83748SJack Wang while (!list_empty(&con->rsp_wr_wait_list)) { 11559cb83748SJack Wang struct rtrs_srv_op *id; 11569cb83748SJack Wang int ret; 11579cb83748SJack Wang 11589cb83748SJack Wang id = list_entry(con->rsp_wr_wait_list.next, 11599cb83748SJack Wang struct rtrs_srv_op, wait_list); 11609cb83748SJack Wang list_del(&id->wait_list); 11619cb83748SJack Wang 11629cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11639cb83748SJack Wang ret = rtrs_srv_resp_rdma(id, id->status); 11649cb83748SJack Wang spin_lock(&con->rsp_wr_wait_lock); 11659cb83748SJack Wang 11669cb83748SJack Wang if (!ret) { 11679cb83748SJack Wang list_add(&id->wait_list, &con->rsp_wr_wait_list); 11689cb83748SJack Wang break; 11699cb83748SJack Wang } 11709cb83748SJack Wang } 11719cb83748SJack Wang spin_unlock(&con->rsp_wr_wait_lock); 11729cb83748SJack Wang } 11739cb83748SJack Wang 11749cb83748SJack Wang static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) 11759cb83748SJack Wang { 11769cb83748SJack Wang struct rtrs_srv_con *con = cq->cq_context; 11779cb83748SJack Wang struct rtrs_sess *s = con->c.sess; 11789cb83748SJack Wang struct rtrs_srv_sess *sess = to_srv_sess(s); 11799cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 11809cb83748SJack Wang u32 imm_type, imm_payload; 11819cb83748SJack Wang int err; 11829cb83748SJack Wang 11839cb83748SJack Wang if (unlikely(wc->status != IB_WC_SUCCESS)) { 11849cb83748SJack Wang if (wc->status != IB_WC_WR_FLUSH_ERR) { 11859cb83748SJack Wang rtrs_err(s, 11869cb83748SJack Wang "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n", 11879cb83748SJack Wang ib_wc_status_msg(wc->status), wc->wr_cqe, 11889cb83748SJack Wang wc->opcode, wc->vendor_err, wc->byte_len); 11899cb83748SJack Wang close_sess(sess); 11909cb83748SJack Wang } 11919cb83748SJack Wang return; 11929cb83748SJack Wang } 11939cb83748SJack Wang 11949cb83748SJack Wang switch (wc->opcode) { 11959cb83748SJack Wang case IB_WC_RECV_RDMA_WITH_IMM: 11969cb83748SJack Wang /* 11979cb83748SJack Wang * post_recv() RDMA write completions of IO reqs (read/write) 11989cb83748SJack Wang * and hb 11999cb83748SJack Wang */ 12009cb83748SJack Wang if (WARN_ON(wc->wr_cqe != &io_comp_cqe)) 12019cb83748SJack Wang return; 12029cb83748SJack Wang err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); 12039cb83748SJack Wang if (unlikely(err)) { 12049cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); 12059cb83748SJack Wang close_sess(sess); 12069cb83748SJack Wang break; 12079cb83748SJack Wang } 12089cb83748SJack Wang rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), 12099cb83748SJack Wang &imm_type, &imm_payload); 12109cb83748SJack Wang if (likely(imm_type == RTRS_IO_REQ_IMM)) { 12119cb83748SJack Wang u32 msg_id, off; 12129cb83748SJack Wang void *data; 12139cb83748SJack Wang 12149cb83748SJack Wang msg_id = imm_payload >> sess->mem_bits; 12159cb83748SJack Wang off = imm_payload & ((1 << sess->mem_bits) - 1); 12169cb83748SJack Wang if (unlikely(msg_id > srv->queue_depth || 12179cb83748SJack Wang off > max_chunk_size)) { 12189cb83748SJack Wang rtrs_err(s, "Wrong msg_id %u, off %u\n", 12199cb83748SJack Wang msg_id, off); 12209cb83748SJack Wang close_sess(sess); 12219cb83748SJack Wang return; 12229cb83748SJack Wang } 12239cb83748SJack Wang if (always_invalidate) { 12249cb83748SJack Wang struct rtrs_srv_mr *mr = &sess->mrs[msg_id]; 12259cb83748SJack Wang 12269cb83748SJack Wang mr->msg_off = off; 12279cb83748SJack Wang mr->msg_id = msg_id; 12289cb83748SJack Wang err = rtrs_srv_inv_rkey(con, mr); 12299cb83748SJack Wang if (unlikely(err)) { 12309cb83748SJack Wang rtrs_err(s, "rtrs_post_recv(), err: %d\n", 12319cb83748SJack Wang err); 12329cb83748SJack Wang close_sess(sess); 12339cb83748SJack Wang break; 12349cb83748SJack Wang } 12359cb83748SJack Wang } else { 12369cb83748SJack Wang data = page_address(srv->chunks[msg_id]) + off; 12379cb83748SJack Wang process_io_req(con, data, msg_id, off); 12389cb83748SJack Wang } 12399cb83748SJack Wang } else if (imm_type == RTRS_HB_MSG_IMM) { 12409cb83748SJack Wang WARN_ON(con->c.cid); 12419cb83748SJack Wang rtrs_send_hb_ack(&sess->s); 12429cb83748SJack Wang } else if (imm_type == RTRS_HB_ACK_IMM) { 12439cb83748SJack Wang WARN_ON(con->c.cid); 12449cb83748SJack Wang sess->s.hb_missed_cnt = 0; 12459cb83748SJack Wang } else { 12469cb83748SJack Wang rtrs_wrn(s, "Unknown IMM type %u\n", imm_type); 12479cb83748SJack Wang } 12489cb83748SJack Wang break; 12499cb83748SJack Wang case IB_WC_RDMA_WRITE: 12509cb83748SJack Wang case IB_WC_SEND: 12519cb83748SJack Wang /* 12529cb83748SJack Wang * post_send() RDMA write completions of IO reqs (read/write) 12539cb83748SJack Wang * and hb 12549cb83748SJack Wang */ 12559cb83748SJack Wang atomic_add(srv->queue_depth, &con->sq_wr_avail); 12569cb83748SJack Wang 12579cb83748SJack Wang if (unlikely(!list_empty_careful(&con->rsp_wr_wait_list))) 12589cb83748SJack Wang rtrs_rdma_process_wr_wait_list(con); 12599cb83748SJack Wang 12609cb83748SJack Wang break; 12619cb83748SJack Wang default: 12629cb83748SJack Wang rtrs_wrn(s, "Unexpected WC type: %d\n", wc->opcode); 12639cb83748SJack Wang return; 12649cb83748SJack Wang } 12659cb83748SJack Wang } 12669cb83748SJack Wang 12679cb83748SJack Wang /** 12689cb83748SJack Wang * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname. 12699cb83748SJack Wang * @srv: Session 12709cb83748SJack Wang * @sessname: Sessname buffer 12719cb83748SJack Wang * @len: Length of sessname buffer 12729cb83748SJack Wang */ 12739cb83748SJack Wang int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) 12749cb83748SJack Wang { 12759cb83748SJack Wang struct rtrs_srv_sess *sess; 12769cb83748SJack Wang int err = -ENOTCONN; 12779cb83748SJack Wang 12789cb83748SJack Wang mutex_lock(&srv->paths_mutex); 12799cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 12809cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTED) 12819cb83748SJack Wang continue; 12829cb83748SJack Wang strlcpy(sessname, sess->s.sessname, 12839cb83748SJack Wang min_t(size_t, sizeof(sess->s.sessname), len)); 12849cb83748SJack Wang err = 0; 12859cb83748SJack Wang break; 12869cb83748SJack Wang } 12879cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 12889cb83748SJack Wang 12899cb83748SJack Wang return err; 12909cb83748SJack Wang } 12919cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_sess_name); 12929cb83748SJack Wang 12939cb83748SJack Wang /** 12949cb83748SJack Wang * rtrs_srv_get_sess_qdepth() - Get rtrs_srv qdepth. 12959cb83748SJack Wang * @srv: Session 12969cb83748SJack Wang */ 12979cb83748SJack Wang int rtrs_srv_get_queue_depth(struct rtrs_srv *srv) 12989cb83748SJack Wang { 12999cb83748SJack Wang return srv->queue_depth; 13009cb83748SJack Wang } 13019cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_get_queue_depth); 13029cb83748SJack Wang 13039cb83748SJack Wang static int find_next_bit_ring(struct rtrs_srv_sess *sess) 13049cb83748SJack Wang { 13059cb83748SJack Wang struct ib_device *ib_dev = sess->s.dev->ib_dev; 13069cb83748SJack Wang int v; 13079cb83748SJack Wang 13089cb83748SJack Wang v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask); 13099cb83748SJack Wang if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors) 13109cb83748SJack Wang v = cpumask_first(&cq_affinity_mask); 13119cb83748SJack Wang return v; 13129cb83748SJack Wang } 13139cb83748SJack Wang 13149cb83748SJack Wang static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess) 13159cb83748SJack Wang { 13169cb83748SJack Wang sess->cur_cq_vector = find_next_bit_ring(sess); 13179cb83748SJack Wang 13189cb83748SJack Wang return sess->cur_cq_vector; 13199cb83748SJack Wang } 13209cb83748SJack Wang 13219cb83748SJack Wang static struct rtrs_srv *__alloc_srv(struct rtrs_srv_ctx *ctx, 13229cb83748SJack Wang const uuid_t *paths_uuid) 13239cb83748SJack Wang { 13249cb83748SJack Wang struct rtrs_srv *srv; 13259cb83748SJack Wang int i; 13269cb83748SJack Wang 13279cb83748SJack Wang srv = kzalloc(sizeof(*srv), GFP_KERNEL); 13289cb83748SJack Wang if (!srv) 13299cb83748SJack Wang return NULL; 13309cb83748SJack Wang 13319cb83748SJack Wang refcount_set(&srv->refcount, 1); 13329cb83748SJack Wang INIT_LIST_HEAD(&srv->paths_list); 13339cb83748SJack Wang mutex_init(&srv->paths_mutex); 13349cb83748SJack Wang mutex_init(&srv->paths_ev_mutex); 13359cb83748SJack Wang uuid_copy(&srv->paths_uuid, paths_uuid); 13369cb83748SJack Wang srv->queue_depth = sess_queue_depth; 13379cb83748SJack Wang srv->ctx = ctx; 13389cb83748SJack Wang 13399cb83748SJack Wang srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks), 13409cb83748SJack Wang GFP_KERNEL); 13419cb83748SJack Wang if (!srv->chunks) 13429cb83748SJack Wang goto err_free_srv; 13439cb83748SJack Wang 13449cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) { 13459cb83748SJack Wang srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL); 13469cb83748SJack Wang if (!srv->chunks[i]) 13479cb83748SJack Wang goto err_free_chunks; 13489cb83748SJack Wang } 13499cb83748SJack Wang list_add(&srv->ctx_list, &ctx->srv_list); 13509cb83748SJack Wang 13519cb83748SJack Wang return srv; 13529cb83748SJack Wang 13539cb83748SJack Wang err_free_chunks: 13549cb83748SJack Wang while (i--) 13559cb83748SJack Wang mempool_free(srv->chunks[i], chunk_pool); 13569cb83748SJack Wang kfree(srv->chunks); 13579cb83748SJack Wang 13589cb83748SJack Wang err_free_srv: 13599cb83748SJack Wang kfree(srv); 13609cb83748SJack Wang 13619cb83748SJack Wang return NULL; 13629cb83748SJack Wang } 13639cb83748SJack Wang 13649cb83748SJack Wang static void free_srv(struct rtrs_srv *srv) 13659cb83748SJack Wang { 13669cb83748SJack Wang int i; 13679cb83748SJack Wang 13689cb83748SJack Wang WARN_ON(refcount_read(&srv->refcount)); 13699cb83748SJack Wang for (i = 0; i < srv->queue_depth; i++) 13709cb83748SJack Wang mempool_free(srv->chunks[i], chunk_pool); 13719cb83748SJack Wang kfree(srv->chunks); 13729cb83748SJack Wang mutex_destroy(&srv->paths_mutex); 13739cb83748SJack Wang mutex_destroy(&srv->paths_ev_mutex); 13749cb83748SJack Wang /* last put to release the srv structure */ 13759cb83748SJack Wang put_device(&srv->dev); 13769cb83748SJack Wang } 13779cb83748SJack Wang 13789cb83748SJack Wang static inline struct rtrs_srv *__find_srv_and_get(struct rtrs_srv_ctx *ctx, 13799cb83748SJack Wang const uuid_t *paths_uuid) 13809cb83748SJack Wang { 13819cb83748SJack Wang struct rtrs_srv *srv; 13829cb83748SJack Wang 13839cb83748SJack Wang list_for_each_entry(srv, &ctx->srv_list, ctx_list) { 13849cb83748SJack Wang if (uuid_equal(&srv->paths_uuid, paths_uuid) && 13859cb83748SJack Wang refcount_inc_not_zero(&srv->refcount)) 13869cb83748SJack Wang return srv; 13879cb83748SJack Wang } 13889cb83748SJack Wang 13899cb83748SJack Wang return NULL; 13909cb83748SJack Wang } 13919cb83748SJack Wang 13929cb83748SJack Wang static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx, 13939cb83748SJack Wang const uuid_t *paths_uuid) 13949cb83748SJack Wang { 13959cb83748SJack Wang struct rtrs_srv *srv; 13969cb83748SJack Wang 13979cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 13989cb83748SJack Wang srv = __find_srv_and_get(ctx, paths_uuid); 13999cb83748SJack Wang if (!srv) 14009cb83748SJack Wang srv = __alloc_srv(ctx, paths_uuid); 14019cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 14029cb83748SJack Wang 14039cb83748SJack Wang return srv; 14049cb83748SJack Wang } 14059cb83748SJack Wang 14069cb83748SJack Wang static void put_srv(struct rtrs_srv *srv) 14079cb83748SJack Wang { 14089cb83748SJack Wang if (refcount_dec_and_test(&srv->refcount)) { 14099cb83748SJack Wang struct rtrs_srv_ctx *ctx = srv->ctx; 14109cb83748SJack Wang 14119cb83748SJack Wang WARN_ON(srv->dev.kobj.state_in_sysfs); 14129cb83748SJack Wang 14139cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 14149cb83748SJack Wang list_del(&srv->ctx_list); 14159cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 14169cb83748SJack Wang free_srv(srv); 14179cb83748SJack Wang } 14189cb83748SJack Wang } 14199cb83748SJack Wang 14209cb83748SJack Wang static void __add_path_to_srv(struct rtrs_srv *srv, 14219cb83748SJack Wang struct rtrs_srv_sess *sess) 14229cb83748SJack Wang { 14239cb83748SJack Wang list_add_tail(&sess->s.entry, &srv->paths_list); 14249cb83748SJack Wang srv->paths_num++; 14259cb83748SJack Wang WARN_ON(srv->paths_num >= MAX_PATHS_NUM); 14269cb83748SJack Wang } 14279cb83748SJack Wang 14289cb83748SJack Wang static void del_path_from_srv(struct rtrs_srv_sess *sess) 14299cb83748SJack Wang { 14309cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 14319cb83748SJack Wang 14329cb83748SJack Wang if (WARN_ON(!srv)) 14339cb83748SJack Wang return; 14349cb83748SJack Wang 14359cb83748SJack Wang mutex_lock(&srv->paths_mutex); 14369cb83748SJack Wang list_del(&sess->s.entry); 14379cb83748SJack Wang WARN_ON(!srv->paths_num); 14389cb83748SJack Wang srv->paths_num--; 14399cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 14409cb83748SJack Wang } 14419cb83748SJack Wang 14429cb83748SJack Wang /* return true if addresses are the same, error other wise */ 14439cb83748SJack Wang static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) 14449cb83748SJack Wang { 14459cb83748SJack Wang switch (a->sa_family) { 14469cb83748SJack Wang case AF_IB: 14479cb83748SJack Wang return memcmp(&((struct sockaddr_ib *)a)->sib_addr, 14489cb83748SJack Wang &((struct sockaddr_ib *)b)->sib_addr, 14499cb83748SJack Wang sizeof(struct ib_addr)) && 14509cb83748SJack Wang (b->sa_family == AF_IB); 14519cb83748SJack Wang case AF_INET: 14529cb83748SJack Wang return memcmp(&((struct sockaddr_in *)a)->sin_addr, 14539cb83748SJack Wang &((struct sockaddr_in *)b)->sin_addr, 14549cb83748SJack Wang sizeof(struct in_addr)) && 14559cb83748SJack Wang (b->sa_family == AF_INET); 14569cb83748SJack Wang case AF_INET6: 14579cb83748SJack Wang return memcmp(&((struct sockaddr_in6 *)a)->sin6_addr, 14589cb83748SJack Wang &((struct sockaddr_in6 *)b)->sin6_addr, 14599cb83748SJack Wang sizeof(struct in6_addr)) && 14609cb83748SJack Wang (b->sa_family == AF_INET6); 14619cb83748SJack Wang default: 14629cb83748SJack Wang return -ENOENT; 14639cb83748SJack Wang } 14649cb83748SJack Wang } 14659cb83748SJack Wang 14669cb83748SJack Wang static bool __is_path_w_addr_exists(struct rtrs_srv *srv, 14679cb83748SJack Wang struct rdma_addr *addr) 14689cb83748SJack Wang { 14699cb83748SJack Wang struct rtrs_srv_sess *sess; 14709cb83748SJack Wang 14719cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 14729cb83748SJack Wang if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr, 14739cb83748SJack Wang (struct sockaddr *)&addr->dst_addr) && 14749cb83748SJack Wang !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr, 14759cb83748SJack Wang (struct sockaddr *)&addr->src_addr)) 14769cb83748SJack Wang return true; 14779cb83748SJack Wang 14789cb83748SJack Wang return false; 14799cb83748SJack Wang } 14809cb83748SJack Wang 14819cb83748SJack Wang static void free_sess(struct rtrs_srv_sess *sess) 14829cb83748SJack Wang { 14839cb83748SJack Wang if (sess->kobj.state_in_sysfs) 14849cb83748SJack Wang kobject_put(&sess->kobj); 14859cb83748SJack Wang else 14869cb83748SJack Wang kfree(sess); 14879cb83748SJack Wang } 14889cb83748SJack Wang 14899cb83748SJack Wang static void rtrs_srv_close_work(struct work_struct *work) 14909cb83748SJack Wang { 14919cb83748SJack Wang struct rtrs_srv_sess *sess; 14929cb83748SJack Wang struct rtrs_srv_con *con; 14939cb83748SJack Wang int i; 14949cb83748SJack Wang 14959cb83748SJack Wang sess = container_of(work, typeof(*sess), close_work); 14969cb83748SJack Wang 14979cb83748SJack Wang rtrs_srv_destroy_sess_files(sess); 14989cb83748SJack Wang rtrs_srv_stop_hb(sess); 14999cb83748SJack Wang 15009cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 15019cb83748SJack Wang if (!sess->s.con[i]) 15029cb83748SJack Wang continue; 15039cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 15049cb83748SJack Wang rdma_disconnect(con->c.cm_id); 15059cb83748SJack Wang ib_drain_qp(con->c.qp); 15069cb83748SJack Wang } 15079cb83748SJack Wang /* Wait for all inflights */ 15089cb83748SJack Wang rtrs_srv_wait_ops_ids(sess); 15099cb83748SJack Wang 15109cb83748SJack Wang /* Notify upper layer if we are the last path */ 15119cb83748SJack Wang rtrs_srv_sess_down(sess); 15129cb83748SJack Wang 15139cb83748SJack Wang unmap_cont_bufs(sess); 15149cb83748SJack Wang rtrs_srv_free_ops_ids(sess); 15159cb83748SJack Wang 15169cb83748SJack Wang for (i = 0; i < sess->s.con_num; i++) { 15179cb83748SJack Wang if (!sess->s.con[i]) 15189cb83748SJack Wang continue; 15199cb83748SJack Wang con = to_srv_con(sess->s.con[i]); 15209cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 15219cb83748SJack Wang rdma_destroy_id(con->c.cm_id); 15229cb83748SJack Wang kfree(con); 15239cb83748SJack Wang } 15249cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 15259cb83748SJack Wang 15269cb83748SJack Wang del_path_from_srv(sess); 15279cb83748SJack Wang put_srv(sess->srv); 15289cb83748SJack Wang sess->srv = NULL; 15299cb83748SJack Wang rtrs_srv_change_state(sess, RTRS_SRV_CLOSED); 15309cb83748SJack Wang 15319cb83748SJack Wang kfree(sess->dma_addr); 15329cb83748SJack Wang kfree(sess->s.con); 15339cb83748SJack Wang free_sess(sess); 15349cb83748SJack Wang } 15359cb83748SJack Wang 15369cb83748SJack Wang static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess, 15379cb83748SJack Wang struct rdma_cm_id *cm_id) 15389cb83748SJack Wang { 15399cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 15409cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15419cb83748SJack Wang struct rdma_conn_param param; 15429cb83748SJack Wang int err; 15439cb83748SJack Wang 15449cb83748SJack Wang param = (struct rdma_conn_param) { 15459cb83748SJack Wang .rnr_retry_count = 7, 15469cb83748SJack Wang .private_data = &msg, 15479cb83748SJack Wang .private_data_len = sizeof(msg), 15489cb83748SJack Wang }; 15499cb83748SJack Wang 15509cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15519cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15529cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15539cb83748SJack Wang .queue_depth = cpu_to_le16(srv->queue_depth), 15549cb83748SJack Wang .max_io_size = cpu_to_le32(max_chunk_size - MAX_HDR_SIZE), 15559cb83748SJack Wang .max_hdr_size = cpu_to_le32(MAX_HDR_SIZE), 15569cb83748SJack Wang }; 15579cb83748SJack Wang 15589cb83748SJack Wang if (always_invalidate) 15599cb83748SJack Wang msg.flags = cpu_to_le32(RTRS_MSG_NEW_RKEY_F); 15609cb83748SJack Wang 15619cb83748SJack Wang err = rdma_accept(cm_id, ¶m); 15629cb83748SJack Wang if (err) 15639cb83748SJack Wang pr_err("rdma_accept(), err: %d\n", err); 15649cb83748SJack Wang 15659cb83748SJack Wang return err; 15669cb83748SJack Wang } 15679cb83748SJack Wang 15689cb83748SJack Wang static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) 15699cb83748SJack Wang { 15709cb83748SJack Wang struct rtrs_msg_conn_rsp msg; 15719cb83748SJack Wang int err; 15729cb83748SJack Wang 15739cb83748SJack Wang msg = (struct rtrs_msg_conn_rsp) { 15749cb83748SJack Wang .magic = cpu_to_le16(RTRS_MAGIC), 15759cb83748SJack Wang .version = cpu_to_le16(RTRS_PROTO_VER), 15769cb83748SJack Wang .errno = cpu_to_le16(errno), 15779cb83748SJack Wang }; 15789cb83748SJack Wang 15799cb83748SJack Wang err = rdma_reject(cm_id, &msg, sizeof(msg)); 15809cb83748SJack Wang if (err) 15819cb83748SJack Wang pr_err("rdma_reject(), err: %d\n", err); 15829cb83748SJack Wang 15839cb83748SJack Wang /* Bounce errno back */ 15849cb83748SJack Wang return errno; 15859cb83748SJack Wang } 15869cb83748SJack Wang 15879cb83748SJack Wang static struct rtrs_srv_sess * 15889cb83748SJack Wang __find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid) 15899cb83748SJack Wang { 15909cb83748SJack Wang struct rtrs_srv_sess *sess; 15919cb83748SJack Wang 15929cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) { 15939cb83748SJack Wang if (uuid_equal(&sess->s.uuid, sess_uuid)) 15949cb83748SJack Wang return sess; 15959cb83748SJack Wang } 15969cb83748SJack Wang 15979cb83748SJack Wang return NULL; 15989cb83748SJack Wang } 15999cb83748SJack Wang 16009cb83748SJack Wang static int create_con(struct rtrs_srv_sess *sess, 16019cb83748SJack Wang struct rdma_cm_id *cm_id, 16029cb83748SJack Wang unsigned int cid) 16039cb83748SJack Wang { 16049cb83748SJack Wang struct rtrs_srv *srv = sess->srv; 16059cb83748SJack Wang struct rtrs_sess *s = &sess->s; 16069cb83748SJack Wang struct rtrs_srv_con *con; 16079cb83748SJack Wang 16089cb83748SJack Wang u16 cq_size, wr_queue_size; 16099cb83748SJack Wang int err, cq_vector; 16109cb83748SJack Wang 16119cb83748SJack Wang con = kzalloc(sizeof(*con), GFP_KERNEL); 16129cb83748SJack Wang if (!con) { 16139cb83748SJack Wang err = -ENOMEM; 16149cb83748SJack Wang goto err; 16159cb83748SJack Wang } 16169cb83748SJack Wang 16179cb83748SJack Wang spin_lock_init(&con->rsp_wr_wait_lock); 16189cb83748SJack Wang INIT_LIST_HEAD(&con->rsp_wr_wait_list); 16199cb83748SJack Wang con->c.cm_id = cm_id; 16209cb83748SJack Wang con->c.sess = &sess->s; 16219cb83748SJack Wang con->c.cid = cid; 16229cb83748SJack Wang atomic_set(&con->wr_cnt, 0); 16239cb83748SJack Wang 16249cb83748SJack Wang if (con->c.cid == 0) { 16259cb83748SJack Wang /* 16269cb83748SJack Wang * All receive and all send (each requiring invalidate) 16279cb83748SJack Wang * + 2 for drain and heartbeat 16289cb83748SJack Wang */ 16299cb83748SJack Wang wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2; 16309cb83748SJack Wang cq_size = wr_queue_size; 16319cb83748SJack Wang } else { 16329cb83748SJack Wang /* 16339cb83748SJack Wang * If we have all receive requests posted and 16349cb83748SJack Wang * all write requests posted and each read request 16359cb83748SJack Wang * requires an invalidate request + drain 16369cb83748SJack Wang * and qp gets into error state. 16379cb83748SJack Wang */ 16389cb83748SJack Wang cq_size = srv->queue_depth * 3 + 1; 16399cb83748SJack Wang /* 16409cb83748SJack Wang * In theory we might have queue_depth * 32 16419cb83748SJack Wang * outstanding requests if an unsafe global key is used 16429cb83748SJack Wang * and we have queue_depth read requests each consisting 16439cb83748SJack Wang * of 32 different addresses. div 3 for mlx5. 16449cb83748SJack Wang */ 16459cb83748SJack Wang wr_queue_size = sess->s.dev->ib_dev->attrs.max_qp_wr / 3; 16469cb83748SJack Wang } 16479cb83748SJack Wang atomic_set(&con->sq_wr_avail, wr_queue_size); 16489cb83748SJack Wang cq_vector = rtrs_srv_get_next_cq_vector(sess); 16499cb83748SJack Wang 16509cb83748SJack Wang /* TODO: SOFTIRQ can be faster, but be careful with softirq context */ 16519cb83748SJack Wang err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size, 16529cb83748SJack Wang wr_queue_size, IB_POLL_WORKQUEUE); 16539cb83748SJack Wang if (err) { 16549cb83748SJack Wang rtrs_err(s, "rtrs_cq_qp_create(), err: %d\n", err); 16559cb83748SJack Wang goto free_con; 16569cb83748SJack Wang } 16579cb83748SJack Wang if (con->c.cid == 0) { 16589cb83748SJack Wang err = post_recv_info_req(con); 16599cb83748SJack Wang if (err) 16609cb83748SJack Wang goto free_cqqp; 16619cb83748SJack Wang } 16629cb83748SJack Wang WARN_ON(sess->s.con[cid]); 16639cb83748SJack Wang sess->s.con[cid] = &con->c; 16649cb83748SJack Wang 16659cb83748SJack Wang /* 16669cb83748SJack Wang * Change context from server to current connection. The other 16679cb83748SJack Wang * way is to use cm_id->qp->qp_context, which does not work on OFED. 16689cb83748SJack Wang */ 16699cb83748SJack Wang cm_id->context = &con->c; 16709cb83748SJack Wang 16719cb83748SJack Wang return 0; 16729cb83748SJack Wang 16739cb83748SJack Wang free_cqqp: 16749cb83748SJack Wang rtrs_cq_qp_destroy(&con->c); 16759cb83748SJack Wang free_con: 16769cb83748SJack Wang kfree(con); 16779cb83748SJack Wang 16789cb83748SJack Wang err: 16799cb83748SJack Wang return err; 16809cb83748SJack Wang } 16819cb83748SJack Wang 16829cb83748SJack Wang static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, 16839cb83748SJack Wang struct rdma_cm_id *cm_id, 16849cb83748SJack Wang unsigned int con_num, 16859cb83748SJack Wang unsigned int recon_cnt, 16869cb83748SJack Wang const uuid_t *uuid) 16879cb83748SJack Wang { 16889cb83748SJack Wang struct rtrs_srv_sess *sess; 16899cb83748SJack Wang int err = -ENOMEM; 16909cb83748SJack Wang 16919cb83748SJack Wang if (srv->paths_num >= MAX_PATHS_NUM) { 16929cb83748SJack Wang err = -ECONNRESET; 16939cb83748SJack Wang goto err; 16949cb83748SJack Wang } 16959cb83748SJack Wang if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) { 16969cb83748SJack Wang err = -EEXIST; 16979cb83748SJack Wang pr_err("Path with same addr exists\n"); 16989cb83748SJack Wang goto err; 16999cb83748SJack Wang } 17009cb83748SJack Wang sess = kzalloc(sizeof(*sess), GFP_KERNEL); 17019cb83748SJack Wang if (!sess) 17029cb83748SJack Wang goto err; 17039cb83748SJack Wang 17049cb83748SJack Wang sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); 17059cb83748SJack Wang if (!sess->stats) 17069cb83748SJack Wang goto err_free_sess; 17079cb83748SJack Wang 17089cb83748SJack Wang sess->stats->sess = sess; 17099cb83748SJack Wang 17109cb83748SJack Wang sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr), 17119cb83748SJack Wang GFP_KERNEL); 17129cb83748SJack Wang if (!sess->dma_addr) 17139cb83748SJack Wang goto err_free_stats; 17149cb83748SJack Wang 17159cb83748SJack Wang sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL); 17169cb83748SJack Wang if (!sess->s.con) 17179cb83748SJack Wang goto err_free_dma_addr; 17189cb83748SJack Wang 17199cb83748SJack Wang sess->state = RTRS_SRV_CONNECTING; 17209cb83748SJack Wang sess->srv = srv; 17219cb83748SJack Wang sess->cur_cq_vector = -1; 17229cb83748SJack Wang sess->s.dst_addr = cm_id->route.addr.dst_addr; 17239cb83748SJack Wang sess->s.src_addr = cm_id->route.addr.src_addr; 17249cb83748SJack Wang sess->s.con_num = con_num; 17259cb83748SJack Wang sess->s.recon_cnt = recon_cnt; 17269cb83748SJack Wang uuid_copy(&sess->s.uuid, uuid); 17279cb83748SJack Wang spin_lock_init(&sess->state_lock); 17289cb83748SJack Wang INIT_WORK(&sess->close_work, rtrs_srv_close_work); 17299cb83748SJack Wang rtrs_srv_init_hb(sess); 17309cb83748SJack Wang 17319cb83748SJack Wang sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); 17329cb83748SJack Wang if (!sess->s.dev) { 17339cb83748SJack Wang err = -ENOMEM; 17349cb83748SJack Wang goto err_free_con; 17359cb83748SJack Wang } 17369cb83748SJack Wang err = map_cont_bufs(sess); 17379cb83748SJack Wang if (err) 17389cb83748SJack Wang goto err_put_dev; 17399cb83748SJack Wang 17409cb83748SJack Wang err = rtrs_srv_alloc_ops_ids(sess); 17419cb83748SJack Wang if (err) 17429cb83748SJack Wang goto err_unmap_bufs; 17439cb83748SJack Wang 17449cb83748SJack Wang __add_path_to_srv(srv, sess); 17459cb83748SJack Wang 17469cb83748SJack Wang return sess; 17479cb83748SJack Wang 17489cb83748SJack Wang err_unmap_bufs: 17499cb83748SJack Wang unmap_cont_bufs(sess); 17509cb83748SJack Wang err_put_dev: 17519cb83748SJack Wang rtrs_ib_dev_put(sess->s.dev); 17529cb83748SJack Wang err_free_con: 17539cb83748SJack Wang kfree(sess->s.con); 17549cb83748SJack Wang err_free_dma_addr: 17559cb83748SJack Wang kfree(sess->dma_addr); 17569cb83748SJack Wang err_free_stats: 17579cb83748SJack Wang kfree(sess->stats); 17589cb83748SJack Wang err_free_sess: 17599cb83748SJack Wang kfree(sess); 17609cb83748SJack Wang err: 17619cb83748SJack Wang return ERR_PTR(err); 17629cb83748SJack Wang } 17639cb83748SJack Wang 17649cb83748SJack Wang static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, 17659cb83748SJack Wang const struct rtrs_msg_conn_req *msg, 17669cb83748SJack Wang size_t len) 17679cb83748SJack Wang { 17689cb83748SJack Wang struct rtrs_srv_ctx *ctx = cm_id->context; 17699cb83748SJack Wang struct rtrs_srv_sess *sess; 17709cb83748SJack Wang struct rtrs_srv *srv; 17719cb83748SJack Wang 17729cb83748SJack Wang u16 version, con_num, cid; 17739cb83748SJack Wang u16 recon_cnt; 17749cb83748SJack Wang int err; 17759cb83748SJack Wang 17769cb83748SJack Wang if (len < sizeof(*msg)) { 17779cb83748SJack Wang pr_err("Invalid RTRS connection request\n"); 17789cb83748SJack Wang goto reject_w_econnreset; 17799cb83748SJack Wang } 17809cb83748SJack Wang if (le16_to_cpu(msg->magic) != RTRS_MAGIC) { 17819cb83748SJack Wang pr_err("Invalid RTRS magic\n"); 17829cb83748SJack Wang goto reject_w_econnreset; 17839cb83748SJack Wang } 17849cb83748SJack Wang version = le16_to_cpu(msg->version); 17859cb83748SJack Wang if (version >> 8 != RTRS_PROTO_VER_MAJOR) { 17869cb83748SJack Wang pr_err("Unsupported major RTRS version: %d, expected %d\n", 17879cb83748SJack Wang version >> 8, RTRS_PROTO_VER_MAJOR); 17889cb83748SJack Wang goto reject_w_econnreset; 17899cb83748SJack Wang } 17909cb83748SJack Wang con_num = le16_to_cpu(msg->cid_num); 17919cb83748SJack Wang if (con_num > 4096) { 17929cb83748SJack Wang /* Sanity check */ 17939cb83748SJack Wang pr_err("Too many connections requested: %d\n", con_num); 17949cb83748SJack Wang goto reject_w_econnreset; 17959cb83748SJack Wang } 17969cb83748SJack Wang cid = le16_to_cpu(msg->cid); 17979cb83748SJack Wang if (cid >= con_num) { 17989cb83748SJack Wang /* Sanity check */ 17999cb83748SJack Wang pr_err("Incorrect cid: %d >= %d\n", cid, con_num); 18009cb83748SJack Wang goto reject_w_econnreset; 18019cb83748SJack Wang } 18029cb83748SJack Wang recon_cnt = le16_to_cpu(msg->recon_cnt); 18039cb83748SJack Wang srv = get_or_create_srv(ctx, &msg->paths_uuid); 18049cb83748SJack Wang if (!srv) { 18059cb83748SJack Wang err = -ENOMEM; 18069cb83748SJack Wang goto reject_w_err; 18079cb83748SJack Wang } 18089cb83748SJack Wang mutex_lock(&srv->paths_mutex); 18099cb83748SJack Wang sess = __find_sess(srv, &msg->sess_uuid); 18109cb83748SJack Wang if (sess) { 18119cb83748SJack Wang struct rtrs_sess *s = &sess->s; 18129cb83748SJack Wang 18139cb83748SJack Wang /* Session already holds a reference */ 18149cb83748SJack Wang put_srv(srv); 18159cb83748SJack Wang 18169cb83748SJack Wang if (sess->state != RTRS_SRV_CONNECTING) { 18179cb83748SJack Wang rtrs_err(s, "Session in wrong state: %s\n", 18189cb83748SJack Wang rtrs_srv_state_str(sess->state)); 18199cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18209cb83748SJack Wang goto reject_w_econnreset; 18219cb83748SJack Wang } 18229cb83748SJack Wang /* 18239cb83748SJack Wang * Sanity checks 18249cb83748SJack Wang */ 18259cb83748SJack Wang if (con_num != sess->s.con_num || cid >= sess->s.con_num) { 18269cb83748SJack Wang rtrs_err(s, "Incorrect request: %d, %d\n", 18279cb83748SJack Wang cid, con_num); 18289cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18299cb83748SJack Wang goto reject_w_econnreset; 18309cb83748SJack Wang } 18319cb83748SJack Wang if (sess->s.con[cid]) { 18329cb83748SJack Wang rtrs_err(s, "Connection already exists: %d\n", 18339cb83748SJack Wang cid); 18349cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18359cb83748SJack Wang goto reject_w_econnreset; 18369cb83748SJack Wang } 18379cb83748SJack Wang } else { 18389cb83748SJack Wang sess = __alloc_sess(srv, cm_id, con_num, recon_cnt, 18399cb83748SJack Wang &msg->sess_uuid); 18409cb83748SJack Wang if (IS_ERR(sess)) { 18419cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18429cb83748SJack Wang put_srv(srv); 18439cb83748SJack Wang err = PTR_ERR(sess); 18449cb83748SJack Wang goto reject_w_err; 18459cb83748SJack Wang } 18469cb83748SJack Wang } 18479cb83748SJack Wang err = create_con(sess, cm_id, cid); 18489cb83748SJack Wang if (err) { 18499cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18509cb83748SJack Wang /* 18519cb83748SJack Wang * Since session has other connections we follow normal way 18529cb83748SJack Wang * through workqueue, but still return an error to tell cma.c 18539cb83748SJack Wang * to call rdma_destroy_id() for current connection. 18549cb83748SJack Wang */ 18559cb83748SJack Wang goto close_and_return_err; 18569cb83748SJack Wang } 18579cb83748SJack Wang err = rtrs_rdma_do_accept(sess, cm_id); 18589cb83748SJack Wang if (err) { 18599cb83748SJack Wang (void)rtrs_rdma_do_reject(cm_id, err); 18609cb83748SJack Wang /* 18619cb83748SJack Wang * Since current connection was successfully added to the 18629cb83748SJack Wang * session we follow normal way through workqueue to close the 18639cb83748SJack Wang * session, thus return 0 to tell cma.c we call 18649cb83748SJack Wang * rdma_destroy_id() ourselves. 18659cb83748SJack Wang */ 18669cb83748SJack Wang err = 0; 18679cb83748SJack Wang goto close_and_return_err; 18689cb83748SJack Wang } 18699cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18709cb83748SJack Wang 18719cb83748SJack Wang return 0; 18729cb83748SJack Wang 18739cb83748SJack Wang reject_w_err: 18749cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, err); 18759cb83748SJack Wang 18769cb83748SJack Wang reject_w_econnreset: 18779cb83748SJack Wang return rtrs_rdma_do_reject(cm_id, -ECONNRESET); 18789cb83748SJack Wang 18799cb83748SJack Wang close_and_return_err: 18809cb83748SJack Wang close_sess(sess); 18819cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 18829cb83748SJack Wang 18839cb83748SJack Wang return err; 18849cb83748SJack Wang } 18859cb83748SJack Wang 18869cb83748SJack Wang static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, 18879cb83748SJack Wang struct rdma_cm_event *ev) 18889cb83748SJack Wang { 18899cb83748SJack Wang struct rtrs_srv_sess *sess = NULL; 18909cb83748SJack Wang struct rtrs_sess *s = NULL; 18919cb83748SJack Wang 18929cb83748SJack Wang if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) { 18939cb83748SJack Wang struct rtrs_con *c = cm_id->context; 18949cb83748SJack Wang 18959cb83748SJack Wang s = c->sess; 18969cb83748SJack Wang sess = to_srv_sess(s); 18979cb83748SJack Wang } 18989cb83748SJack Wang 18999cb83748SJack Wang switch (ev->event) { 19009cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_REQUEST: 19019cb83748SJack Wang /* 19029cb83748SJack Wang * In case of error cma.c will destroy cm_id, 19039cb83748SJack Wang * see cma_process_remove() 19049cb83748SJack Wang */ 19059cb83748SJack Wang return rtrs_rdma_connect(cm_id, ev->param.conn.private_data, 19069cb83748SJack Wang ev->param.conn.private_data_len); 19079cb83748SJack Wang case RDMA_CM_EVENT_ESTABLISHED: 19089cb83748SJack Wang /* Nothing here */ 19099cb83748SJack Wang break; 19109cb83748SJack Wang case RDMA_CM_EVENT_REJECTED: 19119cb83748SJack Wang case RDMA_CM_EVENT_CONNECT_ERROR: 19129cb83748SJack Wang case RDMA_CM_EVENT_UNREACHABLE: 19139cb83748SJack Wang rtrs_err(s, "CM error (CM event: %s, err: %d)\n", 19149cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19159cb83748SJack Wang close_sess(sess); 19169cb83748SJack Wang break; 19179cb83748SJack Wang case RDMA_CM_EVENT_DISCONNECTED: 19189cb83748SJack Wang case RDMA_CM_EVENT_ADDR_CHANGE: 19199cb83748SJack Wang case RDMA_CM_EVENT_TIMEWAIT_EXIT: 19209cb83748SJack Wang close_sess(sess); 19219cb83748SJack Wang break; 19229cb83748SJack Wang case RDMA_CM_EVENT_DEVICE_REMOVAL: 19239cb83748SJack Wang close_sess(sess); 19249cb83748SJack Wang break; 19259cb83748SJack Wang default: 19269cb83748SJack Wang pr_err("Ignoring unexpected CM event %s, err %d\n", 19279cb83748SJack Wang rdma_event_msg(ev->event), ev->status); 19289cb83748SJack Wang break; 19299cb83748SJack Wang } 19309cb83748SJack Wang 19319cb83748SJack Wang return 0; 19329cb83748SJack Wang } 19339cb83748SJack Wang 19349cb83748SJack Wang static struct rdma_cm_id *rtrs_srv_cm_init(struct rtrs_srv_ctx *ctx, 19359cb83748SJack Wang struct sockaddr *addr, 19369cb83748SJack Wang enum rdma_ucm_port_space ps) 19379cb83748SJack Wang { 19389cb83748SJack Wang struct rdma_cm_id *cm_id; 19399cb83748SJack Wang int ret; 19409cb83748SJack Wang 19419cb83748SJack Wang cm_id = rdma_create_id(&init_net, rtrs_srv_rdma_cm_handler, 19429cb83748SJack Wang ctx, ps, IB_QPT_RC); 19439cb83748SJack Wang if (IS_ERR(cm_id)) { 19449cb83748SJack Wang ret = PTR_ERR(cm_id); 19459cb83748SJack Wang pr_err("Creating id for RDMA connection failed, err: %d\n", 19469cb83748SJack Wang ret); 19479cb83748SJack Wang goto err_out; 19489cb83748SJack Wang } 19499cb83748SJack Wang ret = rdma_bind_addr(cm_id, addr); 19509cb83748SJack Wang if (ret) { 19519cb83748SJack Wang pr_err("Binding RDMA address failed, err: %d\n", ret); 19529cb83748SJack Wang goto err_cm; 19539cb83748SJack Wang } 19549cb83748SJack Wang ret = rdma_listen(cm_id, 64); 19559cb83748SJack Wang if (ret) { 19569cb83748SJack Wang pr_err("Listening on RDMA connection failed, err: %d\n", 19579cb83748SJack Wang ret); 19589cb83748SJack Wang goto err_cm; 19599cb83748SJack Wang } 19609cb83748SJack Wang 19619cb83748SJack Wang return cm_id; 19629cb83748SJack Wang 19639cb83748SJack Wang err_cm: 19649cb83748SJack Wang rdma_destroy_id(cm_id); 19659cb83748SJack Wang err_out: 19669cb83748SJack Wang 19679cb83748SJack Wang return ERR_PTR(ret); 19689cb83748SJack Wang } 19699cb83748SJack Wang 19709cb83748SJack Wang static int rtrs_srv_rdma_init(struct rtrs_srv_ctx *ctx, u16 port) 19719cb83748SJack Wang { 19729cb83748SJack Wang struct sockaddr_in6 sin = { 19739cb83748SJack Wang .sin6_family = AF_INET6, 19749cb83748SJack Wang .sin6_addr = IN6ADDR_ANY_INIT, 19759cb83748SJack Wang .sin6_port = htons(port), 19769cb83748SJack Wang }; 19779cb83748SJack Wang struct sockaddr_ib sib = { 19789cb83748SJack Wang .sib_family = AF_IB, 19799cb83748SJack Wang .sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port), 19809cb83748SJack Wang .sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL), 19819cb83748SJack Wang .sib_pkey = cpu_to_be16(0xffff), 19829cb83748SJack Wang }; 19839cb83748SJack Wang struct rdma_cm_id *cm_ip, *cm_ib; 19849cb83748SJack Wang int ret; 19859cb83748SJack Wang 19869cb83748SJack Wang /* 19879cb83748SJack Wang * We accept both IPoIB and IB connections, so we need to keep 19889cb83748SJack Wang * two cm id's, one for each socket type and port space. 19899cb83748SJack Wang * If the cm initialization of one of the id's fails, we abort 19909cb83748SJack Wang * everything. 19919cb83748SJack Wang */ 19929cb83748SJack Wang cm_ip = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sin, RDMA_PS_TCP); 19939cb83748SJack Wang if (IS_ERR(cm_ip)) 19949cb83748SJack Wang return PTR_ERR(cm_ip); 19959cb83748SJack Wang 19969cb83748SJack Wang cm_ib = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sib, RDMA_PS_IB); 19979cb83748SJack Wang if (IS_ERR(cm_ib)) { 19989cb83748SJack Wang ret = PTR_ERR(cm_ib); 19999cb83748SJack Wang goto free_cm_ip; 20009cb83748SJack Wang } 20019cb83748SJack Wang 20029cb83748SJack Wang ctx->cm_id_ip = cm_ip; 20039cb83748SJack Wang ctx->cm_id_ib = cm_ib; 20049cb83748SJack Wang 20059cb83748SJack Wang return 0; 20069cb83748SJack Wang 20079cb83748SJack Wang free_cm_ip: 20089cb83748SJack Wang rdma_destroy_id(cm_ip); 20099cb83748SJack Wang 20109cb83748SJack Wang return ret; 20119cb83748SJack Wang } 20129cb83748SJack Wang 20139cb83748SJack Wang static struct rtrs_srv_ctx *alloc_srv_ctx(struct rtrs_srv_ops *ops) 20149cb83748SJack Wang { 20159cb83748SJack Wang struct rtrs_srv_ctx *ctx; 20169cb83748SJack Wang 20179cb83748SJack Wang ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 20189cb83748SJack Wang if (!ctx) 20199cb83748SJack Wang return NULL; 20209cb83748SJack Wang 20219cb83748SJack Wang ctx->ops = *ops; 20229cb83748SJack Wang mutex_init(&ctx->srv_mutex); 20239cb83748SJack Wang INIT_LIST_HEAD(&ctx->srv_list); 20249cb83748SJack Wang 20259cb83748SJack Wang return ctx; 20269cb83748SJack Wang } 20279cb83748SJack Wang 20289cb83748SJack Wang static void free_srv_ctx(struct rtrs_srv_ctx *ctx) 20299cb83748SJack Wang { 20309cb83748SJack Wang WARN_ON(!list_empty(&ctx->srv_list)); 20319cb83748SJack Wang mutex_destroy(&ctx->srv_mutex); 20329cb83748SJack Wang kfree(ctx); 20339cb83748SJack Wang } 20349cb83748SJack Wang 20359cb83748SJack Wang /** 20369cb83748SJack Wang * rtrs_srv_open() - open RTRS server context 20379cb83748SJack Wang * @ops: callback functions 20389cb83748SJack Wang * @port: port to listen on 20399cb83748SJack Wang * 20409cb83748SJack Wang * Creates server context with specified callbacks. 20419cb83748SJack Wang * 20429cb83748SJack Wang * Return a valid pointer on success otherwise PTR_ERR. 20439cb83748SJack Wang */ 20449cb83748SJack Wang struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) 20459cb83748SJack Wang { 20469cb83748SJack Wang struct rtrs_srv_ctx *ctx; 20479cb83748SJack Wang int err; 20489cb83748SJack Wang 20499cb83748SJack Wang ctx = alloc_srv_ctx(ops); 20509cb83748SJack Wang if (!ctx) 20519cb83748SJack Wang return ERR_PTR(-ENOMEM); 20529cb83748SJack Wang 20539cb83748SJack Wang err = rtrs_srv_rdma_init(ctx, port); 20549cb83748SJack Wang if (err) { 20559cb83748SJack Wang free_srv_ctx(ctx); 20569cb83748SJack Wang return ERR_PTR(err); 20579cb83748SJack Wang } 20589cb83748SJack Wang 20599cb83748SJack Wang return ctx; 20609cb83748SJack Wang } 20619cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_open); 20629cb83748SJack Wang 20639cb83748SJack Wang static void close_sessions(struct rtrs_srv *srv) 20649cb83748SJack Wang { 20659cb83748SJack Wang struct rtrs_srv_sess *sess; 20669cb83748SJack Wang 20679cb83748SJack Wang mutex_lock(&srv->paths_mutex); 20689cb83748SJack Wang list_for_each_entry(sess, &srv->paths_list, s.entry) 20699cb83748SJack Wang close_sess(sess); 20709cb83748SJack Wang mutex_unlock(&srv->paths_mutex); 20719cb83748SJack Wang } 20729cb83748SJack Wang 20739cb83748SJack Wang static void close_ctx(struct rtrs_srv_ctx *ctx) 20749cb83748SJack Wang { 20759cb83748SJack Wang struct rtrs_srv *srv; 20769cb83748SJack Wang 20779cb83748SJack Wang mutex_lock(&ctx->srv_mutex); 20789cb83748SJack Wang list_for_each_entry(srv, &ctx->srv_list, ctx_list) 20799cb83748SJack Wang close_sessions(srv); 20809cb83748SJack Wang mutex_unlock(&ctx->srv_mutex); 20819cb83748SJack Wang flush_workqueue(rtrs_wq); 20829cb83748SJack Wang } 20839cb83748SJack Wang 20849cb83748SJack Wang /** 20859cb83748SJack Wang * rtrs_srv_close() - close RTRS server context 20869cb83748SJack Wang * @ctx: pointer to server context 20879cb83748SJack Wang * 20889cb83748SJack Wang * Closes RTRS server context with all client sessions. 20899cb83748SJack Wang */ 20909cb83748SJack Wang void rtrs_srv_close(struct rtrs_srv_ctx *ctx) 20919cb83748SJack Wang { 20929cb83748SJack Wang rdma_destroy_id(ctx->cm_id_ip); 20939cb83748SJack Wang rdma_destroy_id(ctx->cm_id_ib); 20949cb83748SJack Wang close_ctx(ctx); 20959cb83748SJack Wang free_srv_ctx(ctx); 20969cb83748SJack Wang } 20979cb83748SJack Wang EXPORT_SYMBOL(rtrs_srv_close); 20989cb83748SJack Wang 20999cb83748SJack Wang static int check_module_params(void) 21009cb83748SJack Wang { 21019cb83748SJack Wang if (sess_queue_depth < 1 || sess_queue_depth > MAX_SESS_QUEUE_DEPTH) { 21029cb83748SJack Wang pr_err("Invalid sess_queue_depth value %d, has to be >= %d, <= %d.\n", 21039cb83748SJack Wang sess_queue_depth, 1, MAX_SESS_QUEUE_DEPTH); 21049cb83748SJack Wang return -EINVAL; 21059cb83748SJack Wang } 21069cb83748SJack Wang if (max_chunk_size < 4096 || !is_power_of_2(max_chunk_size)) { 21079cb83748SJack Wang pr_err("Invalid max_chunk_size value %d, has to be >= %d and should be power of two.\n", 21089cb83748SJack Wang max_chunk_size, 4096); 21099cb83748SJack Wang return -EINVAL; 21109cb83748SJack Wang } 21119cb83748SJack Wang 21129cb83748SJack Wang /* 21139cb83748SJack Wang * Check if IB immediate data size is enough to hold the mem_id and the 21149cb83748SJack Wang * offset inside the memory chunk 21159cb83748SJack Wang */ 21169cb83748SJack Wang if ((ilog2(sess_queue_depth - 1) + 1) + 21179cb83748SJack Wang (ilog2(max_chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) { 21189cb83748SJack Wang pr_err("RDMA immediate size (%db) not enough to encode %d buffers of size %dB. Reduce 'sess_queue_depth' or 'max_chunk_size' parameters.\n", 21199cb83748SJack Wang MAX_IMM_PAYL_BITS, sess_queue_depth, max_chunk_size); 21209cb83748SJack Wang return -EINVAL; 21219cb83748SJack Wang } 21229cb83748SJack Wang 21239cb83748SJack Wang return 0; 21249cb83748SJack Wang } 21259cb83748SJack Wang 21269cb83748SJack Wang static int __init rtrs_server_init(void) 21279cb83748SJack Wang { 21289cb83748SJack Wang int err; 21299cb83748SJack Wang 21309cb83748SJack Wang pr_info("Loading module %s, proto %s: (max_chunk_size: %d (pure IO %ld, headers %ld) , sess_queue_depth: %d, always_invalidate: %d)\n", 21319cb83748SJack Wang KBUILD_MODNAME, RTRS_PROTO_VER_STRING, 21329cb83748SJack Wang max_chunk_size, max_chunk_size - MAX_HDR_SIZE, MAX_HDR_SIZE, 21339cb83748SJack Wang sess_queue_depth, always_invalidate); 21349cb83748SJack Wang 21359cb83748SJack Wang rtrs_rdma_dev_pd_init(0, &dev_pd); 21369cb83748SJack Wang 21379cb83748SJack Wang err = check_module_params(); 21389cb83748SJack Wang if (err) { 21399cb83748SJack Wang pr_err("Failed to load module, invalid module parameters, err: %d\n", 21409cb83748SJack Wang err); 21419cb83748SJack Wang return err; 21429cb83748SJack Wang } 21439cb83748SJack Wang chunk_pool = mempool_create_page_pool(sess_queue_depth * CHUNK_POOL_SZ, 21449cb83748SJack Wang get_order(max_chunk_size)); 21459cb83748SJack Wang if (!chunk_pool) 21469cb83748SJack Wang return -ENOMEM; 21479cb83748SJack Wang rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server"); 21489cb83748SJack Wang if (IS_ERR(rtrs_dev_class)) { 21499cb83748SJack Wang err = PTR_ERR(rtrs_dev_class); 21509cb83748SJack Wang goto out_chunk_pool; 21519cb83748SJack Wang } 21529cb83748SJack Wang rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_MEM_RECLAIM, 0); 21539cb83748SJack Wang if (!rtrs_wq) 21549cb83748SJack Wang goto out_dev_class; 21559cb83748SJack Wang 21569cb83748SJack Wang return 0; 21579cb83748SJack Wang 21589cb83748SJack Wang out_dev_class: 21599cb83748SJack Wang class_destroy(rtrs_dev_class); 21609cb83748SJack Wang out_chunk_pool: 21619cb83748SJack Wang mempool_destroy(chunk_pool); 21629cb83748SJack Wang 21639cb83748SJack Wang return err; 21649cb83748SJack Wang } 21659cb83748SJack Wang 21669cb83748SJack Wang static void __exit rtrs_server_exit(void) 21679cb83748SJack Wang { 21689cb83748SJack Wang destroy_workqueue(rtrs_wq); 21699cb83748SJack Wang class_destroy(rtrs_dev_class); 21709cb83748SJack Wang mempool_destroy(chunk_pool); 21719cb83748SJack Wang rtrs_rdma_dev_pd_deinit(&dev_pd); 21729cb83748SJack Wang } 21739cb83748SJack Wang 21749cb83748SJack Wang module_init(rtrs_server_init); 21759cb83748SJack Wang module_exit(rtrs_server_exit); 2176