12e0cbc4dSRam Amrani /* QLogic qedr NIC Driver
22e0cbc4dSRam Amrani * Copyright (c) 2015-2016 QLogic Corporation
32e0cbc4dSRam Amrani *
42e0cbc4dSRam Amrani * This software is available to you under a choice of one of two
52e0cbc4dSRam Amrani * licenses. You may choose to be licensed under the terms of the GNU
62e0cbc4dSRam Amrani * General Public License (GPL) Version 2, available from the file
72e0cbc4dSRam Amrani * COPYING in the main directory of this source tree, or the
82e0cbc4dSRam Amrani * OpenIB.org BSD license below:
92e0cbc4dSRam Amrani *
102e0cbc4dSRam Amrani * Redistribution and use in source and binary forms, with or
112e0cbc4dSRam Amrani * without modification, are permitted provided that the following
122e0cbc4dSRam Amrani * conditions are met:
132e0cbc4dSRam Amrani *
142e0cbc4dSRam Amrani * - Redistributions of source code must retain the above
152e0cbc4dSRam Amrani * copyright notice, this list of conditions and the following
162e0cbc4dSRam Amrani * disclaimer.
172e0cbc4dSRam Amrani *
182e0cbc4dSRam Amrani * - Redistributions in binary form must reproduce the above
192e0cbc4dSRam Amrani * copyright notice, this list of conditions and the following
202e0cbc4dSRam Amrani * disclaimer in the documentation and /or other materials
212e0cbc4dSRam Amrani * provided with the distribution.
222e0cbc4dSRam Amrani *
232e0cbc4dSRam Amrani * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
242e0cbc4dSRam Amrani * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
252e0cbc4dSRam Amrani * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
262e0cbc4dSRam Amrani * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
272e0cbc4dSRam Amrani * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
282e0cbc4dSRam Amrani * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
292e0cbc4dSRam Amrani * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
302e0cbc4dSRam Amrani * SOFTWARE.
312e0cbc4dSRam Amrani */
322e0cbc4dSRam Amrani #ifndef __QEDR_H__
332e0cbc4dSRam Amrani #define __QEDR_H__
342e0cbc4dSRam Amrani
352e0cbc4dSRam Amrani #include <linux/pci.h>
369fd15987SMatthew Wilcox #include <linux/xarray.h>
372e0cbc4dSRam Amrani #include <rdma/ib_addr.h>
382e0cbc4dSRam Amrani #include <linux/qed/qed_if.h>
39ec72fce4SRam Amrani #include <linux/qed/qed_chain.h>
407003cdd6SKalderon, Michal #include <linux/qed/qed_rdma_if.h>
41b262a06eSMichal Kalderon #include <linux/qed/qede_rdma.h>
42be086e7cSMintz, Yuval #include <linux/qed/roce_common.h>
4382af6d19SMichal Kalderon #include <linux/completion.h>
44be086e7cSMintz, Yuval #include "qedr_hsi_rdma.h"
452e0cbc4dSRam Amrani
462e0cbc4dSRam Amrani #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
479de69861SJason Gunthorpe #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev)
48e6a38c54SKalderon, Michal #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
49e6a38c54SKalderon, Michal #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
502e0cbc4dSRam Amrani
512e0cbc4dSRam Amrani #define DP_DEBUG(dev, module, fmt, ...) \
522e0cbc4dSRam Amrani pr_debug("(%s) " module ": " fmt, \
532e0cbc4dSRam Amrani DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
542e0cbc4dSRam Amrani
552e0cbc4dSRam Amrani #define QEDR_MSG_INIT "INIT"
56ac1b36e5SRam Amrani #define QEDR_MSG_MISC "MISC"
57a7efd777SRam Amrani #define QEDR_MSG_CQ " CQ"
58a7efd777SRam Amrani #define QEDR_MSG_MR " MR"
59cecbcddfSRam Amrani #define QEDR_MSG_RQ " RQ"
60cecbcddfSRam Amrani #define QEDR_MSG_SQ " SQ"
61cecbcddfSRam Amrani #define QEDR_MSG_QP " QP"
623491c9e7SYuval Bason #define QEDR_MSG_SRQ " SRQ"
6304886779SRam Amrani #define QEDR_MSG_GSI " GSI"
64e411e058SKalderon, Michal #define QEDR_MSG_IWARP " IW"
65a7efd777SRam Amrani
66a7efd777SRam Amrani #define QEDR_CQ_MAGIC_NUMBER (0x11223344)
67e57bb6beSRam Amrani
68e57bb6beSRam Amrani #define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
69e57bb6beSRam Amrani #define FW_PAGE_SHIFT (12)
702e0cbc4dSRam Amrani
71ec72fce4SRam Amrani struct qedr_dev;
72ec72fce4SRam Amrani
73ec72fce4SRam Amrani struct qedr_cnq {
74ec72fce4SRam Amrani struct qedr_dev *dev;
75ec72fce4SRam Amrani struct qed_chain pbl;
76ec72fce4SRam Amrani struct qed_sb_info *sb;
77ec72fce4SRam Amrani char name[32];
78ec72fce4SRam Amrani u64 n_comp;
79ec72fce4SRam Amrani __le16 *hw_cons_ptr;
80ec72fce4SRam Amrani u8 index;
81ec72fce4SRam Amrani };
82ec72fce4SRam Amrani
83ec72fce4SRam Amrani #define QEDR_MAX_SGID 128
84ec72fce4SRam Amrani
85ec72fce4SRam Amrani struct qedr_device_attr {
86ec72fce4SRam Amrani u32 vendor_id;
87ec72fce4SRam Amrani u32 vendor_part_id;
88ec72fce4SRam Amrani u32 hw_ver;
89ec72fce4SRam Amrani u64 fw_ver;
90ec72fce4SRam Amrani u64 node_guid;
91ec72fce4SRam Amrani u64 sys_image_guid;
92ec72fce4SRam Amrani u8 max_cnq;
93ec72fce4SRam Amrani u8 max_sge;
94ec72fce4SRam Amrani u16 max_inline;
95ec72fce4SRam Amrani u32 max_sqe;
96ec72fce4SRam Amrani u32 max_rqe;
97ec72fce4SRam Amrani u8 max_qp_resp_rd_atomic_resc;
98ec72fce4SRam Amrani u8 max_qp_req_rd_atomic_resc;
99ec72fce4SRam Amrani u64 max_dev_resp_rd_atomic_resc;
100ec72fce4SRam Amrani u32 max_cq;
101ec72fce4SRam Amrani u32 max_qp;
102ec72fce4SRam Amrani u32 max_mr;
103ec72fce4SRam Amrani u64 max_mr_size;
104ec72fce4SRam Amrani u32 max_cqe;
105ec72fce4SRam Amrani u32 max_mw;
106ec72fce4SRam Amrani u32 max_mr_mw_fmr_pbl;
107ec72fce4SRam Amrani u64 max_mr_mw_fmr_size;
108ec72fce4SRam Amrani u32 max_pd;
109ec72fce4SRam Amrani u32 max_ah;
110ec72fce4SRam Amrani u8 max_pkey;
111ec72fce4SRam Amrani u32 max_srq;
112ec72fce4SRam Amrani u32 max_srq_wr;
113ec72fce4SRam Amrani u8 max_srq_sge;
114ec72fce4SRam Amrani u8 max_stats_queues;
115ec72fce4SRam Amrani u32 dev_caps;
116ec72fce4SRam Amrani
117ec72fce4SRam Amrani u64 page_size_caps;
118ec72fce4SRam Amrani u8 dev_ack_delay;
119ec72fce4SRam Amrani u32 reserved_lkey;
120ec72fce4SRam Amrani u32 bad_pkey_counter;
121ec72fce4SRam Amrani struct qed_rdma_events events;
122ec72fce4SRam Amrani };
123ec72fce4SRam Amrani
124f449c7a2SRam Amrani #define QEDR_ENET_STATE_BIT (0)
125f449c7a2SRam Amrani
1262e0cbc4dSRam Amrani struct qedr_dev {
1272e0cbc4dSRam Amrani struct ib_device ibdev;
1282e0cbc4dSRam Amrani struct qed_dev *cdev;
1292e0cbc4dSRam Amrani struct pci_dev *pdev;
1302e0cbc4dSRam Amrani struct net_device *ndev;
1312e0cbc4dSRam Amrani
1322e0cbc4dSRam Amrani enum ib_atomic_cap atomic_cap;
1332e0cbc4dSRam Amrani
134ec72fce4SRam Amrani void *rdma_ctx;
135ec72fce4SRam Amrani struct qedr_device_attr attr;
136ec72fce4SRam Amrani
137ec72fce4SRam Amrani const struct qed_rdma_ops *ops;
138ec72fce4SRam Amrani struct qed_int_info int_info;
139ec72fce4SRam Amrani
140ec72fce4SRam Amrani struct qed_sb_info *sb_array;
141ec72fce4SRam Amrani struct qedr_cnq *cnq_array;
142ec72fce4SRam Amrani int num_cnq;
143ec72fce4SRam Amrani int sb_start;
144ec72fce4SRam Amrani
145ec72fce4SRam Amrani void __iomem *db_addr;
146ec72fce4SRam Amrani u64 db_phys_addr;
147ec72fce4SRam Amrani u32 db_size;
148ec72fce4SRam Amrani u16 dpi;
149ec72fce4SRam Amrani
150ec72fce4SRam Amrani union ib_gid *sgid_tbl;
151ec72fce4SRam Amrani
152ec72fce4SRam Amrani /* Lock for sgid table */
153ec72fce4SRam Amrani spinlock_t sgid_lock;
154ec72fce4SRam Amrani
155ec72fce4SRam Amrani u64 guid;
156ec72fce4SRam Amrani
1572e0cbc4dSRam Amrani u32 dp_module;
1582e0cbc4dSRam Amrani u8 dp_level;
159ec72fce4SRam Amrani u8 num_hwfns;
160443473d2SMichal Kalderon #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1)
161443473d2SMichal Kalderon u8 affin_hwfn_idx;
1620518c12fSMichal Kalderon u8 gsi_ll2_handle;
1630518c12fSMichal Kalderon
164cecbcddfSRam Amrani uint wq_multiplier;
1651d1424c8SRam Amrani u8 gsi_ll2_mac_address[ETH_ALEN];
16604886779SRam Amrani int gsi_qp_created;
16704886779SRam Amrani struct qedr_cq *gsi_sqcq;
16804886779SRam Amrani struct qedr_cq *gsi_rqcq;
16904886779SRam Amrani struct qedr_qp *gsi_qp;
170e6a38c54SKalderon, Michal enum qed_rdma_type rdma_type;
171b6014f9eSMatthew Wilcox struct xarray qps;
1729fd15987SMatthew Wilcox struct xarray srqs;
173e411e058SKalderon, Michal struct workqueue_struct *iwarp_wq;
174e411e058SKalderon, Michal u16 iwarp_max_mtu;
175f449c7a2SRam Amrani
176f449c7a2SRam Amrani unsigned long enet_state;
177ad84dad2SAmrani, Ram
178ad84dad2SAmrani, Ram u8 user_dpm_enabled;
1792e0cbc4dSRam Amrani };
180ec72fce4SRam Amrani
181ec72fce4SRam Amrani #define QEDR_MAX_SQ_PBL (0x8000)
182ec72fce4SRam Amrani #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
183ec72fce4SRam Amrani #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
184ec72fce4SRam Amrani #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
185ec72fce4SRam Amrani QEDR_SQE_ELEMENT_SIZE)
186ec72fce4SRam Amrani #define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
187ec72fce4SRam Amrani QEDR_SQE_ELEMENT_SIZE)
188ec72fce4SRam Amrani #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
189ec72fce4SRam Amrani (RDMA_RING_PAGE_SIZE) / \
190ec72fce4SRam Amrani (QEDR_SQE_ELEMENT_SIZE) /\
191ec72fce4SRam Amrani (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
192ec72fce4SRam Amrani /* RQ */
193ec72fce4SRam Amrani #define QEDR_MAX_RQ_PBL (0x2000)
194ec72fce4SRam Amrani #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
195ec72fce4SRam Amrani #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
196ec72fce4SRam Amrani #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
197ec72fce4SRam Amrani #define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
198ec72fce4SRam Amrani QEDR_RQE_ELEMENT_SIZE)
199ec72fce4SRam Amrani #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
200ec72fce4SRam Amrani (RDMA_RING_PAGE_SIZE) / \
201ec72fce4SRam Amrani (QEDR_RQE_ELEMENT_SIZE) /\
202ec72fce4SRam Amrani (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
203ec72fce4SRam Amrani
204ec72fce4SRam Amrani #define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
205ec72fce4SRam Amrani #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
206ec72fce4SRam Amrani #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
207ec72fce4SRam Amrani sizeof(u64)) - 1)
208ec72fce4SRam Amrani #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
209ec72fce4SRam Amrani (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
210ec72fce4SRam Amrani
211ec72fce4SRam Amrani #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
212ec72fce4SRam Amrani
213ec72fce4SRam Amrani #define QEDR_MAX_PORT (1)
214f449c7a2SRam Amrani #define QEDR_PORT (1)
215ec72fce4SRam Amrani
216ec72fce4SRam Amrani #define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
217ec72fce4SRam Amrani
218ac1b36e5SRam Amrani #define QEDR_ROCE_PKEY_MAX 1
219ac1b36e5SRam Amrani #define QEDR_ROCE_PKEY_TABLE_LEN 1
220ac1b36e5SRam Amrani #define QEDR_ROCE_PKEY_DEFAULT 0xffff
221ac1b36e5SRam Amrani
222a7efd777SRam Amrani struct qedr_pbl {
223a7efd777SRam Amrani struct list_head list_entry;
224a7efd777SRam Amrani void *va;
225a7efd777SRam Amrani dma_addr_t pa;
226a7efd777SRam Amrani };
227a7efd777SRam Amrani
228ac1b36e5SRam Amrani struct qedr_ucontext {
229ac1b36e5SRam Amrani struct ib_ucontext ibucontext;
230ac1b36e5SRam Amrani struct qedr_dev *dev;
231ac1b36e5SRam Amrani struct qedr_pd *pd;
2320058eb58SMichal Kalderon void __iomem *dpi_addr;
2334c6bb02dSMichal Kalderon struct rdma_user_mmap_entry *db_mmap_entry;
234ac1b36e5SRam Amrani u64 dpi_phys_addr;
235ac1b36e5SRam Amrani u32 dpi_size;
236ac1b36e5SRam Amrani u16 dpi;
23797f61250SMichal Kalderon bool db_rec;
238bbe4f424SMichal Kalderon u8 edpm_mode;
239ac1b36e5SRam Amrani };
240ac1b36e5SRam Amrani
241b4bc7660SMichal Kalderon union db_prod32 {
242b4bc7660SMichal Kalderon struct rdma_pwm_val16_data data;
243b4bc7660SMichal Kalderon u32 raw;
244b4bc7660SMichal Kalderon };
245b4bc7660SMichal Kalderon
246a7efd777SRam Amrani union db_prod64 {
247a7efd777SRam Amrani struct rdma_pwm_val32_data data;
248a7efd777SRam Amrani u64 raw;
249a7efd777SRam Amrani };
250a7efd777SRam Amrani
251a7efd777SRam Amrani enum qedr_cq_type {
252a7efd777SRam Amrani QEDR_CQ_TYPE_GSI,
253a7efd777SRam Amrani QEDR_CQ_TYPE_KERNEL,
254a7efd777SRam Amrani QEDR_CQ_TYPE_USER,
255a7efd777SRam Amrani };
256a7efd777SRam Amrani
257a7efd777SRam Amrani struct qedr_pbl_info {
258a7efd777SRam Amrani u32 num_pbls;
259a7efd777SRam Amrani u32 num_pbes;
260a7efd777SRam Amrani u32 pbl_size;
261a7efd777SRam Amrani u32 pbe_size;
262a7efd777SRam Amrani bool two_layered;
263a7efd777SRam Amrani };
264a7efd777SRam Amrani
265a7efd777SRam Amrani struct qedr_userq {
266a7efd777SRam Amrani struct ib_umem *umem;
267a7efd777SRam Amrani struct qedr_pbl_info pbl_info;
268a7efd777SRam Amrani struct qedr_pbl *pbl_tbl;
269a7efd777SRam Amrani u64 buf_addr;
270a7efd777SRam Amrani size_t buf_len;
27197f61250SMichal Kalderon
27297f61250SMichal Kalderon /* doorbell recovery */
27397f61250SMichal Kalderon void __iomem *db_addr;
27497f61250SMichal Kalderon struct qedr_user_db_rec *db_rec_data;
27597f61250SMichal Kalderon struct rdma_user_mmap_entry *db_mmap_entry;
276b4bc7660SMichal Kalderon void __iomem *db_rec_db2_addr;
277b4bc7660SMichal Kalderon union db_prod32 db_rec_db2_data;
278a7efd777SRam Amrani };
279a7efd777SRam Amrani
280a7efd777SRam Amrani struct qedr_cq {
281a7efd777SRam Amrani struct ib_cq ibcq;
282a7efd777SRam Amrani
283a7efd777SRam Amrani enum qedr_cq_type cq_type;
284a7efd777SRam Amrani u32 sig;
285a7efd777SRam Amrani
286a7efd777SRam Amrani u16 icid;
287a7efd777SRam Amrani
288a7efd777SRam Amrani /* Lock to protect multiplem CQ's */
289a7efd777SRam Amrani spinlock_t cq_lock;
290a7efd777SRam Amrani u8 arm_flags;
291a7efd777SRam Amrani struct qed_chain pbl;
292a7efd777SRam Amrani
293a7efd777SRam Amrani void __iomem *db_addr;
294a7efd777SRam Amrani union db_prod64 db;
295a7efd777SRam Amrani
296a7efd777SRam Amrani u8 pbl_toggle;
297a7efd777SRam Amrani union rdma_cqe *latest_cqe;
298a7efd777SRam Amrani union rdma_cqe *toggle_cqe;
299a7efd777SRam Amrani
300a7efd777SRam Amrani u32 cq_cons;
301a7efd777SRam Amrani
302a7efd777SRam Amrani struct qedr_userq q;
3034dd72636SAmrani, Ram u8 destroyed;
3044dd72636SAmrani, Ram u16 cnq_notif;
305a7efd777SRam Amrani };
306a7efd777SRam Amrani
307a7efd777SRam Amrani struct qedr_pd {
308a7efd777SRam Amrani struct ib_pd ibpd;
309a7efd777SRam Amrani u32 pd_id;
310a7efd777SRam Amrani struct qedr_ucontext *uctx;
311a7efd777SRam Amrani };
312a7efd777SRam Amrani
31306e8d1dfSYuval Basson struct qedr_xrcd {
31406e8d1dfSYuval Basson struct ib_xrcd ibxrcd;
31506e8d1dfSYuval Basson u16 xrcd_id;
31606e8d1dfSYuval Basson };
31706e8d1dfSYuval Basson
318cecbcddfSRam Amrani struct qedr_qp_hwq_info {
319cecbcddfSRam Amrani /* WQE Elements */
320cecbcddfSRam Amrani struct qed_chain pbl;
321cecbcddfSRam Amrani u64 p_phys_addr_tbl;
322cecbcddfSRam Amrani u32 max_sges;
323cecbcddfSRam Amrani
324cecbcddfSRam Amrani /* WQE */
325cecbcddfSRam Amrani u16 prod;
326cecbcddfSRam Amrani u16 cons;
327cecbcddfSRam Amrani u16 wqe_cons;
32804886779SRam Amrani u16 gsi_cons;
329cecbcddfSRam Amrani u16 max_wr;
330cecbcddfSRam Amrani
331cecbcddfSRam Amrani /* DB */
332cecbcddfSRam Amrani void __iomem *db;
333cecbcddfSRam Amrani union db_prod32 db_data;
334f5b1b177SKalderon, Michal
335f5b1b177SKalderon, Michal void __iomem *iwarp_db2;
336f5b1b177SKalderon, Michal union db_prod32 iwarp_db2_data;
337cecbcddfSRam Amrani };
338cecbcddfSRam Amrani
339cecbcddfSRam Amrani #define QEDR_INC_SW_IDX(p_info, index) \
340cecbcddfSRam Amrani do { \
341cecbcddfSRam Amrani p_info->index = (p_info->index + 1) & \
342cecbcddfSRam Amrani qed_chain_get_capacity(p_info->pbl) \
343cecbcddfSRam Amrani } while (0)
344cecbcddfSRam Amrani
3453491c9e7SYuval Bason struct qedr_srq_hwq_info {
3463491c9e7SYuval Bason u32 max_sges;
3473491c9e7SYuval Bason u32 max_wr;
3483491c9e7SYuval Bason struct qed_chain pbl;
3493491c9e7SYuval Bason u64 p_phys_addr_tbl;
3503491c9e7SYuval Bason u32 wqe_prod;
3513491c9e7SYuval Bason u32 sge_prod;
3523491c9e7SYuval Bason u32 wr_prod_cnt;
353acca72e2SYuval Basson atomic_t wr_cons_cnt;
3543491c9e7SYuval Bason u32 num_elems;
3553491c9e7SYuval Bason
356acca72e2SYuval Basson struct rdma_srq_producers *virt_prod_pair_addr;
3573491c9e7SYuval Bason dma_addr_t phy_prod_pair_addr;
3583491c9e7SYuval Bason };
3593491c9e7SYuval Bason
3603491c9e7SYuval Bason struct qedr_srq {
3613491c9e7SYuval Bason struct ib_srq ibsrq;
3623491c9e7SYuval Bason struct qedr_dev *dev;
3633491c9e7SYuval Bason
3643491c9e7SYuval Bason struct qedr_userq usrq;
3653491c9e7SYuval Bason struct qedr_srq_hwq_info hw_srq;
3663491c9e7SYuval Bason struct ib_umem *prod_umem;
3673491c9e7SYuval Bason u16 srq_id;
3683491c9e7SYuval Bason u32 srq_limit;
36906e8d1dfSYuval Basson bool is_xrc;
3703491c9e7SYuval Bason /* lock to protect srq recv post */
3713491c9e7SYuval Bason spinlock_t lock;
3723491c9e7SYuval Bason };
3733491c9e7SYuval Bason
374cecbcddfSRam Amrani enum qedr_qp_err_bitmap {
375cecbcddfSRam Amrani QEDR_QP_ERR_SQ_FULL = 1,
376cecbcddfSRam Amrani QEDR_QP_ERR_RQ_FULL = 2,
377cecbcddfSRam Amrani QEDR_QP_ERR_BAD_SR = 4,
378cecbcddfSRam Amrani QEDR_QP_ERR_BAD_RR = 8,
379cecbcddfSRam Amrani QEDR_QP_ERR_SQ_PBL_FULL = 16,
380cecbcddfSRam Amrani QEDR_QP_ERR_RQ_PBL_FULL = 32,
381cecbcddfSRam Amrani };
382cecbcddfSRam Amrani
38382af6d19SMichal Kalderon enum qedr_qp_create_type {
38482af6d19SMichal Kalderon QEDR_QP_CREATE_NONE,
38582af6d19SMichal Kalderon QEDR_QP_CREATE_USER,
38682af6d19SMichal Kalderon QEDR_QP_CREATE_KERNEL,
38782af6d19SMichal Kalderon };
38882af6d19SMichal Kalderon
38982af6d19SMichal Kalderon enum qedr_iwarp_cm_flags {
39082af6d19SMichal Kalderon QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0),
39182af6d19SMichal Kalderon QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1),
39282af6d19SMichal Kalderon };
39382af6d19SMichal Kalderon
394cecbcddfSRam Amrani struct qedr_qp {
395cecbcddfSRam Amrani struct ib_qp ibqp; /* must be first */
396cecbcddfSRam Amrani struct qedr_dev *dev;
397cecbcddfSRam Amrani struct qedr_qp_hwq_info sq;
398cecbcddfSRam Amrani struct qedr_qp_hwq_info rq;
399cecbcddfSRam Amrani
400cecbcddfSRam Amrani u32 max_inline_data;
401cecbcddfSRam Amrani
402cecbcddfSRam Amrani /* Lock for QP's */
403cecbcddfSRam Amrani spinlock_t q_lock;
404cecbcddfSRam Amrani struct qedr_cq *sq_cq;
405cecbcddfSRam Amrani struct qedr_cq *rq_cq;
406cecbcddfSRam Amrani struct qedr_srq *srq;
407cecbcddfSRam Amrani enum qed_roce_qp_state state;
408cecbcddfSRam Amrani u32 id;
409cecbcddfSRam Amrani struct qedr_pd *pd;
410cecbcddfSRam Amrani enum ib_qp_type qp_type;
41182af6d19SMichal Kalderon enum qedr_qp_create_type create_type;
412cecbcddfSRam Amrani struct qed_rdma_qp *qed_qp;
413cecbcddfSRam Amrani u32 qp_id;
414cecbcddfSRam Amrani u16 icid;
415cecbcddfSRam Amrani u16 mtu;
416cecbcddfSRam Amrani int sgid_idx;
417cecbcddfSRam Amrani u32 rq_psn;
418cecbcddfSRam Amrani u32 sq_psn;
419cecbcddfSRam Amrani u32 qkey;
420cecbcddfSRam Amrani u32 dest_qp_num;
421732e73bdSKamal Heib u8 timeout;
422cecbcddfSRam Amrani
423cecbcddfSRam Amrani /* Relevant to qps created from kernel space only (ULPs) */
424cecbcddfSRam Amrani u8 prev_wqe_size;
425cecbcddfSRam Amrani u16 wqe_cons;
426cecbcddfSRam Amrani u32 err_bitmap;
427cecbcddfSRam Amrani bool signaled;
428cecbcddfSRam Amrani
429cecbcddfSRam Amrani /* SQ shadow */
430cecbcddfSRam Amrani struct {
431cecbcddfSRam Amrani u64 wr_id;
432cecbcddfSRam Amrani enum ib_wc_opcode opcode;
433cecbcddfSRam Amrani u32 bytes_len;
434cecbcddfSRam Amrani u8 wqe_size;
435cecbcddfSRam Amrani bool signaled;
436cecbcddfSRam Amrani dma_addr_t icrc_mapping;
437cecbcddfSRam Amrani u32 *icrc;
438cecbcddfSRam Amrani struct qedr_mr *mr;
439cecbcddfSRam Amrani } *wqe_wr_id;
440cecbcddfSRam Amrani
441cecbcddfSRam Amrani /* RQ shadow */
442cecbcddfSRam Amrani struct {
443cecbcddfSRam Amrani u64 wr_id;
444cecbcddfSRam Amrani struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
445cecbcddfSRam Amrani u8 wqe_size;
446cecbcddfSRam Amrani
44704886779SRam Amrani u8 smac[ETH_ALEN];
448efe63c22SAmrani, Ram u16 vlan;
449cecbcddfSRam Amrani int rc;
450cecbcddfSRam Amrani } *rqe_wr_id;
451cecbcddfSRam Amrani
452cecbcddfSRam Amrani /* Relevant to qps created from user space only (applications) */
453cecbcddfSRam Amrani struct qedr_userq usq;
454cecbcddfSRam Amrani struct qedr_userq urq;
45582af6d19SMichal Kalderon
45682af6d19SMichal Kalderon /* synchronization objects used with iwarp ep */
45782af6d19SMichal Kalderon struct kref refcnt;
45882af6d19SMichal Kalderon struct completion iwarp_cm_comp;
45960fab107SPrabhakar Kushwaha struct completion qp_rel_comp;
46082af6d19SMichal Kalderon unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */
461cecbcddfSRam Amrani };
462cecbcddfSRam Amrani
463e0290cceSRam Amrani struct qedr_ah {
464e0290cceSRam Amrani struct ib_ah ibah;
46590898850SDasaratharaman Chandramouli struct rdma_ah_attr attr;
466e0290cceSRam Amrani };
467e0290cceSRam Amrani
468e0290cceSRam Amrani enum qedr_mr_type {
469e0290cceSRam Amrani QEDR_MR_USER,
470e0290cceSRam Amrani QEDR_MR_KERNEL,
471e0290cceSRam Amrani QEDR_MR_DMA,
472e0290cceSRam Amrani QEDR_MR_FRMR,
473e0290cceSRam Amrani };
474e0290cceSRam Amrani
475e0290cceSRam Amrani struct mr_info {
476e0290cceSRam Amrani struct qedr_pbl *pbl_table;
477e0290cceSRam Amrani struct qedr_pbl_info pbl_info;
478e0290cceSRam Amrani struct list_head free_pbl_list;
479e0290cceSRam Amrani struct list_head inuse_pbl_list;
480e0290cceSRam Amrani u32 completed;
481e0290cceSRam Amrani u32 completed_handled;
482e0290cceSRam Amrani };
483e0290cceSRam Amrani
484e0290cceSRam Amrani struct qedr_mr {
485e0290cceSRam Amrani struct ib_mr ibmr;
486e0290cceSRam Amrani struct ib_umem *umem;
487e0290cceSRam Amrani
488e0290cceSRam Amrani struct qed_rdma_register_tid_in_params hw_mr;
489e0290cceSRam Amrani enum qedr_mr_type type;
490e0290cceSRam Amrani
491e0290cceSRam Amrani struct qedr_dev *dev;
492e0290cceSRam Amrani struct mr_info info;
493e0290cceSRam Amrani
494e0290cceSRam Amrani u64 *pages;
495e0290cceSRam Amrani u32 npages;
496e0290cceSRam Amrani };
497e0290cceSRam Amrani
4984c6bb02dSMichal Kalderon struct qedr_user_mmap_entry {
4994c6bb02dSMichal Kalderon struct rdma_user_mmap_entry rdma_entry;
5004c6bb02dSMichal Kalderon struct qedr_dev *dev;
50197f61250SMichal Kalderon union {
5024c6bb02dSMichal Kalderon u64 io_address;
50397f61250SMichal Kalderon void *address;
50497f61250SMichal Kalderon };
5054c6bb02dSMichal Kalderon size_t length;
5064c6bb02dSMichal Kalderon u16 dpi;
5074c6bb02dSMichal Kalderon u8 mmap_flag;
5084c6bb02dSMichal Kalderon };
5094c6bb02dSMichal Kalderon
510afa0e13bSRam Amrani #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
511afa0e13bSRam Amrani
512afa0e13bSRam Amrani #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
513afa0e13bSRam Amrani RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
514afa0e13bSRam Amrani #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
515afa0e13bSRam Amrani RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
516b6acd71fSAmrani, Ram #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
517b6acd71fSAmrani, Ram RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
518afa0e13bSRam Amrani
qedr_inc_sw_cons(struct qedr_qp_hwq_info * info)519afa0e13bSRam Amrani static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
520afa0e13bSRam Amrani {
521afa0e13bSRam Amrani info->cons = (info->cons + 1) % info->max_wr;
522afa0e13bSRam Amrani info->wqe_cons++;
523afa0e13bSRam Amrani }
524afa0e13bSRam Amrani
qedr_inc_sw_prod(struct qedr_qp_hwq_info * info)525afa0e13bSRam Amrani static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
526afa0e13bSRam Amrani {
527afa0e13bSRam Amrani info->prod = (info->prod + 1) % info->max_wr;
528afa0e13bSRam Amrani }
529afa0e13bSRam Amrani
qedr_get_dmac(struct qedr_dev * dev,struct rdma_ah_attr * ah_attr,u8 * mac_addr)530cecbcddfSRam Amrani static inline int qedr_get_dmac(struct qedr_dev *dev,
53190898850SDasaratharaman Chandramouli struct rdma_ah_attr *ah_attr, u8 *mac_addr)
532cecbcddfSRam Amrani {
533cecbcddfSRam Amrani union ib_gid zero_sgid = { { 0 } };
534cecbcddfSRam Amrani struct in6_addr in6;
535d8966fcdSDasaratharaman Chandramouli const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
536d8966fcdSDasaratharaman Chandramouli u8 *dmac;
537cecbcddfSRam Amrani
538d8966fcdSDasaratharaman Chandramouli if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
539cecbcddfSRam Amrani DP_ERR(dev, "Local port GID not supported\n");
540cecbcddfSRam Amrani eth_zero_addr(mac_addr);
541cecbcddfSRam Amrani return -EINVAL;
542cecbcddfSRam Amrani }
543cecbcddfSRam Amrani
544d8966fcdSDasaratharaman Chandramouli memcpy(&in6, grh->dgid.raw, sizeof(in6));
545d8966fcdSDasaratharaman Chandramouli dmac = rdma_ah_retrieve_dmac(ah_attr);
546d8966fcdSDasaratharaman Chandramouli if (!dmac)
547d8966fcdSDasaratharaman Chandramouli return -EINVAL;
548d8966fcdSDasaratharaman Chandramouli ether_addr_copy(mac_addr, dmac);
549cecbcddfSRam Amrani
550cecbcddfSRam Amrani return 0;
551cecbcddfSRam Amrani }
552cecbcddfSRam Amrani
553e411e058SKalderon, Michal struct qedr_iw_listener {
554e411e058SKalderon, Michal struct qedr_dev *dev;
555e411e058SKalderon, Michal struct iw_cm_id *cm_id;
556e411e058SKalderon, Michal int backlog;
557e411e058SKalderon, Michal void *qed_handle;
558e411e058SKalderon, Michal };
559e411e058SKalderon, Michal
560e411e058SKalderon, Michal struct qedr_iw_ep {
561e411e058SKalderon, Michal struct qedr_dev *dev;
562e411e058SKalderon, Michal struct iw_cm_id *cm_id;
563e411e058SKalderon, Michal struct qedr_qp *qp;
564e411e058SKalderon, Michal void *qed_context;
56582af6d19SMichal Kalderon struct kref refcnt;
566e411e058SKalderon, Michal };
567e411e058SKalderon, Michal
568ac1b36e5SRam Amrani static inline
get_qedr_ucontext(struct ib_ucontext * ibucontext)569ac1b36e5SRam Amrani struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
570ac1b36e5SRam Amrani {
571ac1b36e5SRam Amrani return container_of(ibucontext, struct qedr_ucontext, ibucontext);
572ac1b36e5SRam Amrani }
573ac1b36e5SRam Amrani
get_qedr_dev(struct ib_device * ibdev)574ec72fce4SRam Amrani static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
575ec72fce4SRam Amrani {
576ec72fce4SRam Amrani return container_of(ibdev, struct qedr_dev, ibdev);
577ec72fce4SRam Amrani }
578ec72fce4SRam Amrani
get_qedr_pd(struct ib_pd * ibpd)579a7efd777SRam Amrani static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
580a7efd777SRam Amrani {
581a7efd777SRam Amrani return container_of(ibpd, struct qedr_pd, ibpd);
582a7efd777SRam Amrani }
583a7efd777SRam Amrani
get_qedr_xrcd(struct ib_xrcd * ibxrcd)58406e8d1dfSYuval Basson static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd)
58506e8d1dfSYuval Basson {
58606e8d1dfSYuval Basson return container_of(ibxrcd, struct qedr_xrcd, ibxrcd);
58706e8d1dfSYuval Basson }
58806e8d1dfSYuval Basson
get_qedr_cq(struct ib_cq * ibcq)589a7efd777SRam Amrani static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
590a7efd777SRam Amrani {
591a7efd777SRam Amrani return container_of(ibcq, struct qedr_cq, ibcq);
592a7efd777SRam Amrani }
593a7efd777SRam Amrani
get_qedr_qp(struct ib_qp * ibqp)594cecbcddfSRam Amrani static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
595cecbcddfSRam Amrani {
596cecbcddfSRam Amrani return container_of(ibqp, struct qedr_qp, ibqp);
597cecbcddfSRam Amrani }
598e0290cceSRam Amrani
get_qedr_ah(struct ib_ah * ibah)59904886779SRam Amrani static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
60004886779SRam Amrani {
60104886779SRam Amrani return container_of(ibah, struct qedr_ah, ibah);
60204886779SRam Amrani }
60304886779SRam Amrani
get_qedr_mr(struct ib_mr * ibmr)604e0290cceSRam Amrani static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
605e0290cceSRam Amrani {
606e0290cceSRam Amrani return container_of(ibmr, struct qedr_mr, ibmr);
607e0290cceSRam Amrani }
6083491c9e7SYuval Bason
get_qedr_srq(struct ib_srq * ibsrq)6093491c9e7SYuval Bason static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
6103491c9e7SYuval Bason {
6113491c9e7SYuval Bason return container_of(ibsrq, struct qedr_srq, ibsrq);
6123491c9e7SYuval Bason }
6134c6bb02dSMichal Kalderon
qedr_qp_has_srq(struct qedr_qp * qp)61406e8d1dfSYuval Basson static inline bool qedr_qp_has_srq(struct qedr_qp *qp)
61506e8d1dfSYuval Basson {
61606e8d1dfSYuval Basson return qp->srq;
61706e8d1dfSYuval Basson }
61806e8d1dfSYuval Basson
qedr_qp_has_sq(struct qedr_qp * qp)61906e8d1dfSYuval Basson static inline bool qedr_qp_has_sq(struct qedr_qp *qp)
62006e8d1dfSYuval Basson {
62106e8d1dfSYuval Basson if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT)
6221a93e848SJiapeng Chong return false;
62306e8d1dfSYuval Basson
6241a93e848SJiapeng Chong return true;
62506e8d1dfSYuval Basson }
62606e8d1dfSYuval Basson
qedr_qp_has_rq(struct qedr_qp * qp)62706e8d1dfSYuval Basson static inline bool qedr_qp_has_rq(struct qedr_qp *qp)
62806e8d1dfSYuval Basson {
62906e8d1dfSYuval Basson if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI ||
63006e8d1dfSYuval Basson qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp))
6311a93e848SJiapeng Chong return false;
63206e8d1dfSYuval Basson
6331a93e848SJiapeng Chong return true;
63406e8d1dfSYuval Basson }
63506e8d1dfSYuval Basson
6364c6bb02dSMichal Kalderon static inline struct qedr_user_mmap_entry *
get_qedr_mmap_entry(struct rdma_user_mmap_entry * rdma_entry)6374c6bb02dSMichal Kalderon get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
6384c6bb02dSMichal Kalderon {
6394c6bb02dSMichal Kalderon return container_of(rdma_entry, struct qedr_user_mmap_entry,
6404c6bb02dSMichal Kalderon rdma_entry);
6414c6bb02dSMichal Kalderon }
6422e0cbc4dSRam Amrani #endif
643