1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /*          Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6 
7 #ifndef __ERDMA_H__
8 #define __ERDMA_H__
9 
10 #include <linux/bitfield.h>
11 #include <linux/netdevice.h>
12 #include <linux/pci.h>
13 #include <linux/xarray.h>
14 #include <rdma/ib_verbs.h>
15 
16 #include "erdma_hw.h"
17 
18 #define DRV_MODULE_NAME "erdma"
19 #define ERDMA_NODE_DESC "Elastic RDMA(iWARP) stack"
20 
21 struct erdma_eq {
22 	void *qbuf;
23 	dma_addr_t qbuf_dma_addr;
24 
25 	spinlock_t lock;
26 
27 	u32 depth;
28 
29 	u16 ci;
30 	u16 rsvd;
31 
32 	atomic64_t event_num;
33 	atomic64_t notify_num;
34 
35 	void __iomem *db;
36 	u64 *db_record;
37 };
38 
39 struct erdma_cmdq_sq {
40 	void *qbuf;
41 	dma_addr_t qbuf_dma_addr;
42 
43 	spinlock_t lock;
44 
45 	u32 depth;
46 	u16 ci;
47 	u16 pi;
48 
49 	u16 wqebb_cnt;
50 
51 	u64 *db_record;
52 };
53 
54 struct erdma_cmdq_cq {
55 	void *qbuf;
56 	dma_addr_t qbuf_dma_addr;
57 
58 	spinlock_t lock;
59 
60 	u32 depth;
61 	u32 ci;
62 	u32 cmdsn;
63 
64 	u64 *db_record;
65 
66 	atomic64_t armed_num;
67 };
68 
69 enum {
70 	ERDMA_CMD_STATUS_INIT,
71 	ERDMA_CMD_STATUS_ISSUED,
72 	ERDMA_CMD_STATUS_FINISHED,
73 	ERDMA_CMD_STATUS_TIMEOUT
74 };
75 
76 struct erdma_comp_wait {
77 	struct completion wait_event;
78 	u32 cmd_status;
79 	u32 ctx_id;
80 	u16 sq_pi;
81 	u8 comp_status;
82 	u8 rsvd;
83 	u32 comp_data[4];
84 };
85 
86 enum {
87 	ERDMA_CMDQ_STATE_OK_BIT = 0,
88 	ERDMA_CMDQ_STATE_TIMEOUT_BIT = 1,
89 	ERDMA_CMDQ_STATE_CTX_ERR_BIT = 2,
90 };
91 
92 #define ERDMA_CMDQ_TIMEOUT_MS 15000
93 #define ERDMA_REG_ACCESS_WAIT_MS 20
94 #define ERDMA_WAIT_DEV_DONE_CNT 500
95 
96 struct erdma_cmdq {
97 	unsigned long *comp_wait_bitmap;
98 	struct erdma_comp_wait *wait_pool;
99 	spinlock_t lock;
100 
101 	bool use_event;
102 
103 	struct erdma_cmdq_sq sq;
104 	struct erdma_cmdq_cq cq;
105 	struct erdma_eq eq;
106 
107 	unsigned long state;
108 
109 	struct semaphore credits;
110 	u16 max_outstandings;
111 };
112 
113 #define COMPROMISE_CC ERDMA_CC_CUBIC
114 enum erdma_cc_alg {
115 	ERDMA_CC_NEWRENO = 0,
116 	ERDMA_CC_CUBIC,
117 	ERDMA_CC_HPCC_RTT,
118 	ERDMA_CC_HPCC_ECN,
119 	ERDMA_CC_HPCC_INT,
120 	ERDMA_CC_METHODS_NUM
121 };
122 
123 struct erdma_devattr {
124 	u32 fw_version;
125 
126 	unsigned char peer_addr[ETH_ALEN];
127 	unsigned long cap_flags;
128 
129 	int numa_node;
130 	enum erdma_cc_alg cc;
131 	u32 grp_num;
132 	u32 irq_num;
133 
134 	bool disable_dwqe;
135 	u16 dwqe_pages;
136 	u16 dwqe_entries;
137 
138 	u32 max_qp;
139 	u32 max_send_wr;
140 	u32 max_recv_wr;
141 	u32 max_ord;
142 	u32 max_ird;
143 
144 	u32 max_send_sge;
145 	u32 max_recv_sge;
146 	u32 max_sge_rd;
147 	u32 max_cq;
148 	u32 max_cqe;
149 	u64 max_mr_size;
150 	u32 max_mr;
151 	u32 max_pd;
152 	u32 max_mw;
153 	u32 local_dma_key;
154 };
155 
156 #define ERDMA_IRQNAME_SIZE 50
157 
158 struct erdma_irq {
159 	char name[ERDMA_IRQNAME_SIZE];
160 	u32 msix_vector;
161 	cpumask_t affinity_hint_mask;
162 };
163 
164 struct erdma_eq_cb {
165 	bool ready;
166 	void *dev; /* All EQs use this fields to get erdma_dev struct */
167 	struct erdma_irq irq;
168 	struct erdma_eq eq;
169 	struct tasklet_struct tasklet;
170 };
171 
172 struct erdma_resource_cb {
173 	unsigned long *bitmap;
174 	spinlock_t lock;
175 	u32 next_alloc_idx;
176 	u32 max_cap;
177 };
178 
179 enum {
180 	ERDMA_RES_TYPE_PD = 0,
181 	ERDMA_RES_TYPE_STAG_IDX = 1,
182 	ERDMA_RES_CNT = 2,
183 };
184 
185 #define ERDMA_EXTRA_BUFFER_SIZE ERDMA_DB_SIZE
186 #define WARPPED_BUFSIZE(size) ((size) + ERDMA_EXTRA_BUFFER_SIZE)
187 
188 struct erdma_dev {
189 	struct ib_device ibdev;
190 	struct net_device *netdev;
191 	struct pci_dev *pdev;
192 	struct notifier_block netdev_nb;
193 	struct workqueue_struct *reflush_wq;
194 
195 	resource_size_t func_bar_addr;
196 	resource_size_t func_bar_len;
197 	u8 __iomem *func_bar;
198 
199 	struct erdma_devattr attrs;
200 	/* physical port state (only one port per device) */
201 	enum ib_port_state state;
202 	u32 mtu;
203 
204 	/* cmdq and aeq use the same msix vector */
205 	struct erdma_irq comm_irq;
206 	struct erdma_cmdq cmdq;
207 	struct erdma_eq aeq;
208 	struct erdma_eq_cb ceqs[ERDMA_NUM_MSIX_VEC - 1];
209 
210 	spinlock_t lock;
211 	struct erdma_resource_cb res_cb[ERDMA_RES_CNT];
212 	struct xarray qp_xa;
213 	struct xarray cq_xa;
214 
215 	u32 next_alloc_qpn;
216 	u32 next_alloc_cqn;
217 
218 	spinlock_t db_bitmap_lock;
219 	/* We provide max 64 uContexts that each has one SQ doorbell Page. */
220 	DECLARE_BITMAP(sdb_page, ERDMA_DWQE_TYPE0_CNT);
221 	/*
222 	 * We provide max 496 uContexts that each has one SQ normal Db,
223 	 * and one directWQE db.
224 	 */
225 	DECLARE_BITMAP(sdb_entry, ERDMA_DWQE_TYPE1_CNT);
226 
227 	atomic_t num_ctx;
228 	struct list_head cep_list;
229 };
230 
231 static inline void *get_queue_entry(void *qbuf, u32 idx, u32 depth, u32 shift)
232 {
233 	idx &= (depth - 1);
234 
235 	return qbuf + (idx << shift);
236 }
237 
238 static inline struct erdma_dev *to_edev(struct ib_device *ibdev)
239 {
240 	return container_of(ibdev, struct erdma_dev, ibdev);
241 }
242 
243 static inline u32 erdma_reg_read32(struct erdma_dev *dev, u32 reg)
244 {
245 	return readl(dev->func_bar + reg);
246 }
247 
248 static inline u64 erdma_reg_read64(struct erdma_dev *dev, u32 reg)
249 {
250 	return readq(dev->func_bar + reg);
251 }
252 
253 static inline void erdma_reg_write32(struct erdma_dev *dev, u32 reg, u32 value)
254 {
255 	writel(value, dev->func_bar + reg);
256 }
257 
258 static inline void erdma_reg_write64(struct erdma_dev *dev, u32 reg, u64 value)
259 {
260 	writeq(value, dev->func_bar + reg);
261 }
262 
263 static inline u32 erdma_reg_read32_filed(struct erdma_dev *dev, u32 reg,
264 					 u32 filed_mask)
265 {
266 	u32 val = erdma_reg_read32(dev, reg);
267 
268 	return FIELD_GET(filed_mask, val);
269 }
270 
271 int erdma_cmdq_init(struct erdma_dev *dev);
272 void erdma_finish_cmdq_init(struct erdma_dev *dev);
273 void erdma_cmdq_destroy(struct erdma_dev *dev);
274 
275 void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op);
276 int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,
277 			u64 *resp0, u64 *resp1);
278 void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq);
279 
280 int erdma_ceqs_init(struct erdma_dev *dev);
281 void erdma_ceqs_uninit(struct erdma_dev *dev);
282 void notify_eq(struct erdma_eq *eq);
283 void *get_next_valid_eqe(struct erdma_eq *eq);
284 
285 int erdma_aeq_init(struct erdma_dev *dev);
286 void erdma_aeq_destroy(struct erdma_dev *dev);
287 
288 void erdma_aeq_event_handler(struct erdma_dev *dev);
289 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb);
290 
291 #endif
292